diff --git a/BUILDING.md b/BUILDING.md index 54b6021e97..06dec12b15 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -278,6 +278,25 @@ call does not guarantee lint finishes first): The reports persist under each module's `build/reports/`, so re-run lint only when you want fresh lint data in the next scan. +### 5. Optional: include test coverage + +Coverage comes from [Kover](https://github.com/Kotlin/kotlinx-kover). The root +`koverXmlReportCoverage` task runs the unit-test suites of the covered modules +(`quartz`, `commons`, `quic`, `nestsClient`, `geode`, `cli`, `relayBench`, +`desktopApp`, and `amethyst`'s `fdroidDebug` variant) and merges the result +into `build/reports/kover/reportCoverage.xml`, which the scanner imports +automatically when it exists. Same recipe as lint — generate first, scan +second: + +```bash +./gradlew koverXmlReportCoverage +./gradlew sonar +``` + +For a human-readable version of the same data, run +`./gradlew koverHtmlReportCoverage` and open +`build/reports/kover/htmlCoverage/index.html`. + Every `sonar.*` entry in `local.properties` is forwarded to the scanner, so any [analysis parameter](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/analysis-parameters/) can be set there. `sonar.projectKey` / `sonar.projectName` default to the root diff --git a/amethyst/build.gradle.kts b/amethyst/build.gradle.kts index e4df9eb8bf..d7d65cc0cb 100644 --- a/amethyst/build.gradle.kts +++ b/amethyst/build.gradle.kts @@ -6,6 +6,7 @@ plugins { alias(libs.plugins.jetbrainsComposeCompiler) alias(libs.plugins.serialization) alias(libs.plugins.googleKsp) + alias(libs.plugins.kotlinxKover) } fun getCurrentBranch(workingDir: java.io.File): String = diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/AppModules.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/AppModules.kt index 6f421381c1..34ade716a5 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/AppModules.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/AppModules.kt @@ -29,6 +29,7 @@ import com.vitorpamplona.amethyst.commons.model.NoteState import com.vitorpamplona.amethyst.commons.relayClient.BlockedRelayFilteringClient import com.vitorpamplona.amethyst.commons.robohash.CachedRobohash import com.vitorpamplona.amethyst.commons.service.lnurl.OkHttpLnurlEndpointResolver +import com.vitorpamplona.amethyst.commons.service.pow.PoWPolicy import com.vitorpamplona.amethyst.commons.service.pow.PoWPublishQueue import com.vitorpamplona.amethyst.commons.tor.TorSettings import com.vitorpamplona.amethyst.model.Account @@ -67,8 +68,6 @@ import com.vitorpamplona.amethyst.service.okhttp.DualHttpClientManagerForRelays import com.vitorpamplona.amethyst.service.okhttp.EncryptionKeyCache import com.vitorpamplona.amethyst.service.okhttp.OkHttpWebSocket import com.vitorpamplona.amethyst.service.okhttp.OnionLocationCache -import com.vitorpamplona.amethyst.service.okhttp.SurgeDns -import com.vitorpamplona.amethyst.service.okhttp.SurgeDnsStore import com.vitorpamplona.amethyst.service.playback.diskCache.VideoCache import com.vitorpamplona.amethyst.service.playback.diskCache.VideoCacheFactory import com.vitorpamplona.amethyst.service.playback.pip.BackgroundMedia @@ -106,6 +105,8 @@ import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.RelayOfflineT import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.stats.RelayReqStats import com.vitorpamplona.quartz.nip01Core.relay.client.stats.RelayStats import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.CachingEventDecoder +import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.SurgeDns +import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.SurgeDnsStore import com.vitorpamplona.quartz.nip03Timestamp.VerificationStateCache import com.vitorpamplona.quartz.nip03Timestamp.okhttp.OkHttpBitcoinExplorer import com.vitorpamplona.quartz.nip03Timestamp.ots.OtsBlockHeightCache @@ -251,6 +252,23 @@ class AppModules( // path on first lookup. Stored in cacheDir — pure perf data, OK if the OS evicts it. val dnsStore = SurgeDnsStore(File(appContext.safeCacheDir(), SurgeDnsStore.FILE_NAME), surgeDns) + // Network identity change (same trigger as Tor's onNetworkChange above), but for DNS + // the response is deliberately SOFT: most cached answers are still correct on the new + // network, so staleAll() keeps serving every one of them and merely re-verifies — + // positives revalidate in the background on next use, negatives (e.g. hosts that only + // failed because the OLD network / captive portal couldn't resolve them) get re-tried + // on first touch instead of waiting out their TTL. Nothing is dropped, nothing blocks. + init { + applicationIOScope.launch { + connManager.status + .map { (it as? ConnectivityStatus.Active)?.networkId } + .filterNotNull() + .distinctUntilChanged() + .drop(1) + .collect { surgeDns.staleAll() } + } + } + // Shared cache populated by OnionLocationInterceptor from any HTTP/WebSocket // response carrying an Onion-Location header. Consulted by OnionUrlRewriteInterceptor // on Tor-enabled clients to transparently redirect to .onion addresses. @@ -601,9 +619,12 @@ class AppModules( ) // fire-and-forget NIP-13 mining: posts queue here and publish when mined. - // Capped worker pool so a burst of sends never spawns unbounded miners. - // Template jobs checkpoint to disk (restored on login) and every enqueue - // raises the shortService shield so backgrounding doesn't freeze a miner. + // One job mines at a time, racing half the cores over disjoint nonce + // slices — same total CPU budget as the old 2-job pool, but each post + // finishes ~minerThreads× sooner and the other half of the cores stays + // free for the UI. Template jobs checkpoint to disk (restored on login) + // and every enqueue raises the shortService shield so backgrounding + // doesn't freeze a miner. val powJobStore by lazy { PowJobStore(File(appContext.filesDir, PowJobStore.FILE_NAME), applicationIOScope) } @@ -611,7 +632,8 @@ class AppModules( val powPublishQueue by lazy { PoWPublishQueue( scope = applicationIOScope, - maxConcurrent = (Runtime.getRuntime().availableProcessors() / 2).coerceIn(1, 2), + maxConcurrent = 1, + minerThreads = PoWPolicy.minerWorkers(Runtime.getRuntime().availableProcessors()), persistence = powJobStore, onQueueActive = { PowMiningForegroundService.start(appContext) }, ) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt index d24b5878a1..ccd3f37133 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt @@ -875,6 +875,14 @@ class Account( else -> overrideDifficulty } + /** + * Parallel workers a single nonce search should use — the mining queue's + * per-job budget (half the device's cores). 1 when no queue is wired. + * Callers that run [PoWMiner] inside a queued job must pass this so the + * job stays inside the queue's CPU budget. + */ + fun powMinerWorkers(): Int = powQueue()?.minerThreads ?: 1 + /** * Enqueues [work] into the fire-and-forget mining queue. Returns false when * no queue is wired (headless/test accounts): callers must then run their @@ -983,7 +991,9 @@ class Account( mine = { isActive -> // the wrap's ephemeral key is generated inside the wrap build; // the conversion hook hands its pubkey back so the nonce can - // commit to it. + // commit to it. Single-threaded on purpose: the conversion is + // a non-suspend hook deep inside the synchronous NIP-59 wrap + // build, so it can't race PoWMiner.mine workers. val mineWrap: GiftWrapTemplateConversion = { template, ephemeralPubKey -> PoWMiner.run(template, ephemeralPubKey, difficulty, isActive) } @@ -1015,14 +1025,15 @@ class Account( isActive: () -> Boolean, ): NostrSigner { val currentSigner = signer + val workers = powMinerWorkers() return if (currentSigner is NostrSignerWithClientTag) { NostrSignerWithClientTag( - inner = PoWNostrSigner(currentSigner.inner, difficulty, kindsToMine, isActive), + inner = PoWNostrSigner(currentSigner.inner, difficulty, kindsToMine, isActive, workers), clientTag = currentSigner.clientTag, disabled = currentSigner.disabled, ) } else { - PoWNostrSigner(currentSigner, difficulty, kindsToMine, isActive) + PoWNostrSigner(currentSigner, difficulty, kindsToMine, isActive, workers) } } @@ -3249,7 +3260,12 @@ class Account( // process death mid-mine cannot lose the file announcement. val senderMessage = signer.sign(template) val seals = NIP17Factory().createSeals(senderMessage, senderMessage.groupMembers(), signer) - if (mineWrapsInBackground(seals.seals, seals.expirationDelta, powDifficulty)) return + if (mineWrapsInBackground(seals.seals, seals.expirationDelta, powDifficulty)) { + // The wraps publish only after mining, but the user has already + // replied — advance the read marker now. + markDmRoomAsRead(senderMessage) + return + } } broadcastPrivately(NIP17Factory().createEncryptedFileNIP17(template, signer)) @@ -3261,7 +3277,12 @@ class Account( // See sendNip17EncryptedFile: sign inline, queue only wrap mining. val senderMessage = signer.sign(template) val seals = NIP17Factory().createSeals(senderMessage, senderMessage.groupMembers(), signer) - if (mineWrapsInBackground(seals.seals, seals.expirationDelta, powDifficulty)) return + if (mineWrapsInBackground(seals.seals, seals.expirationDelta, powDifficulty)) { + // The wraps publish only after mining, but the user has already + // replied — advance the read marker now. + markDmRoomAsRead(senderMessage) + return + } } broadcastPrivately(NIP17Factory().createMessageNIP17(template, signer)) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/DnsInvalidatingEventListener.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/DnsInvalidatingEventListener.kt index 5696529945..78c1196b06 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/DnsInvalidatingEventListener.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/DnsInvalidatingEventListener.kt @@ -20,9 +20,11 @@ */ package com.vitorpamplona.amethyst.service.okhttp +import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.SurgeDns import okhttp3.Call import okhttp3.EventListener import java.io.IOException +import java.net.UnknownHostException /** * Drops a host's [SurgeDns] entry whenever an OkHttp call to it fails outright. We hook @@ -31,6 +33,12 @@ import java.io.IOException * latter while OkHttp recovers via the next address, and we don't want to invalidate the cache * just because the first IP was unreachable. * + * A failure caused by the DNS LOOKUP ITSELF ([UnknownHostException]) must NOT invalidate: + * this hook exists to drop STALE POSITIVES (rotated IPs that no longer connect), but for a + * resolution failure the freshly-written negative entry IS the useful cache state — deleting + * it milliseconds after creation re-pays the full 10-30s resolver timeout on every re-dial + * and silently defeats the negative TTL entirely. + * * Used by the relay client. The media path uses [MediaCallEventListener], which folds the same * invalidation into its `finish` method alongside its existing timing logging. */ @@ -41,6 +49,7 @@ class DnsInvalidatingEventListener( call: Call, ioe: IOException, ) { + if (ioe.causedByUnknownHost()) return dns.invalidate(call.request().url.host) } @@ -53,3 +62,19 @@ class DnsInvalidatingEventListener( override fun create(call: Call): EventListener = listener } } + +/** + * True when [UnknownHostException] appears anywhere in the cause chain — OkHttp usually + * throws it directly from `callFailed`, but interceptors may wrap it. Depth-capped against + * pathological cause cycles. + */ +fun Throwable.causedByUnknownHost(): Boolean { + var t: Throwable? = this + var depth = 0 + while (t != null && depth < 8) { + if (t is UnknownHostException) return true + t = t.cause + depth++ + } + return false +} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/DualHttpClientManager.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/DualHttpClientManager.kt index 794c3ff703..e012ec2cdd 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/DualHttpClientManager.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/DualHttpClientManager.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.service.okhttp +import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.SurgeDns import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/DualHttpClientManagerForRelays.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/DualHttpClientManagerForRelays.kt index bc9ce6dbfb..862e1ecc32 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/DualHttpClientManagerForRelays.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/DualHttpClientManagerForRelays.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.service.okhttp +import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.SurgeDns import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/MediaCallEventListener.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/MediaCallEventListener.kt index de863490dd..d778cd0db2 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/MediaCallEventListener.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/MediaCallEventListener.kt @@ -21,6 +21,7 @@ package com.vitorpamplona.amethyst.service.okhttp import com.vitorpamplona.amethyst.isDebug +import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.SurgeDns import com.vitorpamplona.quartz.utils.Log import okhttp3.Call import okhttp3.ConnectionPool @@ -129,7 +130,10 @@ class MediaCallEventListener( // Drop the cached entry so the next attempt re-resolves instead of trying the same // dead IPs for up to 24h. Per-attempt connectFailed isn't enough — a multi-A-record // host can have one bad IP and OkHttp will recover by trying the next one. - if (error != null) { + // EXCEPT when the failure is the DNS lookup itself: the freshly-written negative + // entry IS the useful cache state, and deleting it would defeat the negative TTL + // (see DnsInvalidatingEventListener). + if (error != null && !error.causedByUnknownHost()) { dns.invalidate(host) } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/OkHttpClientFactory.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/OkHttpClientFactory.kt index 7c67aafef7..38efd9eb08 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/OkHttpClientFactory.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/OkHttpClientFactory.kt @@ -24,6 +24,7 @@ import com.vitorpamplona.amethyst.service.okhttp.OkHttpClientFactoryForRelays.Co import com.vitorpamplona.amethyst.service.okhttp.OkHttpClientFactoryForRelays.Companion.DEFAULT_SOCKS_PORT import com.vitorpamplona.amethyst.service.okhttp.OkHttpClientFactoryForRelays.Companion.DEFAULT_TIMEOUT_ON_MOBILE_SECS import com.vitorpamplona.amethyst.service.okhttp.OkHttpClientFactoryForRelays.Companion.DEFAULT_TIMEOUT_ON_WIFI_SECS +import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.SurgeDns import okhttp3.ConnectionPool import okhttp3.Dispatcher import okhttp3.OkHttpClient diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/OkHttpClientFactoryForRelays.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/OkHttpClientFactoryForRelays.kt index 8b27d5eee3..fbce710e6f 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/OkHttpClientFactoryForRelays.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/OkHttpClientFactoryForRelays.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.service.okhttp +import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.SurgeDns import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.TcpNoDelaySocketFactory import com.vitorpamplona.quartz.utils.Log import okhttp3.Dispatcher diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/pow/PowDuration.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/pow/PowDuration.kt index fa4eef5166..da95a540b8 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/pow/PowDuration.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/pow/PowDuration.kt @@ -22,10 +22,21 @@ package com.vitorpamplona.amethyst.service.pow import android.content.Context import com.vitorpamplona.amethyst.R +import com.vitorpamplona.amethyst.commons.service.pow.PoWEstimator +import com.vitorpamplona.amethyst.commons.service.pow.PoWPolicy import com.vitorpamplona.amethyst.ui.pluralStringRes import com.vitorpamplona.amethyst.ui.stringRes import kotlin.math.roundToLong +/** + * This device's effective mining rate: the [PoWEstimator] benchmark run with + * the same worker count the mining queue uses (half the cores, see + * [PoWPolicy.minerWorkers] and AppModules.powPublishQueue). Every UI estimate + * must use this rate, or it would describe a single-threaded miner that no + * longer exists. Cached after the first call (~250 ms). + */ +suspend fun deviceHashesPerSecond(): Double = PoWEstimator.hashesPerSecond(PoWPolicy.minerWorkers(Runtime.getRuntime().availableProcessors())) + /** * "45 seconds" / "10 minutes" / "3 hours" — the one human-readable rendering * of a PoW duration estimate, shared by the settings picker, the composer diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/broadcast/BroadcastBanner.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/broadcast/BroadcastBanner.kt index 17ef204ce8..c241f50787 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/broadcast/BroadcastBanner.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/broadcast/BroadcastBanner.kt @@ -73,6 +73,7 @@ import com.vitorpamplona.amethyst.commons.service.broadcast.BroadcastStatus import com.vitorpamplona.amethyst.commons.service.broadcast.RelayResult import com.vitorpamplona.amethyst.commons.service.pow.PoWEstimator import com.vitorpamplona.amethyst.commons.service.pow.PoWJobState +import com.vitorpamplona.amethyst.service.pow.deviceHashesPerSecond import com.vitorpamplona.amethyst.service.pow.formatTimeLeft import com.vitorpamplona.amethyst.service.pow.powKindLabelRes import com.vitorpamplona.amethyst.ui.stringRes @@ -197,7 +198,7 @@ private fun MiningContent( val context = LocalContext.current val hashRate by produceState(initialValue = null) { - value = PoWEstimator.hashesPerSecond() + value = deviceHashesPerSecond() } Column(modifier = Modifier.fillMaxWidth()) { diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/creators/pow/PowOverrideButton.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/creators/pow/PowOverrideButton.kt index 22e0dda3a5..ba040646b2 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/creators/pow/PowOverrideButton.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/creators/pow/PowOverrideButton.kt @@ -48,6 +48,7 @@ import com.vitorpamplona.amethyst.R import com.vitorpamplona.amethyst.commons.icons.symbols.Icon import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols import com.vitorpamplona.amethyst.commons.service.pow.PoWEstimator +import com.vitorpamplona.amethyst.service.pow.deviceHashesPerSecond import com.vitorpamplona.amethyst.service.pow.formatApproxDuration import com.vitorpamplona.amethyst.ui.stringRes import com.vitorpamplona.amethyst.ui.theme.ThemeComparisonRow @@ -117,7 +118,7 @@ fun PowOverrideButton( val context = LocalContext.current val hashRate by produceState(initialValue = null) { - value = PoWEstimator.hashesPerSecond() + value = deviceHashesPerSecond() } fun eta(difficulty: Int): String? = hashRate?.let { formatApproxDuration(context, PoWEstimator.estimateSeconds(difficulty, it)) } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/elements/DisplayPoW.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/elements/DisplayPoW.kt index 7324d6d65d..e257452632 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/elements/DisplayPoW.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/elements/DisplayPoW.kt @@ -64,19 +64,19 @@ fun DisplayPoW(pow: Int) { Modifier .clip(RoundedCornerShape(50)) .background(MaterialTheme.colorScheme.secondaryContainer) - .padding(horizontal = 6.dp, vertical = 1.dp), + .padding(horizontal = 5.dp, vertical = 1.dp), ) { Icon( symbol = MaterialSymbols.Manufacturing, contentDescription = stringRes(R.string.pow_settings_title), tint = MaterialTheme.colorScheme.onSecondaryContainer, - modifier = Modifier.size(12.dp), + modifier = Modifier.size(10.dp), ) Text( text = pow.toString(), color = MaterialTheme.colorScheme.onSecondaryContainer, - fontSize = 12.sp, - lineHeight = 12.sp, + fontSize = 10.sp, + lineHeight = 10.sp, fontWeight = FontWeight.Bold, maxLines = 1, ) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/nip22Comments/CommentPostViewModel.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/nip22Comments/CommentPostViewModel.kt index 740a62cea5..3c6b6d9b79 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/nip22Comments/CommentPostViewModel.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/nip22Comments/CommentPostViewModel.kt @@ -575,7 +575,7 @@ open class CommentPostViewModel : // fresh created_at at mining start (NIP-13 recommendation): // the job may have waited in the queue behind other posts. val fresh = EventTemplate(TimeUtils.now(), template.kind, template.tags, template.content) - val mined = PoWMiner.run(fresh, anonSigner.pubKey, powDifficulty, isActive) + val mined = PoWMiner.mine(fresh, anonSigner.pubKey, powDifficulty, accountViewModel.account.powMinerWorkers(), isActive) accountViewModel.account.signAnonymouslyAndBroadcast(mined, extraNotesToBroadcast, anonSigner) accountViewModel.account.deleteDraftIgnoreErrors(draftToDelete) } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/ShortNotePostViewModel.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/ShortNotePostViewModel.kt index 39434ccf93..3cea485aef 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/ShortNotePostViewModel.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/ShortNotePostViewModel.kt @@ -1056,7 +1056,7 @@ open class ShortNotePostViewModel : // fresh created_at at mining start (NIP-13 recommendation): // the job may have waited in the queue behind other posts. val fresh = EventTemplate(TimeUtils.now(), template.kind, template.tags, template.content) - val mined = PoWMiner.run(fresh, anonSigner.pubKey, powDifficulty, isActive) + val mined = PoWMiner.mine(fresh, anonSigner.pubKey, powDifficulty, accountViewModel.account.powMinerWorkers(), isActive) accountViewModel.account.signAnonymouslyAndBroadcast(mined, extraNotesToBroadcast, anonSigner) accountViewModel.account.deleteDraftIgnoreErrors(draftToDelete) } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/ComposeSettingsScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/ComposeSettingsScreen.kt index e28eeb82bf..c70b623f31 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/ComposeSettingsScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/ComposeSettingsScreen.kt @@ -52,6 +52,7 @@ import com.vitorpamplona.amethyst.commons.service.pow.PoWEstimator import com.vitorpamplona.amethyst.model.AccountPoWPreferences import com.vitorpamplona.amethyst.model.BooleanType import com.vitorpamplona.amethyst.model.UiSettingsFlow +import com.vitorpamplona.amethyst.service.pow.deviceHashesPerSecond import com.vitorpamplona.amethyst.service.pow.formatApproxDuration import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.topbars.TopBarWithBackButton @@ -195,7 +196,7 @@ private fun PowTimeEstimate(difficulty: Int) { val context = LocalContext.current val estimate by produceState(initialValue = null, difficulty) { - val rate = PoWEstimator.hashesPerSecond() + val rate = deviceHashesPerSecond() value = formatApproxDuration(context, PoWEstimator.estimateSeconds(difficulty, rate)) } diff --git a/amethyst/src/main/res/values-ar-rSA/strings.xml b/amethyst/src/main/res/values-ar-rSA/strings.xml index 8af42c8760..fe16a886ea 100644 --- a/amethyst/src/main/res/values-ar-rSA/strings.xml +++ b/amethyst/src/main/res/values-ar-rSA/strings.xml @@ -746,6 +746,8 @@ + + المصدر: %1$s تنزيل حول diff --git a/amethyst/src/main/res/values-bn-rBD/strings.xml b/amethyst/src/main/res/values-bn-rBD/strings.xml index 15704d73e0..edc69f1593 100644 --- a/amethyst/src/main/res/values-bn-rBD/strings.xml +++ b/amethyst/src/main/res/values-bn-rBD/strings.xml @@ -715,6 +715,8 @@ + + উৎস: %1$s ডাউনলোড সম্পর্কে diff --git a/amethyst/src/main/res/values-cs/strings.xml b/amethyst/src/main/res/values-cs/strings.xml index 09d7e868fc..f826000683 100644 --- a/amethyst/src/main/res/values-cs/strings.xml +++ b/amethyst/src/main/res/values-cs/strings.xml @@ -733,6 +733,8 @@ + + Zdroj: %1$s v%1$s Stáhnout diff --git a/amethyst/src/main/res/values-de-rDE/strings.xml b/amethyst/src/main/res/values-de-rDE/strings.xml index 7ac95fcf6f..1f23f55b23 100644 --- a/amethyst/src/main/res/values-de-rDE/strings.xml +++ b/amethyst/src/main/res/values-de-rDE/strings.xml @@ -717,6 +717,8 @@ + + Quelle: %1$s v%1$s Herunterladen diff --git a/amethyst/src/main/res/values-el-rGR/strings.xml b/amethyst/src/main/res/values-el-rGR/strings.xml index 6a5870340a..e989bff3a5 100644 --- a/amethyst/src/main/res/values-el-rGR/strings.xml +++ b/amethyst/src/main/res/values-el-rGR/strings.xml @@ -703,6 +703,8 @@ + + Πηγή: %1$s Λήψη Σχετικά diff --git a/amethyst/src/main/res/values-en-rGB/strings.xml b/amethyst/src/main/res/values-en-rGB/strings.xml index 097ec1a9f5..132d63dc73 100644 --- a/amethyst/src/main/res/values-en-rGB/strings.xml +++ b/amethyst/src/main/res/values-en-rGB/strings.xml @@ -19,6 +19,8 @@ + + + + Fonto: %1$s Elŝuti Pri diff --git a/amethyst/src/main/res/values-es-rES/strings.xml b/amethyst/src/main/res/values-es-rES/strings.xml index 4a2235ccdd..4243cf8549 100644 --- a/amethyst/src/main/res/values-es-rES/strings.xml +++ b/amethyst/src/main/res/values-es-rES/strings.xml @@ -728,6 +728,8 @@ + + Fuente: %1$s v%1$s Descargar diff --git a/amethyst/src/main/res/values-es-rMX/strings.xml b/amethyst/src/main/res/values-es-rMX/strings.xml index b09d5e4088..6df3bf7d0f 100644 --- a/amethyst/src/main/res/values-es-rMX/strings.xml +++ b/amethyst/src/main/res/values-es-rMX/strings.xml @@ -721,6 +721,8 @@ + + Fuente: %1$s v%1$s Descargar diff --git a/amethyst/src/main/res/values-es-rUS/strings.xml b/amethyst/src/main/res/values-es-rUS/strings.xml index 1b72ee2894..05a196bd7d 100644 --- a/amethyst/src/main/res/values-es-rUS/strings.xml +++ b/amethyst/src/main/res/values-es-rUS/strings.xml @@ -721,6 +721,8 @@ + + Fuente: %1$s v%1$s Descargar diff --git a/amethyst/src/main/res/values-fa-rIR/strings.xml b/amethyst/src/main/res/values-fa-rIR/strings.xml index 17198c196e..10dad02bfb 100644 --- a/amethyst/src/main/res/values-fa-rIR/strings.xml +++ b/amethyst/src/main/res/values-fa-rIR/strings.xml @@ -714,6 +714,8 @@ + + منبع: %1$s دانلود درباره diff --git a/amethyst/src/main/res/values-fi-rFI/strings.xml b/amethyst/src/main/res/values-fi-rFI/strings.xml index 449ac2ee05..7ce8e75691 100644 --- a/amethyst/src/main/res/values-fi-rFI/strings.xml +++ b/amethyst/src/main/res/values-fi-rFI/strings.xml @@ -705,6 +705,8 @@ + + Lähde: %1$s Lataa Tietoja diff --git a/amethyst/src/main/res/values-fr-rCA/strings.xml b/amethyst/src/main/res/values-fr-rCA/strings.xml index ab7a83391a..9a98dbfd52 100644 --- a/amethyst/src/main/res/values-fr-rCA/strings.xml +++ b/amethyst/src/main/res/values-fr-rCA/strings.xml @@ -683,6 +683,8 @@ + + Source : %1$s Télécharger À propos diff --git a/amethyst/src/main/res/values-fr-rFR/strings.xml b/amethyst/src/main/res/values-fr-rFR/strings.xml index be94c940bd..5caec58f87 100644 --- a/amethyst/src/main/res/values-fr-rFR/strings.xml +++ b/amethyst/src/main/res/values-fr-rFR/strings.xml @@ -766,6 +766,17 @@ Refuser Oublier cette application + + + Utiliser ce relais + Connectez-vous à votre propre relais + Autoriser une fois + Toujours autoriser ce relais + Toujours transmettre mes messages + Bloquer ce relais + Oublier + Dernière utilisation il y a %1$s + Rien ici pour l\'instant — votre politique globale s\'applique à chaque relais. Autoriser Refuser Source : %1$s @@ -929,8 +940,12 @@ Podcasts Voir les épisodes Aucun épisode trouvé pour le moment + Saison %1$d + Explicite Terminé Premium + Soutenir la série + par %1$s Saison %1$d Vidéo Transcription @@ -982,6 +997,8 @@ Langue en Droit d\'auteur + Titre du trailer + URL du média Votre podcast Sans titre Aucun épisode pour le moment. Appuyez sur nouvel épisode pour publier votre premier épisode. @@ -1669,6 +1686,29 @@ Public Groupe Nouveau Groupe Public ou Privé + Créer + Inviter des gens + Partager ce code pour que les gens puissent rejoindre ce groupe. + Création d\'une invitation… + Code d\'invitation copié + Rejoindre le groupe + Code d\'invitation + Rejoindre + Admin + Modérateur + Membre + Membres + Privé + Sur invitation seulement + Entrer une URL de relais valide (wss://…). + Nom + Description + Photo de groupe + Ajouter une photo + Modifier la photo + Découverte + Localisation (géohash) + Privé Privé À Sujet @@ -3161,6 +3201,17 @@ Enregistre automatiquement un brouillon lorsque vous tapez ou quittez un compositeur avec du texte non envoyé, et l\'envoie à vos relais de boîte d\'envoi privée. Signature Votre signature + + %1$d heure + %1$d heures + + + %1$d jour + %1$d jours + + à tout moment maintenant + Votre %1$s a fini de miner mais n\'a pas pu être publié : %2$s + Signaler Utiliser ceci Ignorer Corriger diff --git a/amethyst/src/main/res/values-hi-rIN/strings.xml b/amethyst/src/main/res/values-hi-rIN/strings.xml index fc2c0e4b1b..a8a851778e 100644 --- a/amethyst/src/main/res/values-hi-rIN/strings.xml +++ b/amethyst/src/main/res/values-hi-rIN/strings.xml @@ -662,9 +662,9 @@ पसंदीदा में जोड़ें पसंदीदा से हटाएँ अपनी विंडो में खोलें - यह ऐप क्या एक्सेस कर सकता है - सैंडबॉक्स में चलता है, आपके खाते तक कोई विशेष पहुँच नहीं। - यह क्या एक्सेस कर सकता है + इस क्रमक के लिए क्या अभिगम्य है + पृथक पर्यावरण में चलता है। आपके लेखा तक कोई विशेष अभिगमन अधिकार नहीं। + इसके लिए क्या अभिगम्य है Tor पर लोड होता है। खुले वेब पर लोड होता है। यह ऐप अभी लोड नहीं हुआ है। इसके कार्ड से खोलें, या कुछ देर बाद फिर प्रयास करें। @@ -783,17 +783,50 @@ पुनःप्रसारक प्रमाणीकरण प्रमाण प्रमाणीकरण पुनःप्रसारक हस्ताक्षरण सत्यापन निप॰४२ परिचय वैश्विक नीति + किसमें प्रवेशांकन करना है सर्वदा प्रमाणीकरण प्रमाणीकरण चुनौतियों को हस्ताक्षर प्रत्येक पुनःप्रसारक के लिए जो इसका अनुरोध करता है प्रमाणीकरण कभी ना करें प्रमाणीकरण चुनौतियों की उपेक्षा सभी पुनःप्रसारकों के प्रति - केवल मेरे पुनःप्रसारक - केवल उन पुनःप्ररसारकों के साथ प्रमाणीकरण जो आपके पुनःप्रसारक सूची में हैं + अनुकूलित + स्पष्ट चयन करें किन पुनःप्रसारकों में प्रवेशांकन करना चाहिए। नीचे जिसको अनुमति नहीं दी गई उन सब के विषय में आप पूछे जाएँगे। + मेरे पुनःप्रसारक तथा आयोजन स्थल + प्रवेशांकन करें आपके अपने पुनःप्रसारक तथा सार्वजनिक चर्चाएँ तथा समुदाय तथा सीधे प्रवाहों को जिनसे आप जुडे हैं अथवा प्रियचिह्नित किए हैं। + मेरे द्वारा अनुचरित लोगों के पत्र पढें + अनुचरित के पुनःप्रसारकों में प्रवेशांकन करें उनके पत्र प्राप्त करने के लिए। + मेरे द्वारा अनुचरित लोगों को सन्देश भेजें + अनुचरित के पुनःप्रसारकों में प्रवेशांकन करें सीधेसन्देश तथा प्रत्युत्तर तथा सूचनाएँ भेजने के लिए। + किसी को भी सन्देश भेजें + अज्ञात लोगों के पुनःप्रसारकों में प्रवेशांकन करें सीधेसन्देश तथा प्रत्युत्तर तथा सूचनाएँ भेजने के लिए। मूलविकल्पतः निष्क्रिय। प्रत्येक बार आप से पूछा जाएगा। + क्या इस पुनःप्रसारक को पुष्टि करें कि यह आप हैं। + यह पुनःप्रसारक पहले पुष्टि करना चाहता हैं कि यह वस्तुतः आप हैं। इसके परिचालक देखेगा आपका लेखा कौनसा है। + + क्या आपका सन्देश %1$s को भेजें। + क्या %1$s को सूचित करें। + क्या %1$s को पत्र भेजें। + क्या %1$s को खोलें। + + यदि आप पुष्टि नहीं करते तो %1$s को आपका सन्देश नहीं भेजा जाएगा। + यदि आप नहीं स्वीकारते तो %1$s को इस विषय पर सूचित नहीं किया जाएगा। + यदि आप स्वीकारते नहीं तो %1$s को आपका सन्देश प्रकाशित नहीं किया जाएगा। + यदि आप स्वीकारते नहीं तो आप यहाँ %1$s को नहीं देखेंगे। + %1$s तथा अन्य + आपका निजी सन्देश भेजें इनको : + इनको सूचित करें : + इस शाला को पत्र भेजें + इस शाला को खोलें + इस पुनःप्रसारक का उपयोग करें + अपने ही पुनःप्रसारक से संयोजन करें + एक बार अनुमत + इस पुनःप्रसारक को सर्वदा अनुमति दें + मेरे सन्देशों को सर्वदा भेजें + इस पुनःप्रसारक को बाधित करें प्रति पुनःप्रसारक आवरक आदेश + भूल जाएँ + पूर्व प्रयुक्त %1$s पहले कोई प्रति पुनःप्रसारक आवरक आदेश नहीं। वैश्विक नीति सर्वत्र लागू अनुमत अस्वीकार - आवरक आदेश हटाएँ स्रोत : %1$s संस्करण %1$s अवरोहण @@ -1739,6 +1772,110 @@ सार्वजनिक समूह नया निजी अथवा सार्वजनिक झुण्ड + पुनःप्रसारक समूह + पंक्ति में + पुनःप्रसारक अनुसार + प्रत्येक समूह को अपने ही वार्तालाप के रूप में दर्शाएँ तथा आपके चर्चाओं में मिश्रित। + प्रत्येक पुनःप्रसारक के समूह को एक ही पंक्ति मे संकुचित करें तथा उसके नवीनतम सन्देश पर रखें। + निप॰२९ समूह दृश्य + पुनःप्रसारक समूह + समूह + सन्देश + समूह बनाएँ + यह पुनःप्रसारक निप॰२९ पुनःप्रसारक समूहों का अवलम्बन करने का विज्ञापन नहीं करता। यहाँ बनाए गए समूह काम नहीं करेंगे। उसका नाम तथा सदस्यों तथा सन्देशों का प्रबन्धन पुनःप्रसारक द्वारा नहीं किया जाएगा। एक पुनःप्रसारक का चयन करें जो पुनःप्रसारक आधारित समूहों का अवलम्बन करता हो। + समूह नाम + विषय (विकल्पात्मक) + निजी (केवल सदस्य पठन) + केवल आमन्त्रित + बनाएँ + लोगों को आमन्त्रण + इस अक्षरराशि को बाँटें जिससे लोग इस समूह से जुड सकें। + आमन्त्रण बनाया जा रहा है… + आमन्त्रण अक्षरराशि अनुकृत + आमन्त्रण अक्षरराशि (इस केवल आमन्त्रित समूह के लिए): + समूह से जुडें + आमन्त्रण अक्षरराशि + यह समूह आमन्त्रण सीमित है। आपको दी गई अक्षरराशि प्रविष्ट करें। + जुडें + अनुरोधित + प्रशासक + नियामक + सदस्य + सदस्य सूची + प्रशासक बनाएँ + नियामक बनाएँ + पद हटाएँ + समूह से हटाएँ + क्या %1$s को इस समूह से हटाएँ। वे अभिगमन अधिकार खो देंगे पुनः जुडने अथवा आमन्त्रित होने तक। + समूह सम्पादन + समूह नाम + विषय (विकल्पात्मक) + निजी (केवल सदस्य पठन) + केवल आमन्त्रित + अभिलेखन + सदस्य सूची + समूह सम्पादन + सूत्र + समूह खोलें + इस पुनःप्रसारक पर कोई समूह नहीं अब तक। + आमन्त्रण उत्पादन… + निजी + केवल आमन्त्रित + प्रविष्ट करें एक मान्य पुनःप्रसारक पता (wss://…)। + नाम + विवरण + समूह चित्र + चित्र जोडें + चित्र परिवर्तन + आविष्करण + लोगों का सहयोग करें इस समूह को ढूँढने में। विषयसूची तथा स्थान से अनुकूल आविष्करण छलनियों द्वारा यह प्राप्त होगा (यदि निवास पुनःप्रसारक उनका अवलम्बन करें तो)। + विषय सूची + द्व्यंकरूप्य नोस्टर कला इत्यादि + स्थान (भूविभेदक) + u0nd + अनुमतियाँ + निजी + केवल सदस्य सन्देशों को पढ सकते हैं। + केवल आमन्त्रित + लोग आमन्त्रित अथवा अनुमत होने चाहिए जुडने के लिए। + केवल सदस्यों द्वारा पत्र प्रकाशन + केवल सदस्य सन्देश भेज सकते हैं। + सूचिबद्ध नहीं + इस समूह को छिपाएँ पुनःप्रसारक की सार्वजनिक निर्देशिका से। + आपके पुनःप्रसारकों में समूहों का खोज… + इस छलनी के अनुकूल कोई समूह प्राप्त नहीं अब तक। + इस पुनःप्रसारक को प्रियचिह्नित करें + कोई सूत्र नहीं अब तक। संकलनचिह्न घुण्डी से एक आरम्भ करें। + नया सूत्र + शीर्षकरहित + शीर्षक + कुछ लिखें… + पत्र प्रकाशन + समूह ढूँढें + पुनःप्रसारक में उपलब्ध समूहों का वीक्षण करें। पुनःप्रसारक पता चिपकाएँ अथवा नीचे चयन करें। + पुनःप्रसारक पता + वीक्षण + पुनःप्रसारक जिन पर आप हैं + लोकप्रिय पुनःप्रसारक + कोई सन्देश नहीं अब तक + इस समूह से जुडें सन्देशों को भेजने के लिए। + यह समूह आमन्त्रण सीमित है। आपके पास एक आमन्त्रण होना चाहिए पत्र प्रकाशन के लिए। + + %1$d सदस्य + %1$d सदस्य + + + %1$d सन्देश + %1$d सन्देश + + + %1$d से अधिक सन्देश + %1$d से अधिक सन्देश + + + %1$d व्यक्ति जिसके आप अनुचर + %1$d व्यक्ति जिसके आप अनुचर + निजी के लिए विषय @@ -2348,6 +2485,7 @@ क्रमक अनुशंसाएँ ज्साप प्राप्त सूचनावली अनुचर सूचनावली + द्व्यंकरूप्य (खण्डश्रृंखलाबद्ध) धनकोष प्रतिक्रिया पंक्ति समाकृति करें कि किन प्रतिक्रिया घुण्डियाँ दिखाए जाएँगे तथा उनके क्रम तथा यदि संकलन सूचक दिखाए जाएँगे। सक्षम @@ -3165,6 +3303,9 @@ %1$s सं॰ %2$d अधिक + पीएस१ स्मृति पट्टि अभिलेखन + खण्ड %1$d + रिक्त छेद रक्षापालिका गति चित्रग्राहक @@ -3296,6 +3437,14 @@ हस्ताक्षर सन्देश के अन्त में जोडा जाएगा नए पत्र प्रत्युत्तर उद्धरण अथवा निबन्ध खोलते समय। रिक्त छोड दें अक्षम करने के लिए। आपका हस्ताक्षर + श्रम प्रमाण + कठिनाई + एक निप॰१३ श्रमप्रमाण का खनन करता है आपके पत्रों के लिए प्रकाशन पूर्व जिससे पुनःप्रसारक तथा पढनेवाले उसका तोलन कर सके कचरालेख के प्रति। उच्चतर मूल्यों के लिए घातांकीय अधिक समय लगेगा। पत्र प्रकाशित होंगे अदृश्यतः खनन पश्चात। + निष्क्रिय + विशिष्ट कठिनाई + लक्ष्य का सूक्ष्मानुकूलन अग्रतम शून्य द्व्यंकों में। + क्या खनन करना चाहिए + समयसंकट घटनाओं (पुनःप्रसारक प्रमाणीकरण ज्साप अनुरोध धनकोष तथा हस्ताक्षरकर्ता सन्देश पाण्डुलिपियाँ सूचियाँ) के लिए खनन कभी नहीं। इसका प्रयोग करें हटाएँ सम्यक diff --git a/amethyst/src/main/res/values-hu-rHU/strings.xml b/amethyst/src/main/res/values-hu-rHU/strings.xml index 8598c5bf53..ef95c83d52 100644 --- a/amethyst/src/main/res/values-hu-rHU/strings.xml +++ b/amethyst/src/main/res/values-hu-rHU/strings.xml @@ -782,18 +782,54 @@ Átjátszó-hitelesítés hitelesítés, átjátszó, aláírás, ellenőrzés, nip-42, azonosság - Globális szabály + Hitelesítéskor + Mibe jelentkezzen be Hitelesítés kérése mindig Minden olyan átjátszó esetében alá kell írni az azonosítási kihívásokat, amely ezt kéri Soha ne végezzen hitelesítést Az összes átjátszó hitelesítési kérésének figyelmen kívül hagyása - Csak saját átjátszók - Csak az átjátszólistában szereplő átjászókkal hitelesítse magát + Egyéni + Válassza ki pontosan, hogy mely átjátszókba jelentkezik be. A lent nem engedélyezett dolgokról kérdést fog kapni. + Saját átjátszók és helyszínek + Bejelentkezés a saját átjátszóiba, valamint azokba a nyilvános csevegésekbe, közösségekbe és élő közvetítésekbe, amelyekhez csatlakozott vagy amelyeket kedvelt. + Követett személyek bejegyzéseinek olvasása + Bejelentkezés egy követett személy átjátszóiba a bejegyzéseinek letöltéséhez. + Üzenet küldése a követett személyeknek + Bejelentkezés egy követett személy átjátszóiba a közvetlen üzenetek, válaszok és értesítések küldéséhez. + Üzenet küldése bárkinek + Bejelentkezés idegenek átjátszóiba a közvetlen üzenetek, válaszok és értesítések küldéséhez. Alapértelmezés szerint ki van kapcsolva, ezért minden alkalommal kérdést fog kapni. + Megerősíti a személyazonosságát ennek az átjátszónak? + Ez az átjátszó először meg szeretné erősíteni, hogy tényleg Ön az. Az üzemeltetője látni fogja, hogy melyik fiókot használja. + + Elküldi az üzenetet neki: %1$s? + Értesíti őt: %1$s? + Betölti a bejegyzéseket tőle: %1$s? + Közzéteszi itt: %1$s? + Megnyitja ezt: %1$s? + + Ha nem teszi meg, az üzenete nem lesz kézbesítve neki: %1$s. + Ha nem teszi meg, %1$s nem kap erről értesítést. + Ha nem teszi meg, nem fogja itt látni a bejegyzéseket tőle: %1$s. + Ha nem teszi meg, az üzenete nem lesz közzétéve itt: %1$s. + Ha nem teszi meg, nem fogja itt látni ezt: %1$s. + %1$s és mások + Privát üzenet küldése neki: + Értesítendő: + Bejegyzések letöltése tőle: + Közzététel ebben a szobában + Szoba megnyitása + Átjátszó használata + Kapcsolódás a saját átjátszójához + Engedélyezés egyszer + Mindig engedélyezze ezt az átjátszót + Mindig kézbesítse az üzeneteimet + Átjátszó letiltása Átjátszónkénti felülírások - Nincs átjátszónkénti felülírás - a globális szabály mindenhol érvényes + Elfelejtés + Utoljára használva: %1$s + Itt még nincs semmi. A globális szabályzat minden átjátszóra vonatkozik. Engedélyezés Megtagadás - Felülírás eltávolítása Forrás: %1$s v%1$s Letöltés @@ -1739,6 +1775,110 @@ Nyilvános Csoport Új nyilvános vagy privát csoport + Átjátszócsoportok + Beágyazott + Átjátszó szerint + Jelenítsen meg minden csoportot saját beszélgetésként, a csevegései közé keverve. + Átjátszók csoportjainak összecsukása egyetlen sorba, a legújabb üzenetnél elhelyezve. + NIP-29 csoport megjelenítése + Átjátszócsoportok + Csoportok + Üzenetek + Csoport létrehozása + Ez az átjátszó nem hirdeti a NIP-29 átjátszócsoportok támogatását. Az itt létrehozott csoport nem fog működni. A nevét, a tagjait és az üzeneteit nem az átjátszó fogja kezelni. Válasszon egy olyan átjátszót, amely támogatja az átjátszóalapú csoportokat. + Csoport neve + Téma (nem kötelező) + Privát (csak a tagok olvashatják) + Csak meghívóval + Létrehozás + Emberek meghívása + Ossza meg ezt a kódot, hogy mások is csatlakozhassanak ehhez a csoporthoz. + Meghívó létrehozása… + Meghívókód másolva + Meghívókód (ehhez a csak meghívóval elérhető csoporthoz): + Csatlakozás a csoporthoz + Meghívókód + Ez a csoport csak meghívóval érhető el. Adja meg a kapott kódot. + Csatlakozás + Kérve + Adminisztrátor + Moderátor + Tag + Tagok + Kinevezés adminisztrátorrá + Kinevezés moderátorrá + Szerepkör eltávolítása + Eltávolítás a csoportból + Eltávolítja őt: %1$s, ebből a csoportból? Elveszíti a hozzáférést, amíg újra hozzá nem adják vagy meg nem hívják. + Csoport szerkesztése + Csoport neve + Téma (nem kötelező) + Privát (csak a tagok olvashatják) + Csak meghívóval + Mentés + Tagok + Csoport szerkesztése + Szálak + Csoport megnyitása + Még nincsenek csoportok ezen az átjátszón. + Meghívó előkészítése… + Privát + Csak meghívóval + Adjon meg egy érvényes átjátszó-webcímet (wss://…). + Név + Leírás + Csoport képe + Fénykép hozzáadása + Fénykép módosítása + Felfedezés + Segítsen másoknak is megtalálni ezt a csoportot. A témák és a helyszín megjelenítik a megfelelő felfedezési szűrők alatt (ha a kiszolgáló átjátszó támogatja azokat). + Témák + bitcoin, nostr, művészet + Helyszín (geo-kivonat) + u0nd + Engedélyek + Privát + Csak a tagok olvashatják az üzeneteket. + Csak meghívóval + Az embereket meg kell hívni vagy jóvá kell hagyni a csatlakozáshoz. + Csak tagok posztolhatnak + Csak a tagok tehetnek közzé üzeneteket. + Nem listázott + Csoport elrejtése az átjátszó nyilvános könyvtárából. + Csoportok keresése az átjátszók között… + Még nem találhatók csoportok ehhez a szűrőhöz. + Átjátszó hozzáadása a kedvencekhez + Még nincsenek szálak. Indítson egyet a + gombbal. + Új szál + Névtelen + Cím + Írjon valamit… + Közzététel + Csoportok keresése + Böngésszen az egy átjátszón tárolt csoportok között. Illesszen be egy átjátszócímet, vagy válasszon egyet alább. + Átjátszó címe + Böngészés + Átjátszók, amelyeken jelen van + Népszerű átjátszók + Még nincsenek üzenetek + Csatlakozzon ehhez a csoporthoz az üzenetek küldéséhez. + Ez a csoport csak meghívóval érhető el. Meghívóra van szüksége a közzétételhez. + + %1$d tag + %1$d tag + + + %1$d üzenet + %1$d üzenet + + + %1$d+ üzenet + %1$d+ üzenet + + + %1$d követett személy + %1$d Ön által követett személy + Privát Neki: Tárgy @@ -3300,6 +3440,84 @@ Aláírás Az üzenet végéhez lesz hozzáadva új bejegyzés, válasz, idézet vagy cikk írásakor. A letiltáshoz hagyja üresen. Saját alírás + Munkabizonyítás + Nehézségi szint + A közzététel előtt egy NIP-13 „munkabizonyítást” bányász a bejegyzéseibe, így az átjátszók és az olvasók mérlegelhetik azokat a kéretlen tartalommal szemben. A magasabb értékek bányászata exponenciálisan több időt vesz igénybe. A bejegyzések a bányászat után a háttérben lesznek közzétéve. + Kikapcsolva + Egyéni nehézségi szint + A cél finomhangolása vezető nulla bitekben. + Mit bányásszon + Az időkritikus események (átjátszóhitelesítés, zapkérések, pénztárca és aláíróüzenetek, piszkozatok, listák) soha nincsenek bányászva. + Rövid jegyzetek és válaszok + Az elsődleges nyilvános kéretlen tartalmi felület + Megjegyzések + Válaszok cikkekre, fájlokra és egyéb tartalmakra + Jelentések + Az átjátszók a költségük alapján mérlegelik a jelentéseket + Hosszú formátum és kiemelések + Cikkek és kiemelések; ritka, a költség elhanyagolható + Hangüzenetek + Nyilvános, hangalapú bejegyzések és válaszok + Újraközzétevések + Nagyszámú aktív felhasználónál + Reakciók + A legnagyobb mennyiségű típus; minden kedvelés bányászata meríti az akkumulátort + Nyilvános és élő csevegés + A bányászati késedelmek rontják a beszélgetés menetét + Privát üzenet burkolók + Lehetővé teszi a kéretlen tartalmak szűrését a beérkező közvetlen üzenetekben az átjátszók számára; csak a külső réteget bányássza, az üzenetet soha + Egyéb nyilvános tartalom + Szavazások, élő állapotok, apróhirdetések és minden egyéb nyilvános dolog + „Munkabizonyítás” bányászata + + „Munkabizonyítás” bányászata… (%1$d bejegyzés a sorban) + Proof of work bányászata… (%1$d bejegyzés a sorban) + + + %1$s • bányászat %2$d biten + %1$s • bányászat %2$d biten + + + %1$s • várakozás a bányászatra %2$d biten + %1$s • várakozás a bányászatra %2$d biten + + + Alapértelmezés (%1$d bit) + Alapértelmezés (%1$d bit) + + Alapértelmezés (kikapcsolva) + Kikapcsolva ehhez a bejegyzéshez + + %1$d bit + %1$d bit + + „Munkabizonyítás” bányászata + Azokat a bejegyzéseket jeleníti meg, amelyek még mindig a NIP-13 szerinti munkabizonyítást végzik, így azok befejeződhetnek, miután kilép az alkalmazásból. + Mégse + ≈ %1$s bejegyzésenként ezen az eszközön + <1 mp + + %1$d mp + %1$d mp + + + %1$d perc + %1$d perc + + + %1$d óra + %1$d óra + + + %1$d nap + %1$d nap + + ≈ %1$s van hátra + mostmár bármelyik pillanatban + A(z) %1$s bányászata befejeződött, de nem sikerült közzétenni: %2$s + A(z) %1$s bányászata befejeződött, de nem sikerült közzétenni. A következő indításkor az alkalmazás újra megpróbálja. + Csevegési üzenet + Jelentés Ennek használata Eltüntetés Javítás diff --git a/amethyst/src/main/res/values-in-rID/strings.xml b/amethyst/src/main/res/values-in-rID/strings.xml index 275cbfdff4..55c2791a6f 100644 --- a/amethyst/src/main/res/values-in-rID/strings.xml +++ b/amethyst/src/main/res/values-in-rID/strings.xml @@ -690,6 +690,8 @@ + + Sumber: %1$s Unduh Tentang diff --git a/amethyst/src/main/res/values-it-rIT/strings.xml b/amethyst/src/main/res/values-it-rIT/strings.xml index 4df3492f7c..11279feaef 100644 --- a/amethyst/src/main/res/values-it-rIT/strings.xml +++ b/amethyst/src/main/res/values-it-rIT/strings.xml @@ -695,6 +695,8 @@ + + Fonte: %1$s Scarica Informazioni diff --git a/amethyst/src/main/res/values-ja-rJP/strings.xml b/amethyst/src/main/res/values-ja-rJP/strings.xml index 844234da4f..033213ceb3 100644 --- a/amethyst/src/main/res/values-ja-rJP/strings.xml +++ b/amethyst/src/main/res/values-ja-rJP/strings.xml @@ -702,6 +702,8 @@ + + ソース: %1$s ダウンロード 概要 diff --git a/amethyst/src/main/res/values-ko-rKR/strings.xml b/amethyst/src/main/res/values-ko-rKR/strings.xml index 64803b3814..d513de6d20 100644 --- a/amethyst/src/main/res/values-ko-rKR/strings.xml +++ b/amethyst/src/main/res/values-ko-rKR/strings.xml @@ -699,6 +699,8 @@ + + 출처: %1$s 다운로드 소개 diff --git a/amethyst/src/main/res/values-lv-rLV/strings.xml b/amethyst/src/main/res/values-lv-rLV/strings.xml index a9b2b3192d..567924d2d6 100644 --- a/amethyst/src/main/res/values-lv-rLV/strings.xml +++ b/amethyst/src/main/res/values-lv-rLV/strings.xml @@ -717,6 +717,8 @@ + + Avots: %1$s Lejupielādēt Par diff --git a/amethyst/src/main/res/values-nl-rNL/strings.xml b/amethyst/src/main/res/values-nl-rNL/strings.xml index 9b19f16dc6..48933a0fb3 100644 --- a/amethyst/src/main/res/values-nl-rNL/strings.xml +++ b/amethyst/src/main/res/values-nl-rNL/strings.xml @@ -774,13 +774,12 @@ Auth-challenges ondertekenen voor elke relay die erom vraagt Nooit authenticeren Auth-challenges van alle relays negeren - Alleen mijn relays - Alleen authenticeren met relays in je relaylijst + + Per-relay-uitzonderingen Geen per-relay-uitzonderingen — globaal beleid geldt overal Toestaan Weigeren - Uitzondering verwijderen Bron: %1$s v%1$s Downloaden diff --git a/amethyst/src/main/res/values-pl-rPL/strings.xml b/amethyst/src/main/res/values-pl-rPL/strings.xml index f26c06a2ad..cf336b07d8 100644 --- a/amethyst/src/main/res/values-pl-rPL/strings.xml +++ b/amethyst/src/main/res/values-pl-rPL/strings.xml @@ -799,17 +799,53 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest Uwierzytelnianie transmitera auth uwierzytelnianie transmiter podpis weryfikacja nip-42 tożsamość Polityka globalna + Jak się zalogować Zawsze uwierzytelniaj Poświadcz autoryzację dla każdego transmitera, który o to poprosi Nigdy nie uwierzytelniaj Ignoruj żądania uwierzytelnienia ze wszystkich transmiterów - Tylko moje transmitery - Uwierzytelniaj tylko z transmiterami z listy transmiterów + Niestandardowa + Wybierz dokładnie, do których transmiterów chcesz się zalogować. Zostaniesz zapytany o wszystko, na co nie wyraziłeś zgody poniżej. + Moje transmitery i obiekty + Zaloguj się do własnych transmiterów oraz do publicznych czatów, społeczności i transmisji na żywo, do których dołączyłeś lub które dodałeś do ulubionych. + Czytaj posty od osób, które obserwuję + Zaloguj się na transmitery obserwowanych, aby pobrać ich posty. + Wyślij wiadomość do obserwowanych osób + Zaloguj się na transmitery obserwowanych, aby wysyłać prywatne wiadomości, odpowiedzi i powiadomienia. + Wiadomość do wszystkich + Zaloguj się do transmiterów nieznanych użytkowników, aby wysyłać prywatne wiadomości, odpowiedzi i powiadomienia. Opcja domyślnie wyłączona; za każdym razem pojawi się pytanie o zgodę. + Czy chcesz potwierdzić, że to właśnie ty łączysz się z tym transmiterem? + Ten transmiter chce najpierw potwierdzić, że to naprawdę ty. Jego operator sprawdzi, z jakiego konta się logujesz. + + Wysłać wiadomość do %1$s? + Powiadom %1$s? + Wczytaj posty z %1$s? + Opublikować do %1$s? + Otworzyć %1$s? + + Jeśli nie, Twoja wiadomość do %1$s nie zostanie dostarczona. + Jeśli nie, %1$s nie zostanie o tym powiadomiony. + Jeśli nie, nie zobaczysz tutaj postów z %1$s. + Jeśli nie, Twoja wiadomość do %1$s nie zostanie wysłana. + Jeśli nie, nie zobaczysz tutaj %1$s. + %1$s i inni + Wyślij prywatną wiadomość do: + Powiadom: + Pobierz posty z: + Opublikuj w tym pokoju + Otwórz ten pokój + Użyj tego transmitera + Podłącz się do własnego transmitera + Zezwól jednorazowo + Zawsze zezwalaj na dostęp do tego transmitera + Zawsze przekazuj moje wiadomości + Zablokuj ten transmiter Nadrzędność dla poszczególnych transmiterów + Zapomnij + Ostatnio użyty %1$s temu Brak nadrzędnych reguł dla transmiterów — polityka globalna obowiązuje wszędzie Zezwól Odmów - Usuń nadpisanie Źródło: %1$s v%1$s Pobierz @@ -1773,6 +1809,118 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest Publiczna Grupa Nowa Grupa Publiczna lub Prywatna + Grupy Transmiterów + Inline + Wg transmitera + Wyświetlaj każdą grupę jako osobną rozmowę, umieszczoną wśród Twoich czatów. + Zwiń grupy poszczególnych transmiterów do jednego wiersza, umieszczając je przy najnowszej wiadomości. + Wyświetlanie grupy NIP-29 + Grupy Transmiterów + Grupy + Wiadomości + Utwórz grupę + Ten transmiter nie obsługuje grup zgodnych ze standardem NIP-29. Grupa utworzona tutaj nie będzie działać — transmiter nie będzie zarządzał jej nazwą, członkami ani wiadomościami. Wybierz transmiter, który obsługuje grupy oparte na rekomendacjach. + Nazwa grupy + Temat (opcjonalnie) + Prywatna (odczyt tylko dla członków) + Tylko zaproszenie + Utwórz + Zaproś osoby + Udostępnij ten kod, aby użytkownicy mogli dołączyć do tej grupy. + Tworzenie zaproszenia… + Kod zaproszenia skopiowany + Kod zaproszenia (dla tej grupy tylko zaproszenia): + Dołącz do grupy + Kod zaproszenia + Ta grupa jest dostępna wyłącznie z zaproszeniem. Wpisz otrzymany kod. + Dołącz + Wysłano prośbę + Admin + Moderator + Członek + Członkowie + Stwórz administratora + Stwórz moderatora + Usuń rolę + Usuń z grupy + Usunąć %1$s z tej grupy? Stracą dostęp do czasu ponownego dodania lub ponownego zaproszenia. + Edytuj grupę + Nazwa grupy + Temat (opcjonalnie) + Prywatna (odczyt tylko dla członków) + Tylko na zaproszenie + Zapisz + Członkowie + Edytuj grupę + Wątki + Grupa otwarta + Brak grup na tym przekaźniku transmiterze. + Przygotowywanie zaproszenia… + Prywatna + Tylko na zaproszenie + Wprowadź poprawny adres URL transmitera (wss://…). + Nazwa + Opis + Zdjęcie grupy + Dodaj zdjęcie + Zmień zdjęcie + Odkryj + Pomóż innym znaleźć tę grupę. Tematy i lokalizacja sprawiają, że grupa pojawia się w wynikach wyszukiwania przy użyciu odpowiednich filtrów (o ile transmiter hosta je obsługuje). + Tematy + bitcoin, nostr, sztuka + Lokalizacja (geohash) + u0nd + Uprawnienia + Prywatna + Tylko członkowie mogą czytać wiadomości. + Tylko na zaproszenie + Aby dołączyć, trzeba otrzymać zaproszenie lub zostać zatwierdzonym. + Publikowanie dostępne tylko dla członków + Tylko członkowie mogą wysyłać wiadomości. + Niewymieniona + Ukryj tę grupę w publicznym katalogu transmitera. + Szukam grup w Twoich transmiterach… + Nie znaleziono jeszcze żadnych grup dla tego filtra. + Faworyzuj ten transmiter + Brak wątków. Zacznij od przycisku +. + Nowy wątek + Bez tytułu + Tytuł + Napisz coś… + Post + Znajdź grupy + Przeglądaj grupy hostowane na transmiterze. Wklej adres transmitera lub wybierz jeden z poniższych. + Adres transmitera + Wybierz + Transmitery, których używasz + Popularne transmitery + Brak wiadomości + Dołącz do tej grupy, aby wysyłać wiadomości. + Ta grupa jest dostępna wyłącznie dla zaproszonych osób — aby móc publikować posty, potrzebujesz zaproszenia. + + %1$d członek + %1$d członków + %1$d członków + %1$d członkowie + + + %1$d wiadomość + %1$d wiadomości + %1$d wiadomości + %1$d wiadomości + + + %1$d+ wiadomość + %1$d+ wiadomości + %1$d+ wiadomości + %1$d+ wiadomości + + + %1$d osoba, którą obserwujesz + %1$d osób, które obserwujesz + %1$d osób, które obserwujesz + %1$d osoby, które obserwujesz + Prywatna Do Temat @@ -3381,6 +3529,101 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest Podpis Dodawane na końcu wiadomości podczas tworzenia nowego wpisu, odpowiedzi, cytatu lub artykułu. Aby wyłączyć tę funkcję, pozostaw to pole puste. Twój podpis + Dowód wykonania + Stopień trudności + Przed opublikowaniem postów generuje się w nich dowód wykonania pracy zgodny ze standardem NIP-13, dzięki czemu transmitery i czytelnicy mogą je zweryfikować pod kątem spamu. Im wyższa wartość, tym wydobycie trwa wykładniczo dłużej. Po wygenerowaniu posty są publikowane w tle. + Wył. + Indywidualny poziom trudności + Dopasuj wartość docelową pod kątem początkowych zerowych bitów. + Co wydobywać + Zdarzenia, w przypadku których czas ma kluczowe znaczenie (autoryzacja transmitera, zlecenia zap, komunikaty dotyczące portfela i sygnatariusza, wersje robocze, listy), nigdy nie są wydobywane. + Krótkie wpisy & odpowiedzi + Główne źródło spamu w sieci + Komentarze + Odpowiedzi na artykuły, pliki i inne treści + Zgłoszenia + Transmitery oceniają raporty pod kątem ich kosztów + Artykuły & główne wydarzenia + Artykuły i główne wydarzenia; pojawiają się sporadycznie, a ich koszt jest znikomy + Wiadomości głosowe + Publiczne wiadomości głosowe i odpowiedzi + Powtórzenia + Duża liczba dla aktywnych użytkowników + Reakcje + Wysoki wolumen; każde polubienie zużywa baterię + Publiczny & czat na żywo + Opóźnienia w wydobyciu zakłócają płynność rozmowy + Zawijanie wiadomości prywatnych + Pozwólmy, aby transmitery DM filtrowały spam w skrzynce odbiorczej; wydobywana jest tylko zewnętrzna otoczka, nigdy Twoja wiadomość + Inne publiczne treści + Ankiety, statusy na żywo, ogłoszenia i wszystkie inne treści publiczne + Wydobywanie dowodu wykonania pracy + + Wydobycie w systemie proof of work… (%1$d wpis w kolejce) + Wydobywanie w systemie proof of work… (%1$d wpisów w kolejce) + Wydobywanie w systemie proof of work… (%1$d wpisów w kolejce) + Dowód pracy w górnictwie… (%1$d posty w kolejce) + + + %1$s • wydobywanie z szybkością %2$d bitów + %1$s • wydobywanie z szybkością %2$d bitów + %1$s • wydobywanie z szybkością %2$d bitów + %1$s • wydobywanie z szybkością %2$d bitów + + + %1$s • oczekuje na wydobycie przy %2$d bitach + %1$s • oczekuje na wydobycie przy %2$d bitach + %1$s • oczekuje na wydobycie przy %2$d bitach + %1$s • oczekuje na wydobycie przy %2$d bitach + + + Domyślnie (%1$d bit) + Domyślnie (%1$d bitów) + Domyślnie (%1$d bitów) + Domyślne (%1$d bity) + + Domyślne (wyłączone) + Wyłączone dla tego postu + + %1$d bit + %1$d bitów + %1$d bitów + %1$d bity + + Wydobywanie dowodów wykonania pracy + Wyświetla posty, które wciąż generują dowód pracy zgodnie ze standardem NIP-13, aby proces ten mógł się zakończyć po zamknięciu aplikacji. + Anuluj + ◆ %1$s na posta na tym urządzeniu + + %1$d sekundę + %1$d sekund + %1$d sekund + %1$d sekund + + + %1$d minutę + %1$s minut + %1$d minut + %1$d minut + + + %1$d godzina + %1$d godzin + %1$d godzin + %1$d godzin + + + %1$d dzień + %1$d dni + %1$d dni + %1$d dni + + ◆ %1$s pozostało + w każdej chwili + Twój %1$s zakończył wydobycie, ale nie mógł zostać opublikowany: %2$s + Twój %1$s zakończył wydobywanie, ale nie udało się go opublikować. Próba zostanie powtórzona przy następnym uruchomieniu aplikacji. + Wiadomość Czatu + Zgłoś Użyj tego Ignoruj Popraw diff --git a/amethyst/src/main/res/values-pt-rBR/strings.xml b/amethyst/src/main/res/values-pt-rBR/strings.xml index b146e6c744..b12e17d4b0 100644 --- a/amethyst/src/main/res/values-pt-rBR/strings.xml +++ b/amethyst/src/main/res/values-pt-rBR/strings.xml @@ -713,6 +713,8 @@ + + Fonte: %1$s v%1$s Baixar diff --git a/amethyst/src/main/res/values-pt-rPT/strings.xml b/amethyst/src/main/res/values-pt-rPT/strings.xml index ee75761c87..d06e8cbdb8 100644 --- a/amethyst/src/main/res/values-pt-rPT/strings.xml +++ b/amethyst/src/main/res/values-pt-rPT/strings.xml @@ -695,6 +695,8 @@ + + Fonte: %1$s Baixar Sobre diff --git a/amethyst/src/main/res/values-ru-rRU/strings.xml b/amethyst/src/main/res/values-ru-rRU/strings.xml index 505ff1d454..541080ba25 100644 --- a/amethyst/src/main/res/values-ru-rRU/strings.xml +++ b/amethyst/src/main/res/values-ru-rRU/strings.xml @@ -727,6 +727,8 @@ + + Источник: %1$s Скачать О приложении diff --git a/amethyst/src/main/res/values-ru-rUA/strings.xml b/amethyst/src/main/res/values-ru-rUA/strings.xml index d2d485490c..63c1339889 100644 --- a/amethyst/src/main/res/values-ru-rUA/strings.xml +++ b/amethyst/src/main/res/values-ru-rUA/strings.xml @@ -715,6 +715,8 @@ + + Источник: %1$s Скачать О приложении diff --git a/amethyst/src/main/res/values-sl-rSI/strings.xml b/amethyst/src/main/res/values-sl-rSI/strings.xml index f458bd62b6..028de5ed97 100644 --- a/amethyst/src/main/res/values-sl-rSI/strings.xml +++ b/amethyst/src/main/res/values-sl-rSI/strings.xml @@ -262,7 +262,7 @@ Za podpisovanje se je potrebno prijaviti s privatnim ključem Poznani Nova prošnja Blokirani uporabniki - Nove objave + Nove nizi objav Razprave Vir vsebin Čakalna vrsta mod @@ -815,13 +815,12 @@ Za podpisovanje se je potrebno prijaviti s privatnim ključem Podpiši avtentikacijske izzive za vsak relej, ki to zahteva Nikoli ne avtenticiraj Prezri avtentikacijske izzive vseh relejev - Samo moji releji - Avtenticiraj samo z releji s seznama + + Prilagoditve po posameznem releju Brez preglasitev na posameznih relejih — globalna politika velja povsod Dovoli Zavrni - Odstrani prilagoditev Vir: %1$s v%1$s Prenos @@ -1785,6 +1784,82 @@ Za podpisovanje se je potrebno prijaviti s privatnim ključem Javno Skupina Nova javna ali zasebna skupina + Skupine relejev + V vrstici + Po releju + Prikaži vsako skupino kot ločen pogovor, pomešan z vašimi klepeti. + Strni skupine posameznega releja v eno vrstico, prikazano pri njegovem najnovejšem sporočilu. + Prikaz skupin NIP-29 + Skupine relejev + Sporočila + Ustvari skupino + Ta relej ne oglašuje podpore za skupine NIP-29. Skupina, ustvarjena tukaj, ne bo delovala – relej ne bo upravljal njenega imena, članov in sporočil. Izberite rele, ki podpira skupine na ravni releja. + Ime supine + Tema (neobvezno) + Zasebno (branje samo za člane) + Samo s povabilom + Ustvari + Povabi ljudi + Delite to kodo, da se lahko ljudje pridružijo skupini. + Ustvari povabilo… + Koda povabila kopirana + Koda povabila (samo za to \"samo s povabilom\" skupino) + Pridruži se skupini + Koda povabila + Ta skupina je \"samo s povabilom\". Vpiši kodo povabila + Pridruži se + Zahtevano + Administrator + Moderator + Član + Člani + Dodeli vlogo skrbnika + Dodeli vlogo moderatorja + Odvzemi vlogo + Odstrani iz skupine + Odstrani %1$s iz te skupine? Izgubili bodo dostop, dokler jih ponovno ne dodate ali povabite. + Uredi skupino + Ime skupine + Tema (nebvezno) + Zasebno (branje samo za člane) + Samo s povabilom + Shrani + Člani + Uredi skupino + Nizi objav + Odpri skupino + Na tem releju še ni skupin. + Pripravljam povabilo… + Zasebno + samo s povabilom + Vnesite veljaven URL releja (wss://…) + Ime + Opis + Slika skupine + Dodaj fotografijo + Zamenjaj sliko + Raziskuj + Pomagajte ljudem najti to skupino. Tematike in lokacija jo bodo prikazale v ustreznih filtrih za odkrivanje (če jih relej podpira). + Teme + bitcoin, nostr, umetnost + Lokacija (geohash) + u0nd + Dovoljenja + Zasebno + Samo člani lahko berejo sporočila. + Samo s povabilom + Za vstop je potrebno povabilo ali odobritev. + Pošiljanje samo za člane + Samo člani lahko pošiljajo sporočila + Skrito + Skrij to skupino iz javnega imenika releja. + Iščem skupine po vaših relejih… + S tem filtrom še ni najdenih nobenih skupin. + Dodaj ta rele med priljubljene + Še ni nizov objav. Začni jih z + gumbom. + Nov niz objav + Brez naslova + Naslov Zasebno Za Zadeva diff --git a/amethyst/src/main/res/values-sr-rSP/strings.xml b/amethyst/src/main/res/values-sr-rSP/strings.xml index b83fd5838f..b068cb0758 100644 --- a/amethyst/src/main/res/values-sr-rSP/strings.xml +++ b/amethyst/src/main/res/values-sr-rSP/strings.xml @@ -710,6 +710,8 @@ + + Izvor: %1$s Preuzmi O programu diff --git a/amethyst/src/main/res/values-sv-rSE/strings.xml b/amethyst/src/main/res/values-sv-rSE/strings.xml index b97106c4bc..ca69e77142 100644 --- a/amethyst/src/main/res/values-sv-rSE/strings.xml +++ b/amethyst/src/main/res/values-sv-rSE/strings.xml @@ -716,6 +716,8 @@ + + Källa: %1$s v%1$s Ladda ner diff --git a/amethyst/src/main/res/values-sw/strings.xml b/amethyst/src/main/res/values-sw/strings.xml index 0ad8934963..fda33de346 100644 --- a/amethyst/src/main/res/values-sw/strings.xml +++ b/amethyst/src/main/res/values-sw/strings.xml @@ -705,6 +705,8 @@ + + Chanzo: %1$s Pakua Kuhusu diff --git a/amethyst/src/main/res/values-ta-rIN/strings.xml b/amethyst/src/main/res/values-ta-rIN/strings.xml index 719fda4992..ac6c3e80b9 100644 --- a/amethyst/src/main/res/values-ta-rIN/strings.xml +++ b/amethyst/src/main/res/values-ta-rIN/strings.xml @@ -709,6 +709,8 @@ + + மூலம்: %1$s பதிவிறக்கவும் பற்றி diff --git a/amethyst/src/main/res/values-th-rTH/strings.xml b/amethyst/src/main/res/values-th-rTH/strings.xml index c22daa661c..7be1e93544 100644 --- a/amethyst/src/main/res/values-th-rTH/strings.xml +++ b/amethyst/src/main/res/values-th-rTH/strings.xml @@ -692,6 +692,8 @@ + + ต้นทาง: %1$s ดาวน์โหลด เกี่ยวกับ diff --git a/amethyst/src/main/res/values-tr-rTR/strings.xml b/amethyst/src/main/res/values-tr-rTR/strings.xml index 8a291ea885..b3b34cb392 100644 --- a/amethyst/src/main/res/values-tr-rTR/strings.xml +++ b/amethyst/src/main/res/values-tr-rTR/strings.xml @@ -709,6 +709,8 @@ + + Kaynak: %1$s İndir Hakkında diff --git a/amethyst/src/main/res/values-uk-rUA/strings.xml b/amethyst/src/main/res/values-uk-rUA/strings.xml index b356b6180a..fe1cde2fcf 100644 --- a/amethyst/src/main/res/values-uk-rUA/strings.xml +++ b/amethyst/src/main/res/values-uk-rUA/strings.xml @@ -724,6 +724,8 @@ + + Джерело: %1$s Завантажити Про застосунок diff --git a/amethyst/src/main/res/values-uz-rUZ/strings.xml b/amethyst/src/main/res/values-uz-rUZ/strings.xml index 2677b870bc..5e41ccb79e 100644 --- a/amethyst/src/main/res/values-uz-rUZ/strings.xml +++ b/amethyst/src/main/res/values-uz-rUZ/strings.xml @@ -712,6 +712,8 @@ + + Manba: %1$s Yuklab olish Haqida diff --git a/amethyst/src/main/res/values-vi-rVN/strings.xml b/amethyst/src/main/res/values-vi-rVN/strings.xml index af67e7afdb..d153949ae2 100644 --- a/amethyst/src/main/res/values-vi-rVN/strings.xml +++ b/amethyst/src/main/res/values-vi-rVN/strings.xml @@ -691,6 +691,8 @@ + + Nguồn: %1$s Tải xuống Giới thiệu diff --git a/amethyst/src/main/res/values-zh-rCN/strings.xml b/amethyst/src/main/res/values-zh-rCN/strings.xml index a5dda0a9b3..3ff2b4c951 100644 --- a/amethyst/src/main/res/values-zh-rCN/strings.xml +++ b/amethyst/src/main/res/values-zh-rCN/strings.xml @@ -774,17 +774,53 @@ 中继身份认证 auth authentication relay sign verify nip-42 identity 全局政策 + 登录什么 总是认证身份 为每个请求身份认证挑战的中继签署它 从不认证 忽略所有中继的认证挑战 - 仅我的中继 - 仅通过您的中继列表中的中继进行身份认证 + 自定义 + 选择登录到哪些中继。会询问你任何下面尚未允许的东西。 + 我的中继和场地 + 登录到您自己的中继和您加入或收藏的公共聊天室、社区和直播。 + 阅读我关注的人的帖子 + 登录到所关注者的中继下载其帖子。 + 给我关注的人留言 + 登录到所关注者的中继来发送私信、回复和通知。 + 给任何人留言 + 登录陌生人的中继来发送私信、回复和通知。默认情况下关闭;每次都要问您。 + 向该中继确认你的身份? + 这个中继想要首先确认真的是你。它的运营者会看到你是哪个帐户。 + + 向 %1$s 发送您的消息吗? + 通知 %1$s? + 从 %1$s 加载帖子? + 发帖到 %1$s? + 打开 %1$s? + + 如果不这么做,您给 %1$s 的消息将不会发送。 + 如果不这样做, %1$s 不会收到关于此内容的通知。 + 如果不这么做,你不会在这里看到 %1$s 的帖子。 + 如果不这么做,您给 %1$s 的消息将不会被发布。 + 如果不这么做,你不会在这里看到 %1$s。 + %1$s 和其他 + 发送您的私信到: + 通知: + 从何处下载帖子: + 发布到此房间 + 打开这个房间 + 使用此中继 + 连接到您自己的中继 + 允许一次 + 总是允许此中继 + 始终发送我的消息 + 屏蔽此中继 不同中继设置覆盖 + 忘记 + 上次使用于 %1$s 前 没有不同中继设置覆盖 — 所有中继使用全局政策 允许 拒绝 - 删除覆盖 来源: %1$s v%1$s 下载 @@ -1721,6 +1757,106 @@ 公开 组 新建公开或私人群组 + 中继组 + 行内 + 根据中继 + 将每个群显示为单独对话,与您的聊天混合。 + 将每个中继的群组折叠为单行,放在最新消息上。 + NIP-29 群组显示 + 中继组 + 群组 + 消息 + 建群 + 此继电器不支持NIP-29中继群。 在这里创建的群将无法工作 — — 其名称、成员和消息将不会由中继管理。选择支持基于中继的群的中继。 + 群名称 + 主题(可选) + 私有(仅成员可读取) + 仅限邀请 + 创建 + 邀请其他人 + 分享这个代码,以便人们可以加入这个群。 + 正在创建邀请… + 已复制邀请码 + 邀请码 (用于此受邀加入群): + 加群 + 邀请码 + 此群只有受邀才能加入。请输入邀请码。 + 加入 + 已请求 + 管理员 + 协管 + 成员 + 成员 + 设为管理员 + 设为协管 + 删除职位 + 从群组中删除 + 从此群中移除 %1$s ?在重新添加或重新邀请之前这些人将失去访问权限。 + 编辑群 + 群名称 + 主题(可选) + 私有(仅成员可读取) + 仅限受邀 + 保存 + 成员 + 编辑群 + 主题帖 + 打开群组 + 此中继上尚没有群。 + 正在准备邀请… + 私密 + 仅限受邀 + 输入一个有效的中继 URL (wss://…)。 + 名称 + 描述 + 群组图片 + 添加照片 + 更改照片 + 发现 + 帮助人们找到这个群。主题和位置会把群置于匹配的发现筛选器下面 (如果托管中继支持)。 + 主题 + 比特币,nostr,艺术品 + 位置 (gehash) + u0nd + 权限 + 私密 + 只有成员可以阅读消息。 + 仅限受邀 + 人们必须被邀请或获批才能加入。 + 仅限成员发布 + 只有成员可以发布消息。 + 未列出的 + 在中继的公共目录中隐藏此群。 + 在您的中继中寻找群组… + 此筛选器尚未找到群组。 + 将该中继加入收藏 + 还没有主题帖。用+按钮开一个。 + 新主题帖 + 无标题 + 标题 + 写点什么… + 发表 + 查找群组 + 浏览中继托管的群组。粘贴一个中继地址,或在下方选择一个。 + 中继地址 + 浏览 + 您正身处的中继 + 热门中继 + 尚无消息 + 加入此群组发送消息。 + 此群只有受邀才能加入 —— 需要邀请才能发帖。 + + %1$d 位成员 + + + %1$d 条消息 + + + 超过 %1$d 条消息 + + + 您关注的 %1$d 人 + 私人 至 主题 @@ -2836,7 +2972,7 @@ 关注包 转发(16) Geohash 关注 - 包装礼物 + 礼物包装 Git 问题 Git 补丁 Git 仓库 @@ -3258,6 +3394,75 @@ 签名 在开启新帖子、回复、引用或文章时在消息末尾添加。留空则禁用。 您的签名 + 工作证明 + 难度 + 发布前将 NIP-13 工作证明开采进你的帖子以便中继和读者可以将帖子和垃圾信息进行权重比较。挖掘较大的值所需的时间会指数级增长。挖好后会在后台发布帖子。 + 关 + 自定义难度 + 微调前导零比特的目标。 + 要挖什么 + 时间紧迫的事件(继中继认证、 zap请求、 钱包和签名者消息、 草稿、 列表)从未被挖掘。 + 短笔记 & 回复 + 主要的公共垃圾邮件表面 + 评论 + 文章回复、文件和其他内容 + 举报 + 中继根据开销对举报进行加权 + 长文 & 重点 + 文章和重点;不经常发生;费用微不足道 + 语音消息 + 公共语音帖子和回复 + 转发 + 活跃用户的高容量 + 回应 + 量最大的类型;挖掘每个赞都会消耗电量 + 公开 & 实时聊天 + 采矿延迟伤害对话流 + 私密消息包装器 + 让私信中继过滤收件箱垃圾邮件;只挖掘外部包装,永远不会挖掘你的消息 + 其他公开内容 + 投票、实时状态、分类和所有其他所有公开信息 + 挖掘工作证明 + + 挖掘工作证明… (队列中有 %1$d 篇帖子) + + + %1$s • 在 %2$d 比特挖掘中 + + + %1$s • 正在等待在 %2$d 位上开采 + + + 默认 (%1$d 比特) + + 默认 (关闭) + 此帖关闭 + + %1$d 比特 + + 工作证明开采 + 显示仍在挖掘 NIP-13 工作证明的帖子,以便在你离开应用后它们可以完成挖掘。 + 取消 + 此设备上每个帖子约为 %1$s + <1 秒 + + %1$d 秒 + + + %1$d 分钟 + + + %1$d 小时 + + + %1$d 天 + + 剩余约 %1$s + 很快 + 您的 %1$s 完成了挖掘,但无法发布: %2$s + 您的 %1$s 完成了挖掘,但无法发布。将在下次应用程序启动时重新尝试。 + 聊天消息 + 举报 使用它 忽略 更正 diff --git a/amethyst/src/main/res/values-zh-rHK/strings.xml b/amethyst/src/main/res/values-zh-rHK/strings.xml index f064a1a2ca..fea511090e 100644 --- a/amethyst/src/main/res/values-zh-rHK/strings.xml +++ b/amethyst/src/main/res/values-zh-rHK/strings.xml @@ -707,6 +707,8 @@ + + 来源: %1$s 下载 关于 diff --git a/amethyst/src/main/res/values-zh-rSG/strings.xml b/amethyst/src/main/res/values-zh-rSG/strings.xml index 85cb0dc794..4d2541c844 100644 --- a/amethyst/src/main/res/values-zh-rSG/strings.xml +++ b/amethyst/src/main/res/values-zh-rSG/strings.xml @@ -707,6 +707,8 @@ + + 来源: %1$s 下载 关于 diff --git a/amethyst/src/main/res/values-zh-rTW/strings.xml b/amethyst/src/main/res/values-zh-rTW/strings.xml index 02bc17159d..8feb85b6b3 100644 --- a/amethyst/src/main/res/values-zh-rTW/strings.xml +++ b/amethyst/src/main/res/values-zh-rTW/strings.xml @@ -701,6 +701,8 @@ + + 來源:%1$s 下載 關於 diff --git a/amethyst/src/main/res/values/strings.xml b/amethyst/src/main/res/values/strings.xml index 833e8e1df7..c0559e4be3 100644 --- a/amethyst/src/main/res/values/strings.xml +++ b/amethyst/src/main/res/values/strings.xml @@ -873,7 +873,7 @@ My relays and venues Log in to your own relays and to public chats, communities and live streams you\'ve joined or favorited. Read posts from people I follow - Log in to a follow\'s relays to download their posts. + Log in to a follow\'s relays to download their profile information, posts and post engagement. Message people I follow Log in to a follow\'s relays to send DMs, replies and notifications. Message anyone @@ -883,19 +883,19 @@ Send your message to %1$s? Notify %1$s? - Load posts from %1$s? + Load profile, posts and engagement from %1$s? Post to %1$s? Open %1$s? If you don\'t, your message to %1$s won\'t be delivered. If you don\'t, %1$s won\'t be notified about this. - If you don\'t, you won\'t see posts from %1$s here. + If you don\'t, you won\'t see the profile, posts or engagement from %1$s here. If you don\'t, your message to %1$s won\'t be posted. If you don\'t, you won\'t see %1$s here. %1$s and others Send your private message to: Notify: - Download posts from: + Download profile information, posts and engagement from: Post to this room Open this room Use this relay diff --git a/build.gradle.kts b/build.gradle.kts index 6b87fbd049..5d6326492d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -45,6 +45,50 @@ plugins { alias(libs.plugins.androidKotlinMultiplatformLibrary) apply false alias(libs.plugins.serialization) alias(libs.plugins.googleKsp) apply false + alias(libs.plugins.kotlinxKover) +} + +// Aggregated code coverage: `./gradlew koverXmlReportCoverage` runs the unit +// test suites of the modules below and merges their coverage into +// build/reports/kover/reportCoverage.xml (JaCoCo-compatible XML, importable by +// SonarQube — see BUILDING.md). `koverHtmlReportCoverage` renders the same +// data for humans. A named variant is used instead of Kover's default total +// report because the total for an Android project unconditionally merges +// EVERY build variant — compiling all release/benchmark variants of +// :amethyst just to measure unit tests. The "coverage" variant below picks +// one representative compilation per project: the `jvm` target for KMP/JVM +// modules and `playDebug` for :amethyst — play is the primary flavor and the +// larger code surface (both `add`s are optional so the one block fits every +// project). :nappletHost stays out — it has no tests. +dependencies { + kover(project(":amethyst")) + kover(project(":quartz")) + kover(project(":commons")) + kover(project(":quic")) + kover(project(":nestsClient")) + kover(project(":geode")) + kover(project(":cli")) + kover(project(":relayBench")) + kover(project(":desktopApp")) +} + +kover { + currentProject { + createVariant("coverage") {} + } +} + +subprojects { + plugins.withId("org.jetbrains.kotlinx.kover") { + configure { + currentProject { + createVariant("coverage") { + add("jvm", optional = true) + add("playDebug", optional = true) + } + } + } + } } // Shared app version for all subprojects — read from gradle/libs.versions.toml. @@ -120,6 +164,15 @@ if (sonarEnabled) { .filter { it.startsWith("sonar.") } .forEach { System.setProperty(it, sonarProperties.getProperty(it)) } + // Import Kover's aggregated coverage (run `./gradlew koverXmlReportCoverage` + // first) unless the operator pointed the scanner somewhere else. + if (System.getProperty("sonar.coverage.jacoco.xmlReportPaths") == null) { + System.setProperty( + "sonar.coverage.jacoco.xmlReportPaths", + layout.buildDirectory.file("reports/kover/reportCoverage.xml").get().asFile.absolutePath, + ) + } + // The scanner's sonarResolver task reads AGP's generated-res-values provider // but doesn't depend on the task that produces it — wire it up in every // module that has both (today only :amethyst enables resValues, but the diff --git a/cli/README.md b/cli/README.md index 0ded3d65d8..928009c996 100644 --- a/cli/README.md +++ b/cli/README.md @@ -216,8 +216,8 @@ Army-knife verbs that operate purely on their arguments. They never touch | `amy encode naddr --kind N --pubkey HEX --identifier D [--relay URL[,URL…]]` | Encode an addressable-event (`a` tag) pointer. | | `amy verify [EVENT-JSON]` | Check an event's id hash and signature. Reads stdin when the argument is omitted or `-`. Reports `id_ok` + `signature_ok` separately. | | `amy pow check EVENT-JSON\|-` | NIP-13 difficulty of a signed event: `actual_bits`, `committed_target`, `has_commitment`, and `effective_pow` (capped at the commitment so lucky low-target spam doesn't over-count), plus `valid` (id + signature). | -| `amy pow mine --target N [--pubkey HEX] [--timeout SECS] TEMPLATE-JSON\|-` | Mine an **unsigned** template to N leading zero bits and print it back with the nonce tag. Ids don't commit to signatures, so amy can mine on behalf of any pubkey (NIP-13 delegated PoW); defaults to the active account. Exit 124 on timeout. | -| `amy pow bench` | Benchmark this machine's hash rate and print expected mining time at 16/20/24/28 bits. | +| `amy pow mine --target N [--pubkey HEX] [--timeout SECS] [--threads N] TEMPLATE-JSON\|-` | Mine an **unsigned** template to N leading zero bits and print it back with the nonce tag. Ids don't commit to signatures, so amy can mine on behalf of any pubkey (NIP-13 delegated PoW); defaults to the active account. Mines on all cores by default (`--threads` to override). Exit 124 on timeout. | +| `amy pow bench` | Benchmark this machine's hash rate — `hashes_per_second` is the all-cores rate `pow mine` uses by default, `hashes_per_second_single_core` the one-thread rate — and print expected mining time at 16/20/24/28 bits. | | `amy key generate` | Mint a fresh keypair (`nsec` + `npub` + hex). Does not persist — use `init`/`login` for that. | | `amy key public NSEC\|HEX` | Derive the public key from a secret key. | | `amy key encrypt NSEC\|HEX --password X` | NIP-49 encrypt a secret key to an `ncryptsec1…`. | @@ -384,7 +384,7 @@ HTTP endpoint. Reuses quartz's `Nip86Client` and the shared `Nip86Retriever` | Command | What it does | |---|---| -| `amy notes post TEXT [--relay URL] [--pow BITS [--pow-timeout SECS]]` | Publish a kind:1 short text note; `--pow` mines a NIP-13 proof of work into it first (blocks while mining, exit 124 on timeout with nothing published; `--json` adds `pow`, `pow_target`, `pow_millis`). | +| `amy notes post TEXT [--relay URL] [--pow BITS [--pow-timeout SECS]]` | Publish a kind:1 short text note; `--pow` mines a NIP-13 proof of work into it first, using all cores (blocks while mining, exit 124 on timeout with nothing published; `--json` adds `pow`, `pow_target`, `pow_millis`). | | `amy notes feed [--author USER \| --following] [--limit N]` | Read recent kind:1 notes (yours, one user's, or your follow set). | | `amy profile show [USER]` | Print kind:0 metadata. USER accepts npub/nprofile/hex/NIP-05; defaults to self. | | `amy profile edit --name … --about … --picture URL …` | Patch and re-publish your kind:0. | diff --git a/cli/build.gradle.kts b/cli/build.gradle.kts index 489e604740..7d26b235b1 100644 --- a/cli/build.gradle.kts +++ b/cli/build.gradle.kts @@ -3,6 +3,7 @@ import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { alias(libs.plugins.jetbrainsKotlinJvm) application + alias(libs.plugins.kotlinxKover) } kotlin { diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt index 4454e0a532..d87970d752 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Config.kt @@ -244,6 +244,13 @@ class DataDir( */ val eventsDbFile: File = File(eventsDir.parentFile ?: root, "events.db") + /** + * Persisted DNS cache (SurgeDns snapshot) under `/shared/`, next to the + * event store — cross-account like the resolver itself: a relay host resolved + * by one account's crawl serves every account's next run. + */ + val dnsCacheFile: File = File(eventsDir.parentFile ?: root, "dns-cache.bin") + /** * Machine-level operator keys for GrapeRank trusted-assertion publishing, * rooted at `~/.amy/operator/` (the account root's parent) so a single @@ -375,12 +382,16 @@ class DataDir( /** * Account names become directory names AND alias keys, so we * keep them to a portable, shell-friendly subset. `shared` is - * reserved for the cross-account events-store sibling, and - * `current` collides with the active-account marker file. + * reserved for the cross-account events-store sibling, `current` + * collides with the active-account marker file, and `operator` is the + * machine-level operator-keys sibling (see [OperatorKeys]) that ANY + * graperank crawl/probe creates — counting it as an account broke + * single-account auto-selection with "multiple accounts (operator, …)". */ private val NAME_REGEX = Regex("^[a-zA-Z0-9_-]{1,64}$") private const val SHARED_DIR_NAME = "shared" - private val RESERVED_NAMES = setOf(SHARED_DIR_NAME, CURRENT_MARKER_NAME) + private const val OPERATOR_DIR_NAME = "operator" + private val RESERVED_NAMES = setOf(SHARED_DIR_NAME, CURRENT_MARKER_NAME, OPERATOR_DIR_NAME) fun validateName(name: String): String { require(NAME_REGEX.matches(name)) { diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index 22c849df1e..b478ef8b85 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.cli +import com.sun.management.UnixOperatingSystemMXBean import com.vitorpamplona.amethyst.cli.stores.FileCashuKeysetCounterStore import com.vitorpamplona.amethyst.cli.stores.FileKeyPackageBundleStore import com.vitorpamplona.amethyst.cli.stores.FileMarmotMessageStore @@ -54,6 +55,8 @@ import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer import com.vitorpamplona.quartz.nip01Core.relay.normalizer.toHttp import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.BasicOkHttpWebSocket +import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.SurgeDns +import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.SurgeDnsStore import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.TcpNoDelaySocketFactory import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal @@ -88,6 +91,7 @@ import kotlinx.coroutines.withTimeoutOrNull import okhttp3.Dispatcher import okhttp3.OkHttpClient import okhttp3.Request +import java.lang.management.ManagementFactory import java.util.concurrent.TimeUnit /** @@ -135,6 +139,18 @@ class Context( */ val anonymous: Boolean = false, ) : AutoCloseable { + // Shared resolver — the SAME SurgeDns the Android app runs (stale-while-revalidate, + // single-flight, jittered 24-48h positive TTL, 10-min negative TTL), persisted under + // `/shared/` so repeat crawls start with the whole relay universe pre-resolved: + // every previously-seen host serves its cached answer instantly and re-verifies in the + // background instead of paying a blocking getaddrinfo per host. + private val surgeDns = SurgeDns() + private val dnsStore = + SurgeDnsStore(dataDir.dnsCacheFile, surgeDns).also { + // Best-effort warm start (~25KB read); a corrupt/missing blob just means a cold cache. + runCatching { it.load() } + } + private val okhttp = OkHttpClient .Builder() @@ -149,14 +165,26 @@ class Context( // already-open sockets, bounded by AdaptiveRelayLimiter), so it can't // trip a relay's REQ rate-limit — it only speeds connection setup. The // executor thread pool is unbounded on demand, so raising maxRequests - // just lets more of those short-lived handshakes proceed at once. 7s + // just lets more of those short-lived handshakes proceed at once + // (1 platform thread per in-flight handshake — ~1k is fine on a JVM, + // 10k is not). Sized against the process FD budget: every pending + // handshake and every open socket is one file descriptor. 7s // (not 5s): a 5s cap struck too many merely-busy relays as connect // failures — the crawl treats a connect *timeout* as retryable anyway, // but the extra headroom lets slow-but-alive relays finish the handshake. .connectTimeout(7, TimeUnit.SECONDS) + // DNS dominates a mass connection ramp without help: Dns.SYSTEM blocks a + // dispatcher thread per lookup, the JVM's own cache lasts ~30s (useless + // over a 30-minute crawl), a dead domain burns 10-30s of resolver + // timeouts on EVERY re-dial, and the outbox model mints hundreds of + // per-user URLs on one host — each a fresh lookup. SurgeDns (shared with + // the Android app) collapses all of it: single-flight per host, + // stale-while-revalidate positives, 10-min negative TTL, persisted + // across runs via [dnsStore]. + .dns(surgeDns) .dispatcher( Dispatcher().apply { - maxRequests = 256 + maxRequests = maxParallelHandshakes maxRequestsPerHost = 16 }, ).build() @@ -212,7 +240,11 @@ class Context( // on a from-scratch GrapeRank crawl than the old 100 (which drowned // damus/nos.lol in 100 concurrent giant REQs) at equal completeness, and // is still generous for the single-user fetches other amy commands do. - startCap = 16, + // AMY_RELAY_SUB_CAP overrides for experiments — the crawl's Phase-B + // throughput plateaus on hot-relay permit queues, and the 16-vs-100 + // benchmark predates the multithreaded-crawl fix, so the sweet spot may + // sit higher; relays that complain still get demoted down the ladder. + startCap = System.getenv("AMY_RELAY_SUB_CAP")?.toIntOrNull()?.coerceIn(1, 100) ?: 16, ).also { client.addConnectionListener(it) } /** @@ -1030,6 +1062,10 @@ class Context( override fun close() { // Nothing to persist for an anonymous run (no account dir to write into). if (!anonymous) dataDir.saveRunState(state) + // Persist the DNS cache (shared dir, account-independent) so the next run's + // connection storm starts with every known host pre-resolved. Best-effort: + // a failed save must never break the run it is summarizing. + runCatching { dnsStore.save() } (signer as? NostrSignerRemote)?.let { try { it.closeSubscription() @@ -1058,6 +1094,39 @@ class Context( */ private const val GIFT_WRAP_LOOKBACK_SECS: Long = 2L * 24 * 60 * 60 + /** FDs reserved for everything that isn't a relay socket (store, jars, pipes, DNS). */ + private const val FD_RESERVE = 256L + + /** + * The process's max-open-files limit (`ulimit -n`), the hard ceiling on + * concurrent sockets: every open WebSocket AND every in-flight handshake is + * one file descriptor, and the JVM cannot raise its own rlimit. Falls back + * to the conservative 1024 (the common soft default) when the platform bean + * doesn't expose it. + */ + val maxFileDescriptors: Long = + (ManagementFactory.getOperatingSystemMXBean() as? UnixOperatingSystemMXBean) + ?.maxFileDescriptorCount ?: 1024L + + /** + * Concurrent WS-upgrade handshakes (OkHttp Dispatcher.maxRequests): a quarter + * of the FD budget, so pending dials can never crowd out the sockets already + * held open (warm pool, drains, parked subs). Also bounds the transient + * platform threads OkHttp spawns — one per in-flight handshake. + */ + val maxParallelHandshakes: Int = + ((maxFileDescriptors - FD_RESERVE) / 4).coerceIn(64, 1024).toInt() + + /** + * Default cap for the crawl's mass pre-connect (warm pool): half the FD + * budget goes to held-open relay sockets, leaving the other half for + * in-flight handshakes, drain/parked subscriptions and headroom. At the + * common 1024-FD soft limit this is ~384; `ulimit -n 16384` unlocks the + * full 4000. Overridable per-run with --preconnect-cap. + */ + val defaultPreconnectCap: Int = + ((maxFileDescriptors - FD_RESERVE) / 2).coerceIn(100, 4000).toInt() + /** * Build a Context but require an account with a usable identity — * signing verbs can't run without one. Throws [IllegalArgumentException] diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index 0171bac900..8fe4996c7a 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -624,6 +624,14 @@ private fun printUsage() { | new/changed ranks >= --min-rank (default 2), skips | unchanged, and retracts (kind:5) any card whose | target left the graph or fell below the cutoff. + | graperank crawl [OBSERVER] network only: crawl the WoT graph (kind 3/10000/ + | [--max-hops N] [--preconnect-cap N] 1984/10002) into the local store without scoring. + | [--no-preconnect] Pre-connects every known-live relay in one parallel + | storm (seeded from the reachability cache). + | graperank probe [--timeout SECS] relay census: mass-connect every relay the store + | [--concurrency N] knows and record live/dead + measured rtt-open into + | the reachability cache (NIP-66 kind:30166), so the + | next crawl skips dead relays and waits once. | graperank update [--down] [--up] refresh every locally-known author's WoT record kinds | [--no-sync-deletions] [--timeout SECS] (0/3/10002/1984) from their own outbox: reads all | [--relay-concurrency N] [--author-chunk N] kind:10002 in the store, groups authors by write diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt index e9a7177e36..243b88264d 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GrapeRankCommand.kt @@ -46,6 +46,7 @@ import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip09Deletions.DeletionIndex import com.vitorpamplona.quartz.nip51Lists.muteList.MuteListEvent import com.vitorpamplona.quartz.nip56Reports.ReportEvent +import com.vitorpamplona.quartz.nip66RelayMonitor.reachability.RelayProber import com.vitorpamplona.quartz.nip85TrustedAssertions.list.TrustProviderListEvent import com.vitorpamplona.quartz.nip85TrustedAssertions.list.serviceProviders import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ProviderTypes @@ -92,6 +93,9 @@ import kotlin.math.roundToInt * - `amy graperank score [OBSERVER]` — local only: build the graph from the store * and score (same as bare `--offline`). Instant and param-tunable; repeat with * different `--rigor`/`--attenuation`/cutoffs without re-crawling. + * - `amy graperank probe` — the relay census: mass-connect every relay the store + * knows and record live/dead + measured RTT into the reachability cache, so the + * next crawl skips the dead and pre-connects the living in one parallel storm. * - bare `amy graperank [OBSERVER]` — the convenience combo: crawl then score. * * Sub-verbs complete the NIP-85 provider experience — the discovery layer that @@ -133,6 +137,10 @@ object GrapeRankCommand { private const val PROBE_TIMEOUT_MS = 2000 + // args.bool on a mistyped flag silently returns false, so the one flag read from + // three different functions goes through a compile-time-checked name. + private const val NO_REACHABILITY_CACHE_FLAG = "no-reachability-cache" + // The probe does BLOCKING DNS + TCP connect, and dead-domain DNS lookups can hang // far past the connect timeout. On the shared Dispatchers.IO those hanging lookups // starve the crawl's own IO — measured +462s on the finishing drain at hop-3. Run @@ -193,6 +201,7 @@ object GrapeRankCommand { // `sync` is the pre-rename name kept as a back-compat alias; `crawl` is // canonical (disambiguates from negentropy `amy sync` / `graperank update`). "crawl", "sync" -> crawl(dataDir, tail.drop(1).toTypedArray()) + "probe" -> probe(dataDir, tail.drop(1).toTypedArray()) "update" -> update(dataDir, tail.drop(1).toTypedArray()) "score" -> run(dataDir, tail.drop(1).toTypedArray(), forceOffline = true) else -> run(dataDir, tail) @@ -441,11 +450,25 @@ object GrapeRankCommand { // Aggregator kind:3 recovery for stragglers is on by default; --no-aggregators // disables it for A/B comparison. val aggregators = if (args.bool("no-aggregators")) emptySet() else CONTENT_AGGREGATOR_RELAYS - // Seed the crawl with relays a prior run/monitor proved dead within the cache's - // TTL, so the WS path never re-pays their connect timeouts (--no-reachability-cache - // to skip). The crawl's own final live/dead set is flushed back by the caller. - val knownDead = - if (args.bool("no-reachability-cache")) emptySet() else ctx.reachability.snapshot().dead + // Seed the crawl from the reachability cache (--no-reachability-cache to skip): + // proven-dead relays within the TTL are never dialed (their connect timeouts + // aren't re-paid), and the proven-live universe is pre-connected in one + // parallel storm at crawl start so the connect wait is paid once, up front. + // The crawl's own final live/dead set is flushed back by the caller. + val reachability = + if (args.bool(NO_REACHABILITY_CACHE_FLAG)) null else ctx.reachability.snapshot() + val knownDead = reachability?.dead ?: emptySet() + // Mass pre-connect sizing: every warm socket is one FD, so the default is + // derived from the process's ulimit (--preconnect-cap to override, + // --no-preconnect to fall back to the top-20 warm pool). + val preconnectCap = + if (args.bool("no-preconnect")) 0 else args.intFlag("preconnect-cap", Context.defaultPreconnectCap) + if (preconnectCap > 0 && Context.maxFileDescriptors < 4096) { + System.err.println( + "[graperank] open-files limit is ${Context.maxFileDescriptors} → pre-connect capped at " + + "$preconnectCap sockets; `ulimit -n 16384` before running unlocks a faster crawl", + ) + } return GrapeRankCrawler( client = ctx.client, store = ctx.store, @@ -454,6 +477,8 @@ object GrapeRankCommand { GrapeRankCrawler.Config( relayListDiscoveryRelays = discoveryRelays, knownDeadRelays = knownDead, + knownLiveRelays = if (preconnectCap > 0) reachability?.live.orEmpty() else emptySet(), + preconnectCap = preconnectCap, contentFallbackRelays = contentFallback, contentAggregatorRelays = aggregators, maxRounds = args.intFlag("max-rounds", Int.MAX_VALUE), @@ -462,7 +487,7 @@ object GrapeRankCommand { parkTimeoutMs = args.longFlag("park-timeout", 40L) * 1000, diagnose = args.bool("diagnose"), insertBatchSize = args.intFlag("insert-batch", 500), - drainConcurrency = args.intFlag("drain-concurrency", 24), + drainConcurrency = args.intFlag("drain-concurrency", 48), timeoutEvictStrikes = args.intFlag("timeout-evict", 3), // Cheap TCP reachability pre-probe (--no-probe to disable). No Tor // transport here, so .onion relays are skipped on sight. @@ -471,10 +496,22 @@ object GrapeRankCommand { // shedDeadDiscovery / shardRotations keep their benchmarked-best // Config defaults. ), - log = { System.err.println(it) }, + log = crawlLogger(), ) } + /** + * Crawl progress logger with a `[t+SSSs]` elapsed prefix, so a saved log + * attributes wall time to rounds/phases without external timestamps. + */ + private fun crawlLogger(): (String) -> Unit { + val start = System.nanoTime() + return { line -> + val secs = (System.nanoTime() - start) / 1_000_000_000 + System.err.println("[t+${secs}s] $line") + } + } + /** Echo any relay NOTICE/CLOSED feedback + adaptive throttling the crawl saw. */ private fun reportRelayFeedback(ctx: Context) { if (ctx.relayDiagnostics.hadFeedback()) { @@ -496,7 +533,7 @@ object GrapeRankCommand { args: Args, stats: GrapeRankCrawler.Stats, ) { - if (args.bool("no-reachability-cache")) return + if (args.bool(NO_REACHABILITY_CACHE_FLAG)) return runCatching { ctx.reachability.record(reachable = stats.liveRelays, dead = stats.deadRelays) System.err.println( @@ -548,6 +585,86 @@ object GrapeRankCommand { return 0 } + /** + * `amy graperank probe [--timeout SECS] [--concurrency N]` — + * the relay census. Mass-connects the ENTIRE relay universe the local store knows + * (every relay advertised in any stored kind:10002, deduped per host, plus + * everything already in the reachability cache) in parallel waves with a no-op + * REQ, so the "is this relay alive, and how slow?" wait is paid once, up front, + * concurrently — then records per-relay verdicts with real measured `rtt-open` + * into the NIP-66 reachability cache (kind:30166). + * + * The next `graperank crawl` reads that cache to (a) skip the dead set without + * dialing it and (b) pre-connect the live set in one storm — separating "working + * but slow" (kept; the crawler's patient park path waits for them) from "not + * working" (skipped entirely). Typical flow the first time: + * `graperank crawl --max-hops 2` (cheap, saves the relay lists) → `graperank + * probe` → full `graperank crawl`. + */ + private suspend fun probe( + dataDir: DataDir, + rest: Array, + ): Int { + val args = Args(rest) + val timeoutMs = args.longFlag("timeout", 15L) * 1000 + val waveSize = args.intFlag("concurrency", Context.defaultPreconnectCap) + + Context.openOrAnonymous(dataDir).use { ctx -> + ctx.prepare() + val cached = ctx.reachability.snapshot() + val universe = RelayProber.knownRelayUniverse(ctx.store) + cached.live + cached.dead + if (universe.isEmpty()) { + Output.emit( + linkedMapOf( + "probed" to 0, + "note" to "no relays known locally — run `amy graperank crawl` first to gather kind:10002 relay lists", + ), + ) + return 0 + } + + System.err.println( + "[relay-probe] probing ${universe.size} relays in waves of $waveSize " + + "(${timeoutMs / 1000}s per wave; open-files limit ${Context.maxFileDescriptors})", + ) + val result = + RelayProber(ctx.client) { System.err.println(it) } + .probe(universe, timeoutMs, waveSize) + + ctx.reachability.recordProbed(result.reachableRttMs(), result.deadRelays()) + + val rtts = + result.reachable + .map { it.rttOpenMs } + .filter { it >= 0 } + .sorted() + + fun pct(p: Int): Long? = if (rtts.isEmpty()) null else rtts[(rtts.size - 1) * p / 100] + val slowest = + result.reachable + .filter { it.rttOpenMs >= 0 } + .sortedByDescending { it.rttOpenMs } + .take(10) + .map { mapOf("relay" to it.relay.url, "rtt_open_ms" to it.rttOpenMs) } + val authWalled = result.reachable.count { it.error?.startsWith("closed:") == true } + + Output.emit( + linkedMapOf( + "probed" to result.verdicts.size, + "reachable" to result.reachable.size, + "dead" to result.dead.size, + "closed_by_policy" to authWalled, + "elapsed_ms" to result.elapsedMs, + "rtt_open_p50_ms" to pct(50), + "rtt_open_p90_ms" to pct(90), + "rtt_open_p99_ms" to pct(99), + "slowest" to slowest, + ), + ) + } + return 0 + } + /** * `amy graperank update [flags]` — refresh every locally-known author's WoT * record kinds (0 / 3 / 10002 / 1984) straight from their own outbox, so the @@ -586,7 +703,7 @@ object GrapeRankCommand { // Live author-advertised relays are always synced (--no-reachability-cache // to reconcile every relay regardless). val knownDead = - if (args.bool("no-reachability-cache")) emptySet() else ctx.reachability.snapshot().dead + if (args.bool(NO_REACHABILITY_CACHE_FLAG)) emptySet() else ctx.reachability.snapshot().dead val updater = GrapeRankUpdater( diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PostCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PostCommand.kt index b94ead611d..d26283d735 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PostCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PostCommand.kt @@ -81,13 +81,14 @@ object PostCommand { var powMillis: Long? = null val readyToSign = if (powTarget != null) { - System.err.println("mining $powTarget bits…") + val threads = Runtime.getRuntime().availableProcessors().coerceAtLeast(1) + System.err.println("mining $powTarget bits… ($threads threads)") val deadlineNanos = powTimeoutSec?.let { System.nanoTime() + it * 1_000_000_000L } val startedAt = System.nanoTime() val mined = try { withContext(Dispatchers.Default) { - PoWMiner.run(template, ctx.signer.pubKey, powTarget) { + PoWMiner.mine(template, ctx.signer.pubKey, powTarget, threads) { deadlineNanos == null || System.nanoTime() < deadlineNanos } } diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PowCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PowCommands.kt index bc6a1e73bb..293b3d7428 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PowCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PowCommands.kt @@ -108,13 +108,18 @@ object PowCommands { rest: Array, ): Int { val args = Args(rest) - val usage = "pow mine --target N [--pubkey HEX] [--timeout SECS] " + val usage = "pow mine --target N [--pubkey HEX] [--timeout SECS] [--threads N] " val target = args.flags["target"]?.toIntOrNull() ?: return Output.error("bad_args", usage) if (target < 1 || target > MAX_DIFFICULTY) { return Output.error("bad_args", "--target must be between 1 and $MAX_DIFFICULTY") } + val threads = args.intFlag("threads", defaultThreads()) + if (threads < 1) { + return Output.error("bad_args", "--threads must be >= 1") + } + val json = readPayload(args.positional.toTypedArray()) ?: return Output.error("bad_args", usage) val template = try { @@ -142,13 +147,13 @@ object PowCommands { val timeoutSec = args.flags["timeout"]?.toLongOrNull() val deadlineNanos = timeoutSec?.let { System.nanoTime() + it * 1_000_000_000L } - System.err.println("mining $target bits for ${pubKey.take(8)}…") + System.err.println("mining $target bits for ${pubKey.take(8)}… ($threads threads)") val startedAt = System.nanoTime() val mined = try { withContext(Dispatchers.Default) { - PoWMiner.run(template, pubKey, target) { + PoWMiner.mine(template, pubKey, target, threads) { deadlineNanos == null || System.nanoTime() < deadlineNanos } } @@ -176,27 +181,38 @@ object PowCommands { "pow" to PoWRankEvaluator.calculatePowRankOf(id), "pow_target" to target, "pow_millis" to elapsedMs, + "threads" to threads, "template_json" to mined.toJson(), ), ) return 0 } - /** `amy pow bench` — hash rate + expected mining time per common target. */ + /** + * `amy pow bench` — single-core and all-cores hash rate, plus expected + * mining time per common target at the all-cores rate (what `pow mine` + * uses by default). + */ private suspend fun bench(): Int { - val rate = PoWEstimator.hashesPerSecond() + val threads = defaultThreads() + val singleRate = PoWEstimator.hashesPerSecond() + val parallelRate = PoWEstimator.hashesPerSecond(threads) Output.emit( mapOf( - "hashes_per_second" to rate.roundToLong(), + "hashes_per_second" to parallelRate.roundToLong(), + "hashes_per_second_single_core" to singleRate.roundToLong(), + "threads" to threads, "expected_seconds" to listOf(16, 20, 24, 28).associate { bits -> - bits.toString() to PoWEstimator.estimateSeconds(bits, rate) + bits.toString() to PoWEstimator.estimateSeconds(bits, parallelRate) }, ), ) return 0 } + private fun defaultThreads(): Int = Runtime.getRuntime().availableProcessors().coerceAtLeast(1) + private fun readPayload(rest: Array): String? { val arg = rest.firstOrNull() ?: return null val payload = if (arg == "-") System.`in`.readBytes().decodeToString() else arg diff --git a/commons/build.gradle.kts b/commons/build.gradle.kts index 8182a3604b..b71358533c 100644 --- a/commons/build.gradle.kts +++ b/commons/build.gradle.kts @@ -25,6 +25,7 @@ plugins { alias(libs.plugins.jetbrainsComposeCompiler) alias(libs.plugins.composeMultiplatform) alias(libs.plugins.serialization) + alias(libs.plugins.kotlinxKover) } kotlin { diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayauth/RelayAuthPermissionStore.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayauth/RelayAuthPermissionStore.kt index 3b33a0666f..68dbb5d6cf 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayauth/RelayAuthPermissionStore.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayauth/RelayAuthPermissionStore.kt @@ -50,7 +50,9 @@ interface RelayAuthPermissionStore { suspend fun recordUse( relayUrl: String, additions: Map>, - ) {} + ) { + // no-op by default: stores that don't track rationale ignore recorded uses. + } /** The accumulated grant rationale for [relayUrl] (purpose → counterparty pubkeys). */ suspend fun loadRationale(relayUrl: String): Map> = emptyMap() @@ -59,7 +61,9 @@ interface RelayAuthPermissionStore { suspend fun allRationales(): Map>> = emptyMap() /** Forgets the accumulated grant rationale for [relayUrl] (does not touch the ALLOW/DENY override). */ - suspend fun clearRationale(relayUrl: String) {} + suspend fun clearRationale(relayUrl: String) { + // no-op by default: stores that don't track rationale have nothing to clear. + } /** Epoch-second timestamp of the last time each relay was authenticated with (for display). */ suspend fun allLastUsed(): Map = emptyMap() diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWEstimator.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWEstimator.kt index e12843c5fc..5caef9b9d2 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWEstimator.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWEstimator.kt @@ -20,9 +20,12 @@ */ package com.vitorpamplona.amethyst.commons.service.pow -import com.vitorpamplona.quartz.utils.sha256.sha256 +import com.vitorpamplona.quartz.utils.sha256.sha256Into import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext @@ -48,18 +51,41 @@ object PoWEstimator { private const val BATCH = 2_000 private val BENCH_DURATION = 250.milliseconds - private var cachedRate: Double? = null + // cached per worker count; guarded by benchLock (single-flight so + // concurrent first callers — e.g. the settings screen recomposing while + // the composer chip opens — share one ~250 ms benchmark instead of each + // burning the cores). + private val cachedRates = mutableMapOf() private val benchLock = Mutex() - suspend fun hashesPerSecond(dispatcher: CoroutineDispatcher = Dispatchers.Default): Double = - cachedRate ?: withContext(dispatcher) { - // single-flight: concurrent first callers (e.g. the settings screen - // recomposing while the composer chip opens) share one ~250 ms - // benchmark instead of each burning a core. - benchLock.withLock { - cachedRate ?: benchmark().also { cachedRate = it } + suspend fun hashesPerSecond(dispatcher: CoroutineDispatcher = Dispatchers.Default): Double = hashesPerSecond(1, dispatcher) + + /** + * Aggregate hash rate with [workers] concurrent miners — what + * [com.vitorpamplona.quartz.nip13Pow.miner.PoWMiner.mine] achieves when + * racing that many workers. Each worker runs its own ~250 ms benchmark + * loop concurrently and the rates are summed, so contention between + * cores is priced in. + */ + suspend fun hashesPerSecond( + workers: Int, + dispatcher: CoroutineDispatcher = Dispatchers.Default, + ): Double { + val count = workers.coerceAtLeast(1) + benchLock.withLock { + cachedRates[count]?.let { return it } + return withContext(dispatcher) { + val rate = + if (count == 1) { + benchmark() + } else { + coroutineScope { List(count) { async { benchmark() } }.awaitAll().sum() } + } + cachedRates[count] = rate + rate } } + } fun estimateSeconds( difficulty: Int, @@ -68,14 +94,16 @@ object PoWEstimator { private fun benchmark(): Double { val payload = ByteArray(PAYLOAD_BYTES) { (it % 251).toByte() } + // same allocation-free hashing the miner's hot loop uses + val out = ByteArray(32) // warm up JIT/caches so the measured window reflects steady state - repeat(3 * BATCH) { sha256(payload) } + repeat(3 * BATCH) { sha256Into(out, payload, payload.size) } val mark = TimeSource.Monotonic.markNow() var count = 0L while (mark.elapsedNow() < BENCH_DURATION) { - repeat(BATCH) { sha256(payload) } + repeat(BATCH) { sha256Into(out, payload, payload.size) } count += BATCH } return count / mark.elapsedNow().toDouble(DurationUnit.SECONDS) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWPolicy.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWPolicy.kt index 158ccb7adb..d4c0da8079 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWPolicy.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWPolicy.kt @@ -86,6 +86,13 @@ object PoWPolicy { */ const val MAX_DIFFICULTY = 40 + /** + * How many parallel miner workers one mining job should use on a device + * with [availableProcessors] cores: half of them, so a running miner never + * starves the UI/render threads or the relay client. At least 1. + */ + fun minerWorkers(availableProcessors: Int): Int = (availableProcessors / 2).coerceAtLeast(1) + private const val LONG_FORM_DRAFT_KIND = 30024 /** NIP-51 sets and other settings-like addressable kinds. */ diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWPublishQueue.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWPublishQueue.kt index 4f69808bea..d590b9e344 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWPublishQueue.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWPublishQueue.kt @@ -96,7 +96,10 @@ data class PoWJobFailure( * to the normal sign+broadcast continuation captured at enqueue time. * * FIFO with at most [maxConcurrent] concurrent miners so a burst of posts - * queues up instead of spawning unbounded CPU work. Jobs are cancellable + * queues up instead of spawning unbounded CPU work. Each job's nonce search + * runs on [minerThreads] parallel workers (see PoWMiner.mine) — callers that + * bring their own mine lambda via [enqueueStaged]/[enqueueWork] should reuse + * this value to stay inside the queue's CPU budget. Jobs are cancellable * while QUEUED or MINING; once PUBLISHING starts, cancel is a no-op. * * Template jobs enqueued with a [PersistedPoWJob] record are checkpointed to @@ -115,6 +118,7 @@ data class PoWJobFailure( class PoWPublishQueue( private val scope: CoroutineScope, maxConcurrent: Int = 1, + val minerThreads: Int = 1, miningDispatcher: CoroutineDispatcher = Dispatchers.Default, private val persistence: PoWJobPersistence? = null, private val onQueueActive: () -> Unit = {}, @@ -198,7 +202,7 @@ class PoWPublishQueue( } else { template } - PoWMiner.run(toMine, pubKey, difficulty, isActive) + PoWMiner.mine(toMine, pubKey, difficulty, minerThreads, isActive) }, publish = onMined, ) diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWPolicyTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWPolicyTest.kt index d1822331fa..db53f1570d 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWPolicyTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWPolicyTest.kt @@ -28,6 +28,16 @@ import kotlin.test.assertTrue class PoWPolicyTest { val allCategories = PoWCategory.entries.toSet() + @Test + fun minerWorkersIsHalfTheCoresButNeverZero() { + assertEquals(1, PoWPolicy.minerWorkers(1)) + assertEquals(1, PoWPolicy.minerWorkers(2)) + assertEquals(2, PoWPolicy.minerWorkers(4)) + assertEquals(3, PoWPolicy.minerWorkers(6)) + assertEquals(4, PoWPolicy.minerWorkers(8)) + assertEquals(1, PoWPolicy.minerWorkers(0)) + } + @Test fun difficultyOffMinesNothing() { assertNull(PoWPolicy.shouldMine(1, 0, allCategories)) diff --git a/desktopApp/build.gradle.kts b/desktopApp/build.gradle.kts index 10ca2b3d48..f49cad7ba2 100644 --- a/desktopApp/build.gradle.kts +++ b/desktopApp/build.gradle.kts @@ -6,6 +6,7 @@ plugins { alias(libs.plugins.jetbrainsKotlinJvm) alias(libs.plugins.composeMultiplatform) alias(libs.plugins.jetbrainsComposeCompiler) + alias(libs.plugins.kotlinxKover) } // RPM rejects dashes in version strings — replace with tilde (~) which RPM uses diff --git a/docs/changelog/translators.json b/docs/changelog/translators.json index 15f1e60516..88e2ab67dc 100644 --- a/docs/changelog/translators.json +++ b/docs/changelog/translators.json @@ -135,9 +135,15 @@ ] }, { - "user": "crowdin.pretended462", + "user": "summoner001", "languages": [ - "German" + "Hungarian" + ] + }, + { + "user": "hypnotichemionus4", + "languages": [ + "Chinese Simplified" ] }, { @@ -153,15 +159,9 @@ ] }, { - "user": "summoner001", + "user": "crowdin.pretended462", "languages": [ - "Hungarian" - ] - }, - { - "user": "hypnotichemionus4", - "languages": [ - "Chinese Simplified" + "German" ] }, { diff --git a/geode/build.gradle.kts b/geode/build.gradle.kts index ee636965c7..46e9f1cf23 100644 --- a/geode/build.gradle.kts +++ b/geode/build.gradle.kts @@ -5,6 +5,7 @@ plugins { alias(libs.plugins.serialization) application `java-test-fixtures` + alias(libs.plugins.kotlinxKover) } application { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 3d768731a6..cc1cf2bad1 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -35,6 +35,7 @@ kmpTorResource = "409.5.0" junit = "4.13.2" kchesslib = "1.0.5" kotlin = "2.4.0" +kover = "0.9.8" kotlinxCollectionsImmutable = "0.5.0" kotlinxCoroutinesCore = "1.11.0" kotlinxSerialization = "1.11.0" @@ -246,6 +247,7 @@ jetbrainsKotlinJvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } jetbrainsComposeCompiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } serialization = { id = 'org.jetbrains.kotlin.plugin.serialization', version.ref = 'kotlin' } kotlinMultiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } +kotlinxKover = { id = "org.jetbrains.kotlinx.kover", version.ref = "kover" } androidKotlinMultiplatformLibrary = { id = "com.android.kotlin.multiplatform.library", version.ref = "agp" } vanniktech-mavenPublish = { id = "com.vanniktech.maven.publish", version.ref = "mavenPublish" } composeMultiplatform = { id = "org.jetbrains.compose", version.ref = "composeMultiplatform" } diff --git a/nestsClient/build.gradle.kts b/nestsClient/build.gradle.kts index 3cb05ae465..bb5acd43a2 100644 --- a/nestsClient/build.gradle.kts +++ b/nestsClient/build.gradle.kts @@ -5,6 +5,7 @@ plugins { alias(libs.plugins.kotlinMultiplatform) alias(libs.plugins.androidKotlinMultiplatformLibrary) alias(libs.plugins.serialization) + alias(libs.plugins.kotlinxKover) } kotlin { diff --git a/quartz/build.gradle.kts b/quartz/build.gradle.kts index 10c0817ee8..108587341f 100644 --- a/quartz/build.gradle.kts +++ b/quartz/build.gradle.kts @@ -9,6 +9,7 @@ plugins { alias(libs.plugins.androidKotlinMultiplatformLibrary) alias(libs.plugins.serialization) alias(libs.plugins.vanniktech.mavenPublish) + alias(libs.plugins.kotlinxKover) } kotlin { diff --git a/quartz/plans/2026-07-10-graperank-connect-storm.md b/quartz/plans/2026-07-10-graperank-connect-storm.md new file mode 100644 index 0000000000..600cb8d684 --- /dev/null +++ b/quartz/plans/2026-07-10-graperank-connect-storm.md @@ -0,0 +1,176 @@ +# GrapeRank crawl: connect once, wait once (connection storm) + +## Problem + +A from-scratch `amy graperank crawl` spends most of its wall clock **waiting to +connect** to relays, not downloading. Three compounding causes, all on our side: + +1. **Sockets are torn down between drains.** `NostrClient` reconciles the relay + pool against the union of *active* subscriptions (`RelayPool.updatePool`, + sampled every 300ms). A Phase-B drain unit unsubscribes when it finishes, so + an outbox relay with no other in-flight drain is **disconnected ~300ms + later** — and the next round re-pays DNS + TCP + TLS + WS-upgrade for the + same relay. Only the top-20 "warm pool" relays kept a do-nothing + subscription open. A crawl touches ~10–15k distinct relay URLs; most get + dialed many times. +2. **Handshake concurrency was capped at 256** (OkHttp `Dispatcher.maxRequests`; + the OkHttp default is 64). Every WS upgrade is an async call holding a + dispatcher slot for its whole DNS + TCP + TLS + upgrade; a dead relay holds + a slot for the full 7s `connectTimeout`. At 256 slots, dialing 5k relays + where half are dead is ~20 serialized waves — minutes of pure waiting that + could be one parallel wait. +3. **DNS is blocking, uncached and duplicated.** OkHttp's `Dns.SYSTEM` calls + `InetAddress.getAllByName` (glibc `getaddrinfo`) on the dispatcher thread. + The JVM's own cache is ~30s positive / ~10s negative — useless across a + 30-minute crawl that re-dials the same hosts every round. Worse, the outbox + model mints hundreds of per-user *URLs* on one *host* + (`filter.nostr.wine/npubA`, `/npubB`, …) and each URL is a fresh lookup. A + dead domain is the worst case: resolver timeouts (5s × 2 attempts per + nameserver) can hold a thread 10–30s, and the crawl re-resolves it from + every straggler's outbox, every round. + +## Single-server limits (what bounds "1000s of connections at once") + +Measured/derived for a JVM CLI on one Linux box: + +| Resource | Limit | Consequence | +|---|---|---| +| File descriptors | `ulimit -n` soft (often 1024–4096; systemd default 1024 soft / 512k+ hard) | 1 FD per socket. The JVM cannot raise its own rlimit — detect via `UnixOperatingSystemMXBean` and size the storm to fit; advise `ulimit -n 16384` for big crawls. | +| Ephemeral ports | ~28k (`ip_local_port_range`) **per destination tuple** | Non-issue: connections go to thousands of *distinct* relays, each gets its own tuple space. | +| Concurrent TCP connects | Kernel: effectively unbounded (a pending SYN is just a socket in `SYN_SENT`) | Dead-IP dials cost nothing in CPU but hold their FD + dispatcher slot for the connect timeout. Parallelism is the fix; refused/reset fails in 1 RTT. | +| DNS | glibc `getaddrinfo` is blocking, ~1 thread per in-flight lookup; dead domains 10–30s | Cache positives AND negatives in-process; dedupe concurrent lookups for the same host (path-URL explosion). | +| Threads | 1 platform thread per in-flight handshake (OkHttp) | 1–2k concurrent handshakes ≈ 1–2k transient threads — fine on a 64-bit JVM (~stack is virtual), but don't go to 10k. | +| TLS | CPU per handshake (~1ms) | Negligible next to network RTTs; OkHttp shares an SSL session cache per client. | +| Middleboxes | Home-router NAT/conntrack tables (~4–16k entries) | Operational caveat for residential operators; server/VPS operators unaffected (`nf_conntrack_max` default ~256k). | + +Conclusion: with FDs raised and DNS cached, a single server comfortably opens +**a few thousand concurrent connections**; the binding constraints are the FD +soft limit and the dispatcher cap, both of which we now size/raise explicitly. + +## Design + +Three changes, all keeping completeness identical (same relays asked the same +filters — only *when* sockets open changes): + +1. **Crawl-wide warm pool = mass pre-connect.** The existing warm-pool trick + (a never-matching `ids` filter that EOSEs instantly and just holds the + socket) is extended from the top-20 relays to **every candidate relay we + know** — seeded at crawl start from the NIP-66 reachability cache's live + set, and refreshed every round with the relays learned from kind:10002s — + capped by `preconnectCap` (FD-budget-aware, busiest relays first, dead + relays excluded). Connections open in one parallel storm at start (and in + the background as new relays are learned), stay up for the whole crawl, and + every drain hits an already-open socket. This also implements the "crawl + once with --max-hops to save the relay list" flow: the first (even shallow) + crawl populates the store with 10002s and the reachability cache with + live/dead verdicts; the next crawl pre-connects that whole universe and + waits once. +2. **Transport ceilings raised (CLI).** `Dispatcher.maxRequests` 256 → 1024 + (per-host stays 16 — politeness to path-multiplexed hosts is a *server* + property), and the app's `SurgeDns` (moved to quartz; stale-while-revalidate, + single-flight, 10-min negative TTL, persisted under `~/.amy/shared/`) + replaces `Dns.SYSTEM`. FD budget is detected at startup and sizes both the + dispatcher and the default `preconnectCap`. Caveat: in a proxied environment + (HTTP CONNECT), OkHttp sends the hostname to the proxy and never consults + the client-side resolver — so the DNS layer only pays off on direct-connect + deployments, and none of the A/B numbers below depend on it. +3. **`amy graperank probe` — the relay census.** Reads the full known relay + universe (every relay in stored kind:10002s + everything in the + reachability cache), mass-connects it in waves with a never-matching REQ, + and records per-relay verdicts with **real** `rtt-open` into the NIP-66 + store: connected → live (with measured RTT, however slow), cannot-connect / + never-completed → dead (TTL'd, re-probed after expiry). Separates + "working but slow" (kept, given the crawler's patient park path) from + "not working" (skipped entirely) without burning crawl time on the + distinction. + +## Measured A/B results (cold hop-3 crawl, fresh store per leg, same observer) + +Sequential single-variable legs on the same 4-core / 4096-FD box. Completeness +gate: `contact_lists_fed` and `users_discovered` must not drop (±2% network +noise between runs is normal — relays come and go). + +| leg | wall | lists fed | users found | verdict | +|---|---|---|---|---| +| baseline (all connect-storm work, 24 workers) | 638s | 18,519 | 165k | reference | +| + background parks don't gate convergence | 579s | 18,829 | 167k | **keep** | +| + `--drain-concurrency 48` | 559s | 18,817 | 169k | keep (confirmed below) | +| + `--timeout 5` (shorter fast window) | 735s | 18,524 | 164k | **reject** — more parks/retries | +| + sharded-sweep dedup | 675s* | 18,801 | 168k | keep — round wall ↓ (301s vs 336s); total noise | +| + overlap tail (no convergence park-wait, agg ∥ deletions) | 474s | 18,816 | 167k | **keep** | +| + `--drain-concurrency 48` on top | **396s** | 18,816 | 167k | **keep → new default** | +| + per-relay sub cap 16→32 (`AMY_RELAY_SUB_CAP`) | 602s | 18,478 | 164k | **reject** — 27 relays demoted us + 7 rate-limited (vs 5+1 at 16); the Phase-B plateau is relay-side service rate, not client permits | + +### Reproducibility + the recovery fixpoint + +A cold hop-11 rerun matched hops 1–4 within ~3% but lost the entire hop-6/7 +cluster (394k vs 621k discovered at −7.5% lists): one weak backbone hour at +round 5 shifted ~30k lists from in-round arrival to the terminal aggregator +pass, whose discoveries were never crawled. Fixed by iterating rounds + +recovery to a fixpoint (recovery reveals in-budget pending → resume rounds → +recover again). Validation, cold hop-8 on the fixpoint build: **796,459 +discovered / 253,213 lists** — the deepest crawl of the series (hop 5–8 +populations 107k/167k/131k/72k vs 67k/110k/127k/2k before); the fixpoint fired +live (+25 recovered lists → 1 pending → extra round → +0 → converge). Known +follow-ups: each fixpoint iteration re-asks aggregators for the FULL straggler +set (~4 min per pass at 470k stragglers — dedupe to newly-added stragglers), +and the retry rounds' backbone re-fan remains the dominant tail cost (rounds +8–10 ≈ 80 min for +6.6k lists). + +Final validation — **cold hop-5, fresh store, winning stack** (the workload that +never completed pre-fixes): **85.4 min, 203,903 contact lists, 391,549 users +discovered, 626,599 events, 5,794 relays**. Per-hop list completeness: 90% +(hop 2), 71% (hop 3), 56% (hop 4). Remaining known lever for a future pass: +attempt>0 retries re-fan every user to the whole backbone inside per-batch +units, so the two retry rounds cost nearly as much as first passes at a +fraction of the yield — route retry backbone re-asks as round-wide chunked +per-relay units instead. + +### Retry routing: the round-wide communal sweep (Phase C) + +That "remaining lever" landed as a restructure of Phase B's shared-relay +coverage. Before: every 256-user batch with retry (attempt>0) or no-outbox +users fanned the SAME ~40 backbone+fallback relays, queueing thousands of +small REQs against their 16-permit gates — measured as rounds 8–10 of a hop-8 +crawl burning ~80 min for +6.6k lists, nearly all hot-relay head-of-line +blocking. After: `routeByOutbox` routes only a user's OWN-signal relays +(write relays; harvested hints when there's no outbox) and flags +communal-coverage users into a per-round set; a new **Phase C** +(`communalSweep`) asks the shared relays ONCE per round for all flagged users +in author-chunked units — the same (relay, user) pairs, ~8x fewer REQs — and +records answered-empty pairs so each subsequent round shrinks. Attempt +accounting moves to Phase C (a user burns an attempt only after both its own +relays and the shared set missed that round), `shardedSweep` now records +answered-empty pairs too, and each recovery-fixpoint iteration only asks the +aggregators for stragglers ADDED since the previous pass (a repeat full-set +pass cost ~4 min at 470k stragglers for zero new answers). + +Validation, cold hop-8, same observer (reference = the fixpoint build's +232.6 min / 253,213 lists / 796,459 users): **160.7 min, 292,768 lists, +1,169,194 users discovered, 7,240 relays** — 31% less wall clock with +15.6% +lists and +47% users. The depth cliff moved out two hops: hop-7/8 populations +319,878/292,268 vs 131,051/72,389, hop-7 lists 29,445 vs 6,117. Round-level +profile: the retry rounds that defined the old tail now run in minutes +(round 8: 247s total, of which Phase C fed 5,695 of its 6,080 lists), and +Phase C itself costs 10–170s per round, folded out of the per-batch hot path. + +\* total inflated by a slow round-3 network sample + the then-serial tail; the +change's own effect (straggler-round Phase A 20s → 20ms) is visible directly. + +Not measurable in these legs but fixed on the way (validated separately): +- the whole crawl ran on `runBlocking`'s single-threaded event loop — thousands + of Phase-B unit coroutines starved one core while three idled, inflating + batch walls ~5x (`crawl()` now hops to `Dispatchers.IO`); this also + invalidated the old "64 workers is 2x slower" A/B; +- per-user store point queries (~8ms each on a multi-GB store) in + harvestFromStore / sweep / consumer / aggregator paths — now one chunked + author query per 300 users (hop-3 replay Phase A: 176s → 36s at identical + fed counts). + +## Non-goals + +- No change to REQ concurrency per relay (`AdaptiveRelayLimiter`) or the + drain worker count — the 64-worker A/B showed the *REQ* fan-out re-floods + busy hubs; this work only parallelizes and amortizes *connection setup*. +- No async-DNS library dependency; cached blocking lookups on OkHttp's + existing threads are sufficient once deduped and negative-cached. diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankCrawler.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankCrawler.kt index 63ebda32bf..70ef709b72 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankCrawler.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankCrawler.kt @@ -44,6 +44,9 @@ import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap import com.vitorpamplona.quartz.utils.concurrent.ConcurrentSet import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO +import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll @@ -57,8 +60,11 @@ import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.selects.select import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeoutOrNull +import kotlin.concurrent.Volatile import kotlin.concurrent.atomics.AtomicLong +import kotlin.concurrent.atomics.AtomicReference import kotlin.concurrent.atomics.ExperimentalAtomicApi import kotlin.coroutines.coroutineContext import kotlin.time.TimeSource @@ -107,6 +113,18 @@ class GrapeRankCrawler( private val insertNanos = AtomicLong(0) private val eventsStored = AtomicLong(0) + // Store-path attribution the summed [insertNanos] can't give: [insertNanos] + // counts every concurrent caller's wall INCLUDING time queued on the store's + // single-writer mutex, so at high fan-out it wildly overstates write cost. + // Batches that started with no other insert in flight ([insertInFlight] == 1) + // approximate the UNCONTENDED write rate; reads get their own count + wall so + // a point-query pathology (thousands of tiny queries) is visible at a glance. + private val insertInFlight = AtomicLong(0) + private val insertSoloNanos = AtomicLong(0) + private val insertSoloEvents = AtomicLong(0) + private val queryCount = AtomicLong(0) + private val queryNanos = AtomicLong(0) + /** * Relay policy + crawl bounds. The relay sets come from the caller because the * aggregator/bootstrap defaults live outside quartz. @@ -144,10 +162,12 @@ class GrapeRankCrawler( * per-transaction + writer-mutex cost across the batch (coerced to `>= 1`). * @param drainConcurrency how many outbox batches drain at once (the worker * pool size). A GLOBAL bound (memory / open sockets); the per-relay - * concurrent-sub cap is enforced separately by [AdaptiveRelayLimiter]. Keep it - * moderate: a higher global fan-out re-floods busy hubs faster than demotion - * catches up (an A/B at 64 ran ~2x slower with more dead relays), so 24 is the - * validated default and raising it is a probe, not a speedup. + * concurrent-sub cap is enforced separately by [AdaptiveRelayLimiter]. An + * early A/B that made 64 look ~2x slower than 24 was an artifact of the crawl + * running on a single-threaded event loop (more coroutines on one starved + * thread); re-run on the multithreaded dispatcher, 48 beat 24 twice at + * identical completeness (hop-3 cold: 579s→559s and 474s→396s), so 48 is the + * validated default. * @param timeoutEvictStrikes evict a relay after this many drains that timed out * (connect timeout or park idle-cut) having delivered NOTHING. Unlike * [classifyDrainFailure] — which never marks a timeout dead, since one slow @@ -166,7 +186,7 @@ class GrapeRankCrawler( val parkTimeoutMs: Long = 40_000, val diagnose: Boolean = false, val insertBatchSize: Int = 500, - val drainConcurrency: Int = 24, + val drainConcurrency: Int = 48, val timeoutEvictStrikes: Int = 3, /** * Also skip proven-dead relays in the kind:10002 discovery sweep @@ -208,6 +228,27 @@ class GrapeRankCrawler( * permanently ignores an author's advertised home. See RelayReachabilityStore. */ val knownDeadRelays: Set = emptySet(), + /** + * Relays a prior run (or another monitor) proved REACHABLE within the + * reachability cache's TTL. Seeded into the crawl-wide warm pool at start, so + * their DNS + TCP + TLS + WS-upgrade cost is paid ONCE, in one parallel + * connection storm, before the first round — instead of serially inside each + * drain that first routes to them. See [preconnectCap]. + */ + val knownLiveRelays: Set = emptySet(), + /** + * Cap on the crawl-wide warm pool — the mass pre-connect. The client tears a + * relay's socket down ~300ms after its last subscription closes, so without a + * warm hold every round re-pays the full connect latency per outbox relay. + * The warm pool keeps a do-nothing subscription (never-matching filter, EOSEs + * instantly, streams nothing) open to up to this many relays for the whole + * crawl: busiest proven relays first, then next-round outbox candidates, then + * [knownLiveRelays]. Each warm relay holds one socket (one file descriptor) + * and one dormant sub — size against the process FD budget, and keep it above + * the transport's concurrent-handshake cap only if FDs allow. `<= 0` falls + * back to warming just the busiest [WARM_POOL_SIZE] relays (the old behavior). + */ + val preconnectCap: Int = 2500, ) /** What the crawl fetched — the counters the caller reports and the graph is built from. */ @@ -257,7 +298,20 @@ class GrapeRankCrawler( verifyNanos.store(0) insertNanos.store(0) eventsStored.store(0) - return CrawlRun(observer, builder).run() + insertInFlight.store(0) + insertSoloNanos.store(0) + insertSoloEvents.store(0) + queryCount.store(0) + queryNanos.store(0) + // Hop off the caller's dispatcher: a CLI calls this from runBlocking's + // SINGLE-THREADED event loop, and Phase B runs thousands of concurrent + // drain-unit coroutines (each with a timeout timer, a channel, REQ JSON + // encoding, signature verifies and store writes). On one thread those + // timers fire late and batch walls inflate ~5x — measured 111 users/s + // with the lone thread pegged at 100% while 3 cores idled. IO (not + // Default): the store's blocking SQLite calls must not starve the + // small cores-sized pool. + return withContext(Dispatchers.IO) { CrawlRun(observer, builder).run() } } /** @@ -279,6 +333,15 @@ class GrapeRankCrawler( val hopOf = hashMapOf(observer to 0) val done = hashSetOf() + // Users already offered to the sharded backbone sweep. The top-relay set and + // a user's answer don't change between adjacent rounds, so re-sweeping the + // same straggler set every round only re-pays the rotation barriers + // (measured: ~20s of Phase A per straggler round for ~zero yield). New users + // (frontier growth) still sweep on their first pending round; the outbox, + // broadcast and aggregator paths keep covering the swept-but-missing rest. + // Single-writer: only the round loop's [shardedSweep] touches it. + val shardSwept = hashSetOf() + // Users we've already run the STATIC-relay outbox-discovery sweep for (the // indexer set in [ensureRelayLists]). That relay set never changes, so re-asking // it for the same never-had-a-10002 user each round it recirculates is pure waste @@ -303,6 +366,17 @@ class GrapeRankCrawler( // future adaptive controller reads to size filters / cap / strangle per relay. val telemetry = RelayTelemetry() + // Per-round Phase-B batch-wall distribution (diagnose only): reset by the + // round loop, recorded by the drain workers, dumped after each round. + val batchWalls = BatchWallStats() + + // Phase-B churn the done-counter can't see: users VISITED (every batch member + // processed by the consumer, found or not) and batches completed. The ticker + // derives rates from these so a retry round over list-less users reads as + // "working, zero yield" instead of "stalled". + val usersVisited = AtomicLong(0) + val batchesDone = AtomicLong(0) + // Concurrent: touched by more than one of producer/consumer/drain-workers. val relayHints = ConcurrentMap>() val attempts = ConcurrentMap() @@ -312,6 +386,10 @@ class GrapeRankCrawler( // more as it goes and flushes the union back at the end. val deadRelays = ConcurrentSet().apply { config.knownDeadRelays.forEach { add(it) } } + // --diagnose [slow-relay] lines emitted per authority, for the suppression cap + // in logSlow (the telemetry table keeps the complete per-relay counts). + val slowLogCounts = ConcurrentMap() + // Unproductive-TIMEOUT strikes, keyed by relay AUTHORITY (host[:port]), not the // full URL. [classifyDrainFailure] treats a READ timeout or a park idle-cut — the // relay answered the handshake but is slow — as "busy, retry" and never dead, @@ -356,6 +434,14 @@ class GrapeRankCrawler( // ones that timed out — which unlike a clean EOSE might just be slow). val askedEmpty = ConcurrentMap>() + // Stragglers already asked of the content aggregators, so each fixpoint + // iteration of [recoverStragglersFromAggregators] only queries the users + // ADDED since the last pass. Without this every iteration re-asked the full + // straggler set (~4 min of store scans + aggregator REQs per pass at 470k + // stragglers) for zero new answers — the aggregators already EOSE'd on them. + // Single-writer: only the terminal recovery pass (round loop context) touches it. + val aggregatorAsked = HashSet() + // Contact lists delivered LATE by parked (slow-but-alive) relays. A parked // unit persists its events, then pushes any kind:3 it found here; the round // loop (the single graph-writer) folds these into hopOf/done/builder between @@ -484,6 +570,41 @@ class GrapeRankCrawler( .map { it.key } .toList() + /** + * Refresh the crawl-wide warm pool — the mass pre-connect. Re-subscribing the + * same [WARM_SUB_ID] with a new relay set just updates the desired-relay set: + * relays already warm stay connected, new ones start their handshake NOW (in + * parallel, in the background), so by the time a drain routes to them the + * socket is already open. Priority under [Config.preconnectCap]: busiest + * proven-live relays, then advertised-but-not-yet-contacted outboxes (exactly + * the relays the next rounds will dial), then the prior run's known-live + * universe. Dead relays are always excluded so the pool never redials them. + */ + fun refreshWarmPool() { + val massCap = config.preconnectCap + val cap = if (massCap > 0) massCap else WARM_POOL_SIZE + val warm = LinkedHashSet(cap * 2) + warm.addAll(topLiveRelays(cap)) + if (massCap > 0) { + if (warm.size < cap) { + // Advertised write relays not yet proven live — next drains' targets. + val advertised = writeRelayFreq.snapshot().entries.sortedByDescending { it.value } + for ((relay, _) in advertised) { + if (warm.size >= cap) break + if (!isDead(relay)) warm.add(relay) + } + } + if (warm.size < cap) { + for (relay in config.knownLiveRelays) { + if (warm.size >= cap) break + if (!isDead(relay)) warm.add(relay) + } + } + } + if (warm.isEmpty()) return + client.subscribe(WARM_SUB_ID, warm.associateWith { WARM_FILTERS }, null) + } + /** * Background reachability culler. Cheaply TCP-probes the relays we've learned — * COLD TAIL FIRST — and drops the unreachable ones into [deadHosts] so the WS @@ -574,17 +695,13 @@ class GrapeRankCrawler( * whole still-missing set. Returns the count newly fed. */ suspend fun harvest(events: List>): Int { - var got = 0 + val authors = LinkedHashSet() for ((_, ev) in events) { - if (ev !is ContactListEvent) continue - val pk = ev.pubKey - if (pk in done) continue - val contacts = contactsOf(pk) ?: continue - done += pk - ingest(pk, contacts) - got++ + if (ev is ContactListEvent && ev.pubKey !in done) authors.add(ev.pubKey) } - return got + // The store's canonical latest for each delivered author wins over the + // delivered copy itself — same rule as before, now one query per chunk. + return harvestFromStore(authors) } /** @@ -592,17 +709,25 @@ class GrapeRankCrawler( * previous round's [ensureRelayLists] co-fetch, a late parked delivery, or a * prior run's data — folding it into the graph so Phase B never spends an outbox * drain re-pulling a contact list we already hold. Single-writer: called only - * from the round loop at Phase-A time, before the drain workers start. Returns - * the count newly fed. + * from the round loop at Phase-A time, before the drain workers start. + * + * BATCHED: one chunked author query per [AUTHORS_PER_FILTER] users, folding as + * each chunk streams so peak memory is one chunk's lists. The obvious + * per-user `contactsOf` point query costs ~8ms each on a multi-GB store under + * concurrent writers — 100k pending users made that serial scan dominate a + * round's Phase A wall (measured minutes), so batching here is structural. + * Returns the count newly fed. */ suspend fun harvestFromStore(authors: Collection): Int { var got = 0 - for (pk in authors) { - if (pk in done) continue - val contacts = contactsOf(pk) ?: continue - done += pk - ingest(pk, contacts) - got++ + val pending = authors.filterTo(ArrayList()) { it !in done } + for (chunk in pending.chunked(AUTHORS_PER_FILTER)) { + for ((pk, contacts) in latestContactsFor(chunk)) { + if (pk in done) continue + done += pk + ingest(pk, contacts) + got++ + } } return got } @@ -614,12 +739,20 @@ class GrapeRankCrawler( * still missing onto a different relay for up to SHARD_ROTATIONS passes. * Once the remainder is small it's cheap to broadcast it to every top relay * at once. Returns lists fed. + * + * Precondition: the caller ran [harvestFromStore] over [authors] first, so + * `!in done` alone means "no kind:3 in the store" — no per-author store + * point-queries here (they cost ~8ms each at crawl scale). */ suspend fun shardedSweep(authors: Collection): Int { val top = topLiveRelays(SHARD_RELAYS) if (top.isEmpty()) return 0 val n = top.size - var missing = authors.filter { it !in done && contactsOf(it) == null } + var missing = authors.filter { it !in done && it !in shardSwept } + // Only remember the sweep once the backbone is mature (full shard width): + // users swept against round 2's two-relay proto-backbone deserve a re-ask + // when the real top-10 exists by round 3. + if (n >= SHARD_RELAYS) shardSwept.addAll(missing) var got = 0 var rotation = 0 while (missing.size > SHARD_BROADCAST_THRESHOLD && rotation < config.shardRotations) { @@ -640,17 +773,30 @@ class GrapeRankCrawler( // drains must not share a mutable HashMap. async { val dead = HashMap() + val answered = HashSet() val filters = mapOf(relay to shard.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = FETCH_KINDS, authors = it) }) - drainGated(filters, dead) to dead + val events = drainGated(filters, dead, answered) + ShardDrain(relay, shard, events, relay in answered, dead) } } }.awaitAll() } - for ((_, dead) in results) recordDead(dead) + for (r in results) recordDead(r.dead) relaysContacted += top - val flat = results.flatMap { it.first } + val flat = results.flatMap { it.events } for ((relay, _) in flat) liveRelays.add(relay) + // A shard relay that cleanly EOSE'd without a user's kind:3 doesn't + // have it — record the pair so later passes (outbox routing, communal + // sweep) never re-ask it. Serial here, after the concurrent drains. + for (r in results) { + if (!r.answered) continue + val returned = HashSet() + for ((_, ev) in r.events) if (ev is ContactListEvent) returned.add(ev.pubKey) + for (pk in r.shard) { + if (pk !in returned) askedEmpty.getOrPut(pk) { ConcurrentSet() }.add(r.relay) + } + } got += harvest(flat) missing = missing.filter { it !in done } rotation++ @@ -666,12 +812,24 @@ class GrapeRankCrawler( val live = topLiveRelays(BROADCAST_RELAYS) if (live.isNotEmpty()) { val dead = HashMap() + val answered = HashSet() val filters = live.associateWith { missing.chunked(AUTHORS_PER_FILTER).map { Filter(kinds = FETCH_KINDS, authors = it) } } - val events = drainGated(filters, dead) + val events = drainGated(filters, dead, answered) recordDead(dead) relaysContacted += live for ((relay, _) in events) liveRelays.add(relay) + // Same answered-empty pruning as the rotations, per broadcast relay. + val returnedByRelay = HashMap>() + for ((relay, ev) in events) { + if (ev is ContactListEvent) returnedByRelay.getOrPut(relay) { HashSet() }.add(ev.pubKey) + } + for (relay in answered) { + val returned = returnedByRelay[relay].orEmpty() + for (pk in missing) { + if (pk !in returned) askedEmpty.getOrPut(pk) { ConcurrentSet() }.add(relay) + } + } got += harvest(events) } } @@ -701,6 +859,7 @@ class GrapeRankCrawler( authors: List, relays: Set, kinds: List, + background: Boolean = false, ) { if (relays.isEmpty() || authors.isEmpty()) return val filters = @@ -709,7 +868,7 @@ class GrapeRankCrawler( Filter(kinds = kinds, authors = chunk) } } - drainGated(filters, null) + drainGated(filters, null, background = background) } val discovery = @@ -749,8 +908,11 @@ class GrapeRankCrawler( (olderStillMissing.isNotEmpty() && newWide.isNotEmpty()) if (hasWork) { bgScope.launch { - query(freshStillMissing, wide, listOf(AdvertisedRelayListEvent.KIND)) - query(olderStillMissing, newWide, listOf(AdvertisedRelayListEvent.KIND)) + // background=true: these sweeps' parked units must never gate + // round convergence — they are cancelled at crawl end instead, + // and incremental persist keeps whatever they delivered. + query(freshStillMissing, wide, listOf(AdvertisedRelayListEvent.KIND), background = true) + query(olderStillMissing, newWide, listOf(AdvertisedRelayListEvent.KIND), background = true) } } } @@ -764,8 +926,9 @@ class GrapeRankCrawler( * touch our reports. The events land in the store for the caller to apply. */ suspend fun fetchReportDeletions(backbone: Set) { + val stageMark = TimeSource.Monotonic.markNow() val idsByAuthor = HashMap>() - for (ev in store.query(Filter(kinds = listOf(ReportEvent.KIND)))) { + for (ev in queryTimed(Filter(kinds = listOf(ReportEvent.KIND)))) { if (ev is ReportEvent) idsByAuthor.getOrPut(ev.pubKey) { ArrayList() }.add(ev.id) } if (idsByAuthor.isEmpty()) return @@ -794,16 +957,30 @@ class GrapeRankCrawler( } } drainGated(filters, null) + log( + "[graperank] report deletions: ${idsByAuthor.size} reporters via ${perRelayAuthors.size} relays " + + "in ${stageMark.elapsedNow().inWholeMilliseconds}ms", + ) } /** - * Group [pubkeys] by the relays we should query for their events: - * - first try: the user's own kind:10002 write relays (the outbox model); - * - a retry (`attempts[pk] > 0`, its outbox already failed): outbox + - * [backbone] — the known-good relays other people write to; - * - no outbox at all: harvested hints + backbone + the general fallback. + * Group [pubkeys] by the user's OWN-SIGNAL relays for a Phase-B batch drain: + * - the user's own kind:10002 write relays (the outbox model), retried as-is + * on later attempts (a timeout there may succeed now; clean-empty answers + * are pruned via [askedEmpty]); + * - no outbox at all: the per-user relay hints harvested from follows. * - * The content aggregators are deliberately NOT mixed in here: this path's + * The SHARED sets — backbone + fallback — are deliberately NOT fanned out + * here anymore. They used to ride inside every 256-user batch, so a + * retry-heavy round queued thousands of small REQs against the same ~40 hot + * relays' 16-permit gates (measured: rounds 8–10 of a hop-8 crawl burned + * ~80 minutes for +6.6k lists, nearly all of it hot-relay head-of-line + * blocking). Users needing shared coverage (no outbox, or attempt > 0) are + * flagged into [communalOut] instead, and [communalSweep] asks the shared + * relays ONCE per round in big author-chunked units — the same (relay, user) + * pairs, ~8x fewer REQs. + * + * The content aggregators are deliberately NOT mixed in either: this path's * multi-kind [FETCH_KINDS] query loses their kind:3 to their per-REQ result * cap (a big indexer fills the response with the abundant kind:10002 and * returns no kind:3), so recovering from them is done separately — kind:3-only, @@ -814,9 +991,8 @@ class GrapeRankCrawler( */ suspend fun routeByOutbox( pubkeys: Set, - backbone: Set, + communalOut: MutableSet, ): Map> { - val fallback = config.contentFallbackRelays val perRelay = HashMap>() for (pk in pubkeys) { @@ -824,8 +1000,14 @@ class GrapeRankCrawler( write?.forEach { writeRelayFreq.merge(it, 1) { a, b -> a + b } } val relays = when { - write == null -> relayHints[pk]?.snapshot().orEmpty() + backbone + fallback - (attempts[pk] ?: 0) > 0 -> write + backbone + write == null -> { + communalOut.add(pk) + relayHints[pk]?.snapshot().orEmpty() + } + (attempts[pk] ?: 0) > 0 -> { + communalOut.add(pk) + write + } else -> write } // Skip relays proven dead (routing to them only burns the drain @@ -846,6 +1028,58 @@ class GrapeRankCrawler( } } + /** + * Round-wide sweep of the SHARED relay sets (backbone + fallback) for the + * users Phase B flagged for communal coverage — see [routeByOutbox] for why + * this is not fanned out per batch. Asks each shared relay for its pending + * users in [AUTHORS_PER_FILTER] chunks, skipping (user, relay) pairs already + * answered empty, records new answered-empty pairs so the next round's pass + * shrinks, and folds delivered lists. Returns lists fed. + */ + suspend fun communalSweep( + users: Collection, + backbone: Set, + ): Int { + if (users.isEmpty()) return 0 + val shared = backbone + config.contentFallbackRelays + val perRelay = HashMap>() + for (pk in users) { + if (pk in done) continue + val emptied = askedEmpty[pk] + for (relay in shared) { + if (isDead(relay)) continue + if (emptied != null && relay in emptied) continue + perRelay.getOrPut(relay) { ArrayList() }.add(pk) + } + } + if (perRelay.isEmpty()) return 0 + val filters = + perRelay.mapValues { (_, authors) -> + authors.chunked(AUTHORS_PER_FILTER).map { chunk -> Filter(kinds = FETCH_KINDS, authors = chunk) } + } + val dead = HashMap() + val answered = HashSet() + val events = drainGated(filters, dead, answered) + recordDead(dead) + relaysContacted += filters.keys + for ((relay, _) in events) liveRelays.add(relay) + + // Same answered-empty rule as the Phase-B consumer: a clean EOSE without + // a user's kind:3 means this relay lacks it — never re-ask that pair. + val returnedByRelay = HashMap>() + for ((relay, ev) in events) { + if (ev is ContactListEvent) returnedByRelay.getOrPut(relay) { HashSet() }.add(ev.pubKey) + } + for (relay in answered) { + val asked = perRelay[relay] ?: continue + val returned = returnedByRelay[relay].orEmpty() + for (pk in asked) { + if (pk !in returned) askedEmpty.getOrPut(pk) { ConcurrentSet() }.add(relay) + } + } + return harvest(events) + } + /** * Final patient pass for the stragglers the outbox model couldn't resolve. * A large tail of reachable users have no kind:3 on their own advertised @@ -879,10 +1113,22 @@ class GrapeRankCrawler( // Stragglers = crawled users we still have no kind:3 for. Most are already // in `done` (their outbox attempts were exhausted), which is exactly why // [harvest]/[ingestLate] can't be reused — they skip `done` users — so we - // fold these directly. - val stragglers = hopOf.keys.filterTo(HashSet()) { (hopOf[it] ?: 0) < config.maxHops && contactsOf(it) == null } + // fold these directly. Batched: the candidate set is EVERY discovered user + // within the hop budget (hundreds of thousands), so a per-user store point + // query here would take tens of minutes on its own. + // Skip users a previous fixpoint pass already asked the aggregators for: + // they EOSE'd on those, so re-asking is pure wall-clock (measured ~4 min + // per repeat pass at 470k stragglers). Only newly-added stragglers count. + val candidates = hopOf.keys.filterTo(ArrayList()) { (hopOf[it] ?: 0) < config.maxHops && it !in aggregatorAsked } + val stragglers = HashSet(candidates.size) + for (chunk in candidates.chunked(AUTHORS_PER_FILTER)) { + val have = latestContactsFor(chunk).keys + for (pk in chunk) if (pk !in have) stragglers.add(pk) + } if (stragglers.isEmpty()) return + aggregatorAsked.addAll(stragglers) val before = contactListsFed + val stageMark = TimeSource.Monotonic.markNow() log("[graperank] aggregator recovery: ${stragglers.size} stragglers via ${aggregators.size} aggregators") // Build the query against the full straggler set BEFORE any folding (the @@ -916,15 +1162,22 @@ class GrapeRankCrawler( relaysContacted += aggregators // Fast deliveries fold immediately; a slow aggregator parks and its late - // kind:3 arrives on [lateHarvest], which we drain until the parked units - // finish — so a big aggregator that can't answer within the fast window is - // still fully harvested here instead of being abandoned. + // kind:3 arrives on [lateHarvest]. Fold that trickle until the channel + // stays QUIET for one full park window: our own parked units' idle + // discipline guarantees anything still coming lands within it. This + // deliberately does NOT wait for parkedInFlight to reach zero — that + // inherited the lifetime of every old round-park still draining in the + // background (measured 222s of pure waiting at hop-3) while this pass's + // own relays were long done. foldAgg(drainGated(filters, null)) - while (parkedInFlight.load() > 0L) { - withTimeoutOrNull(PARK_POLL_MS) { lateHarvest.receive() }?.let { foldAgg(listOf(it)) } + while (true) { + val late = withTimeoutOrNull(config.parkTimeoutMs) { lateHarvest.receive() } ?: break + foldAgg(listOf(late)) } - while (true) foldAgg(listOf(lateHarvest.tryReceive().getOrNull() ?: break)) - log("[graperank] aggregator recovery: +${contactListsFed - before} contact lists") + log( + "[graperank] aggregator recovery: +${contactListsFed - before} contact lists " + + "in ${stageMark.elapsedNow().inWholeMilliseconds}ms", + ) } /** @@ -945,9 +1198,21 @@ class GrapeRankCrawler( suspend fun flush() { if (buffer.isEmpty()) return + // Solo = no other insert was in flight when this one started, so its + // wall is (approximately) pure write, not writer-mutex queueing. + val solo = insertInFlight.addAndFetch(1) == 1L val mark = TimeSource.Monotonic.markNow() - store.batchInsert(buffer) - insertNanos.addAndFetch(mark.elapsedNow().inWholeNanoseconds) + try { + store.batchInsert(buffer) + val ns = mark.elapsedNow().inWholeNanoseconds + insertNanos.addAndFetch(ns) + if (solo) { + insertSoloNanos.addAndFetch(ns) + insertSoloEvents.addAndFetch(buffer.size.toLong()) + } + } finally { + insertInFlight.addAndFetch(-1) + } eventsStored.addAndFetch(buffer.size.toLong()) buffer.clear() } @@ -970,34 +1235,6 @@ class GrapeRankCrawler( return fresh } - /** - * Wait for a subscription's terminal ([done]: EOSE/CLOSED/cannot), resetting - * the [idleMs] window every time an event pings [activity]. So the wait ends - * with "timeout" only after [idleMs] of actual SILENCE — a relay that keeps - * streaming (however long its result set) is never cut mid-flight; only a - * genuinely stalled one is. Used for the patient park window. - */ - private suspend fun awaitTerminalOrIdle( - done: CompletableDeferred, - activity: Channel, - idleMs: Long, - ): String { - while (true) { - val r = - withTimeoutOrNull(idleMs) { - select { - done.onAwait { it } - activity.onReceive { ACTIVITY } - } - } - when (r) { - null -> return "timeout" // idleMs elapsed with no event and no terminal - ACTIVITY -> Unit // an event arrived — reset the idle window and keep waiting - else -> return r // terminal reason - } - } - } - /** * Fold one late-delivered event from a parked relay into the graph. Only the * round loop calls this (directly or via [foldLateHarvest]), so graph state @@ -1038,6 +1275,8 @@ class GrapeRankCrawler( */ private suspend fun progressTicker() { var lastFed = 0 + var lastVisited = 0L + var lastBatches = 0L var lastMark = TimeSource.Monotonic.markNow() while (true) { delay(PROGRESS_INTERVAL_MS) @@ -1047,6 +1286,14 @@ class GrapeRankCrawler( val fed = contactListsFed val rate = (fed - lastFed) * 1000L / dtMs // lists/sec over this interval lastFed = fed + // Churn rates the done-counter can't see: a retry round over list-less + // users shows 0 lists/s while visiting thousands of users per second. + val visited = usersVisited.load() + val visitedRate = (visited - lastVisited) * 1000L / dtMs + lastVisited = visited + val batches = batchesDone.load() + val batchRate10 = (batches - lastBatches) * 10_000L / dtMs // batches/s ×10 (one decimal) + lastBatches = batches val events = eventsStored.load() val parked = parkedInFlight.load() when { @@ -1064,7 +1311,8 @@ class GrapeRankCrawler( // the pool full (raise concurrency) or starved (producer/relays bound)? val sat = if (config.diagnose) { - " · ${activeWorkers.load()}/${config.drainConcurrency}w · ${throttled.load()} rl" + " · ${activeWorkers.load()}/${config.drainConcurrency}w · ${throttled.load()} rl" + + " · ${visitedRate}v/s ${batchRate10 / 10}.${batchRate10 % 10}b/s" } else { "" } @@ -1083,21 +1331,24 @@ class GrapeRankCrawler( * background with `until` cursors ([fetchAllPages], starting at the page's * oldest event, inclusive) to drain whatever the cap hid, streaming the extra * events to [lateHarvest] just like a parked slow relay. Tracked by - * [parkedInFlight] so the round waits for it; gated by [limiter] and dropped if - * we have no [bgScope]. The boundary second is re-fetched and its already-seen - * events are dropped by [persist]'s crawl-wide dedup, so nothing double-counts. - * A no-op unless the page hit the threshold, so only dense units pay for it. + * [parkedInFlight] so the round waits for it — unless [background], in which + * case (like every background unit) it must never gate convergence; gated by + * [limiter] and dropped if we have no [bgScope]. The boundary second is + * re-fetched and its already-seen events are dropped by [persist]'s crawl-wide + * dedup, so nothing double-counts. A no-op unless the page hit + * [FULL_PAGE_THRESHOLD], so only dense units pay for it. */ private fun paginateIfCapped( relay: NormalizedRelayUrl, groupFilters: List, - page: List>, + pageSize: Int, + pageOldest: Long, + background: Boolean, ) { - if (page.size < FULL_PAGE_THRESHOLD) return + if (pageSize < FULL_PAGE_THRESHOLD) return val scope = bgScope ?: return - val oldest = page.minOf { it.second.createdAt } - val contFilters = groupFilters.map { it.copy(until = oldest) } - parkedInFlight.addAndFetch(1) + val contFilters = groupFilters.map { it.copy(until = pageOldest) } + if (!background) parkedInFlight.addAndFetch(1) scope.launch { try { val more = ArrayList>() @@ -1106,7 +1357,7 @@ class GrapeRankCrawler( } for (pair in persist(more)) lateHarvest.trySend(pair) } finally { - parkedInFlight.addAndFetch(-1) + if (!background) parkedInFlight.addAndFetch(-1) } } } @@ -1130,6 +1381,21 @@ class GrapeRankCrawler( filters: Map>, deadOut: MutableMap?, answeredOut: MutableSet? = null, + /** + * A fire-and-forget completeness sweep (Tier-2 relay-list recovery), not + * work a round resolves on. Its parked units do NOT count toward + * [parkedInFlight] — a hop-3 crawl measured ~500s (of 594s total!) just + * waiting out background parks' idle windows at convergence — and are + * simply cancelled at crawl end; incremental persist (see the park loop) + * means everything they delivered up to that instant is already stored. + */ + background: Boolean = false, + /** + * Diagnose hook: updated with the unit that took longest to resolve its + * FAST outcome (terminal, park handoff, or bare timeout), so a slow batch + * can name the relay that held its awaitAll open. + */ + slowestOut: AtomicReference? = null, ): List> { if (filters.isEmpty()) return emptyList() @@ -1154,12 +1420,11 @@ class GrapeRankCrawler( if (group.isNotEmpty()) units.add(relay to group) } - // Per-relay failure classification (HARD wins over TRANSIENT); which relays - // stalled past the fast window; and which did NOT cleanly EOSE (timed out, - // parked, closed, or couldn't connect) — a relay absent from that set - // answered definitively, so an author it didn't return is one it lacks. + // Per-relay failure classification (HARD wins over TRANSIENT), and which + // relays did NOT cleanly EOSE (timed out, parked, closed, or couldn't + // connect) — a relay absent from that set answered definitively, so an + // author it didn't return is one it lacks. val failures = ConcurrentMap() - val timedOut = ConcurrentSet() val notAnswered = ConcurrentSet() fun classify( @@ -1183,11 +1448,18 @@ class GrapeRankCrawler( groupFilters: List, ) { if (!config.diagnose) return + // The tail re-visits the same dead authorities hundreds of times; after + // a few examples per host these lines are pure log bloat (a hop-8 log + // was ~1MB of them). The per-relay telemetry table keeps the full + // counts; here we keep the first few occurrences as samples. + val n = slowLogCounts.merge(authorityOf(relay.url), 1) { a, b -> a + b } + if (n > SLOW_LOG_MAX_PER_AUTHORITY) return + val suffix = if (n == SLOW_LOG_MAX_PER_AUTHORITY) " (further lines for this host suppressed)" else "" val authors = groupFilters.flatMap { it.authors.orEmpty() } val kinds = groupFilters.flatMap { it.kinds.orEmpty() }.distinct() log( "[slow-relay] ${relay.url} $reason in ${elapsedMs}ms | kinds=$kinds authors=${authors.size}: " + - authors.take(30).joinToString(",") + (if (authors.size > 30) ",…" else ""), + authors.take(3).joinToString(",") + (if (authors.size > 3) ",…" else "") + suffix, ) } @@ -1196,7 +1468,12 @@ class GrapeRankCrawler( units .map { (subRelay, groupFilters) -> async { + // Queue time for this relay's limiter slot (permit + rate + // gate) — spent BEFORE any drain timer starts, so it is + // invisible to every latency metric unless ledgered here. + val permitMark = TimeSource.Monotonic.markNow() limiter.withPermit(subRelay) { + telemetry.recordPermitWait(subRelay, permitMark.elapsedNow().inWholeMilliseconds) val subId = newSubId() val done = CompletableDeferred() val unitEvents = Channel>(Channel.UNLIMITED) @@ -1253,6 +1530,7 @@ class GrapeRankCrawler( if (reason != null) { // Terminal within the fast window — resolve this round. val elapsedMs = mark.elapsedNow().inWholeMilliseconds + bumpSlowest(slowestOut, subRelay, elapsedMs, reason) if (reason != "eose") notAnswered.add(subRelay) classify(reason, subRelay, failures) if (isRateLimit(reason)) throttled.addAndFetch(1) @@ -1273,7 +1551,9 @@ class GrapeRankCrawler( val persisted = persist(drained) // A full page from a clean EOSE may be the relay's cap, not the // whole answer — background-paginate the remainder into lateHarvest. - if (reason == "eose") paginateIfCapped(subRelay, groupFilters, drained) + if (reason == "eose" && drained.isNotEmpty()) { + paginateIfCapped(subRelay, groupFilters, drained.size, drained.minOf { it.second.createdAt }, background) + } // Alive if it EOSE'd or handed us anything; a connect-timeout that // gave nothing (classifyDrainFailure leaves it retryable forever) // earns a strike toward eviction instead. @@ -1286,17 +1566,50 @@ class GrapeRankCrawler( } else { // Still streaming — hand off and let the round move on. burnedFastWindow.addAndFetch(1) + bumpSlowest(slowestOut, subRelay, mark.elapsedNow().inWholeMilliseconds, "parked") notAnswered.add(subRelay) - timedOut.add(subRelay) val scope = bgScope if (scope != null && config.parkTimeoutMs > config.timeoutMs) { - parkedInFlight.addAndFetch(1) + if (!background) parkedInFlight.addAndFetch(1) scope.launch { + // Incremental drain: persist + hand to lateHarvest whatever + // has queued so far. Called on every activity ping and again + // (NonCancellable) on the way out, so a park cancelled at + // crawl end loses nothing it already received. + var lateCount = 0 + var lateOldest = Long.MAX_VALUE + + suspend fun drainChunk() { + val chunk = ArrayList>() + while (true) chunk.add(unitEvents.tryReceive().getOrNull() ?: break) + if (chunk.isEmpty()) return + lateCount += chunk.size + for ((_, ev) in chunk) if (ev.createdAt < lateOldest) lateOldest = ev.createdAt + for (pair in persist(chunk)) lateHarvest.trySend(pair) + } try { // Idle timeout, not absolute: only cut after parkTimeoutMs // of SILENCE (no event, no terminal), so a relay still // streaming a large result set is never chopped mid-flight. - val late = awaitTerminalOrIdle(done, activity, config.parkTimeoutMs) + var late = "timeout" + while (true) { + val r = + withTimeoutOrNull(config.parkTimeoutMs) { + select { + done.onAwait { it } + activity.onReceive { ACTIVITY } + } + } + drainChunk() + when { + r == null -> break // idle window elapsed in silence + r == ACTIVITY -> continue // streamed — reset the window + else -> { + late = r + break + } + } + } val lateMs = mark.elapsedNow().inWholeMilliseconds logSlow(subRelay, "parked→$late", lateMs, groupFilters) // A parked relay that ends in a hard/transient failure (not a @@ -1304,24 +1617,27 @@ class GrapeRankCrawler( val lateDead = ConcurrentMap() classify(late, subRelay, lateDead) recordDead(lateDead.snapshot()) - unitEvents.close() - val lateDrained = buildList { for (e in unitEvents) add(e) } - telemetry.record(subRelay, RelayTelemetry.outcomeOf(late, parked = true), lateMs, authorsIn(groupFilters), lateDrained.size) - for (pair in persist(lateDrained)) lateHarvest.trySend(pair) + telemetry.record(subRelay, RelayTelemetry.outcomeOf(late, parked = true), lateMs, authorsIn(groupFilters), lateCount) // A full parked page from a clean EOSE may also be capped — // paginate its remainder in the background, same as the fast path. - if (late == "eose") paginateIfCapped(subRelay, groupFilters, lateDrained) + if (late == "eose" && lateCount > 0) { + paginateIfCapped(subRelay, groupFilters, lateCount, lateOldest, background) + } // Same liveness rule as the fast path: a park that ended // in a clean EOSE or delivered anything clears the relay; // one that idle-cut ("timeout") with nothing strikes it. - if (late == "eose" || lateDrained.isNotEmpty()) { + if (late == "eose" || lateCount > 0) { clearTimeoutStrikes(subRelay) } else if (late == "timeout" || isTimeoutReason(late)) { strikeUnproductiveTimeout(subRelay) } } finally { - client.unsubscribe(subId) - parkedInFlight.addAndFetch(-1) + withContext(NonCancellable) { + drainChunk() // keep whatever arrived since the last ping + unitEvents.close() + client.unsubscribe(subId) + if (!background) parkedInFlight.addAndFetch(-1) + } } } // Parked: events are persisted into lateHarvest by the @@ -1334,6 +1650,7 @@ class GrapeRankCrawler( // it exactly like the fast path — the other two branches persist, // this one must too or those events are lost and re-queried. val toMs = mark.elapsedNow().inWholeMilliseconds + bumpSlowest(slowestOut, subRelay, toMs, "timeout") logSlow(subRelay, "timeout", toMs, groupFilters) unitEvents.close() client.unsubscribe(subId) @@ -1351,9 +1668,6 @@ class GrapeRankCrawler( .flatten() } - if (config.diagnose && timedOut.size() > 0) { - log("[drain] parked ${timedOut.size()} slow relay(s) past ${config.timeoutMs}ms") - } deadOut?.putAll(failures.snapshot()) answeredOut?.addAll(filters.keys.filter { it !in notAnswered }) return fast @@ -1406,25 +1720,15 @@ class GrapeRankCrawler( log("[relay-class-sum] relays=${rows.size} live=$live throttled=$throttled unreachable=$unreachable concurrency_caps=$capsStr") } - suspend fun run(): Stats { - val crawlMark = TimeSource.Monotonic.markNow() - // Scope owning parked (slow-relay) subscriptions and the fire-and-forget - // Tier-2 relay-list sweeps. SupervisorJob so one failure never cancels the - // others; cancelled once the crawl converges. Published to [bgScope] so - // drainGated can hand slow subs to it. - val scope = CoroutineScope(coroutineContext + SupervisorJob()) - bgScope = scope - - // Heartbeat: keeps a long, silent round feeling alive with live % + ETA. - // Runs on [scope], so scope.cancel() at crawl end stops it. - scope.launch { progressTicker() } - - // Background reachability culler: cheaply TCP-probes the cold tail of - // learned relays and drops the unreachable ones into deadHosts before the - // WS path pays the full connectTimeout on them. Runs on [scope], stopped - // by scope.cancel() at crawl end. - config.reachabilityProbe?.let { probe -> scope.launch { cullUnreachable(probe) } } - + /** + * The round loop: expand the frontier until every in-budget user is done (or + * [Config.maxRounds] hits). RE-ENTRANT: the terminal aggregator recovery calls + * it again when its folded lists reveal users the previous rounds never saw. + */ + private suspend fun runRounds() { + // Background scope for parked handoffs + Tier-2 sweeps; set by [run] + // before the first call and alive until the crawl's very end. + val scope = checkNotNull(bgScope) { "runRounds requires bgScope" } while (rounds < config.maxRounds) { // Fold in whatever the parked (slow-but-alive) relays have delivered // since the last round — their late contact lists expand the frontier @@ -1435,15 +1739,17 @@ class GrapeRankCrawler( // in the graph as follow targets, we just don't fetch their lists. val pending = hopOf.keys.filter { it !in done && (hopOf[it] ?: 0) < config.maxHops } if (pending.isEmpty()) { - // Frontier drained. If no slow relay is still streaming, a final - // fold catches any last-moment delivery and we're done; otherwise - // wait for a parked relay to deliver (completeness) and loop. + // Frontier drained. Parked relays still streaming can no longer + // change this loop's outcome: pending empty means every in-budget + // user is done, and parked filters only ever asked for in-budget + // users, so ingestLate would skip 100% of their late lists. Their + // events still persist incrementally to the store, where the + // aggregator pass re-reads them — blocking here on parkedInFlight + // only serialized that wait (measured 222s at hop-3) in front of + // the final passes. A last fold catches any already-queued + // delivery; if it (unexpectedly) fed someone, loop once more. progConverging = true - if (parkedInFlight.load() == 0L) { - if (foldLateHarvest() == 0) break else continue - } - withTimeoutOrNull(PARK_POLL_MS) { lateHarvest.receive() }?.let { ingestLate(it.first, it.second) } - continue + if (foldLateHarvest() == 0) break else continue } rounds++ // Frame this round for the heartbeat ticker: its target is the pending @@ -1453,32 +1759,33 @@ class GrapeRankCrawler( progBaseDone = done.size progConverging = false - // Refresh the warm pool to this round's busiest relays and keep that - // subscription open — reusing the same subId just updates the - // desired-relay set, so these sockets stay up across the round. - topLiveRelays(WARM_POOL_SIZE).takeIf { it.isNotEmpty() }?.let { warm -> - client.subscribe(WARM_SUB_ID, warm.associateWith { WARM_FILTERS }, null) - } + // Refresh the warm pool for this round and keep that subscription open — + // reusing the same subId just updates the desired-relay set, so warm + // sockets survive across rounds and newly-learned outbox relays start + // connecting in the background before Phase B routes to them. + refreshWarmPool() val discoveredBefore = hopOf.size val fedBefore = contactListsFed val roundMark = TimeSource.Monotonic.markNow() - // Phase A — bulk-fetch from the busiest relays via the sharded sweep. - // Most users' kind:3 lives on the big popular relays, so this clears - // the majority cheaply (early rounds no-op until a backbone is learned). + // Phase A — fold any kind:3 already sitting in the store FIRST (a prior + // round's ensureRelayLists co-fetch, a late parked delivery, or a + // previous run), batched by author chunk; then bulk-fetch the genuinely + // missing rest from the busiest relays via the sharded sweep (which per + // its precondition trusts `done` instead of re-checking the store). val fedBeforeA = contactListsFed - shardedSweep(pending) - // Fold any kind:3 already sitting in the store — a prior round's - // ensureRelayLists co-fetch, a late parked delivery, or a previous run — - // so Phase B doesn't re-drain contact lists we already hold. harvestFromStore(pending) + shardedSweep(pending) val phaseAMs = roundMark.elapsedNow().inWholeMilliseconds val phaseAFed = contactListsFed - fedBeforeA // Phase B — whoever the popular relays didn't have (niche outboxes): // resolve their kind:10002, then fetch from their own write relays, // drained a few at a time and skipping dead relays. + if (config.diagnose) batchWalls.reset() + var phaseCMs = 0L + var phaseCFed = 0 val stragglers = pending.filter { it !in done } if (stragglers.isNotEmpty()) { val backbone = topLiveRelays(BACKBONE_SIZE).toSet() @@ -1493,6 +1800,10 @@ class GrapeRankCrawler( // on the producer (keeps writeRelayFreq serial) and ingest runs // only on the consumer (keeps done/builder/hopOf serial), now // overlapped with draining instead of blocked behind each batch. + // Users needing shared-relay coverage (no outbox, or retrying): + // flagged by routeByOutbox on the producer (serial), swept ONCE + // per round in Phase C instead of fanned into every batch. + val communalUsers = HashSet() val routed = Channel, Map>>>(config.drainConcurrency * 2) val drainedOut = Channel(Channel.UNLIMITED) coroutineScope { @@ -1501,7 +1812,7 @@ class GrapeRankCrawler( val producer = launch { for (batch in stragglers.chunked(USER_BATCH)) { - val filters = routeByOutbox(batch.toSet(), backbone) + val filters = routeByOutbox(batch.toSet(), communalUsers) routed.send(batch to filters) } routed.close() @@ -1518,9 +1829,21 @@ class GrapeRankCrawler( try { val dead = HashMap() val answered = HashSet() - val events = drainGated(filters, dead, answered) + val slowest = if (config.diagnose) AtomicReference(null) else null + val batchMark = TimeSource.Monotonic.markNow() + val events = drainGated(filters, dead, answered, slowestOut = slowest) recordDead(dead) - drainedOut.send(DrainedBatch(batch, filters, answered, events)) + val drained = + DrainedBatch( + batch, + filters, + answered, + events, + wallMs = batchMark.elapsedNow().inWholeMilliseconds, + slowest = slowest?.load(), + ) + if (config.diagnose) batchWalls.record(drained) + drainedOut.send(drained) } finally { activeWorkers.addAndFetch(-1) } @@ -1549,17 +1872,22 @@ class GrapeRankCrawler( } } + usersVisited.addAndFetch(d.batch.size.toLong()) + batchesDone.addAndFetch(1) + + // One chunked store query for the whole batch (256 + // authors) instead of a point query per user. + val stored = latestContactsFor(d.batch.filter { it !in done }) for (pk in d.batch) { if (pk in done) continue - val contacts = contactsOf(pk) + val contacts = stored[pk] if (contacts != null) { done += pk ingest(pk, contacts) - } else { - val tries = (attempts[pk] ?: 0) + 1 - attempts[pk] = tries - if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk } + // Not found on own relays: no attempt burned yet. + // Accounting runs after Phase C, once the round's + // FULL coverage (own + shared relays) has run. } } } @@ -1568,6 +1896,22 @@ class GrapeRankCrawler( drainedOut.close() consumer.join() } + + // Phase C — one round-wide communal sweep of the shared relays + // (backbone + fallback) for the flagged users, then attempt + // accounting: a user only burns an attempt after BOTH its own + // relays (Phase B) and the shared set (here) missed this round. + val phaseCMark = TimeSource.Monotonic.markNow() + val fedBeforeC = contactListsFed + communalSweep(communalUsers, backbone) + for (pk in stragglers) { + if (pk in done) continue + val tries = (attempts[pk] ?: 0) + 1 + attempts[pk] = tries + if (tries >= MAX_OUTBOX_ATTEMPTS) done += pk + } + phaseCMs = phaseCMark.elapsedNow().inWholeMilliseconds + phaseCFed = contactListsFed - fedBeforeC } val roundMs = roundMark.elapsedNow().inWholeMilliseconds @@ -1575,25 +1919,107 @@ class GrapeRankCrawler( "[graperank] round $rounds: pending=${pending.size}, " + "gotList=${contactListsFed - fedBefore}, newUsers=${hopOf.size - discoveredBefore}, " + "discovered=${hopOf.size}, done=${done.size}, dead=${deadRelays.size()}, " + - "time=${roundMs}ms (phaseA=${phaseAMs}ms fed=$phaseAFed, phaseB=${roundMs - phaseAMs}ms fed=${contactListsFed - fedBefore - phaseAFed})", + "time=${roundMs}ms (phaseA=${phaseAMs}ms fed=$phaseAFed, " + + "phaseB=${roundMs - phaseAMs - phaseCMs}ms fed=${contactListsFed - fedBefore - phaseAFed - phaseCFed}, " + + "phaseC=${phaseCMs}ms fed=$phaseCFed)", ) + if (config.diagnose) { + // Batch-wall distribution vs the fast window that supposedly bounds + // it — the direct measure of pre-drain queueing — plus the limiter's + // running demotion counts, so relay push-back is visible per round. + batchWalls.dump(log, config.timeoutMs) + val snap = limiter.snapshot() + log( + "[graperank] limiter: capped=${snap["concurrency_capped_relays"]} " + + "rate_limited=${snap["rate_limited_relays"]}", + ) + } + } + } + + suspend fun run(): Stats { + val crawlMark = TimeSource.Monotonic.markNow() + // Scope owning parked (slow-relay) subscriptions and the fire-and-forget + // Tier-2 relay-list sweeps. SupervisorJob so one failure never cancels the + // others; cancelled once the crawl converges. Published to [bgScope] so + // drainGated can hand slow subs to it. + val scope = CoroutineScope(coroutineContext + SupervisorJob()) + bgScope = scope + + // Heartbeat: keeps a long, silent round feeling alive with live % + ETA. + // Runs on [scope], so scope.cancel() at crawl end stops it. + scope.launch { progressTicker() } + + // Background reachability culler: cheaply TCP-probes the cold tail of + // learned relays and drops the unreachable ones into deadHosts before the + // WS path pays the full connectTimeout on them. Runs on [scope], stopped + // by scope.cancel() at crawl end. + config.reachabilityProbe?.let { probe -> scope.launch { cullUnreachable(probe) } } + + // Mass pre-connect: start the handshake to every relay a prior run proved + // live NOW, in one parallel storm, so the connect wait is paid once — + // concurrently, before the first drain — instead of serially inside each + // drain that first routes to a relay. The subscribe returns immediately; + // sockets ramp in the background while round 1's discovery queries run. + if (config.preconnectCap > 0 && config.knownLiveRelays.isNotEmpty()) { + log( + "[graperank] pre-connecting up to ${config.preconnectCap} of " + + "${config.knownLiveRelays.size} known-live relays", + ) + refreshWarmPool() } - // Crawl done — drop the warm pool. - client.unsubscribe(WARM_SUB_ID) + runRounds() - // Patient final pass: recover the stragglers the outbox model couldn't + // Patient final passes, run CONCURRENTLY — they are independent (one + // folds stragglers' kind:3 into the graph, the other lands kind:5 + // retractions in the store; graph state stays single-writer inside the + // aggregator pass), so serially they cost sum instead of max (report + // deletions alone measured 137s at hop-3). + // + // Aggregator pass: recover the stragglers the outbox model couldn't // resolve by asking the content aggregators for their kind:3 ALONE, no // longer racing the full fan-out (which cut the aggregators short during // the rounds). Runs before [scope] is cancelled so slow aggregators park. - recoverStragglersFromAggregators() + // + // Report deletions: reports can be retracted. Ask each reporter's outbox + // for NIP-09 kind:5 deletions that cite the reports we gathered + // (#e-filtered to our report ids). The events land in the store; the + // caller decides which reports they actually retract. Runs before + // [scope] is cancelled so it can still park slow relays. + coroutineScope { + launch { recoverStragglersFromAggregators() } + launch { fetchReportDeletions(topLiveRelays(BACKBONE_SIZE).toSet()) } + } - // Reports can be retracted. Ask each reporter's outbox for NIP-09 kind:5 - // deletions that cite the reports we gathered (#e-filtered to our report - // ids). The events land in the store; the caller decides which reports - // they actually retract. Run before cancelling [scope] so it can still - // park slow relays. - fetchReportDeletions(topLiveRelays(BACKBONE_SIZE).toSet()) + // FIXPOINT: recovery discoveries must be CRAWLED, not just counted. A + // recovered straggler's list reveals users the rounds never saw; without + // re-entering the round loop those users dead-end, making deep-hop + // completeness hostage to WHEN a list arrived (in-round = crawlable, + // terminal recovery = ignored). Measured: one weak backbone hour at + // round 5 shifted ~30k lists to the terminal pass and silently + // amputated ~220k hop-6/7 users vs an otherwise-identical run. So: + // while recovery keeps revealing in-budget pending users, run more + // rounds and recover again — each pass folds each straggler at most + // once and the hop budget bounds depth, so this converges (and + // maxRounds backstops it). The warm pool is still up, so resumed + // rounds start on warm sockets. + var extraRounds = false + while (rounds < config.maxRounds && + hopOf.keys.any { it !in done && (hopOf[it] ?: 0) < config.maxHops } + ) { + extraRounds = true + log("[graperank] recovery expanded the frontier — resuming rounds") + runRounds() + recoverStragglersFromAggregators() + } + // Extra rounds can gather new kind:1984 reports whose deletions the + // first pass never saw; top up (store-deduped, so re-asking is cheap). + // Skipped entirely on the common no-extra-rounds path. + if (extraRounds) fetchReportDeletions(topLiveRelays(BACKBONE_SIZE).toSet()) + + // Crawl done — drop the warm pool. + client.unsubscribe(WARM_SUB_ID) // Stop any parked subscriptions + Tier-2 relay-list sweeps still in flight // (whatever they fetched already landed in the store). @@ -1626,10 +2052,14 @@ class GrapeRankCrawler( "$rounds rounds in $downloadMs ms; " + "by hop: " + hopHistogram.entries.joinToString(" ") { "${it.key}=${it.value}" }, ) + val soloEvents = insertSoloEvents.load() + val soloUsPerEvent = if (soloEvents > 0) insertSoloNanos.load() / 1_000 / soloEvents else 0 log( - "[graperank] write path: $stored events stored, verify ${verifyMs}ms + insert ${insertMs}ms " + - "(summed across all drains, batch=${config.insertBatchSize})", + "[graperank] write path: $stored events stored, verify ${verifyMs}ms; insert wall ${insertMs}ms " + + "summed across concurrent drains — writer-mutex wait INCLUDED " + + "(uncontended writes: $soloEvents events ≈ ${soloUsPerEvent}µs/event; batch=${config.insertBatchSize})", ) + log("[graperank] store reads: ${queryCount.load()} queries in ${queryNanos.load() / 1_000_000}ms") if (config.diagnose) { val n = drainSamples.load().coerceAtLeast(1) val wall = drainWallSumMs.load().coerceAtLeast(1) @@ -1665,8 +2095,9 @@ class GrapeRankCrawler( /** * One Phase-B batch after draining: the users asked for, the relay->filters map - * they were routed through, the relays that cleanly EOSE'd ([answered]), and the - * fresh events. Carries enough for the consumer to attribute "answered but + * they were routed through, the relays that cleanly EOSE'd ([answered]), the + * fresh events, and (diagnose) how long the drain held its worker plus the unit + * that resolved last. Carries enough for the consumer to attribute "answered but * empty" per user without re-deriving the routing. */ private class DrainedBatch( @@ -1674,8 +2105,77 @@ class GrapeRankCrawler( val filters: Map>, val answered: Set, val events: List>, + val wallMs: Long = 0, + val slowest: SlowestUnit? = null, ) + /** + * One sharded-sweep drain after completion: the relay asked, the shard of users + * it was asked for, the fresh events, and whether the relay cleanly EOSE'd — + * enough to attribute "answered but empty" per (user, relay) pair serially + * after the concurrent rotation drains join. + */ + private class ShardDrain( + val relay: NormalizedRelayUrl, + val shard: List, + val events: List>, + val answered: Boolean, + val dead: Map, + ) + + /** The drain unit that took longest to reach its fast outcome within one drainGated call. */ + class SlowestUnit( + val relay: NormalizedRelayUrl, + val elapsedMs: Long, + val reason: String, + ) + + /** + * Per-round Phase-B batch-wall statistics (diagnose only). A batch's wall is the + * awaitAll over its units, so its distribution — against the fast window that + * supposedly bounds it — is the direct measure of pre-drain queueing (permits, + * rate gates, dispatcher). Reset by the round loop before Phase B; recorded + * concurrently by the drain workers; dumped at round end with the slowest + * batches' last-resolving units named. + */ + private class BatchWallStats { + private val seq = AtomicLong(0) + + // Swapped whole on reset (ConcurrentMap exposes no clear); reset only runs + // from the round loop between rounds, when no drain worker is recording. + @Volatile private var samples = ConcurrentMap() + + fun reset() { + samples = ConcurrentMap() + } + + fun record(batch: DrainedBatch) { + samples[seq.addAndFetch(1)] = batch + } + + fun dump( + log: (String) -> Unit, + fastWindowMs: Long, + ) { + val snap = samples.snapshot().values.toList() + if (snap.isEmpty()) return + val walls = snap.map { it.wallMs }.sorted() + + fun pct(p: Int) = walls[(walls.size - 1) * p / 100] + val over = walls.count { it > fastWindowMs } + val slowest = snap.sortedByDescending { it.wallMs }.take(3) + val detail = + slowest.joinToString(" ; ") { b -> + "${b.wallMs}ms ${b.filters.size}relays ${b.batch.size}authors" + + (b.slowest?.let { " last=${it.relay.url} ${it.elapsedMs}ms(${it.reason})" } ?: "") + } + log( + "[batch-walls] n=${walls.size} p50=${pct(50)}ms p90=${pct(90)}ms max=${walls.last()}ms · " + + "$over over the ${fastWindowMs}ms fast window | slowest: $detail", + ) + } + } + /** * Per-relay outcome + latency + yield accounting, accumulated across the whole * crawl from every drain unit (fast and parked). This is the ground truth for @@ -1728,6 +2228,15 @@ class GrapeRankCrawler( val latSumMs = AtomicLong(0) val latMaxMs = AtomicLong(0) + /** + * Wall time units spent QUEUED for this relay's limiter slot (concurrency + * permit + rate gate) BEFORE their drain timer starts. Invisible in every + * latency number above — [latSumMs] starts after the permit is acquired — + * yet it is exactly where a hot relay serializes the whole fan-out, so it + * gets its own ledger. + */ + val permitWaitMs = AtomicLong(0) + fun bump(outcome: Outcome) = byOutcome.getOrPut(outcome) { AtomicLong(0) }.addAndFetch(1) fun count(outcome: Outcome): Long = byOutcome[outcome]?.load() ?: 0 @@ -1735,6 +2244,15 @@ class GrapeRankCrawler( val rows = ConcurrentMap() + /** Add [waitMs] of pre-drain limiter queueing (permit + rate gate) to [relay]'s row. */ + fun recordPermitWait( + relay: NormalizedRelayUrl, + waitMs: Long, + ) { + if (waitMs <= 0) return + rows.getOrPut(relay) { Row() }.permitWaitMs.addAndFetch(waitMs) + } + fun record( relay: NormalizedRelayUrl, outcome: Outcome, @@ -1801,7 +2319,19 @@ class GrapeRankCrawler( return " $url att=$a eose=${r.count(Outcome.FAST_EOSE)}/${r.count(Outcome.SLOW_EOSE)} " + "to=${r.count(Outcome.FAST_TIMEOUT) + r.count(Outcome.PARK_TIMEOUT)} cannot=${r.count(Outcome.CANNOT)} " + "rate=${r.count(Outcome.CLOSED_RATE)} auth=${r.count(Outcome.CLOSED_AUTH)} " + - "ask=$ask got=${r.eventsReturned.load()} yield=$yield% lat=$meanLat/${r.latMaxMs.load()}ms wasted=${wastedMs(r)}ms" + "ask=$ask got=${r.eventsReturned.load()} yield=$yield% lat=$meanLat/${r.latMaxMs.load()}ms " + + "pwait=${r.permitWaitMs.load()}ms wasted=${wastedMs(r)}ms" + } + + // Total pre-drain limiter queueing and the single worst offender — the + // signal that a hot relay's permit gate is serializing the fan-out. + val totalPermitWait = snap.values.sumOf { it.permitWaitMs.load() } + if (totalPermitWait > 0) { + val worst = snap.entries.maxByOrNull { it.value.permitWaitMs.load() } + log( + "[relay-telemetry] permit/rate-gate wait total ${totalPermitWait}ms" + + (worst?.let { " (worst: ${it.key.url} ${it.value.permitWaitMs.load()}ms)" } ?: ""), + ) } val ordered = snap.entries.sortedByDescending { wastedMs(it.value) } @@ -1809,6 +2339,15 @@ class GrapeRankCrawler( for ((relay, r) in ordered.take(25)) log(line(relay.url, r)) if (full) { + val byPermitWait = + snap.entries + .filter { it.value.permitWaitMs.load() > 0 } + .sortedByDescending { it.value.permitWaitMs.load() } + .take(10) + if (byPermitWait.isNotEmpty()) { + log("[relay-telemetry] top permit/rate-gate queues:") + for ((relay, r) in byPermitWait) log(line(relay.url, r)) + } log("[relay-telemetry] FULL per-relay table (${snap.size} relays, worst-first):") for ((relay, r) in ordered) log(line(relay.url, r)) } @@ -1838,16 +2377,42 @@ class GrapeRankCrawler( } } + /** [IEventStore.query] with crawl-wide count + wall-time accounting. */ + private suspend fun queryTimed(filter: Filter): List { + val mark = TimeSource.Monotonic.markNow() + val result = store.query(filter) + queryNanos.addAndFetch(mark.elapsedNow().inWholeNanoseconds) + queryCount.addAndFetch(1) + return result + } + /** Latest known kind:3 contact list for [pubKey] from the local store, or null. */ private suspend fun contactsOf(pubKey: HexKey): ContactListEvent? = - store - .query(Filter(authors = listOf(pubKey), kinds = listOf(ContactListEvent.KIND), limit = 1)) + queryTimed(Filter(authors = listOf(pubKey), kinds = listOf(ContactListEvent.KIND), limit = 1)) .firstOrNull() as? ContactListEvent + /** + * Latest stored kind:3 per author for a CHUNK of authors, in ONE store query. + * The batched sibling of [contactsOf]: a point query costs ~8ms on a multi-GB + * store under concurrent writers, so every crawl path that resolves contact + * lists for thousands of users at once must go through this instead. Newest + * `created_at` per author wins, matching the replaceable-event rule. + * Callers chunk to [AUTHORS_PER_FILTER] to keep the SQL parameter list sane. + */ + private suspend fun latestContactsFor(authors: Collection): Map { + if (authors.isEmpty()) return emptyMap() + val latest = HashMap() + for (ev in queryTimed(Filter(kinds = listOf(ContactListEvent.KIND), authors = authors.toList()))) { + if (ev !is ContactListEvent) continue + val cur = latest[ev.pubKey] + if (cur == null || ev.createdAt > cur.createdAt) latest[ev.pubKey] = ev + } + return latest + } + /** Latest known kind:10002 advertised relay list for [pubKey] from the store, or null. */ private suspend fun relaysOf(pubKey: HexKey): AdvertisedRelayListEvent? = - store - .query(Filter(authors = listOf(pubKey), kinds = listOf(AdvertisedRelayListEvent.KIND), limit = 1)) + queryTimed(Filter(authors = listOf(pubKey), kinds = listOf(AdvertisedRelayListEvent.KIND), limit = 1)) .firstOrNull() as? AdvertisedRelayListEvent companion object { @@ -1881,6 +2446,9 @@ class GrapeRankCrawler( // timeout) is logged with its relay + filter, so slow relays can be replayed. private const val SLOW_DRAIN_LOG_MS = 4000L + // --diagnose: [slow-relay] sample lines kept per authority before suppression. + private const val SLOW_LOG_MAX_PER_AUTHORITY = 3 + /** * Is a drain terminal reason a connect/read TIMEOUT — the class * [classifyDrainFailure] leaves retryable forever? The reason shape is @@ -1961,10 +2529,12 @@ class GrapeRankCrawler( // Most-used write relays kept as the known-good backbone for retrying users. private const val BACKBONE_SIZE = 30 - // Warm pool: hold a do-nothing subscription open to the busiest relays for - // the whole crawl, so the connections we reuse every round survive the - // between-round routing gaps. The filter matches an impossible event id, so - // the relay EOSEs immediately and streams nothing — it only keeps sockets warm. + // Warm pool: hold a do-nothing subscription open to relays for the whole + // crawl, so their connections survive the between-round routing gaps. The + // filter matches an impossible event id, so the relay EOSEs immediately and + // streams nothing — it only keeps sockets warm. With Config.preconnectCap > 0 + // the pool covers the whole candidate universe (mass pre-connect); this + // constant is the busiest-relays fallback size when that is disabled. private const val WARM_POOL_SIZE = 20 private const val WARM_SUB_ID = "graperank-warm" private val WARM_FILTERS = listOf(Filter(ids = listOf("0".repeat(64)))) @@ -1980,6 +2550,21 @@ class GrapeRankCrawler( /** Total author slots across a unit's filters — what we asked a relay for. */ private fun authorsIn(filters: List): Int = filters.sumOf { it.authors?.size ?: 0 } + /** CAS-update [out] with this unit if it beats the current slowest. Null out = diagnose off. */ + private fun bumpSlowest( + out: AtomicReference?, + relay: NormalizedRelayUrl, + elapsedMs: Long, + reason: String, + ) { + if (out == null) return + while (true) { + val cur = out.load() + if (cur != null && cur.elapsedMs >= elapsedMs) return + if (out.compareAndSet(cur, SlowestUnit(relay, elapsedMs, reason))) return + } + } + /** Count the size-driving entries in a filter: authors, ids, and tag values. */ private fun filterEntries(f: Filter): Int = (f.authors?.size ?: 0) + diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/metadata/TolerantPrimitiveSerializers.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/metadata/TolerantPrimitiveSerializers.kt new file mode 100644 index 0000000000..728fb8aaf3 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/metadata/TolerantPrimitiveSerializers.kt @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.metadata + +import com.vitorpamplona.quartz.utils.Log +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.KSerializer +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.descriptors.nullable +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import kotlinx.serialization.json.JsonDecoder +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.booleanOrNull + +/** + * Tolerant serializer for kind-0 string fields. + * + * Some clients publish structurally wrong values for NIP-01/NIP-24 profile + * fields — seen in the wild: `"nip05":{}`. With the default serializer that + * type mismatch throws, and because [MetadataEvent.contactMetaData] turns any + * parse exception into `null`, one malformed field would discard the **entire** + * profile (name, picture, about…). + * + * This serializer accepts any JSON primitive (matching the pre-existing lenient + * behavior, where a bare number or boolean decodes into a string field) and + * treats objects, arrays, and JSON null as absent rather than failing. + * + * Same rationale as [BirthdayTolerantSerializer]. + */ +object TolerantStringSerializer : KSerializer { + override val descriptor: SerialDescriptor = + PrimitiveSerialDescriptor("com.vitorpamplona.quartz.nip01Core.metadata.TolerantString", PrimitiveKind.STRING).nullable + + override fun deserialize(decoder: Decoder): String? { + require(decoder is JsonDecoder) { "This serializer can only be used with Json format" } + + val element = decoder.decodeJsonElement() + return when { + element is JsonNull -> null + element is JsonPrimitive -> element.content + else -> { + // Log the JSON kind only, not the raw (untrusted, network-sourced) value. + Log.w("TolerantStringSerializer") { "Ignoring non-primitive string field (${element::class.simpleName})" } + null + } + } + } + + @OptIn(ExperimentalSerializationApi::class) + override fun serialize( + encoder: Encoder, + value: String?, + ) { + if (value == null) { + encoder.encodeNull() + } else { + encoder.encodeString(value) + } + } +} + +/** + * Tolerant serializer for the kind-0 `bot` flag: accepts `true`/`false` and their + * quoted string forms; anything else (objects, arrays, other strings, JSON null) + * is treated as absent rather than failing the whole profile parse. + */ +object TolerantBooleanSerializer : KSerializer { + override val descriptor: SerialDescriptor = + PrimitiveSerialDescriptor("com.vitorpamplona.quartz.nip01Core.metadata.TolerantBoolean", PrimitiveKind.BOOLEAN).nullable + + override fun deserialize(decoder: Decoder): Boolean? { + require(decoder is JsonDecoder) { "This serializer can only be used with Json format" } + + val element = decoder.decodeJsonElement() + if (element is JsonPrimitive && element !is JsonNull) { + val parsed = element.booleanOrNull + if (parsed != null) return parsed + } + if (element !is JsonNull) { + Log.w("TolerantBooleanSerializer") { "Ignoring non-boolean bot field (${element::class.simpleName})" } + } + return null + } + + @OptIn(ExperimentalSerializationApi::class) + override fun serialize( + encoder: Encoder, + value: Boolean?, + ) { + if (value == null) { + encoder.encodeNull() + } else { + encoder.encodeBoolean(value) + } + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/metadata/UserMetadata.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/metadata/UserMetadata.kt index b902f1604b..fc59e9540f 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/metadata/UserMetadata.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/metadata/UserMetadata.kt @@ -35,31 +35,58 @@ class Birthday { var day: Int? = null } +// Every field uses a tolerant serializer: clients in the wild publish +// structurally wrong values (e.g. "nip05":{}, birthday as a string), and one +// bad field must not discard the whole profile — see TolerantStringSerializer. @Stable @Serializable class UserMetadata { + @Serializable(with = TolerantStringSerializer::class) var name: String? = null @SerialName("display_name") + @Serializable(with = TolerantStringSerializer::class) var displayName: String? = null + + @Serializable(with = TolerantStringSerializer::class) var picture: String? = null + + @Serializable(with = TolerantStringSerializer::class) var banner: String? = null + + @Serializable(with = TolerantStringSerializer::class) var website: String? = null + + @Serializable(with = TolerantStringSerializer::class) var about: String? = null + + @Serializable(with = TolerantBooleanSerializer::class) var bot: Boolean? = null + + @Serializable(with = TolerantStringSerializer::class) var pronouns: String? = null @Serializable(with = BirthdayTolerantSerializer::class) var birthday: Birthday? = null + + @Serializable(with = TolerantStringSerializer::class) var nip05: String? = null + + @Serializable(with = TolerantStringSerializer::class) var domain: String? = null + + @Serializable(with = TolerantStringSerializer::class) var lud06: String? = null + + @Serializable(with = TolerantStringSerializer::class) var lud16: String? = null /** CLINK Offers pointer (`noffer1…`) the user advertises to receive payments over Nostr. */ @SerialName("clink_offer") + @Serializable(with = TolerantStringSerializer::class) var clinkOffer: String? = null + @Serializable(with = TolerantStringSerializer::class) var twitter: String? = null fun anyName(): String? = displayName ?: name diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/listeners/RelayConnectionListener.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/listeners/RelayConnectionListener.kt index c3e4be1810..feefbb532f 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/listeners/RelayConnectionListener.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/listeners/RelayConnectionListener.kt @@ -80,7 +80,9 @@ interface RelayConnectionListener { fun onEventGaveUp( relay: IRelayClient, event: Event, - ) {} + ) { + // no-op by default: listeners that don't surface delivery failures ignore it. + } } object EmptyConnectionListener : RelayConnectionListener diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip13Pow/miner/PoWMiner.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip13Pow/miner/PoWMiner.kt index 280d38309f..cb47baebf2 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip13Pow/miner/PoWMiner.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip13Pow/miner/PoWMiner.kt @@ -25,19 +25,30 @@ import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.crypto.EventHasherSerializer import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate import com.vitorpamplona.quartz.nip13Pow.tags.PoWTag -import com.vitorpamplona.quartz.utils.sha256.sha256 +import com.vitorpamplona.quartz.utils.sha256.sha256Into +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch import kotlin.coroutines.cancellation.CancellationException class PoWMiner( val buffer: MiningBuffer, val desiredPoW: Int, val isActive: () -> Boolean = { true }, + // first nonce byte the search is allowed to change; bytes between + // nonceStarts and this index stay fixed (a parallel worker's prefix). + val searchFrom: Int = buffer.nonceStarts, ) { val emptyBytesForDesiredPoW = desiredPoW / 8 - fun reachedDesiredPoW(byteArray: ByteArray) = PoWRankEvaluator.atLeastPowRank(sha256(byteArray), desiredPoW, emptyBytesForDesiredPoW) + // sha256Into writes every attempt's hash here instead of allocating a fresh + // 32-byte array per hash, keeping the hot loop allocation-free. + private val hashOut = ByteArray(32) - fun run() = runDigit(buffer.nonceStarts) + fun reachedDesiredPoW(byteArray: ByteArray) = PoWRankEvaluator.atLeastPowRank(sha256Into(hashOut, byteArray, byteArray.size), desiredPoW, emptyBytesForDesiredPoW) + + fun run() = runDigit(searchFrom) private fun runDigit(index: Int): Boolean { // checks once every VALID_BYTES.size^2 hashes: cheap enough to not slow @@ -74,6 +85,10 @@ class PoWMiner( * The miner creates a stringified json template and changes the nonce directly in the UTF-8 ByteArray representation * to avoid having to recompute the json objects and stringify it. * + * The search enumerates the nonce space deterministically ([VALID_BYTES] + * in order at every position), so for a given template and pubkey every + * call hashes the same sequence of candidates. + * * [isActive] is polled while mining; returning false aborts the search with a * [CancellationException] so callers can cancel long-running jobs cooperatively. */ @@ -82,6 +97,19 @@ class PoWMiner( pubKey: HexKey, desiredPoW: Int, isActive: () -> Boolean = { true }, + ): EventTemplate = search(template, pubKey, desiredPoW, isActive, "") + + /** + * [noncePrefix] is kept verbatim at the front of the nonce while only the + * bytes after it are enumerated — parallel workers get distinct prefixes + * so their search spaces never overlap. + */ + private fun search( + template: EventTemplate, + pubKey: HexKey, + desiredPoW: Int, + isActive: () -> Boolean, + noncePrefix: String, ): EventTemplate { // sha256 ids have 256 bits; anything outside would index past the // hash (or never terminate) deep inside the hot loop. @@ -90,7 +118,7 @@ class PoWMiner( var nextSize = STARTING_NONCE_SIZE do { - val initialNonce = randomBase(nextSize) + val initialNonce = noncePrefix + randomBase(nextSize) val bytes = EventHasherSerializer @@ -104,9 +132,9 @@ class PoWMiner( val startIndex = bytes.indexOf(initialNonce.encodeToByteArray()) - val buffer = MiningBuffer(bytes, startIndex, startIndex + nextSize) + val buffer = MiningBuffer(bytes, startIndex, startIndex + initialNonce.length) - if (PoWMiner(buffer, desiredPoW, isActive).run()) { + if (PoWMiner(buffer, desiredPoW, isActive, startIndex + noncePrefix.length).run()) { return EventTemplate( template.createdAt, template.kind, @@ -120,5 +148,79 @@ class PoWMiner( throw RuntimeException("Could not find PoW") } + + /** + * Distinct fixed nonce prefix for each racing worker, encoding the worker + * index in base-[VALID_CHARS].size at the smallest width that fits + * [workers] — one char up to 73 workers. + */ + private fun workerPrefix( + worker: Int, + workers: Int, + ): String { + var width = 1 + var capacity = VALID_CHARS.size + while (capacity < workers) { + width++ + capacity *= VALID_CHARS.size + } + + var remaining = worker + val prefix = CharArray(width) + for (i in width - 1 downTo 0) { + prefix[i] = VALID_CHARS[remaining % VALID_CHARS.size] + remaining /= VALID_CHARS.size + } + return prefix.concatToString() + } + + /** + * Multi-core variant of [run]: races [workers] searches over disjoint + * slices of the nonce space (each worker's nonce carries a distinct fixed + * prefix) and returns the first template to reach [desiredPoW]. Workers + * share nothing but the finish flag, so the hash rate scales roughly + * linearly with cores. + * + * Throws the same [CancellationException] as [run] when [isActive] flips + * false before a nonce is found. + */ + suspend fun mine( + template: EventTemplate, + pubKey: HexKey, + desiredPoW: Int, + workers: Int = 1, + isActive: () -> Boolean = { true }, + ): EventTemplate { + require(workers >= 1) { "workers must be >= 1, was $workers" } + if (workers == 1) return run(template, pubKey, desiredPoW, isActive) + + return coroutineScope { + val winner = CompletableDeferred>() + val race = + launch { + repeat(workers) { worker -> + launch(Dispatchers.Default) { + winner.complete( + search(template, pubKey, desiredPoW, { + isActive() && !winner.isCompleted + }, workerPrefix(worker, workers)), + ) + } + } + } + // every worker aborting without a win means the caller's isActive + // flipped: rethrow the CancellationException the workers swallowed + // (a losing worker's complete() is a no-op, so this can't clobber + // a real result). + race.invokeOnCompletion { + winner.completeExceptionally(CancellationException("PoW mining was cancelled")) + } + try { + winner.await() + } finally { + race.cancel() + } + } + } } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip13Pow/signer/PoWNostrSigner.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip13Pow/signer/PoWNostrSigner.kt index b114259885..0aeb529e1e 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip13Pow/signer/PoWNostrSigner.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip13Pow/signer/PoWNostrSigner.kt @@ -38,12 +38,16 @@ import com.vitorpamplona.quartz.nip57Zaps.LnZapRequestEvent * Only events whose kind is in [kindsToMine] are mined; anything else (e.g. the * seal and rumor of a gift-wrapped flow) passes through untouched. Templates that * already carry a nonce tag are not mined again. + * + * [workers] parallel searches race over disjoint nonce slices (see + * [PoWMiner.mine]); 1 keeps the historical single-threaded behavior. */ class PoWNostrSigner( val signer: NostrSigner, val desiredPoW: Int, val kindsToMine: Set, val isActive: () -> Boolean = { true }, + val workers: Int = 1, ) : NostrSigner(signer.pubKey) { override fun isWriteable(): Boolean = signer.isWriteable() @@ -55,10 +59,11 @@ class PoWNostrSigner( ): T = if (kind in kindsToMine && tags.none { PoWTag.hasTagWithContent(it) }) { val mined = - PoWMiner.run( + PoWMiner.mine( template = EventTemplate(createdAt, kind, tags, content), pubKey = pubKey, desiredPoW = desiredPoW, + workers = workers, isActive = isActive, ) signer.sign(mined.createdAt, mined.kind, mined.tags, mined.content) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt new file mode 100644 index 0000000000..a95da3f70a --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip66RelayMonitor.reachability + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.listeners.RelayConnectionListener +import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener +import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient +import com.vitorpamplona.quartz.nip01Core.relay.client.single.newSubId +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent +import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.withTimeoutOrNull +import kotlin.time.TimeSource + +/** + * Mass relay census: dials a whole relay universe in parallel waves and returns a + * per-relay [Verdict] — reachable (with the measured WebSocket-open RTT and, when the + * relay answered a no-op REQ, its time-to-EOSE) or dead (with the failure reason). + * + * The point is to pay the "is this relay alive, and how slow is it?" wait ONCE, up + * front and concurrently, instead of rediscovering it serially inside a crawl: feed + * the verdicts to [RelayReachabilityStore.recordProbed] and the next crawl skips the + * dead set entirely and pre-connects the live set in one storm. + * + * Mechanics per wave: + * - subscribe a never-matching filter (impossible event id) to every relay in the + * wave at once — the client dials them all in parallel (bounded by the transport's + * handshake concurrency), a healthy relay EOSEs immediately, and no event payload + * is ever streamed; + * - a connection listener captures the real WS-upgrade RTT per relay; + * - any app-level answer (EOSE, or even a CLOSED — an auth-walled relay is still a + * WORKING relay) marks it reachable; a connect failure marks it dead; + * - a relay with no terminal by the wave deadline is dead if it never opened the + * socket, reachable-but-slow if it did. + * + * Wave size should stay at or below the transport's concurrent-handshake cap (and + * well below the process's file-descriptor budget) — beyond that, extra relays in a + * wave just queue and eat the wave deadline without dialing. + */ +class RelayProber( + private val client: INostrClient, + private val log: (String) -> Unit = {}, +) { + /** One relay's probe outcome. [rttOpenMs]/[rttEoseMs] are -1 when not observed. */ + class Verdict( + val relay: NormalizedRelayUrl, + val reachable: Boolean, + val rttOpenMs: Long, + val rttEoseMs: Long, + val error: String?, + ) + + class Result( + val verdicts: List, + val elapsedMs: Long, + ) { + val reachable: List get() = verdicts.filter { it.reachable } + + val dead: List get() = verdicts.filter { !it.reachable } + + /** Per-relay open RTT for the reachable set, 0 (= "live, latency unknown") when unobserved. */ + fun reachableRttMs(): Map = reachable.associate { it.relay to it.rttOpenMs.coerceAtLeast(0) } + + fun deadRelays(): Set = dead.mapTo(HashSet()) { it.relay } + } + + /** + * Probe every relay in [relays], [waveSize] at a time, giving each wave up to + * [timeoutMs] to reach terminals. Returns one [Verdict] per input relay. + */ + suspend fun probe( + relays: Collection, + timeoutMs: Long = 15_000, + waveSize: Int = 1000, + ): Result { + val mark = TimeSource.Monotonic.markNow() + val all = ArrayList(relays.size) + val distinct = relays.toSet() + var done = 0 + for (wave in distinct.chunked(waveSize.coerceAtLeast(1))) { + all += probeWave(wave, timeoutMs) + done += wave.size + if (distinct.size > wave.size) { + val liveSoFar = all.count { it.reachable } + log("[relay-probe] $done/${distinct.size} probed · $liveSoFar reachable") + } + } + return Result(all, mark.elapsedNow().inWholeMilliseconds) + } + + private suspend fun probeWave( + wave: List, + timeoutMs: Long, + ): List { + val mark = TimeSource.Monotonic.markNow() + val waveSet = wave.toHashSet() + val openRtt = ConcurrentMap() + val eoseMs = ConcurrentMap() + val errors = ConcurrentMap() + // Every terminal (EOSE / CLOSED / cannot-connect) pings this with its relay so + // the wait loop can stop as soon as the whole wave has resolved. + val terminals = Channel(Channel.UNLIMITED) + + val connListener = + object : RelayConnectionListener { + override fun onConnected( + relay: IRelayClient, + pingMillis: Int, + compressed: Boolean, + ) { + if (relay.url in waveSet) openRtt.getOrPut(relay.url) { pingMillis.toLong() } + } + } + + val subId = newSubId() + val subListener = + object : SubscriptionListener { + override fun onEvent( + event: Event, + isLive: Boolean, + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + // The filter matches nothing; any stray event still proves liveness. + eoseMs.getOrPut(relay) { mark.elapsedNow().inWholeMilliseconds } + } + + override fun onEose( + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + eoseMs.getOrPut(relay) { mark.elapsedNow().inWholeMilliseconds } + terminals.trySend(relay) + } + + override fun onClosed( + message: String, + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + // A CLOSED is an app-level ANSWER (auth wall, policy, …): the relay + // is working. Record why so the caller can see the wall. + errors.getOrPut(relay) { "closed:$message" } + terminals.trySend(relay) + } + + override fun onCannotConnect( + relay: NormalizedRelayUrl, + message: String, + forFilters: List?, + ) { + errors.getOrPut(relay) { "cannot:$message" } + terminals.trySend(relay) + } + } + + client.addConnectionListener(connListener) + try { + client.subscribe(subId, wave.associateWith { PROBE_FILTERS }, subListener) + val remaining = wave.toMutableSet() + withTimeoutOrNull(timeoutMs) { + while (remaining.isNotEmpty()) { + remaining.remove(terminals.receive()) + } + } + } finally { + client.unsubscribe(subId) + client.removeConnectionListener(connListener) + terminals.close() + } + + return wave.map { relay -> + val opened = openRtt[relay] + val answered = eoseMs[relay] + val error = errors[relay] + // Reachable = the socket opened OR the relay answered at the app level + // (EOSE, or a CLOSED — an auth/policy wall is still a working relay). + // Only a connect failure, or silence with no socket, is dead. + val cannot = error?.startsWith("cannot:") == true + val reachable = !cannot && (opened != null || answered != null || error != null) + Verdict( + relay = relay, + reachable = reachable, + rttOpenMs = opened ?: -1, + rttEoseMs = answered ?: -1, + error = error, + ) + } + } + + companion object { + // A filter no event can match (ids are 64-hex of a hash): the relay answers + // with an immediate EOSE and never streams a payload. Same trick as the + // crawler's warm pool. + private val PROBE_FILTERS = listOf(Filter(ids = listOf("0".repeat(64)))) + + /** + * The relay universe the local store knows: every read/write relay advertised + * in any stored kind:10002. Callers typically union this with the reachability + * cache's live+dead sets so previously-probed relays are re-checked too. + * `.onion` relays are excluded unless [includeOnion] — without a Tor transport + * they'd only burn a wave slot. [maxPerAuthority] bounds how many distinct + * URLs are kept per host[:port]: paid/filter relays mint one path URL per user + * (`wss://filter.example/npubA`, `/npubB`, …), and probing hundreds of paths + * of ONE server is redundant (liveness is a server property) and rude. + */ + suspend fun knownRelayUniverse( + store: IEventStore, + includeOnion: Boolean = false, + maxPerAuthority: Int = 3, + ): Set { + val out = HashSet() + val perAuthority = HashMap() + for (ev in store.query(Filter(kinds = listOf(AdvertisedRelayListEvent.KIND)))) { + if (ev !is AdvertisedRelayListEvent) continue + for (relay in ev.relaysNorm()) { + if (!includeOnion && RelayUrlNormalizer.isOnion(relay.url)) continue + if (relay in out) continue + val authority = authorityOf(relay.url) + val count = perAuthority[authority] ?: 0 + if (count >= maxPerAuthority) continue + perAuthority[authority] = count + 1 + out.add(relay) + } + } + return out + } + + /** host[:port] between the ws/wss scheme and the first path slash. */ + private fun authorityOf(url: String): String { + val afterScheme = + when { + url.startsWith("wss://") -> url.substring(6) + url.startsWith("ws://") -> url.substring(5) + else -> url + } + val slash = afterScheme.indexOf('/') + return if (slash >= 0) afterScheme.substring(0, slash) else afterScheme + } + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt index 93f6648639..3d3631a5f0 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt @@ -142,6 +142,21 @@ class RelayReachabilityStore( for (relay in dead) if (relay !in reachable) writeOne(relay, up = false, now, rttOpenMs) } + /** + * Like [record], but with a real, per-relay measured open round-trip — the shape a + * dedicated probe (see RelayProber) produces. Each reachable relay's record carries + * ITS OWN `rtt-open`, so the cache doubles as a latency census: a later reader can + * separate fast relays from working-but-slow ones instead of only live from dead. + */ + suspend fun recordProbed( + reachableRttMs: Map, + dead: Set, + now: Long = TimeUtils.now(), + ) { + for ((relay, rtt) in reachableRttMs) writeOne(relay, up = true, now, rtt.coerceAtLeast(0)) + for (relay in dead) if (relay !in reachableRttMs) writeOne(relay, up = false, now, 0) + } + private suspend fun writeOne( relay: NormalizedRelayUrl, up: Boolean, diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/metadata/TolerantPrimitiveSerializersTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/metadata/TolerantPrimitiveSerializersTest.kt new file mode 100644 index 0000000000..84ba9d9c72 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/metadata/TolerantPrimitiveSerializersTest.kt @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.metadata + +import com.vitorpamplona.quartz.nip01Core.core.JsonMapper +import com.vitorpamplona.quartz.utils.EventFactory +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertIs +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class TolerantPrimitiveSerializersTest { + private fun metaWith(content: String): MetadataEvent = + EventFactory.create( + id = "ed269c23907649461da4b0fe109eed689ed1a562d33873b97ed01496dd02b87c", + pubKey = "932614571afcbad4d17a191ee281e39eebbb41b93fac8fd87829622aeb112f4d", + createdAt = 1L, + kind = MetadataEvent.KIND, + tags = emptyArray(), + content = content, + sig = "00".repeat(64), + ) as MetadataEvent + + /** + * Regression for profiles seen in the wild that publish `"nip05":{}` + * (an empty object where NIP-05 mandates a string). Before the tolerant + * serializer this threw and [MetadataEvent.contactMetaData] returned null, + * dropping the whole profile. + */ + @Test + fun objectNip05DoesNotDropTheProfile() { + val meta = + metaWith( + """{"name":"alice","about":"welcome to follow me","picture":"https://example.com/a.jpg","nip05":{},"lud06":null,"lud16":null}""", + ).contactMetaData() + + assertIs(meta, "profile must still parse despite the malformed nip05") + assertEquals("alice", meta.name) + assertEquals("welcome to follow me", meta.about) + assertEquals("https://example.com/a.jpg", meta.picture) + assertNull(meta.nip05, "non-string nip05 must be ignored, not fatal") + assertNull(meta.lud06) + assertNull(meta.lud16) + } + + @Test + fun otherNonStringValuesAreIgnoredFieldByField() { + val meta = + metaWith( + """{"name":{"first":"A"},"display_name":["A"],"picture":null,"about":"bio","website":{}}""", + ).contactMetaData() + + assertIs(meta) + assertNull(meta.name) + assertNull(meta.displayName) + assertNull(meta.picture) + assertNull(meta.website) + assertEquals("bio", meta.about, "well-formed fields must survive the malformed ones") + } + + /** Pre-existing lenient behavior: bare primitives still decode into string fields. */ + @Test + fun numberAndBooleanPrimitivesStillDecodeIntoStringFields() { + val meta = metaWith("""{"name":123,"about":true}""").contactMetaData() + + assertIs(meta) + assertEquals("123", meta.name) + assertEquals("true", meta.about) + } + + @Test + fun malformedBotFlagIsIgnored() { + listOf( + """{"name":"A","bot":{}}""", + """{"name":"A","bot":[true]}""", + """{"name":"A","bot":"maybe"}""", + """{"name":"A","bot":null}""", + ).forEach { json -> + val meta = metaWith(json).contactMetaData() + assertIs(meta, "profile must survive bot field in $json") + assertEquals("A", meta.name) + assertNull(meta.bot) + } + } + + @Test + fun stringBotFlagStillDecodes() { + val meta = metaWith("""{"name":"A","bot":"true"}""").contactMetaData() + assertIs(meta) + assertEquals(true, meta.bot) + } + + /** + * Content that is not JSON at all (some relays hand back things like + * "Relay initialized") cannot be salvaged: contactMetaData must return + * null without throwing. + */ + @Test + fun nonJsonContentReturnsNull() { + assertNull(metaWith("Relay initialized").contactMetaData()) + assertNull(metaWith("").contactMetaData()) + } + + /** Dropped fields must not leak back into the serialized profile as nulls. */ + @Test + fun droppedFieldsAreOmittedOnSerialization() { + val meta = metaWith("""{"name":"A","nip05":{}}""").contactMetaData() + assertIs(meta) + val serialized = JsonMapper.toJson(meta) + assertTrue("nip05" !in serialized, "a dropped nip05 should not be serialized back out: $serialized") + assertTrue("\"name\":\"A\"" in serialized, "valid fields must round-trip: $serialized") + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip13Pow/PoWMinerParallelTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip13Pow/PoWMinerParallelTest.kt new file mode 100644 index 0000000000..d877a481ec --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip13Pow/PoWMinerParallelTest.kt @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip13Pow + +import com.vitorpamplona.quartz.nip01Core.crypto.EventHasherSerializer +import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate +import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.nip13Pow.miner.PoWMiner +import com.vitorpamplona.quartz.nip13Pow.miner.PoWRankEvaluator +import com.vitorpamplona.quartz.nip13Pow.tags.PoWTag +import com.vitorpamplona.quartz.utils.sha256.sha256 +import kotlinx.coroutines.test.runTest +import kotlin.coroutines.cancellation.CancellationException +import kotlin.test.Test +import kotlin.test.assertFailsWith +import kotlin.test.assertNotNull +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.TimeSource + +class PoWMinerParallelTest { + val pubKey = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c" + + val baseTemplate = + EventTemplate( + 1683596206, + TextNoteEvent.KIND, + emptyArray(), + "A note to mine", + ) + + @Test + fun parallelMinerFindsAValidPoW() = + runTest { + val desiredPoW = 12 + val mined = PoWMiner.mine(baseTemplate, pubKey, desiredPoW, workers = 4) + + val powTag = mined.tags.firstNotNullOfOrNull { PoWTag.parse(it) } + assertNotNull(powTag, "mined template must carry a nonce tag") + + val id = + sha256( + EventHasherSerializer.fastMakeJsonForId( + pubKey = pubKey, + createdAt = mined.createdAt, + kind = mined.kind, + tags = mined.tags, + content = mined.content, + ), + ) + + assertTrue( + PoWRankEvaluator.atLeastPowRank(id, desiredPoW, desiredPoW / 8), + "mined id must reach the desired PoW", + ) + } + + @Test + fun cancellationAbortsAllParallelWorkers() = + runTest { + val start = TimeSource.Monotonic.markNow() + // 256 bits never completes; only the isActive deadline can end the run. + assertFailsWith { + PoWMiner.mine(baseTemplate, pubKey, 256, workers = 4) { + start.elapsedNow() < 150.milliseconds + } + } + assertTrue( + start.elapsedNow() < 5_000.milliseconds, + "all workers must stop shortly after isActive flips, took ${start.elapsedNow()}", + ) + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberUniverseTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberUniverseTest.kt new file mode 100644 index 0000000000..846f4ac3e9 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberUniverseTest.kt @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip66RelayMonitor.reachability + +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent +import com.vitorpamplona.quartz.nip65RelayList.tags.AdvertisedRelayInfo +import com.vitorpamplona.quartz.nip65RelayList.tags.AdvertisedRelayType +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class RelayProberUniverseTest { + private fun norm(url: String) = RelayUrlNormalizer.normalizeOrNull(url)!! + + private fun relayInfo(url: String) = AdvertisedRelayInfo(norm(url), AdvertisedRelayType.BOTH) + + @Test + fun collectsAdvertisedRelaysAcrossAuthors() = + runBlocking { + val store = EventStore(null) + store.insert(AdvertisedRelayListEvent.create(listOf(relayInfo("wss://a.example"), relayInfo("wss://b.example")), NostrSignerSync())) + store.insert(AdvertisedRelayListEvent.create(listOf(relayInfo("wss://b.example"), relayInfo("wss://c.example")), NostrSignerSync())) + + val universe = RelayProber.knownRelayUniverse(store) + assertEquals(setOf(norm("wss://a.example"), norm("wss://b.example"), norm("wss://c.example")), universe) + store.close() + } + + @Test + fun capsPathUrlsPerAuthority() = + runBlocking { + val store = EventStore(null) + // One paid/filter host advertised as one path URL per user. + val paths = (1..10).map { relayInfo("wss://filter.example/user$it") } + store.insert(AdvertisedRelayListEvent.create(paths + relayInfo("wss://solo.example"), NostrSignerSync())) + + val universe = RelayProber.knownRelayUniverse(store, maxPerAuthority = 3) + assertEquals(3, universe.count { it.url.contains("filter.example") }, "per-authority URLs must be capped") + assertTrue(norm("wss://solo.example") in universe) + store.close() + } + + @Test + fun onionRelaysAreExcludedByDefault() = + runBlocking { + val store = EventStore(null) + store.insert( + AdvertisedRelayListEvent.create( + listOf(relayInfo("wss://clear.example"), relayInfo("ws://someonionaddressabcdefghijklmnop.onion")), + NostrSignerSync(), + ), + ) + + val universe = RelayProber.knownRelayUniverse(store) + assertEquals(setOf(norm("wss://clear.example")), universe) + + val withOnion = RelayProber.knownRelayUniverse(store, includeOnion = true) + assertEquals(2, withOnion.size) + store.close() + } + + @Test + fun recordProbedKeepsPerRelayRttAndLiveWins() = + runBlocking { + val store = EventStore(null) + val reach = RelayReachabilityStore(store, NostrSignerInternal(KeyPair())) + + val fast = norm("wss://fast.example") + val slow = norm("wss://slow.example") + val dead = norm("wss://dead.example") + reach.recordProbed(mapOf(fast to 42L, slow to 9000L), setOf(dead, slow)) + + val snapshot = reach.snapshot() + // A relay both probed-live and reported dead stays live (live wins). + assertEquals(setOf(fast, slow), snapshot.live) + assertEquals(setOf(dead), snapshot.dead) + store.close() + } +} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDns.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/okhttp/SurgeDns.kt similarity index 88% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDns.kt rename to quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/okhttp/SurgeDns.kt index 16de1276d8..463c2a22ea 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDns.kt +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/okhttp/SurgeDns.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.okhttp +package com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp import okhttp3.Dns import java.net.InetAddress @@ -74,7 +74,16 @@ class SurgeDns( private val maxEntries: Int = 2000, private val positiveTtlMs: Long = TimeUnit.HOURS.toMillis(24), private val positiveTtlJitterMs: Long = TimeUnit.HOURS.toMillis(24), - private val negativeTtlMs: Long = TimeUnit.SECONDS.toMillis(10), + /** + * How long a resolution FAILURE is served from cache. 10 minutes: a dead domain's + * `getaddrinfo` burns 10-30s of resolver timeouts, and both the app's relay pool and a + * GrapeRank crawl re-dial dead hosts continuously — the relay pool's own reconnect + * backoff already reaches 5 minutes, so a 10-minute DNS negative doesn't change relay + * behavior, it only makes the retries that do happen cost microseconds. Recovery paths + * for a WRONG negative (transient blip): [staleAll] on a network-identity change + * re-tries it on next use, and the entry ages out naturally. + */ + private val negativeTtlMs: Long = TimeUnit.MINUTES.toMillis(10), private val refreshExecutor: Executor = DEFAULT_REFRESH_EXECUTOR, ) : Dns { private val cache = ConcurrentHashMap() @@ -205,7 +214,33 @@ class SurgeDns( } } - /** Drop all cached entries. Call when the network changes (e.g. WiFi <-> mobile). */ + /** + * Soft-expire every entry WITHOUT discarding anything — the right response to a network + * change (WiFi <-> mobile, captive portal cleared). Most cached answers are still correct + * on the new network, so keep serving them under a "double-check" regime instead of + * assuming either way: + * + * - positives fall into the stale-while-revalidate path on their next use: the old + * answer is served instantly and a background refresh re-verifies it (and demotes it + * to negative if the new network really can't resolve it); + * - negatives are re-tried synchronously on their next use, so a host that only failed + * because of the OLD network recovers on first touch instead of waiting out its TTL. + * + * Contrast [invalidate], which throws every answer away and re-pays a blocking + * `getaddrinfo` per host — a self-inflicted cold start on what is usually a healthy cache. + */ + fun staleAll() { + val now = System.currentTimeMillis() + for ((host, entry) in cache) { + if (entry.expiresAtMillis > now) { + // replace(k, old, new): a concurrent fresh resolution wins over the staling. + cache.replace(host, entry, Entry(entry.addresses, now)) + } + } + dirty.set(true) + } + + /** Drop all cached entries. Prefer [staleAll] for network changes — this forces a blocking re-resolve per host. */ fun invalidate() { cache.clear() dirty.set(true) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDnsStore.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/okhttp/SurgeDnsStore.kt similarity index 99% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDnsStore.kt rename to quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/okhttp/SurgeDnsStore.kt index 34fc03c772..a2113832f4 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDnsStore.kt +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/okhttp/SurgeDnsStore.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.okhttp +package com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp import com.vitorpamplona.quartz.utils.Log import java.io.BufferedInputStream diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDnsStoreTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/okhttp/SurgeDnsStoreTest.kt similarity index 98% rename from amethyst/src/test/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDnsStoreTest.kt rename to quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/okhttp/SurgeDnsStoreTest.kt index 730bdbb2f1..45a98c6aad 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDnsStoreTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/okhttp/SurgeDnsStoreTest.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.okhttp +package com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp import okhttp3.Dns import org.junit.Assert.assertArrayEquals diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDnsTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/okhttp/SurgeDnsTest.kt similarity index 93% rename from amethyst/src/test/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDnsTest.kt rename to quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/okhttp/SurgeDnsTest.kt index 8e4fd72f56..53f531843a 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/okhttp/SurgeDnsTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/okhttp/SurgeDnsTest.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.okhttp +package com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp import okhttp3.Dns import org.junit.Assert.assertArrayEquals @@ -93,6 +93,45 @@ class SurgeDnsTest { assertEquals(1, upstream.calls("missing.example")) } + @Test + fun `staleAll serves stale positives and revalidates on next use`() { + val upstream = CountingDns(mapOf("a.example" to listOf(ip("1.2.3.4")))) + val syncRefresh = Executor { it.run() } + val dns = SurgeDns(delegate = upstream, refreshExecutor = syncRefresh) + + dns.lookup("a.example") + assertEquals(1, upstream.calls("a.example")) + + dns.staleAll() + + // Nothing was dropped: the answer still comes straight from cache — but the + // touch double-checks upstream (background refresh, synchronous here). + assertEquals(listOf(ip("1.2.3.4")), dns.lookup("a.example")) + assertEquals(2, upstream.calls("a.example")) + + // The refresh re-armed the TTL: back to a plain cache hit. + dns.lookup("a.example") + assertEquals(2, upstream.calls("a.example")) + } + + @Test + fun `staleAll retries negatives on next use instead of waiting out the TTL`() { + val upstream = CountingDns(emptyMap()) + val dns = SurgeDns(delegate = upstream) + + assertThrows(UnknownHostException::class.java) { dns.lookup("missing.example") } + assertThrows(UnknownHostException::class.java) { dns.lookup("missing.example") } + assertEquals(1, upstream.calls("missing.example")) + + dns.staleAll() + + // The negative no longer short-circuits: the next use goes back upstream (sync + // path — negatives never serve stale), so a host that only failed because of + // the OLD network recovers on first touch after a network change. + assertThrows(UnknownHostException::class.java) { dns.lookup("missing.example") } + assertEquals(2, upstream.calls("missing.example")) + } + @Test fun `expired positive entry serves stale and refreshes in background`() { val upstream = CountingDns(mapOf("a.example" to listOf(ip("1.2.3.4")))) diff --git a/quic/build.gradle.kts b/quic/build.gradle.kts index def6dad697..55c28f88d1 100644 --- a/quic/build.gradle.kts +++ b/quic/build.gradle.kts @@ -23,6 +23,7 @@ import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { alias(libs.plugins.kotlinMultiplatform) alias(libs.plugins.androidKotlinMultiplatformLibrary) + alias(libs.plugins.kotlinxKover) } kotlin { diff --git a/relayBench/build.gradle.kts b/relayBench/build.gradle.kts index 8a02cc1e96..831f524883 100644 --- a/relayBench/build.gradle.kts +++ b/relayBench/build.gradle.kts @@ -3,6 +3,7 @@ import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { alias(libs.plugins.jetbrainsKotlinJvm) application + alias(libs.plugins.kotlinxKover) } application {