diff --git a/cli/README.md b/cli/README.md index 0ded3d65d8..928009c996 100644 --- a/cli/README.md +++ b/cli/README.md @@ -216,8 +216,8 @@ Army-knife verbs that operate purely on their arguments. They never touch | `amy encode naddr --kind N --pubkey HEX --identifier D [--relay URL[,URL…]]` | Encode an addressable-event (`a` tag) pointer. | | `amy verify [EVENT-JSON]` | Check an event's id hash and signature. Reads stdin when the argument is omitted or `-`. Reports `id_ok` + `signature_ok` separately. | | `amy pow check EVENT-JSON\|-` | NIP-13 difficulty of a signed event: `actual_bits`, `committed_target`, `has_commitment`, and `effective_pow` (capped at the commitment so lucky low-target spam doesn't over-count), plus `valid` (id + signature). | -| `amy pow mine --target N [--pubkey HEX] [--timeout SECS] TEMPLATE-JSON\|-` | Mine an **unsigned** template to N leading zero bits and print it back with the nonce tag. Ids don't commit to signatures, so amy can mine on behalf of any pubkey (NIP-13 delegated PoW); defaults to the active account. Exit 124 on timeout. | -| `amy pow bench` | Benchmark this machine's hash rate and print expected mining time at 16/20/24/28 bits. | +| `amy pow mine --target N [--pubkey HEX] [--timeout SECS] [--threads N] TEMPLATE-JSON\|-` | Mine an **unsigned** template to N leading zero bits and print it back with the nonce tag. Ids don't commit to signatures, so amy can mine on behalf of any pubkey (NIP-13 delegated PoW); defaults to the active account. Mines on all cores by default (`--threads` to override). Exit 124 on timeout. | +| `amy pow bench` | Benchmark this machine's hash rate — `hashes_per_second` is the all-cores rate `pow mine` uses by default, `hashes_per_second_single_core` the one-thread rate — and print expected mining time at 16/20/24/28 bits. | | `amy key generate` | Mint a fresh keypair (`nsec` + `npub` + hex). Does not persist — use `init`/`login` for that. | | `amy key public NSEC\|HEX` | Derive the public key from a secret key. | | `amy key encrypt NSEC\|HEX --password X` | NIP-49 encrypt a secret key to an `ncryptsec1…`. | @@ -384,7 +384,7 @@ HTTP endpoint. Reuses quartz's `Nip86Client` and the shared `Nip86Retriever` | Command | What it does | |---|---| -| `amy notes post TEXT [--relay URL] [--pow BITS [--pow-timeout SECS]]` | Publish a kind:1 short text note; `--pow` mines a NIP-13 proof of work into it first (blocks while mining, exit 124 on timeout with nothing published; `--json` adds `pow`, `pow_target`, `pow_millis`). | +| `amy notes post TEXT [--relay URL] [--pow BITS [--pow-timeout SECS]]` | Publish a kind:1 short text note; `--pow` mines a NIP-13 proof of work into it first, using all cores (blocks while mining, exit 124 on timeout with nothing published; `--json` adds `pow`, `pow_target`, `pow_millis`). | | `amy notes feed [--author USER \| --following] [--limit N]` | Read recent kind:1 notes (yours, one user's, or your follow set). | | `amy profile show [USER]` | Print kind:0 metadata. USER accepts npub/nprofile/hex/NIP-05; defaults to self. | | `amy profile edit --name … --about … --picture URL …` | Patch and re-publish your kind:0. | diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PostCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PostCommand.kt index b94ead611d..d26283d735 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PostCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PostCommand.kt @@ -81,13 +81,14 @@ object PostCommand { var powMillis: Long? = null val readyToSign = if (powTarget != null) { - System.err.println("mining $powTarget bits…") + val threads = Runtime.getRuntime().availableProcessors().coerceAtLeast(1) + System.err.println("mining $powTarget bits… ($threads threads)") val deadlineNanos = powTimeoutSec?.let { System.nanoTime() + it * 1_000_000_000L } val startedAt = System.nanoTime() val mined = try { withContext(Dispatchers.Default) { - PoWMiner.run(template, ctx.signer.pubKey, powTarget) { + PoWMiner.mine(template, ctx.signer.pubKey, powTarget, threads) { deadlineNanos == null || System.nanoTime() < deadlineNanos } } diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PowCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PowCommands.kt index bc6a1e73bb..293b3d7428 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PowCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/PowCommands.kt @@ -108,13 +108,18 @@ object PowCommands { rest: Array, ): Int { val args = Args(rest) - val usage = "pow mine --target N [--pubkey HEX] [--timeout SECS] " + val usage = "pow mine --target N [--pubkey HEX] [--timeout SECS] [--threads N] " val target = args.flags["target"]?.toIntOrNull() ?: return Output.error("bad_args", usage) if (target < 1 || target > MAX_DIFFICULTY) { return Output.error("bad_args", "--target must be between 1 and $MAX_DIFFICULTY") } + val threads = args.intFlag("threads", defaultThreads()) + if (threads < 1) { + return Output.error("bad_args", "--threads must be >= 1") + } + val json = readPayload(args.positional.toTypedArray()) ?: return Output.error("bad_args", usage) val template = try { @@ -142,13 +147,13 @@ object PowCommands { val timeoutSec = args.flags["timeout"]?.toLongOrNull() val deadlineNanos = timeoutSec?.let { System.nanoTime() + it * 1_000_000_000L } - System.err.println("mining $target bits for ${pubKey.take(8)}…") + System.err.println("mining $target bits for ${pubKey.take(8)}… ($threads threads)") val startedAt = System.nanoTime() val mined = try { withContext(Dispatchers.Default) { - PoWMiner.run(template, pubKey, target) { + PoWMiner.mine(template, pubKey, target, threads) { deadlineNanos == null || System.nanoTime() < deadlineNanos } } @@ -176,27 +181,38 @@ object PowCommands { "pow" to PoWRankEvaluator.calculatePowRankOf(id), "pow_target" to target, "pow_millis" to elapsedMs, + "threads" to threads, "template_json" to mined.toJson(), ), ) return 0 } - /** `amy pow bench` — hash rate + expected mining time per common target. */ + /** + * `amy pow bench` — single-core and all-cores hash rate, plus expected + * mining time per common target at the all-cores rate (what `pow mine` + * uses by default). + */ private suspend fun bench(): Int { - val rate = PoWEstimator.hashesPerSecond() + val threads = defaultThreads() + val singleRate = PoWEstimator.hashesPerSecond() + val parallelRate = PoWEstimator.hashesPerSecond(threads) Output.emit( mapOf( - "hashes_per_second" to rate.roundToLong(), + "hashes_per_second" to parallelRate.roundToLong(), + "hashes_per_second_single_core" to singleRate.roundToLong(), + "threads" to threads, "expected_seconds" to listOf(16, 20, 24, 28).associate { bits -> - bits.toString() to PoWEstimator.estimateSeconds(bits, rate) + bits.toString() to PoWEstimator.estimateSeconds(bits, parallelRate) }, ), ) return 0 } + private fun defaultThreads(): Int = Runtime.getRuntime().availableProcessors().coerceAtLeast(1) + private fun readPayload(rest: Array): String? { val arg = rest.firstOrNull() ?: return null val payload = if (arg == "-") System.`in`.readBytes().decodeToString() else arg diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWEstimator.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWEstimator.kt index e12843c5fc..1ed00fc1ff 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWEstimator.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/pow/PoWEstimator.kt @@ -20,9 +20,11 @@ */ package com.vitorpamplona.amethyst.commons.service.pow -import com.vitorpamplona.quartz.utils.sha256.sha256 +import com.vitorpamplona.quartz.utils.sha256.sha256Into import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext @@ -61,6 +63,25 @@ object PoWEstimator { } } + /** + * Aggregate hash rate with [workers] concurrent miners — what + * [com.vitorpamplona.quartz.nip13Pow.miner.PoWMiner.mine] achieves when + * racing that many workers. Measured live (not cached): each worker runs + * its own ~250 ms benchmark loop concurrently and the rates are summed, + * so contention between cores is priced in. + */ + suspend fun hashesPerSecond( + workers: Int, + dispatcher: CoroutineDispatcher = Dispatchers.Default, + ): Double = + if (workers <= 1) { + hashesPerSecond(dispatcher) + } else { + withContext(dispatcher) { + List(workers) { async { benchmark() } }.awaitAll().sum() + } + } + fun estimateSeconds( difficulty: Int, hashesPerSecond: Double, @@ -68,14 +89,16 @@ object PoWEstimator { private fun benchmark(): Double { val payload = ByteArray(PAYLOAD_BYTES) { (it % 251).toByte() } + // same allocation-free hashing the miner's hot loop uses + val out = ByteArray(32) // warm up JIT/caches so the measured window reflects steady state - repeat(3 * BATCH) { sha256(payload) } + repeat(3 * BATCH) { sha256Into(out, payload, payload.size) } val mark = TimeSource.Monotonic.markNow() var count = 0L while (mark.elapsedNow() < BENCH_DURATION) { - repeat(BATCH) { sha256(payload) } + repeat(BATCH) { sha256Into(out, payload, payload.size) } count += BATCH } return count / mark.elapsedNow().toDouble(DurationUnit.SECONDS) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip13Pow/miner/PoWMiner.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip13Pow/miner/PoWMiner.kt index 280d38309f..cb47baebf2 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip13Pow/miner/PoWMiner.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip13Pow/miner/PoWMiner.kt @@ -25,19 +25,30 @@ import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.crypto.EventHasherSerializer import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate import com.vitorpamplona.quartz.nip13Pow.tags.PoWTag -import com.vitorpamplona.quartz.utils.sha256.sha256 +import com.vitorpamplona.quartz.utils.sha256.sha256Into +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch import kotlin.coroutines.cancellation.CancellationException class PoWMiner( val buffer: MiningBuffer, val desiredPoW: Int, val isActive: () -> Boolean = { true }, + // first nonce byte the search is allowed to change; bytes between + // nonceStarts and this index stay fixed (a parallel worker's prefix). + val searchFrom: Int = buffer.nonceStarts, ) { val emptyBytesForDesiredPoW = desiredPoW / 8 - fun reachedDesiredPoW(byteArray: ByteArray) = PoWRankEvaluator.atLeastPowRank(sha256(byteArray), desiredPoW, emptyBytesForDesiredPoW) + // sha256Into writes every attempt's hash here instead of allocating a fresh + // 32-byte array per hash, keeping the hot loop allocation-free. + private val hashOut = ByteArray(32) - fun run() = runDigit(buffer.nonceStarts) + fun reachedDesiredPoW(byteArray: ByteArray) = PoWRankEvaluator.atLeastPowRank(sha256Into(hashOut, byteArray, byteArray.size), desiredPoW, emptyBytesForDesiredPoW) + + fun run() = runDigit(searchFrom) private fun runDigit(index: Int): Boolean { // checks once every VALID_BYTES.size^2 hashes: cheap enough to not slow @@ -74,6 +85,10 @@ class PoWMiner( * The miner creates a stringified json template and changes the nonce directly in the UTF-8 ByteArray representation * to avoid having to recompute the json objects and stringify it. * + * The search enumerates the nonce space deterministically ([VALID_BYTES] + * in order at every position), so for a given template and pubkey every + * call hashes the same sequence of candidates. + * * [isActive] is polled while mining; returning false aborts the search with a * [CancellationException] so callers can cancel long-running jobs cooperatively. */ @@ -82,6 +97,19 @@ class PoWMiner( pubKey: HexKey, desiredPoW: Int, isActive: () -> Boolean = { true }, + ): EventTemplate = search(template, pubKey, desiredPoW, isActive, "") + + /** + * [noncePrefix] is kept verbatim at the front of the nonce while only the + * bytes after it are enumerated — parallel workers get distinct prefixes + * so their search spaces never overlap. + */ + private fun search( + template: EventTemplate, + pubKey: HexKey, + desiredPoW: Int, + isActive: () -> Boolean, + noncePrefix: String, ): EventTemplate { // sha256 ids have 256 bits; anything outside would index past the // hash (or never terminate) deep inside the hot loop. @@ -90,7 +118,7 @@ class PoWMiner( var nextSize = STARTING_NONCE_SIZE do { - val initialNonce = randomBase(nextSize) + val initialNonce = noncePrefix + randomBase(nextSize) val bytes = EventHasherSerializer @@ -104,9 +132,9 @@ class PoWMiner( val startIndex = bytes.indexOf(initialNonce.encodeToByteArray()) - val buffer = MiningBuffer(bytes, startIndex, startIndex + nextSize) + val buffer = MiningBuffer(bytes, startIndex, startIndex + initialNonce.length) - if (PoWMiner(buffer, desiredPoW, isActive).run()) { + if (PoWMiner(buffer, desiredPoW, isActive, startIndex + noncePrefix.length).run()) { return EventTemplate( template.createdAt, template.kind, @@ -120,5 +148,79 @@ class PoWMiner( throw RuntimeException("Could not find PoW") } + + /** + * Distinct fixed nonce prefix for each racing worker, encoding the worker + * index in base-[VALID_CHARS].size at the smallest width that fits + * [workers] — one char up to 73 workers. + */ + private fun workerPrefix( + worker: Int, + workers: Int, + ): String { + var width = 1 + var capacity = VALID_CHARS.size + while (capacity < workers) { + width++ + capacity *= VALID_CHARS.size + } + + var remaining = worker + val prefix = CharArray(width) + for (i in width - 1 downTo 0) { + prefix[i] = VALID_CHARS[remaining % VALID_CHARS.size] + remaining /= VALID_CHARS.size + } + return prefix.concatToString() + } + + /** + * Multi-core variant of [run]: races [workers] searches over disjoint + * slices of the nonce space (each worker's nonce carries a distinct fixed + * prefix) and returns the first template to reach [desiredPoW]. Workers + * share nothing but the finish flag, so the hash rate scales roughly + * linearly with cores. + * + * Throws the same [CancellationException] as [run] when [isActive] flips + * false before a nonce is found. + */ + suspend fun mine( + template: EventTemplate, + pubKey: HexKey, + desiredPoW: Int, + workers: Int = 1, + isActive: () -> Boolean = { true }, + ): EventTemplate { + require(workers >= 1) { "workers must be >= 1, was $workers" } + if (workers == 1) return run(template, pubKey, desiredPoW, isActive) + + return coroutineScope { + val winner = CompletableDeferred>() + val race = + launch { + repeat(workers) { worker -> + launch(Dispatchers.Default) { + winner.complete( + search(template, pubKey, desiredPoW, { + isActive() && !winner.isCompleted + }, workerPrefix(worker, workers)), + ) + } + } + } + // every worker aborting without a win means the caller's isActive + // flipped: rethrow the CancellationException the workers swallowed + // (a losing worker's complete() is a no-op, so this can't clobber + // a real result). + race.invokeOnCompletion { + winner.completeExceptionally(CancellationException("PoW mining was cancelled")) + } + try { + winner.await() + } finally { + race.cancel() + } + } + } } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip13Pow/PoWMinerParallelTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip13Pow/PoWMinerParallelTest.kt new file mode 100644 index 0000000000..d877a481ec --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip13Pow/PoWMinerParallelTest.kt @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip13Pow + +import com.vitorpamplona.quartz.nip01Core.crypto.EventHasherSerializer +import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate +import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.nip13Pow.miner.PoWMiner +import com.vitorpamplona.quartz.nip13Pow.miner.PoWRankEvaluator +import com.vitorpamplona.quartz.nip13Pow.tags.PoWTag +import com.vitorpamplona.quartz.utils.sha256.sha256 +import kotlinx.coroutines.test.runTest +import kotlin.coroutines.cancellation.CancellationException +import kotlin.test.Test +import kotlin.test.assertFailsWith +import kotlin.test.assertNotNull +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.TimeSource + +class PoWMinerParallelTest { + val pubKey = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c" + + val baseTemplate = + EventTemplate( + 1683596206, + TextNoteEvent.KIND, + emptyArray(), + "A note to mine", + ) + + @Test + fun parallelMinerFindsAValidPoW() = + runTest { + val desiredPoW = 12 + val mined = PoWMiner.mine(baseTemplate, pubKey, desiredPoW, workers = 4) + + val powTag = mined.tags.firstNotNullOfOrNull { PoWTag.parse(it) } + assertNotNull(powTag, "mined template must carry a nonce tag") + + val id = + sha256( + EventHasherSerializer.fastMakeJsonForId( + pubKey = pubKey, + createdAt = mined.createdAt, + kind = mined.kind, + tags = mined.tags, + content = mined.content, + ), + ) + + assertTrue( + PoWRankEvaluator.atLeastPowRank(id, desiredPoW, desiredPoW / 8), + "mined id must reach the desired PoW", + ) + } + + @Test + fun cancellationAbortsAllParallelWorkers() = + runTest { + val start = TimeSource.Monotonic.markNow() + // 256 bits never completes; only the isActive deadline can end the run. + assertFailsWith { + PoWMiner.mine(baseTemplate, pubKey, 256, workers = 4) { + start.elapsedNow() < 150.milliseconds + } + } + assertTrue( + start.elapsedNow() < 5_000.milliseconds, + "all workers must stop shortly after isActive flips, took ${start.elapsedNow()}", + ) + } +}