From ccfb9fdb615fee935b5674e307a53d7c40edc703 Mon Sep 17 00:00:00 2001 From: Craig Raw Date: Mon, 24 Aug 2026 13:04:43 +0200 Subject: [PATCH] support additional electrum server rpc methods and configurable batch paging --- build.gradle | 21 +- drongo | 2 +- .../sparrow/net/BatchedElectrumServerRpc.java | 90 +++++++++ .../sparrow/net/BlockHeaders.java | 25 +++ .../sparrow/net/ElectrumServer.java | 19 +- .../sparrow/net/ElectrumServerRpc.java | 68 +++++++ .../sparrow/net/PagedBatchRequestBuilder.java | 145 ++++++++++---- .../sparrow/net/ServerCapability.java | 10 + .../sparrow/net/SimpleElectrumServerRpc.java | 72 +++++++ .../sparrow/net/TransactionMerkleProof.java | 25 +++ .../net/UnsupportedMethodException.java | 19 ++ .../cormorant/bitcoind/BitcoindClient.java | 4 +- .../sparrow/net/ElectrumServerRpcTest.java | 76 +++++++ .../net/HeaderCheckpointVerificationTest.java | 149 ++++++++++++++ .../net/PagedBatchRequestBuilderTest.java | 185 ++++++++++++++++++ 15 files changed, 863 insertions(+), 47 deletions(-) create mode 100644 src/main/java/com/sparrowwallet/sparrow/net/BlockHeaders.java create mode 100644 src/main/java/com/sparrowwallet/sparrow/net/TransactionMerkleProof.java create mode 100644 src/main/java/com/sparrowwallet/sparrow/net/UnsupportedMethodException.java create mode 100644 src/test/java/com/sparrowwallet/sparrow/net/ElectrumServerRpcTest.java create mode 100644 src/test/java/com/sparrowwallet/sparrow/net/HeaderCheckpointVerificationTest.java create mode 100644 src/test/java/com/sparrowwallet/sparrow/net/PagedBatchRequestBuilderTest.java diff --git a/build.gradle b/build.gradle index 7fe200e7..1ab50fa5 100644 --- a/build.gradle +++ b/build.gradle @@ -126,10 +126,29 @@ compileJava { } test { - useJUnitPlatform() + useJUnitPlatform { + //Checkpoint verification needs the network and checks compiled in data rather than code, so it is a release checklist step + excludeTags 'checkpoint' + } jvmArgs = ["--enable-native-access=ALL-UNNAMED"] } +tasks.register('verifyCheckpoint', Test) { + description = 'Verifies the compiled in header checkpoints against live public Electrum servers' + group = 'verification' + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + useJUnitPlatform { + includeTags 'checkpoint' + } + jvmArgs = ["--enable-native-access=ALL-UNNAMED"] + outputs.upToDateWhen { false } + testLogging { + showStandardStreams = true + exceptionFormat = 'full' + } +} + application { mainModule = 'com.sparrowwallet.sparrow' mainClass = 'com.sparrowwallet.sparrow.SparrowWallet' diff --git a/drongo b/drongo index b3856107..96c91f50 160000 --- a/drongo +++ b/drongo @@ -1 +1 @@ -Subproject commit b38561072dab79188b3ec845a440735dbe0455ec +Subproject commit 96c91f506ef96114cdd7414c5fb8cdd33224fb3d diff --git a/src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java b/src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java index 1f3b8516..5493114f 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/BatchedElectrumServerRpc.java @@ -5,6 +5,8 @@ import com.github.arteam.simplejsonrpc.client.Transport; import com.github.arteam.simplejsonrpc.client.exception.JsonRpcBatchException; import com.github.arteam.simplejsonrpc.client.exception.JsonRpcException; import com.sparrowwallet.drongo.protocol.Sha256Hash; +import com.sparrowwallet.drongo.protocol.VerificationException; +import com.sparrowwallet.drongo.wallet.BlockTransactionHash; import com.sparrowwallet.drongo.wallet.Wallet; import com.sparrowwallet.sparrow.EventManager; import com.sparrowwallet.sparrow.event.WalletHistoryStatusEvent; @@ -22,6 +24,7 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc { private static final Logger log = LoggerFactory.getLogger(BatchedElectrumServerRpc.class); static final int DEFAULT_MAX_ATTEMPTS = 5; static final int RETRY_DELAY_SECS = 1; + static final int HEADERS_BATCH_PAGE_SIZE = 4; //Four difficulty periods of headers is ~1.3MB of hex, within every server's max response size private final AtomicLong idCounter; private final int maxTargetBlocks; @@ -287,6 +290,93 @@ public class BatchedElectrumServerRpc implements ElectrumServerRpc { } } + @Override + @SuppressWarnings("unchecked") + public Map getTransactionMerkleProofs(Transport transport, Wallet wallet, Collection references) { + PagedBatchRequestBuilder batchRequest = PagedBatchRequestBuilder.create(transport, idCounter).keysType(String.class).returnType(TransactionMerkleProof.class); + EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Verifying " + references.size() + " transactions")); + + for(BlockTransactionHash reference : references) { + //Keyed by the exact pair: the same txid may legitimately be requested at two heights in one batch, and each must be answered on its own + batchRequest.add(reference.getHashAsString() + ":" + reference.getHeight(), "blockchain.transaction.get_merkle", reference.getHashAsString(), reference.getHeight()); + } + + try { + //Every page is sent, per-page errors are substituted with the sentinel, and the merged map covers every requested pair + return batchRequest.executeTolerant(DEFAULT_MAX_ATTEMPTS, TransactionMerkleProof.ERROR_PROOF, e -> { + //Method not found is a whole batch property: either the server implements get_merkle or it does not + if(ElectrumServerRpc.isMethodNotFound(e)) { + throw new UnsupportedMethodException("blockchain.transaction.get_merkle", e); + } + }); + } catch(UnsupportedMethodException e) { + throw e; //before the generic catch, or it is rewrapped and the caller cannot tell an unsupported server from a refusal + } catch(Exception e) { + throw new ElectrumServerRpcException("Failed to retrieve merkle proofs", e); + } + } + + @Override + public BlockHeaders getBlockHeadersChunk(Transport transport, int startHeight, int count) { + try { + JsonRpcClient client = new JsonRpcClient(transport); + BlockHeaders blockHeaders = new RetryLogic(DEFAULT_MAX_ATTEMPTS, RETRY_DELAY_SECS, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(() -> + client.createRequest().returnAs(BlockHeaders.class).method("blockchain.block.headers").id(idCounter.incrementAndGet()).params(startHeight, count).execute()); + + return ElectrumServerRpc.checkBlockHeaders(blockHeaders, startHeight, count); + } catch(UnsupportedMethodException | VerificationException e) { + throw e; //before the generic catch, so callers can treat an unsupported server and a malformed response on their own terms + } catch(JsonRpcException e) { + if(ElectrumServerRpc.isMethodNotFound(e)) { + throw new UnsupportedMethodException("blockchain.block.headers", e); + } + + throw new ElectrumServerRpcException("Failed to retrieve " + count + " block headers from height " + startHeight, e); + } catch(Exception e) { + throw new ElectrumServerRpcException("Failed to retrieve " + count + " block headers from height " + startHeight, e); + } + } + + @Override + public Map getBlockHeadersChunks(Transport transport, Map startHeightCounts) { + //A page of the default size would be tens of megabytes of hex, far beyond what any server returns in one response + PagedBatchRequestBuilder batchRequest = PagedBatchRequestBuilder.create(transport, idCounter).keysType(Integer.class).returnType(BlockHeaders.class) + .pageSize(HEADERS_BATCH_PAGE_SIZE); + + for(Map.Entry startHeightCount : startHeightCounts.entrySet()) { + batchRequest.add(startHeightCount.getKey(), "blockchain.block.headers", startHeightCount.getKey(), startHeightCount.getValue()); + } + + Map result; + try { + result = batchRequest.executeTolerant(DEFAULT_MAX_ATTEMPTS, BlockHeaders.ERROR_HEADERS, e -> { + if(ElectrumServerRpc.isMethodNotFound(e)) { + throw new UnsupportedMethodException("blockchain.block.headers", e); + } + }); + } catch(UnsupportedMethodException e) { + throw e; + } catch(Exception e) { + throw new ElectrumServerRpcException("Failed to retrieve block headers for start heights: " + startHeightCounts.keySet(), e); + } + + //A range the server errored on, or whose response fails the checks, is omitted rather than failing the ranges that succeeded + Map checked = new LinkedHashMap<>(); + for(Map.Entry entry : result.entrySet()) { + if(entry.getValue() == BlockHeaders.ERROR_HEADERS) { + continue; + } + + try { + checked.put(entry.getKey(), ElectrumServerRpc.checkBlockHeaders(entry.getValue(), entry.getKey(), startHeightCounts.get(entry.getKey()))); + } catch(VerificationException e) { + log.warn("Omitting block headers from height " + entry.getKey() + ": " + e.getMessage()); + } + } + + return checked; + } + @Override public Map getFeeEstimates(Transport transport, List targetBlocks) { PagedBatchRequestBuilder batchRequest = PagedBatchRequestBuilder.create(transport, idCounter).keysType(Integer.class).returnType(Double.class); diff --git a/src/main/java/com/sparrowwallet/sparrow/net/BlockHeaders.java b/src/main/java/com/sparrowwallet/sparrow/net/BlockHeaders.java new file mode 100644 index 00000000..49a174e1 --- /dev/null +++ b/src/main/java/com/sparrowwallet/sparrow/net/BlockHeaders.java @@ -0,0 +1,25 @@ +package com.sparrowwallet.sparrow.net; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * The blockchain.block.headers response: a run of consecutive block headers as concatenated hex, with the maximum number of headers the server will return. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class BlockHeaders { + public static final int HEADER_HEX_LENGTH = 160; + + /** + * Substituted for a range the server returned an error for, and filtered out before a batched result is returned. + */ + public static final BlockHeaders ERROR_HEADERS = new BlockHeaders(); + + public int count; + public String hex; + public int max; + + @Override + public String toString() { + return "BlockHeaders{count=" + count + ", max=" + max + '}'; + } +} diff --git a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java index de262555..db91f239 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java @@ -53,7 +53,7 @@ import java.util.stream.Stream; public class ElectrumServer { private static final Logger log = LoggerFactory.getLogger(ElectrumServer.class); - private static final String[] SUPPORTED_VERSIONS = new String[]{"1.3", "1.4.2"}; + static final String[] SUPPORTED_VERSIONS = new String[]{"1.3", "1.4.2"}; private static final Version ELECTRS_MIN_BATCHING_VERSION = new Version("0.9.0"); @@ -1882,7 +1882,7 @@ public class ElectrumServer { } if(server.startsWith("cormorant")) { - return new ServerCapability(true, false, true, false, true); + return new ServerCapability(true, false, true, false, true).withMerkleProofs(false); } if(server.startsWith("electrs/")) { @@ -2094,6 +2094,10 @@ public class ElectrumServer { } } + if(isVerificationMandatory() && !serverCapability.supportsMerkleProofs()) { + throw new ServerException("Server does not support transaction verification (blockchain.transaction.get_merkle)"); + } + BlockHeaderTip tip; if(subscribe) { tip = electrumServer.subscribeBlockHeaders(); @@ -2101,6 +2105,13 @@ public class ElectrumServer { if(tipError != null) { throw new ServerException(tipError); } + if(isVerificationMandatory()) { + //A server below the last pinned header cannot serve the header sync, and would report every proof as refused + int maxCheckpointHeight = Network.get().getHeaderCheckpoints().getMaxHeight(); + if(tip.height < maxCheckpointHeight) { + throw new ServerException("Server is at height " + tip.height + ", below the last verified checkpoint at height " + maxCheckpointHeight); + } + } initializeTip(tip); subscribedScriptHashes.clear(); } else { @@ -2142,6 +2153,10 @@ public class ElectrumServer { }; } + private boolean isVerificationMandatory() { + return Config.get().getServerType() == ServerType.PUBLIC_ELECTRUM_SERVER && Network.get() == Network.MAINNET; + } + private void checkTipStaleness() { if(subscribe && Network.get() == Network.MAINNET && lastTipReceivedAt > 0 && !staleTipWarned && System.currentTimeMillis() - lastTipReceivedAt > STALE_TIP_WARNING_AGE_MILLIS) { staleTipWarned = true; diff --git a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServerRpc.java b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServerRpc.java index 1082aadf..440e8e9f 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServerRpc.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServerRpc.java @@ -1,13 +1,23 @@ package com.sparrowwallet.sparrow.net; import com.github.arteam.simplejsonrpc.client.Transport; +import com.github.arteam.simplejsonrpc.client.exception.JsonRpcBatchException; +import com.github.arteam.simplejsonrpc.client.exception.JsonRpcException; +import com.sparrowwallet.drongo.protocol.HeaderChainState; +import com.sparrowwallet.drongo.protocol.VerificationException; +import com.sparrowwallet.drongo.wallet.BlockTransactionHash; import com.sparrowwallet.drongo.wallet.Wallet; +import com.sparrowwallet.sparrow.AppServices; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; public interface ElectrumServerRpc { + /** The JSON-RPC standard code for a method the server does not implement, which bitcoind returns as well as Electrum servers. */ + int METHOD_NOT_FOUND = -32601; + void ping(Transport transport); List getServerVersion(Transport transport, String clientName, String[] supportedVersions); @@ -38,6 +48,24 @@ public interface ElectrumServerRpc { Map getVerboseTransactions(Transport transport, Set txids, String scriptHash); + /** + * Retrieves the merkle inclusion proof for each of the given transactions at the height it is claimed to be confirmed at. + * The result is keyed by the exact pair as "txid:height", since the same transaction may legitimately be requested at two heights in one call, + * and carries TransactionMerkleProof.ERROR_PROOF for every pair the server returned an error for. + */ + Map getTransactionMerkleProofs(Transport transport, Wallet wallet, Collection references); + + /** + * Retrieves a run of consecutive block headers, throwing VerificationException if the response is malformed or short of the server's own tip. + */ + BlockHeaders getBlockHeadersChunk(Transport transport, int startHeight, int count); + + /** + * Retrieves several runs of consecutive block headers, keyed by start height. A range the server errors on, or whose response fails the + * checks getBlockHeadersChunk applies, is omitted from the result rather than failing the others. + */ + Map getBlockHeadersChunks(Transport transport, Map startHeightCounts); + Map getFeeEstimates(Transport transport, List targetBlocks); Map getFeeRateHistogram(Transport transport); @@ -47,4 +75,44 @@ public interface ElectrumServerRpc { String broadcastTransaction(Transport transport, String txHex); long getIdCounterValue(); + + /** Whether every error in a batch reports the method as not found, which is a property of the server rather than of any one request. */ + static boolean isMethodNotFound(JsonRpcBatchException e) { + return !e.getErrors().isEmpty() && e.getErrors().values().stream().allMatch(error -> error.getCode() == METHOD_NOT_FOUND); + } + + static boolean isMethodNotFound(JsonRpcException e) { + return e.getErrorMessage() != null && e.getErrorMessage().getCode() == METHOD_NOT_FOUND; + } + + /** + * Applies Electrum's checks to a blockchain.block.headers response. A server returning at most a partial difficulty period cannot serve the + * header sync at all, and a response shorter than requested is only explicable by the server having reached its own announced tip. + */ + static BlockHeaders checkBlockHeaders(BlockHeaders blockHeaders, int startHeight, int count) throws VerificationException { + return checkBlockHeaders(blockHeaders, startHeight, count, AppServices.getCurrentBlockHeight()); + } + + static BlockHeaders checkBlockHeaders(BlockHeaders blockHeaders, int startHeight, int count, Integer tip) throws VerificationException { + if(blockHeaders == null || blockHeaders.hex == null || blockHeaders.count < 0) { + throw new VerificationException("Malformed response to a request for " + count + " block headers from height " + startHeight); + } + if(blockHeaders.max < HeaderChainState.RETARGET_INTERVAL) { + throw new VerificationException("Server returns at most " + blockHeaders.max + " block headers per request, too few to cover a difficulty period"); + } + if(blockHeaders.count > count) { + throw new VerificationException("Requested " + count + " block headers from height " + startHeight + " but the server returned " + blockHeaders.count); + } + if(blockHeaders.hex.length() != blockHeaders.count * BlockHeaders.HEADER_HEX_LENGTH) { + throw new VerificationException("Response to a request for block headers from height " + startHeight + " contains " + blockHeaders.hex.length() / 2 + + " bytes for " + blockHeaders.count + " headers"); + } + + if(blockHeaders.count < count && tip != null && startHeight + blockHeaders.count - 1 < tip) { + throw new VerificationException("Requested " + count + " block headers from height " + startHeight + " but the server returned only " + blockHeaders.count + + " while announcing a tip at height " + tip); + } + + return blockHeaders; + } } diff --git a/src/main/java/com/sparrowwallet/sparrow/net/PagedBatchRequestBuilder.java b/src/main/java/com/sparrowwallet/sparrow/net/PagedBatchRequestBuilder.java index 41545c07..56a8bbd9 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/PagedBatchRequestBuilder.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/PagedBatchRequestBuilder.java @@ -14,6 +14,7 @@ import org.jetbrains.annotations.Nullable; import java.util.*; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; import static com.sparrowwallet.sparrow.net.BatchedElectrumServerRpc.DEFAULT_MAX_ATTEMPTS; import static com.sparrowwallet.sparrow.net.BatchedElectrumServerRpc.RETRY_DELAY_SECS; @@ -40,6 +41,12 @@ public class PagedBatchRequestBuilder extends AbstractBuilder { @Nullable private final Class returnType; + /** + * Overrides the configured page size, for methods whose responses are far larger than a typical request + */ + @Nullable + private Integer pageSize; + /** * Creates a new batch request builder in an initial state * @@ -84,7 +91,9 @@ public class PagedBatchRequestBuilder extends AbstractBuilder { * @return a new builder */ public PagedBatchRequestBuilder keysType(@NotNull Class keysClass) { - return new PagedBatchRequestBuilder(transport, mapper, new ArrayList>(), keysClass, returnType, counter); + PagedBatchRequestBuilder builder = new PagedBatchRequestBuilder(transport, mapper, new ArrayList>(), keysClass, returnType, counter); + builder.pageSize = pageSize; + return builder; } /** @@ -96,7 +105,21 @@ public class PagedBatchRequestBuilder extends AbstractBuilder { * @return a new builder */ public PagedBatchRequestBuilder returnType(@NotNull Class valuesClass) { - return new PagedBatchRequestBuilder(transport, mapper, requests, keysType, valuesClass, counter); + PagedBatchRequestBuilder builder = new PagedBatchRequestBuilder(transport, mapper, requests, keysType, valuesClass, counter); + builder.pageSize = pageSize; + return builder; + } + + /** + * Sets the number of requests sent in each batch, overriding the configured maximum page size. + * + * @param pageSize number of requests per batch + * @return the current builder + */ + @NotNull + public PagedBatchRequestBuilder pageSize(int pageSize) { + this.pageSize = pageSize; + return this; } public Map execute() throws Exception { @@ -114,54 +137,94 @@ public class PagedBatchRequestBuilder extends AbstractBuilder { Map allResults = new HashMap<>(); JsonRpcClient client = new JsonRpcClient(transport); - List>> pages = Lists.partition(requests, getPageSize()); - for(List> page : pages) { - if(counter != null) { - Map counterIdMap = new HashMap<>(); - BatchRequestBuilder batchRequest = client.createBatchRequest().keysType(Long.class).returnType(returnType); - for(Request request : page) { - counterIdMap.put(request.counterId, request.id); - batchRequest.add(request.counterId, request.method, request.params); - } + for(List> page : Lists.partition(requests, getPageSize())) { + allResults.putAll(executePage(client, page, maxAttempts)); + } - try { - Map pageResult = new RetryLogic>(maxAttempts, RETRY_DELAY_SECS, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute); - for(Map.Entry pageEntry : pageResult.entrySet()) { - allResults.put(counterIdMap.get(pageEntry.getKey()), pageEntry.getValue()); - } - } catch(JsonRpcBatchException e) { - Map mappedSuccesess = new HashMap<>(); - for(Map.Entry successEntry : e.getSuccesses().entrySet()) { - mappedSuccesess.put(counterIdMap.get((Long)successEntry.getKey()), successEntry.getValue()); - } - Map mappedErrors = new HashMap<>(); - for(Map.Entry errorEntry : e.getErrors().entrySet()) { - mappedErrors.put(counterIdMap.get((Long)errorEntry.getKey()), errorEntry.getValue()); - } - throw new JsonRpcBatchException(e.getMessage(), mappedSuccesess, mappedErrors); - } - } else { - BatchRequestBuilder batchRequest = client.createBatchRequest().keysType(keysType).returnType(returnType); - for(Request request : page) { - if(request.id instanceof String strReq) { - batchRequest.add(strReq, request.method, request.params); - } else if(request.id instanceof Integer intReq) { - batchRequest.add(intReq, request.method, request.params); - } else { - throw new IllegalArgumentException("Id of class " + request.id.getClass().getName() + " not supported"); - } - } + return allResults; + } - Map pageResult = new RetryLogic>(maxAttempts, RETRY_DELAY_SECS, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute); - allResults.putAll(pageResult); + /** + * Validates and executes the request, substituting the given value for the keys any page reports an error for so that the returned map + * covers every requested key. Unlike execute(), a JSON-RPC error in one page neither discards the results of earlier pages nor prevents + * later pages from being sent, which is what makes a per-key error meaningful when the request spans more than one page. + * + * @param maxAttempts number of times to try each page + * @param errorValue the value to record for keys the server returned an error for + * @param batchErrorCheck applied to each page's errors before they are substituted, to let the caller classify whole-batch failures + * @return map of responses by request ids, covering every requested id + */ + @NotNull + @SuppressWarnings("unchecked") + public Map executeTolerant(int maxAttempts, V errorValue, Consumer batchErrorCheck) throws Exception { + Map allResults = new HashMap<>(); + JsonRpcClient client = new JsonRpcClient(transport); + + for(List> page : Lists.partition(requests, getPageSize())) { + try { + allResults.putAll(executePage(client, page, maxAttempts)); + } catch(JsonRpcBatchException e) { + batchErrorCheck.accept(e); + allResults.putAll((Map)e.getSuccesses()); + for(Object key : e.getErrors().keySet()) { + allResults.put((K)key, errorValue); + } } } return allResults; } + /** + * Executes a single page, mapping any batch exception back to the request ids the caller supplied + */ + @NotNull + private Map executePage(JsonRpcClient client, List> page, int maxAttempts) throws Exception { + if(counter != null) { + Map counterIdMap = new HashMap<>(); + BatchRequestBuilder batchRequest = client.createBatchRequest().keysType(Long.class).returnType(returnType); + for(Request request : page) { + counterIdMap.put(request.counterId, request.id); + batchRequest.add(request.counterId, request.method, request.params); + } + + try { + Map pageResult = new RetryLogic>(maxAttempts, RETRY_DELAY_SECS, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute); + Map mappedResult = new HashMap<>(); + for(Map.Entry pageEntry : pageResult.entrySet()) { + mappedResult.put(counterIdMap.get(pageEntry.getKey()), pageEntry.getValue()); + } + + return mappedResult; + } catch(JsonRpcBatchException e) { + Map mappedSuccesess = new HashMap<>(); + for(Map.Entry successEntry : e.getSuccesses().entrySet()) { + mappedSuccesess.put(counterIdMap.get((Long)successEntry.getKey()), successEntry.getValue()); + } + Map mappedErrors = new HashMap<>(); + for(Map.Entry errorEntry : e.getErrors().entrySet()) { + mappedErrors.put(counterIdMap.get((Long)errorEntry.getKey()), errorEntry.getValue()); + } + throw new JsonRpcBatchException(e.getMessage(), mappedSuccesess, mappedErrors); + } + } else { + BatchRequestBuilder batchRequest = client.createBatchRequest().keysType(keysType).returnType(returnType); + for(Request request : page) { + if(request.id instanceof String strReq) { + batchRequest.add(strReq, request.method, request.params); + } else if(request.id instanceof Integer intReq) { + batchRequest.add(intReq, request.method, request.params); + } else { + throw new IllegalArgumentException("Id of class " + request.id.getClass().getName() + " not supported"); + } + } + + return new RetryLogic>(maxAttempts, RETRY_DELAY_SECS, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(batchRequest::execute); + } + } + private int getPageSize() { - int pageSize = Config.get().getMaxPageSize(); + int pageSize = this.pageSize != null ? this.pageSize : Config.get().getMaxPageSize(); if(pageSize < 1) { pageSize = DEFAULT_PAGE_SIZE; } diff --git a/src/main/java/com/sparrowwallet/sparrow/net/ServerCapability.java b/src/main/java/com/sparrowwallet/sparrow/net/ServerCapability.java index 4946e7ef..7c1f969e 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/ServerCapability.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/ServerCapability.java @@ -12,6 +12,7 @@ public class ServerCapability { private final boolean supportsBlockStats; private final boolean supportsUnsubscribe; private final boolean supportsServerFeatures; + private boolean supportsMerkleProofs = true; private List supportedSilentPaymentsVersions = Collections.emptyList(); public ServerCapability(boolean supportsBatching, boolean supportsUnsubscribe, boolean supportsServerFeatures) { @@ -64,6 +65,10 @@ public class ServerCapability { return supportsServerFeatures; } + public boolean supportsMerkleProofs() { + return supportsMerkleProofs; + } + public List getSupportedSilentPaymentsVersions() { return supportedSilentPaymentsVersions; } @@ -72,6 +77,11 @@ public class ServerCapability { return supportedSilentPaymentsVersions.contains(0); } + public ServerCapability withMerkleProofs(boolean supportsMerkleProofs) { + this.supportsMerkleProofs = supportsMerkleProofs; + return this; + } + public ServerCapability withServerFeatures(ServerFeatures features) { if(features != null && features.silent_payments != null) { this.supportedSilentPaymentsVersions = List.copyOf(features.silent_payments); diff --git a/src/main/java/com/sparrowwallet/sparrow/net/SimpleElectrumServerRpc.java b/src/main/java/com/sparrowwallet/sparrow/net/SimpleElectrumServerRpc.java index b909e7e7..30b76c0a 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/SimpleElectrumServerRpc.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/SimpleElectrumServerRpc.java @@ -6,6 +6,8 @@ import com.github.arteam.simplejsonrpc.client.exception.JsonRpcException; import com.sparrowwallet.drongo.Utils; import com.sparrowwallet.drongo.protocol.Sha256Hash; import com.sparrowwallet.drongo.protocol.Transaction; +import com.sparrowwallet.drongo.protocol.VerificationException; +import com.sparrowwallet.drongo.wallet.BlockTransactionHash; import com.sparrowwallet.drongo.wallet.Wallet; import com.sparrowwallet.sparrow.AppServices; import com.sparrowwallet.sparrow.EventManager; @@ -325,6 +327,76 @@ public class SimpleElectrumServerRpc implements ElectrumServerRpc { return result; } + @Override + public Map getTransactionMerkleProofs(Transport transport, Wallet wallet, Collection references) { + JsonRpcClient client = new JsonRpcClient(transport); + + Map result = new LinkedHashMap<>(); + for(BlockTransactionHash reference : references) { + EventManager.get().post(new WalletHistoryStatusEvent(wallet, true, "Verifying transaction [" + reference.getHashAsString().substring(0, 6) + "]")); + //Keyed by the exact pair: the same txid may legitimately be requested at two heights, and each must be answered on its own + String key = reference.getHashAsString() + ":" + reference.getHeight(); + try { + TransactionMerkleProof proof = new RetryLogic(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(() -> + client.createRequest().returnAs(TransactionMerkleProof.class).method("blockchain.transaction.get_merkle").id(idCounter.incrementAndGet()) + .params(reference.getHashAsString(), reference.getHeight()).execute()); + result.put(key, proof); + } catch(ServerException e) { + //If there is an error with the server connection, don't keep trying - this may take too long given many transactions + throw new ElectrumServerRpcException("Failed to retrieve merkle proof for transaction [" + reference.getHashAsString().substring(0, 6) + "]", e); + } catch(JsonRpcException e) { + //Method not found is a property of the server rather than of this transaction, so stop rather than recording a refusal for every one + if(ElectrumServerRpc.isMethodNotFound(e)) { + throw new UnsupportedMethodException("blockchain.transaction.get_merkle", e); + } + + result.put(key, TransactionMerkleProof.ERROR_PROOF); + } catch(Exception e) { + result.put(key, TransactionMerkleProof.ERROR_PROOF); + } + } + + return result; + } + + @Override + public BlockHeaders getBlockHeadersChunk(Transport transport, int startHeight, int count) { + try { + JsonRpcClient client = new JsonRpcClient(transport); + BlockHeaders blockHeaders = new RetryLogic(MAX_RETRIES, RETRY_DELAY, List.of(IllegalStateException.class, IllegalArgumentException.class)).getResult(() -> + client.createRequest().returnAs(BlockHeaders.class).method("blockchain.block.headers").id(idCounter.incrementAndGet()).params(startHeight, count).execute()); + + return ElectrumServerRpc.checkBlockHeaders(blockHeaders, startHeight, count); + } catch(UnsupportedMethodException | VerificationException e) { + throw e; //before the generic catch, so callers can treat an unsupported server and a malformed response on their own terms + } catch(JsonRpcException e) { + if(ElectrumServerRpc.isMethodNotFound(e)) { + throw new UnsupportedMethodException("blockchain.block.headers", e); + } + + throw new ElectrumServerRpcException("Failed to retrieve " + count + " block headers from height " + startHeight, e); + } catch(Exception e) { + throw new ElectrumServerRpcException("Failed to retrieve " + count + " block headers from height " + startHeight, e); + } + } + + @Override + public Map getBlockHeadersChunks(Transport transport, Map startHeightCounts) { + Map result = new LinkedHashMap<>(); + for(Map.Entry startHeightCount : startHeightCounts.entrySet()) { + try { + result.put(startHeightCount.getKey(), getBlockHeadersChunk(transport, startHeightCount.getKey(), startHeightCount.getValue())); + } catch(UnsupportedMethodException e) { + throw e; + } catch(VerificationException | ElectrumServerRpcException e) { + //A range that fails is omitted rather than failing the ranges that succeeded + log.warn("Omitting block headers from height " + startHeightCount.getKey() + ": " + e.getMessage()); + } + } + + return result; + } + @Override public Map getFeeEstimates(Transport transport, List targetBlocks) { JsonRpcClient client = new JsonRpcClient(transport); diff --git a/src/main/java/com/sparrowwallet/sparrow/net/TransactionMerkleProof.java b/src/main/java/com/sparrowwallet/sparrow/net/TransactionMerkleProof.java new file mode 100644 index 00000000..b34cca08 --- /dev/null +++ b/src/main/java/com/sparrowwallet/sparrow/net/TransactionMerkleProof.java @@ -0,0 +1,25 @@ +package com.sparrowwallet.sparrow.net; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.util.List; + +/** + * The blockchain.transaction.get_merkle response: the sibling path from a transaction to the merkle root of the block containing it. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class TransactionMerkleProof { + /** + * Substituted for a transaction the server returned an error for, so that a batch covering many transactions can partially succeed. + */ + public static final TransactionMerkleProof ERROR_PROOF = new TransactionMerkleProof(); + + public int block_height; + public List merkle; + public int pos; + + @Override + public String toString() { + return "TransactionMerkleProof{block_height=" + block_height + ", pos=" + pos + ", merkle=" + merkle + '}'; + } +} diff --git a/src/main/java/com/sparrowwallet/sparrow/net/UnsupportedMethodException.java b/src/main/java/com/sparrowwallet/sparrow/net/UnsupportedMethodException.java new file mode 100644 index 00000000..b013ec03 --- /dev/null +++ b/src/main/java/com/sparrowwallet/sparrow/net/UnsupportedMethodException.java @@ -0,0 +1,19 @@ +package com.sparrowwallet.sparrow.net; + +/** + * Thrown when the connected server does not implement a method, as reported by the JSON-RPC method not found error code. + * Unlike other errors this is a property of the server rather than of the request, so callers disable the feature for the session + * rather than treating the response as a refusal. + */ +public class UnsupportedMethodException extends ElectrumServerRpcException { + private final String method; + + public UnsupportedMethodException(String method, Throwable cause) { + super("Server does not support " + method, cause); + this.method = method; + } + + public String getMethod() { + return method; + } +} diff --git a/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClient.java b/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClient.java index 7ff33f27..68890ee0 100644 --- a/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClient.java +++ b/src/main/java/com/sparrowwallet/sparrow/net/cormorant/bitcoind/BitcoindClient.java @@ -20,6 +20,7 @@ import com.sparrowwallet.sparrow.io.Config; import com.sparrowwallet.sparrow.net.Bwt; import com.sparrowwallet.sparrow.net.ConfigurationException; import com.sparrowwallet.sparrow.net.CoreAuthType; +import com.sparrowwallet.sparrow.net.ElectrumServerRpc; import com.sparrowwallet.sparrow.net.cormorant.Cormorant; import com.sparrowwallet.drongo.address.Address; import com.sparrowwallet.drongo.address.InvalidAddressException; @@ -51,7 +52,6 @@ public class BitcoindClient { private static final long PRUNED_RESCAN_TIMEGAP_MILLIS = 7200*1000; //Error codes from https://github.com/bitcoin/bitcoin/blob/master/src/rpc/protocol.h - public static final int RPC_METHOD_NOT_FOUND = -32601; public static final int RPC_WALLET_NOT_FOUND = -18; public static final String WALLET_ALREADY_LOADING_MESSAGE = "Wallet already loading."; @@ -159,7 +159,7 @@ public class BitcoindClient { } legacyWalletExists = loadedWallets.contains(Bwt.DEFAULT_CORE_WALLET); } catch(JsonRpcException e) { - if(e.getErrorMessage().getCode() == RPC_METHOD_NOT_FOUND) { + if(ElectrumServerRpc.isMethodNotFound(e)) { throw new BitcoinRPCException("Wallet support must be enabled in Bitcoin Core"); } else { throw e; diff --git a/src/test/java/com/sparrowwallet/sparrow/net/ElectrumServerRpcTest.java b/src/test/java/com/sparrowwallet/sparrow/net/ElectrumServerRpcTest.java new file mode 100644 index 00000000..3fd8258a --- /dev/null +++ b/src/test/java/com/sparrowwallet/sparrow/net/ElectrumServerRpcTest.java @@ -0,0 +1,76 @@ +package com.sparrowwallet.sparrow.net; + +import com.sparrowwallet.drongo.protocol.HeaderChainState; +import com.sparrowwallet.drongo.protocol.VerificationException; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The checks Electrum applies to a blockchain.block.headers response. Every failure is refusal class rather than a session failure, so the + * header sync stops advancing on a server answering nonsense instead of feeding it to the chain state. + */ +public class ElectrumServerRpcTest { + private static final int TIP = 900000; + + @Test + public void acceptsAFullResponse() { + assertDoesNotThrow(() -> ElectrumServerRpc.checkBlockHeaders(headers(2016, 2016, 2016), 800000, 2016, TIP)); + } + + @Test + public void acceptsAShortResponseThatReachesTheTip() { + //The only reason a server may send fewer headers than asked for + assertDoesNotThrow(() -> ElectrumServerRpc.checkBlockHeaders(headers(17, 17, 2016), TIP - 16, 2016, TIP)); + } + + @Test + public void rejectsAShortResponseBelowTheTip() { + assertThrows(VerificationException.class, () -> ElectrumServerRpc.checkBlockHeaders(headers(17, 17, 2016), 800000, 2016, TIP)); + } + + @Test + public void acceptsAShortResponseWhenTheTipIsUnknown() { + //Before the application has a chain view there is nothing to judge the response against, so the caller's own linkage check decides + assertDoesNotThrow(() -> ElectrumServerRpc.checkBlockHeaders(headers(17, 17, 2016), 800000, 2016, null)); + } + + @Test + public void rejectsMoreHeadersThanRequested() { + assertThrows(VerificationException.class, () -> ElectrumServerRpc.checkBlockHeaders(headers(2017, 2017, 2016), 800000, 2016, TIP)); + } + + @Test + public void rejectsAHexLengthInconsistentWithTheCount() { + assertThrows(VerificationException.class, () -> ElectrumServerRpc.checkBlockHeaders(headers(2016, 2015, 2016), 800000, 2016, TIP)); + assertThrows(VerificationException.class, () -> ElectrumServerRpc.checkBlockHeaders(headers(2016, 2017, 2016), 800000, 2016, TIP)); + } + + @Test + public void rejectsAServerThatCannotReturnAWholeDifficultyPeriod() { + //A period must always fit one call, or the sync cannot advance across a retarget + assertThrows(VerificationException.class, () -> ElectrumServerRpc.checkBlockHeaders(headers(2016, 2016, HeaderChainState.RETARGET_INTERVAL - 1), 800000, 2016, TIP)); + } + + @Test + public void rejectsMalformedResponses() { + assertThrows(VerificationException.class, () -> ElectrumServerRpc.checkBlockHeaders(null, 800000, 2016, TIP)); + assertThrows(VerificationException.class, () -> ElectrumServerRpc.checkBlockHeaders(headers(2016, -1, 2016), 800000, 2016, TIP)); + assertThrows(VerificationException.class, () -> ElectrumServerRpc.checkBlockHeaders(headers(-1, 0, 2016), 800000, 2016, TIP)); + } + + /** + * @param count the count the server reports + * @param hexHeaders the number of headers actually present in the hex + * @param max the maximum number of headers the server reports it will return + */ + private static BlockHeaders headers(int count, int hexHeaders, int max) { + BlockHeaders blockHeaders = new BlockHeaders(); + blockHeaders.count = count; + blockHeaders.hex = hexHeaders < 0 ? null : "0".repeat(hexHeaders * BlockHeaders.HEADER_HEX_LENGTH); + blockHeaders.max = max; + + return blockHeaders; + } +} diff --git a/src/test/java/com/sparrowwallet/sparrow/net/HeaderCheckpointVerificationTest.java b/src/test/java/com/sparrowwallet/sparrow/net/HeaderCheckpointVerificationTest.java new file mode 100644 index 00000000..0adb4cde --- /dev/null +++ b/src/test/java/com/sparrowwallet/sparrow/net/HeaderCheckpointVerificationTest.java @@ -0,0 +1,149 @@ +package com.sparrowwallet.sparrow.net; + +import com.sparrowwallet.drongo.Network; +import com.sparrowwallet.drongo.Utils; +import com.sparrowwallet.drongo.protocol.BlockHeader; +import com.sparrowwallet.drongo.protocol.HeaderChainState; +import com.sparrowwallet.drongo.protocol.HeaderCheckpoints; +import com.sparrowwallet.sparrow.SparrowWallet; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Verifies the pinned header hashes compiled into drongo against live public Electrum servers. This is a release checklist step rather than a unit + * test - it needs the network and it is checking data, not code - so it is tagged out of the default test task and run with the verifyCheckpoint task. + *

+ * A server that cannot be reached is skipped; a server that answers with a different chain fails the run. Checkpoint generation is a derivation from a + * locally validated header chain, so this confirms independently that what was derived matches what the network says, and that the last pin is buried. + */ +@Tag("checkpoint") +public class HeaderCheckpointVerificationTest { + /** + * The number of blocks the last pinned header must be buried by. A pin that is later orphaned would have every install refuse the real chain at + * that height, so a period is only pinned once it is beyond any plausible reorg. + */ + private static final int CHECKPOINT_BURIAL_DEPTH = 6; + + private static final int PINS_VERIFIED = 3; + private static final int MINIMUM_RESPONDERS = 3; + + @TempDir + private static Path tempHome; + + @BeforeAll + public static void setUp() { + //Config.get() caches its instance statically for the life of the JVM, so keep this test from loading the developer's real config + System.setProperty(SparrowWallet.APP_HOME_PROPERTY, tempHome.toString()); + } + + @AfterAll + public static void tearDownAll() { + System.clearProperty(SparrowWallet.APP_HOME_PROPERTY); + } + + @Test + public void verifyMainnetCheckpoints() { + verifyCheckpoints(Network.MAINNET); + } + + @Test + public void verifyTestnetCheckpoints() { + verifyCheckpoints(Network.TESTNET); + } + + @Test + public void verifyTestnet4Checkpoints() { + verifyCheckpoints(Network.TESTNET4); + } + + @Test + public void verifySignetCheckpoints() { + verifyCheckpoints(Network.SIGNET); + } + + private void verifyCheckpoints(Network network) { + Network.set(network); + + HeaderCheckpoints checkpoints = network.getHeaderCheckpoints(); + List servers = PublicElectrumServer.getServers(); + assertTrue(!servers.isEmpty(), "No public servers configured for " + network); + + List unreachable = new ArrayList<>(); + int responders = 0; + for(PublicElectrumServer publicServer : servers) { + try { + verifyAgainstServer(publicServer, checkpoints); + responders++; + } catch(AssertionError e) { + throw new AssertionError(publicServer.getServer().getHostAndPort() + " disagrees with the " + network + " checkpoints: " + e.getMessage(), e); + } catch(Exception e) { + unreachable.add(publicServer.getServer().getHostAndPort() + " (" + e.getMessage() + ")"); + } + } + + int required = Math.min(MINIMUM_RESPONDERS, servers.size()); + assertTrue(responders >= required, "Only " + responders + " of " + servers.size() + " " + network + " servers responded, at least " + required + + " are needed to confirm the checkpoints. Unreachable: " + unreachable); + } + + private void verifyAgainstServer(PublicElectrumServer publicServer, HeaderCheckpoints checkpoints) throws Exception { + try(CloseableTransport transport = publicServer.getServer().getProtocol().getTransport(publicServer.getServer().getHostAndPort())) { + transport.connect(); + Thread reader = new Thread(() -> { + try { + ((TcpTransport)transport).readInputLoop(); + } catch(ServerException e) { + //Expected once the transport is closed + } + }, "CheckpointVerificationReadThread"); + reader.setDaemon(true); + reader.start(); + + ElectrumServerRpc rpc = new SimpleElectrumServerRpc(); + rpc.getServerVersion(transport, "Sparrow", ElectrumServer.SUPPORTED_VERSIONS); + + //The last pin must be buried, which is established by the server having the headers above it rather than by its announced tip + int maxHeight = checkpoints.getMaxHeight(); + BlockHeaders buried = rpc.getBlockHeadersChunk(transport, maxHeight, CHECKPOINT_BURIAL_DEPTH + 1); + assertEquals(CHECKPOINT_BURIAL_DEPTH + 1, buried.count, "The last pinned header at height " + maxHeight + " is not buried by " + CHECKPOINT_BURIAL_DEPTH + " blocks"); + + for(int i = 0; i < PINS_VERIFIED; i++) { + int pinnedHeight = maxHeight - (i * HeaderChainState.RETARGET_INTERVAL); + if(pinnedHeight <= 0) { + break; + } + + BlockHeaders pair = rpc.getBlockHeadersChunk(transport, pinnedHeight, 2); + assertEquals(2, pair.count, "Server did not return the pinned header at height " + pinnedHeight + " and the one above it"); + BlockHeader pinned = header(pair, 0); + BlockHeader above = header(pair, 1); + + assertTrue(pinned.verifyProofOfWork(), "Header at height " + pinnedHeight + " does not meet its claimed proof of work target"); + assertTrue(above.verifyProofOfWork(), "Header at height " + (pinnedHeight + 1) + " does not meet its claimed proof of work target"); + assertEquals(pinned.getHash(), above.getPrevBlockHash(), "Header at height " + (pinnedHeight + 1) + " does not link to the header below it"); + assertEquals(checkpoints.getHash(pinnedHeight), pinned.getHash(), "Pinned hash at height " + pinnedHeight); + assertEquals(checkpoints.getBitsAfter(pinnedHeight), above.getDifficultyTarget(), "Pinned target following height " + pinnedHeight); + } + } + } + + private BlockHeader header(BlockHeaders blockHeaders, int index) { + return new BlockHeader(Utils.hexToBytes(blockHeaders.hex.substring(index * BlockHeaders.HEADER_HEX_LENGTH, (index + 1) * BlockHeaders.HEADER_HEX_LENGTH))); + } + + @AfterEach + public void tearDown() { + Network.set(null); + } +} diff --git a/src/test/java/com/sparrowwallet/sparrow/net/PagedBatchRequestBuilderTest.java b/src/test/java/com/sparrowwallet/sparrow/net/PagedBatchRequestBuilderTest.java new file mode 100644 index 00000000..df624ff1 --- /dev/null +++ b/src/test/java/com/sparrowwallet/sparrow/net/PagedBatchRequestBuilderTest.java @@ -0,0 +1,185 @@ +package com.sparrowwallet.sparrow.net; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.arteam.simplejsonrpc.client.Transport; +import com.github.arteam.simplejsonrpc.client.exception.JsonRpcBatchException; +import com.sparrowwallet.sparrow.SparrowWallet; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Regression tests for cross page partial success. The existing execute() rethrows a batch exception carrying only the failing page's results, so + * earlier pages are discarded and later pages are never sent. Merkle proof verification reads a per transaction error as a refusal by that server, + * so with execute() a single genuinely refused proof in a multi page request would have been reported as a refusal of every other transaction too. + */ +public class PagedBatchRequestBuilderTest { + @TempDir + private static Path tempHome; + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @BeforeAll + public static void setUp() { + //Config.get() caches its instance statically for the life of the JVM, so keep these tests from loading the developer's real config + System.setProperty(SparrowWallet.APP_HOME_PROPERTY, tempHome.toString()); + } + + @AfterAll + public static void tearDown() { + System.clearProperty(SparrowWallet.APP_HOME_PROPERTY); + } + + @Test + public void tolerantExecutionCoversEveryKeyAcrossPages() throws Exception { + //Six requests over three pages, with one error in the first page and one in the last + FakeBatchTransport transport = new FakeBatchTransport(Set.of("b", "f"), 0); + Map result = builder(transport).executeTolerant(1, "ERROR", e -> {}); + + assertEquals(Set.of("a", "b", "c", "d", "e", "f"), result.keySet()); + assertEquals("result:a", result.get("a")); + assertEquals("ERROR", result.get("b")); + assertEquals("result:c", result.get("c")); + assertEquals("result:e", result.get("e")); + assertEquals("ERROR", result.get("f")); + assertEquals(3, transport.requests.size(), "Every page must be sent, including those after a page that errored"); + } + + @Test + public void tolerantExecutionKeepsEveryResultWhenNothingFails() throws Exception { + FakeBatchTransport transport = new FakeBatchTransport(Set.of(), 0); + Map result = builder(transport).executeTolerant(1, "ERROR", e -> {}); + + assertEquals(6, result.size()); + assertFalse(result.containsValue("ERROR")); + } + + @Test + public void existingExecutionDiscardsResultsOutsideTheFailingPage() { + //The behaviour executeTolerant exists to avoid, asserted so that a change to either is a deliberate one + FakeBatchTransport transport = new FakeBatchTransport(Set.of("b"), 0); + JsonRpcBatchException e = assertThrows(JsonRpcBatchException.class, () -> builder(transport).execute(1)); + + assertEquals(Set.of("a"), new HashSet<>(e.getSuccesses().keySet())); + assertEquals(1, transport.requests.size(), "Pages after the failing one are never sent"); + } + + @Test + public void batchErrorCheckSeesEveryFailingPage() throws Exception { + FakeBatchTransport transport = new FakeBatchTransport(Set.of("b", "f"), 0); + List seen = new ArrayList<>(); + builder(transport).executeTolerant(1, "ERROR", seen::add); + + assertEquals(2, seen.size()); + assertEquals(Set.of("b"), new HashSet<>(seen.get(0).getErrors().keySet())); + assertEquals(Set.of("f"), new HashSet<>(seen.get(1).getErrors().keySet())); + } + + @Test + public void batchErrorCheckCanAbortTheWholeRequest() { + //A method the server does not implement is a property of the server, so the caller stops rather than recording an error per key + FakeBatchTransport transport = new FakeBatchTransport(Set.of("b"), ElectrumServerRpc.METHOD_NOT_FOUND); + UnsupportedMethodException e = assertThrows(UnsupportedMethodException.class, () -> builder(transport).executeTolerant(1, "ERROR", batchException -> { + if(ElectrumServerRpc.isMethodNotFound(batchException)) { + throw new UnsupportedMethodException("test.method", batchException); + } + })); + + assertEquals("test.method", e.getMethod()); + assertEquals(1, transport.requests.size(), "No further pages are sent once the method is known to be unsupported"); + } + + @Test + public void methodNotFoundRequiresEveryErrorToReportIt() { + assertTrue(ElectrumServerRpc.isMethodNotFound(batchException(Map.of("a", ElectrumServerRpc.METHOD_NOT_FOUND, "b", ElectrumServerRpc.METHOD_NOT_FOUND)))); + assertFalse(ElectrumServerRpc.isMethodNotFound(batchException(Map.of("a", ElectrumServerRpc.METHOD_NOT_FOUND, "b", -32603)))); + assertFalse(ElectrumServerRpc.isMethodNotFound(batchException(Map.of("a", -32603)))); + //An empty error map would make allMatch vacuously true, so it is excluded + assertFalse(ElectrumServerRpc.isMethodNotFound(batchException(Map.of()))); + } + + @Test + public void pageSizeSurvivesTheBuilderCopies() throws Exception { + FakeBatchTransport transport = new FakeBatchTransport(Set.of(), 0); + PagedBatchRequestBuilder batchRequest = + (PagedBatchRequestBuilder)PagedBatchRequestBuilder.create(transport, new AtomicLong()).pageSize(3).keysType(String.class).returnType(String.class); + for(String id : List.of("a", "b", "c", "d", "e", "f")) { + batchRequest.add(id, "test.method", id); + } + batchRequest.executeTolerant(1, "ERROR", e -> {}); + + assertEquals(2, transport.requests.size(), "pageSize set before keysType and returnType must be carried onto the copies they return"); + } + + @SuppressWarnings("unchecked") + private PagedBatchRequestBuilder builder(Transport transport) { + PagedBatchRequestBuilder batchRequest = + (PagedBatchRequestBuilder)PagedBatchRequestBuilder.create(transport, new AtomicLong()).keysType(String.class).returnType(String.class).pageSize(2); + for(String id : List.of("a", "b", "c", "d", "e", "f")) { + batchRequest.add(id, "test.method", id); + } + + return batchRequest; + } + + private static JsonRpcBatchException batchException(Map errorCodes) { + Map errors = new java.util.HashMap<>(); + errorCodes.forEach((key, code) -> errors.put(key, new com.github.arteam.simplejsonrpc.core.domain.ErrorMessage(code, "error", null))); + + return new JsonRpcBatchException("test", Map.of(), errors); + } + + /** + * Answers a JSON-RPC batch request with a result for every request, except those whose single parameter is named as failing. + */ + private static class FakeBatchTransport implements Transport { + private final Set failingParams; + private final int errorCode; + private final List requests = new ArrayList<>(); + + FakeBatchTransport(Set failingParams, int errorCode) { + this.failingParams = failingParams; + this.errorCode = errorCode == 0 ? -32603 : errorCode; + } + + @Override + public String pass(String request) throws IOException { + requests.add(request); + ArrayNode responses = MAPPER.createArrayNode(); + for(JsonNode node : MAPPER.readTree(request)) { + String param = node.get("params").get(0).asText(); + ObjectNode response = responses.addObject(); + response.put("jsonrpc", "2.0"); + response.set("id", node.get("id")); + if(failingParams.contains(param)) { + ObjectNode error = response.putObject("error"); + error.put("code", errorCode); + error.put("message", "failed for " + param); + } else { + response.put("result", "result:" + param); + } + } + + return MAPPER.writeValueAsString(responses); + } + } +}