improve deep wallet load performance by adding a setting to watch only the last x used addresses

src/main/java/com/sparrowwallet/sparrow/event/SettingsChangedEvent.java

@@ -20,6 +20,6 @@ public class SettingsChangedEvent {
     }
 
     public enum Type {
-        POLICY, SCRIPT_TYPE, MUTLISIG_THRESHOLD, MULTISIG_TOTAL, KEYSTORE_LABEL, KEYSTORE_FINGERPRINT, KEYSTORE_DERIVATION, KEYSTORE_XPUB, GAP_LIMIT, BIRTH_DATE;
+        POLICY, SCRIPT_TYPE, MUTLISIG_THRESHOLD, MULTISIG_TOTAL, KEYSTORE_LABEL, KEYSTORE_FINGERPRINT, KEYSTORE_DERIVATION, KEYSTORE_XPUB, GAP_LIMIT, BIRTH_DATE, WATCH_LAST;
     }
 }

src/main/java/com/sparrowwallet/sparrow/event/WalletWatchLastChangedEvent.java

@@ -0,0 +1,16 @@
+package com.sparrowwallet.sparrow.event;
+
+import com.sparrowwallet.drongo.wallet.Wallet;
+
+public class WalletWatchLastChangedEvent extends WalletSettingsChangedEvent {
+    private final Integer watchLast;
+
+    public WalletWatchLastChangedEvent(Wallet wallet, Wallet pastWallet, String walletId, Integer watchLast) {
+        super(wallet, pastWallet, walletId);
+        this.watchLast = watchLast;
+    }
+
+    public Integer getWatchLast() {
+        return watchLast;
+    }
+}

src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java

@@ -274,6 +274,10 @@ public class DbPersistence implements Persistence {
                     walletDao.updateGapLimit(wallet.getId(), dirtyPersistables.gapLimit);
                 }
 
+                if(dirtyPersistables.watchLast != null) {
+                    walletDao.updateWatchLast(wallet.getId(), dirtyPersistables.watchLast);
+                }
+
                 if(!dirtyPersistables.labelEntries.isEmpty()) {
                     BlockTransactionDao blockTransactionDao = handle.attach(BlockTransactionDao.class);
                     WalletNodeDao walletNodeDao = handle.attach(WalletNodeDao.class);
@@ -763,6 +767,13 @@ public class DbPersistence implements Persistence {
         }
     }
 
+    @Subscribe
+    public void walletWatchLastChanged(WalletWatchLastChangedEvent event) {
+        if(persistsFor(event.getWallet())) {
+            updateExecutor.execute(() -> dirtyPersistablesMap.computeIfAbsent(event.getWallet(), key -> new DirtyPersistables()).watchLast = event.getWatchLast());
+        }
+    }
+
     private static class DirtyPersistables {
         public boolean deleteAccount;
         public boolean clearHistory;
@@ -770,6 +781,7 @@ public class DbPersistence implements Persistence {
         public String label;
         public Integer blockHeight = null;
         public Integer gapLimit = null;
+        public Integer watchLast = null;
         public final List<Entry> labelEntries = new ArrayList<>();
         public final List<BlockTransactionHashIndex> utxoStatuses = new ArrayList<>();
         public boolean mixConfig;
@@ -786,6 +798,7 @@ public class DbPersistence implements Persistence {
                     "\nLabel:" + label +
                     "\nBlockHeight:" + blockHeight +
                     "\nGap limit:" + gapLimit +
+                    "\nWatch last:" + watchLast +
                     "\nTx labels:" + labelEntries.stream().filter(entry -> entry instanceof TransactionEntry).map(entry -> ((TransactionEntry)entry).getBlockTransaction().getHash().toString()).collect(Collectors.toList()) +
                     "\nAddress labels:" + labelEntries.stream().filter(entry -> entry instanceof NodeEntry).map(entry -> ((NodeEntry)entry).getNode().toString() + " " + entry.getLabel()).collect(Collectors.toList()) +
                     "\nUTXO labels:" + labelEntries.stream().filter(entry -> entry instanceof HashIndexEntry).map(entry -> ((HashIndexEntry)entry).getHashIndex().toString()).collect(Collectors.toList()) +

src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java

@@ -36,21 +36,21 @@ public interface WalletDao {
     @CreateSqlObject
     UtxoMixDataDao createUtxoMixDataDao();
 
-    @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id")
+    @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.watchLast, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id")
     @RegisterRowMapper(WalletMapper.class)
     List<Wallet> loadAllWallets();
 
-    @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id where wallet.id = 1")
+    @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.watchLast, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id where wallet.id = 1")
     @RegisterRowMapper(WalletMapper.class)
     Wallet loadMainWallet();
 
-    @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id where wallet.id != 1")
+    @SqlQuery("select wallet.id, wallet.name, wallet.label, wallet.network, wallet.policyType, wallet.scriptType, wallet.storedBlockHeight, wallet.gapLimit, wallet.watchLast, wallet.birthDate, policy.id, policy.name, policy.script from wallet left join policy on wallet.defaultPolicy = policy.id where wallet.id != 1")
     @RegisterRowMapper(WalletMapper.class)
     List<Wallet> loadChildWallets();
 
-    @SqlUpdate("insert into wallet (name, label, network, policyType, scriptType, storedBlockHeight, gapLimit, birthDate, defaultPolicy) values (?, ?, ?, ?, ?, ?, ?, ?, ?)")
+    @SqlUpdate("insert into wallet (name, label, network, policyType, scriptType, storedBlockHeight, gapLimit, watchLast, birthDate, defaultPolicy) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
     @GetGeneratedKeys("id")
-    long insert(String name, String label, int network, int policyType, int scriptType, Integer storedBlockHeight, Integer gapLimit, Date birthDate, long defaultPolicy);
+    long insert(String name, String label, int network, int policyType, int scriptType, Integer storedBlockHeight, Integer gapLimit, Integer watchLast, Date birthDate, long defaultPolicy);
 
     @SqlUpdate("update wallet set label = :label where id = :id")
     void updateLabel(@Bind("id") long id, @Bind("label") String label);
@@ -61,6 +61,9 @@ public interface WalletDao {
     @SqlUpdate("update wallet set gapLimit = :gapLimit where id = :id")
     void updateGapLimit(@Bind("id") long id, @Bind("gapLimit") Integer gapLimit);
 
+    @SqlUpdate("update wallet set watchLast = :watchLast where id = :id")
+    void updateWatchLast(@Bind("id") long id, @Bind("watchLast") Integer watchLast);
+
     @SqlUpdate("set schema ?")
     int setSchema(String schema);
 
@@ -111,7 +114,7 @@ public interface WalletDao {
             setSchema(schema);
             createPolicyDao().addPolicy(wallet.getDefaultPolicy());
 
-            long id = insert(truncate(wallet.getName()), truncate(wallet.getLabel()), wallet.getNetwork().ordinal(), wallet.getPolicyType().ordinal(), wallet.getScriptType().ordinal(), wallet.getStoredBlockHeight(), wallet.gapLimit(), wallet.getBirthDate(), wallet.getDefaultPolicy().getId());
+            long id = insert(truncate(wallet.getName()), truncate(wallet.getLabel()), wallet.getNetwork().ordinal(), wallet.getPolicyType().ordinal(), wallet.getScriptType().ordinal(), wallet.getStoredBlockHeight(), wallet.gapLimit(), wallet.getWatchLast(), wallet.getBirthDate(), wallet.getDefaultPolicy().getId());
             wallet.setId(id);
 
             createKeystoreDao().addKeystores(wallet);

src/main/java/com/sparrowwallet/sparrow/io/db/WalletMapper.java

@@ -31,6 +31,8 @@ public class WalletMapper implements RowMapper<Wallet> {
 
         int gapLimit = rs.getInt("wallet.gapLimit");
         wallet.gapLimit(rs.wasNull() ? null : gapLimit);
+        int watchLast = rs.getInt("wallet.watchLast");
+        wallet.setWatchLast(rs.wasNull() ? null : watchLast);
         wallet.setBirthDate(rs.getTimestamp("wallet.birthDate"));
 
         return wallet;

src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java

@@ -298,7 +298,7 @@ public class ElectrumServer {
     public void getHistory(Wallet wallet, KeyPurpose keyPurpose, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap) throws ServerException {
         WalletNode purposeNode = wallet.getNode(keyPurpose);
         //Subscribe to all existing address WalletNodes and add them to nodeTransactionMap as keys to empty sets if they have history that needs to be fetched
-        subscribeWalletNodes(wallet, purposeNode.getChildren(), nodeTransactionMap, 0);
+        subscribeWalletNodes(wallet, getAddressNodes(wallet, purposeNode), nodeTransactionMap, 0);
         //All WalletNode keys in nodeTransactionMap need to have their history fetched (nodes without history will not be keys in the map yet)
         getReferences(wallet, nodeTransactionMap.keySet(), nodeTransactionMap, 0);
         //Fetch all referenced transaction to wallet transactions map. We do this now even though it is done again later to get it done before too many script hashes are subscribed
@@ -309,7 +309,7 @@ public class ElectrumServer {
         log.debug("Fetched history for: " + nodeTransactionMap.keySet());
 
         //Set the remaining WalletNode keys in nodeTransactionMap to empty sets to indicate no history (if no script hash history has already been retrieved in a previous call)
-        purposeNode.getChildren().stream().filter(node -> !nodeTransactionMap.containsKey(node) && retrievedScriptHashes.get(getScriptHash(wallet, node)) == null).forEach(node -> nodeTransactionMap.put(node, Collections.emptySet()));
+        getAddressNodes(wallet, purposeNode).stream().filter(node -> !nodeTransactionMap.containsKey(node) && retrievedScriptHashes.get(getScriptHash(wallet, node)) == null).forEach(node -> nodeTransactionMap.put(node, Collections.emptySet()));
     }
 
     private void getHistoryToGapLimit(Wallet wallet, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap, WalletNode purposeNode) throws ServerException {
@@ -319,7 +319,7 @@ public class ElectrumServer {
         int gapLimitSize = getGapLimitSize(wallet, nodeTransactionMap);
         while(historySize < gapLimitSize) {
             purposeNode.fillToIndex(gapLimitSize - 1);
-            subscribeWalletNodes(wallet, purposeNode.getChildren(), nodeTransactionMap, historySize);
+            subscribeWalletNodes(wallet, getAddressNodes(wallet, purposeNode), nodeTransactionMap, historySize);
             getReferences(wallet, nodeTransactionMap.keySet(), nodeTransactionMap, historySize);
             getReferencedTransactions(wallet, nodeTransactionMap);
             historySize = purposeNode.getChildren().size();
@@ -327,6 +327,17 @@ public class ElectrumServer {
         }
     }
 
+    private Set<WalletNode> getAddressNodes(Wallet wallet, WalletNode purposeNode) {
+        Integer watchLast = wallet.getWatchLast();
+        if(watchLast == null || watchLast < wallet.getGapLimit() || wallet.getStoredBlockHeight() == 0 || wallet.getTransactions().isEmpty()) {
+            return purposeNode.getChildren();
+        }
+
+        int highestUsedIndex = purposeNode.getChildren().stream().filter(WalletNode::isUsed).mapToInt(WalletNode::getIndex).max().orElse(0);
+        int startFromIndex = highestUsedIndex - watchLast;
+        return purposeNode.getChildren().stream().filter(walletNode -> walletNode.getIndex() >= startFromIndex).collect(Collectors.toCollection(TreeSet::new));
+    }
+
     private int getGapLimitSize(Wallet wallet, Map<WalletNode, Set<BlockTransactionHash>> nodeTransactionMap) {
         int highestIndex = nodeTransactionMap.keySet().stream().map(WalletNode::getIndex).max(Comparator.comparing(Integer::valueOf)).orElse(-1);
         return highestIndex + wallet.getGapLimit() + 1;

src/main/java/com/sparrowwallet/sparrow/wallet/AdvancedController.java

@@ -4,24 +4,35 @@ import com.sparrowwallet.drongo.wallet.Wallet;
 import com.sparrowwallet.sparrow.EventManager;
 import com.sparrowwallet.sparrow.control.DateStringConverter;
 import com.sparrowwallet.sparrow.event.SettingsChangedEvent;
+import javafx.collections.FXCollections;
+import javafx.collections.ObservableList;
 import javafx.fxml.FXML;
 import javafx.fxml.Initializable;
+import javafx.scene.control.ComboBox;
 import javafx.scene.control.DatePicker;
 import javafx.scene.control.Spinner;
 import javafx.scene.control.SpinnerValueFactory;
+import javafx.util.StringConverter;
 
 import java.net.URL;
 import java.time.ZoneId;
 import java.util.Date;
+import java.util.List;
 import java.util.ResourceBundle;
+import java.util.stream.Collectors;
 
 public class AdvancedController implements Initializable {
+    private static final List<Integer> DEFAULT_WATCH_LIST_ITEMS = List.of(-1, 100, 500, 1000, 5000, 10000);
+
     @FXML
     private DatePicker birthDate;
 
     @FXML
     private Spinner<Integer> gapLimit;
 
+    @FXML
+    private ComboBox<Integer> watchLast;
+
     @Override
     public void initialize(URL location, ResourceBundle resources) {
 
@@ -39,10 +50,37 @@ public class AdvancedController implements Initializable {
             }
         });
 
-        gapLimit.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(Wallet.DEFAULT_LOOKAHEAD, 10000, wallet.getGapLimit()));
+        gapLimit.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(Wallet.DEFAULT_LOOKAHEAD, 9999, wallet.getGapLimit()));
         gapLimit.valueProperty().addListener((observable, oldValue, newValue) -> {
             wallet.setGapLimit(newValue);
+            if(!watchLast.getItems().equals(getWatchListItems(wallet))) {
+                Integer value = watchLast.getValue();
+                watchLast.setItems(getWatchListItems(wallet));
+                watchLast.setValue(watchLast.getItems().contains(value) ? value : DEFAULT_WATCH_LIST_ITEMS.stream().filter(val -> val > wallet.getGapLimit()).findFirst().orElse(-1));
+            }
             EventManager.get().post(new SettingsChangedEvent(wallet, SettingsChangedEvent.Type.GAP_LIMIT));
         });
+
+        watchLast.setItems(getWatchListItems(wallet));
+        watchLast.setConverter(new StringConverter<>() {
+            @Override
+            public String toString(Integer value) {
+                return value == null ? "" : (value < 0 ? "All" : "Last " + value + " only");
+            }
+
+            @Override
+            public Integer fromString(String string) {
+                return null;
+            }
+        });
+        watchLast.setValue(wallet.getWatchLast() == null || !watchLast.getItems().contains(wallet.getWatchLast()) ? -1 : wallet.getWatchLast());
+        watchLast.valueProperty().addListener((observable, oldValue, newValue) -> {
+            wallet.setWatchLast(newValue == null || newValue < 0 ? -1 : newValue);
+            EventManager.get().post(new SettingsChangedEvent(wallet, SettingsChangedEvent.Type.WATCH_LAST));
+        });
+    }
+
+    private ObservableList<Integer> getWatchListItems(Wallet wallet) {
+        return FXCollections.observableList(DEFAULT_WATCH_LIST_ITEMS.stream().filter(val -> val < 0 || val > wallet.getGapLimit()).collect(Collectors.toList()));
     }
 }

src/main/java/com/sparrowwallet/sparrow/wallet/SettingsWalletForm.java

@@ -7,10 +7,7 @@ import com.sparrowwallet.drongo.wallet.MasterPrivateExtendedKey;
 import com.sparrowwallet.drongo.wallet.Wallet;
 import com.sparrowwallet.sparrow.AppServices;
 import com.sparrowwallet.sparrow.EventManager;
-import com.sparrowwallet.sparrow.event.KeystoreEncryptionChangedEvent;
-import com.sparrowwallet.sparrow.event.KeystoreLabelsChangedEvent;
-import com.sparrowwallet.sparrow.event.WalletAddressesChangedEvent;
-import com.sparrowwallet.sparrow.event.WalletPasswordChangedEvent;
+import com.sparrowwallet.sparrow.event.*;
 import com.sparrowwallet.sparrow.io.Storage;
 import com.sparrowwallet.sparrow.io.StorageException;
 
@@ -90,6 +87,10 @@ public class SettingsWalletForm extends WalletForm {
                 EventManager.get().post(new KeystoreLabelsChangedEvent(wallet, pastWallet, getWalletId(), labelChangedKeystores));
             }
 
+            if(!Objects.equals(wallet.getWatchLast(), walletCopy.getWatchLast())) {
+                EventManager.get().post(new WalletWatchLastChangedEvent(wallet, pastWallet, getWalletId(), walletCopy.getWatchLast()));
+            }
+
             List<Keystore> encryptionChangedKeystores = getEncryptionChangedKeystores(wallet, walletCopy);
             if(!encryptionChangedKeystores.isEmpty()) {
                 EventManager.get().post(new KeystoreEncryptionChangedEvent(wallet, pastWallet, getWalletId(), encryptionChangedKeystores));

src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java

@@ -391,6 +391,13 @@ public class WalletForm {
         }
     }
 
+    @Subscribe
+    public void walletWatchLastChanged(WalletWatchLastChangedEvent event) {
+        if(event.getWalletId().equals(getWalletId())) {
+            Platform.runLater(() -> EventManager.get().post(new WalletDataChangedEvent(wallet)));
+        }
+    }
+
     @Subscribe
     public void keystoreEncryptionChanged(KeystoreEncryptionChangedEvent event) {
         if(event.getWalletId().equals(getWalletId())) {