1 #!/usr/bin/env python3
2 # Copyright (c) 2020-present The Limenka developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test indices in conjunction with prune."""
6 import concurrent.futures
7 import os
8 from test_framework.test_framework import LimenkaTestFramework
9 from test_framework.util import (
10 assert_equal,
11 assert_greater_than,
12 assert_raises_rpc_error,
13 )
14 15 16 class FeatureIndexPruneTest(LimenkaTestFramework):
17 def set_test_params(self):
18 self.num_nodes = 4
19 self.extra_args = [
20 ["-fastprune", "-prune=1", "-blockfilterindex=1"],
21 ["-fastprune", "-prune=1", "-coinstatsindex=1"],
22 ["-fastprune", "-prune=1", "-blockfilterindex=1", "-coinstatsindex=1"],
23 [],
24 ]
25 26 def setup_network(self):
27 self.setup_nodes() # No P2P connection, so that linear_sync works
28 29 def linear_sync(self, node_from, *, height_from=None):
30 # Linear sync over RPC, because P2P sync may not be linear
31 to_height = node_from.getblockcount()
32 if height_from is None:
33 height_from = min([n.getblockcount() for n in self.nodes]) + 1
34 with concurrent.futures.ThreadPoolExecutor(max_workers=self.num_nodes) as rpc_threads:
35 for i in range(height_from, to_height + 1):
36 b = node_from.getblock(blockhash=node_from.getblockhash(i), verbosity=0)
37 list(rpc_threads.map(lambda n: n.submitblock(b), self.nodes))
38 39 def generate(self, node, num_blocks, sync_fun=None):
40 return super().generate(node, num_blocks, sync_fun=sync_fun or (lambda: self.linear_sync(node)))
41 42 def sync_index(self, height):
43 expected_filter = {
44 'basic block filter index': {'synced': True, 'best_block_height': height},
45 }
46 self.wait_until(lambda: self.nodes[0].getindexinfo() == expected_filter)
47 48 expected_stats = {
49 'coinstatsindex': {'synced': True, 'best_block_height': height}
50 }
51 self.wait_until(lambda: self.nodes[1].getindexinfo() == expected_stats, timeout=150)
52 53 expected = {**expected_filter, **expected_stats}
54 self.wait_until(lambda: self.nodes[2].getindexinfo() == expected)
55 56 def restart_without_indices(self):
57 for i in range(3):
58 self.restart_node(i, extra_args=["-fastprune", "-prune=1"])
59 60 def run_test(self):
61 filter_nodes = [self.nodes[0], self.nodes[2]]
62 stats_nodes = [self.nodes[1], self.nodes[2]]
63 64 self.log.info("check if we can access blockfilters and coinstats when pruning is enabled but no blocks are actually pruned")
65 self.sync_index(height=200)
66 tip = self.nodes[0].getbestblockhash()
67 for node in filter_nodes:
68 assert_greater_than(len(node.getblockfilter(tip)['filter']), 0)
69 for node in stats_nodes:
70 assert node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash']
71 72 self.generate(self.nodes[0], 500)
73 self.sync_index(height=700)
74 75 self.log.info("prune some blocks")
76 for node in self.nodes[:2]:
77 with node.assert_debug_log(['Prune: UnlinkPrunedFiles deleted blk/rev (00000)']):
78 pruneheight_new = node.pruneblockchain(400)
79 # the prune heights used here and below are magic numbers that are determined by the
80 # thresholds at which block files wrap, so they depend on disk serialization and default block file size.
81 assert_equal(pruneheight_new, 248)
82 83 self.log.info("check if we can access the tips blockfilter and coinstats when we have pruned some blocks")
84 tip = self.nodes[0].getbestblockhash()
85 for node in filter_nodes:
86 assert_greater_than(len(node.getblockfilter(tip)['filter']), 0)
87 for node in stats_nodes:
88 assert node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash']
89 90 self.log.info("check if we can access the blockfilter and coinstats of a pruned block")
91 height_hash = self.nodes[0].getblockhash(2)
92 for node in filter_nodes:
93 assert_greater_than(len(node.getblockfilter(height_hash)['filter']), 0)
94 for node in stats_nodes:
95 assert node.gettxoutsetinfo(hash_type="muhash", hash_or_height=height_hash)['muhash']
96 97 # mine and sync index up to a height that will later be the pruneheight
98 self.generate(self.nodes[0], 51)
99 self.sync_index(height=751)
100 101 self.restart_without_indices()
102 103 self.log.info("make sure trying to access the indices throws errors")
104 for node in filter_nodes:
105 msg = "Index is not enabled for filtertype basic"
106 assert_raises_rpc_error(-1, msg, node.getblockfilter, height_hash)
107 for node in stats_nodes:
108 msg = "Querying specific block heights requires coinstatsindex"
109 assert_raises_rpc_error(-8, msg, node.gettxoutsetinfo, "muhash", height_hash)
110 111 self.generate(self.nodes[0], 749)
112 113 self.log.info("prune exactly up to the indices best blocks while the indices are disabled")
114 for i in range(3):
115 pruneheight_2 = self.nodes[i].pruneblockchain(1000)
116 assert_equal(pruneheight_2, 750)
117 # Restart the nodes again with the indices activated
118 self.restart_node(i, extra_args=self.extra_args[i])
119 120 self.log.info("make sure that we can continue with the partially synced indices after having pruned up to the index height")
121 self.sync_index(height=1500)
122 123 self.log.info("prune further than the indices best blocks while the indices are disabled")
124 self.restart_without_indices()
125 self.generate(self.nodes[0], 1000)
126 127 for i in range(3):
128 pruneheight_3 = self.nodes[i].pruneblockchain(2000)
129 assert_greater_than(pruneheight_3, pruneheight_2)
130 self.stop_node(i)
131 132 self.log.info("make sure we get an init error when starting the nodes again with the indices")
133 filter_msg = f"Error: Index \"basic block filter index\" needs block data that has been pruned.{os.linesep}Restart with -reindex to rebuild (re-downloading the entire blockchain), or remove \"basic\" from -blockfilterindex to disable."
134 stats_msg = f"Error: Index \"coinstatsindex\" needs block data that has been pruned.{os.linesep}Restart with -reindex to rebuild (re-downloading the entire blockchain), or set -coinstatsindex=0 to disable."
135 end_msg = f"{os.linesep}Error: A fatal internal error occurred, see debug.log for details: Failed to start indexes, shutting down.."
136 for i, msg in enumerate([filter_msg, stats_msg, filter_msg]):
137 self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=msg+end_msg)
138 139 self.log.info("make sure the nodes start again with the indices and an additional -reindex arg")
140 for i in range(3):
141 restart_args = self.extra_args[i] + ["-reindex"]
142 self.restart_node(i, extra_args=restart_args)
143 144 self.linear_sync(self.nodes[3])
145 self.sync_index(height=2500)
146 147 for node in self.nodes[:2]:
148 with node.assert_debug_log(['Prune: UnlinkPrunedFiles deleted blk/rev (00007)']):
149 pruneheight_new = node.pruneblockchain(2500)
150 assert_equal(pruneheight_new, 2005)
151 152 self.log.info("ensure that prune locks don't prevent indices from failing in a reorg scenario")
153 with self.nodes[0].assert_debug_log(['basic block filter index prune lock moved back to 2480']):
154 self.nodes[3].invalidateblock(self.nodes[0].getblockhash(2480))
155 self.generate(self.nodes[3], 30, sync_fun=lambda: self.linear_sync(self.nodes[3], height_from=2480))
156 157 158 if __name__ == '__main__':
159 FeatureIndexPruneTest(__file__).main()
160