1 #!/usr/bin/env python3
2 # Copyright (c) 2020-present The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test indices in conjunction with prune."""
6 import concurrent.futures
7 import os
8 from test_framework.test_framework import BitcoinTestFramework
9 from test_framework.test_node import TestNode
10 from test_framework.util import (
11 assert_equal,
12 assert_greater_than,
13 assert_raises_rpc_error,
14 JSONRPCException,
15 )
16 17 from typing import List, Any
18 19 def send_batch_request(node: TestNode, method: str, params: List[Any]) -> List[Any]:
20 """Send batch request and parse all results"""
21 data = [getattr(node, method).get_request(*p) for p in params]
22 response = node.batch(data)
23 result = []
24 for item in response:
25 assert "error" not in item, item["error"]
26 result.append(item["result"])
27 28 return result
29 30 31 class FeatureIndexPruneTest(BitcoinTestFramework):
32 def set_test_params(self):
33 self.num_nodes = 4
34 self.extra_args = [
35 ["-fastprune", "-prune=1", "-blockfilterindex=1"],
36 ["-fastprune", "-prune=1", "-coinstatsindex=1"],
37 ["-fastprune", "-prune=1", "-blockfilterindex=1", "-coinstatsindex=1"],
38 [],
39 ]
40 41 def setup_network(self):
42 self.setup_nodes() # No P2P connection, so that linear_sync works
43 44 def linear_sync(self, node_from, *, height_from=None):
45 # Linear sync over RPC, because P2P sync may not be linear
46 to_height = node_from.getblockcount()
47 if height_from is None:
48 height_from = min([n.getblockcount() for n in self.nodes]) + 1
49 with concurrent.futures.ThreadPoolExecutor(max_workers=self.num_nodes) as rpc_threads:
50 for i in range(height_from, to_height + 1):
51 b = node_from.getblock(blockhash=node_from.getblockhash(i), verbosity=0)
52 list(rpc_threads.map(lambda n: n.submitblock(b), self.nodes))
53 54 def generate(self, node, num_blocks, sync_fun=None):
55 return super().generate(node, num_blocks, sync_fun=sync_fun or (lambda: self.linear_sync(node)))
56 57 def sync_index(self, height):
58 expected_filter = {
59 'basic block filter index': {'synced': True, 'best_block_height': height},
60 }
61 self.wait_until(lambda: self.nodes[0].getindexinfo() == expected_filter)
62 63 expected_stats = {
64 'coinstatsindex': {'synced': True, 'best_block_height': height}
65 }
66 self.wait_until(lambda: self.nodes[1].getindexinfo() == expected_stats, timeout=150)
67 68 expected = {**expected_filter, **expected_stats}
69 self.wait_until(lambda: self.nodes[2].getindexinfo() == expected)
70 71 def restart_without_indices(self):
72 for i in range(3):
73 self.restart_node(i, extra_args=["-fastprune", "-prune=1"])
74 75 def check_for_block(self, node, hash):
76 try:
77 self.nodes[node].getblock(hash)
78 return True
79 except JSONRPCException:
80 return False
81 82 def run_test(self):
83 filter_nodes = [self.nodes[0], self.nodes[2]]
84 stats_nodes = [self.nodes[1], self.nodes[2]]
85 86 self.log.info("check if we can access blockfilters and coinstats when pruning is enabled but no blocks are actually pruned")
87 self.sync_index(height=200)
88 tip = self.nodes[0].getbestblockhash()
89 for node in filter_nodes:
90 assert_greater_than(len(node.getblockfilter(tip)['filter']), 0)
91 for node in stats_nodes:
92 assert node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash']
93 94 self.generate(self.nodes[0], 500)
95 self.sync_index(height=700)
96 97 self.log.info("prune some blocks")
98 for node in self.nodes[:2]:
99 with node.assert_debug_log(['limited pruning to height 689']):
100 pruneheight_new = node.pruneblockchain(400)
101 # the prune heights used here and below are magic numbers that are determined by the
102 # thresholds at which block files wrap, so they depend on disk serialization and default block file size.
103 assert_equal(pruneheight_new, 249)
104 105 self.log.info("check if we can access the tips blockfilter and coinstats when we have pruned some blocks")
106 tip = self.nodes[0].getbestblockhash()
107 for node in filter_nodes:
108 assert_greater_than(len(node.getblockfilter(tip)['filter']), 0)
109 for node in stats_nodes:
110 assert node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash']
111 112 self.log.info("check if we can access the blockfilter and coinstats of a pruned block")
113 height_hash = self.nodes[0].getblockhash(2)
114 for node in filter_nodes:
115 assert_greater_than(len(node.getblockfilter(height_hash)['filter']), 0)
116 for node in stats_nodes:
117 assert node.gettxoutsetinfo(hash_type="muhash", hash_or_height=height_hash)['muhash']
118 119 # mine and sync index up to a height that will later be the pruneheight
120 self.generate(self.nodes[0], 54)
121 self.sync_index(height=754)
122 123 self.restart_without_indices()
124 125 self.log.info("make sure trying to access the indices throws errors")
126 for node in filter_nodes:
127 msg = "Index is not enabled for filtertype basic"
128 assert_raises_rpc_error(-1, msg, node.getblockfilter, height_hash)
129 for node in stats_nodes:
130 msg = "Querying specific block heights requires coinstatsindex"
131 assert_raises_rpc_error(-8, msg, node.gettxoutsetinfo, "muhash", height_hash)
132 133 self.generate(self.nodes[0], 746)
134 135 self.log.info("prune exactly up to the indices best blocks while the indices are disabled")
136 for i in range(3):
137 pruneheight_2 = self.nodes[i].pruneblockchain(1000)
138 assert_equal(pruneheight_2, 753)
139 # Restart the nodes again with the indices activated
140 self.restart_node(i, extra_args=self.extra_args[i])
141 142 self.log.info("make sure that we can continue with the partially synced indices after having pruned up to the index height")
143 self.sync_index(height=1500)
144 145 self.log.info("prune further than the indices best blocks while the indices are disabled")
146 self.restart_without_indices()
147 self.generate(self.nodes[0], 1000)
148 149 for i in range(3):
150 pruneheight_3 = self.nodes[i].pruneblockchain(2000)
151 assert_greater_than(pruneheight_3, pruneheight_2)
152 self.stop_node(i)
153 154 self.log.info("make sure we get an init error when starting the nodes again with the indices")
155 filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data (including undo data). Please disable the index or reindex (which will download the whole blockchain again)"
156 stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data (including undo data). Please disable the index or reindex (which will download the whole blockchain again)"
157 end_msg = f"{os.linesep}Error: A fatal internal error occurred, see debug.log for details: Failed to start indexes, shutting down…"
158 for i, msg in enumerate([filter_msg, stats_msg, filter_msg]):
159 self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=msg+end_msg)
160 161 self.log.info("fetching the missing blocks with getblockfrompeer doesn't work for block filter index and coinstatsindex")
162 # Only checking the first two nodes since this test takes a long time
163 # and the third node is kind of redundant in this context
164 for i, msg in enumerate([filter_msg, stats_msg]):
165 self.restart_node(i, extra_args=["-prune=1", "-fastprune"])
166 node = self.nodes[i]
167 prune_height = node.getblockchaininfo()["pruneheight"]
168 self.connect_nodes(i, 3)
169 peers = node.getpeerinfo()
170 assert_equal(len(peers), 1)
171 peer_id = peers[0]["id"]
172 173 # 1500 is the height to where the indices were able to sync previously
174 hashes = send_batch_request(node, "getblockhash", [[a] for a in range(1500, prune_height)])
175 send_batch_request(node, "getblockfrompeer", [[bh, peer_id] for bh in hashes])
176 # Ensure all necessary blocks have been fetched before proceeding
177 for bh in hashes:
178 self.wait_until(lambda: self.check_for_block(i, bh), timeout=10)
179 180 # Upon restart we expect the same errors as previously although all
181 # necessary blocks have been fetched. Both indices need the undo
182 # data of the blocks to be available as well and getblockfrompeer
183 # can not provide that.
184 self.stop_node(i)
185 node.assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=msg+end_msg)
186 187 self.log.info("make sure the nodes start again with the indices and an additional -reindex arg")
188 for i in range(3):
189 restart_args = self.extra_args[i] + ["-reindex"]
190 self.restart_node(i, extra_args=restart_args)
191 192 self.linear_sync(self.nodes[3])
193 self.sync_index(height=2500)
194 195 for node in self.nodes[:2]:
196 with node.assert_debug_log(['limited pruning to height 2489']):
197 pruneheight_new = node.pruneblockchain(2500)
198 assert_equal(pruneheight_new, 2013)
199 200 self.log.info("ensure that prune locks don't prevent indices from failing in a reorg scenario")
201 with self.nodes[0].assert_debug_log(['basic block filter index prune lock moved back to 2480']):
202 self.nodes[3].invalidateblock(self.nodes[0].getblockhash(2480))
203 self.generate(self.nodes[3], 30, sync_fun=lambda: self.linear_sync(self.nodes[3], height_from=2480))
204 205 206 if __name__ == '__main__':
207 FeatureIndexPruneTest(__file__).main()
208