1 #!/usr/bin/env python3
2 # Copyright (c) 2014-2022 The Limenka developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test the pruning code.
6 7 WARNING:
8 This test uses 4GB of disk space.
9 This test takes 30 mins or more (up to 2 hours)
10 """
11 import os
12 13 from test_framework.blocktools import (
14 MIN_BLOCKS_TO_KEEP,
15 create_block,
16 create_coinbase,
17 )
18 from test_framework.script import (
19 CScript,
20 OP_NOP,
21 OP_RETURN,
22 )
23 from test_framework.test_framework import LimenkaTestFramework
24 from test_framework.util import (
25 assert_equal,
26 assert_greater_than,
27 assert_raises_rpc_error,
28 try_rpc,
29 )
30 31 # Rescans start at the earliest block up to 2 hours before a key timestamp, so
32 # the manual prune RPC avoids pruning blocks in the same window to be
33 # compatible with pruning based on key creation time.
34 TIMESTAMP_WINDOW = 2 * 60 * 60
35 36 def mine_large_blocks(node, n):
37 # Make a large scriptPubKey for the coinbase transaction. This is OP_RETURN
38 # followed by 950k of OP_NOP. This would be non-standard in a non-coinbase
39 # transaction but is consensus valid.
40 41 # Set the nTime if this is the first time this function has been called.
42 # A static variable ensures that time is monotonicly increasing and is therefore
43 # different for each block created => blockhash is unique.
44 if "nTime" not in mine_large_blocks.__dict__:
45 mine_large_blocks.nTime = 0
46 47 # Get the block parameters for the first block
48 big_script = CScript([OP_RETURN] + [OP_NOP] * 950000)
49 best_block = node.getblock(node.getbestblockhash())
50 height = int(best_block["height"]) + 1
51 mine_large_blocks.nTime = max(mine_large_blocks.nTime, int(best_block["time"])) + 1
52 previousblockhash = int(best_block["hash"], 16)
53 54 for _ in range(n):
55 block = create_block(hashprev=previousblockhash, ntime=mine_large_blocks.nTime, coinbase=create_coinbase(height, script_pubkey=big_script))
56 block.solve()
57 58 # Submit to the node
59 node.submitblock(block.serialize().hex())
60 61 previousblockhash = block.sha256
62 height += 1
63 mine_large_blocks.nTime += 1
64 65 def calc_usage(blockdir):
66 return sum(os.path.getsize(blockdir + f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.)
67 68 class PruneTest(LimenkaTestFramework):
69 def add_options(self, parser):
70 self.add_wallet_options(parser)
71 72 def set_test_params(self):
73 self.setup_clean_chain = True
74 self.num_nodes = 6
75 self.supports_cli = False
76 77 # Create nodes 0 and 1 to mine.
78 # Create node 2 to test pruning.
79 self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5"]
80 # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later)
81 # Create nodes 5 to test wallet in prune mode, but do not connect
82 self.extra_args = [
83 self.full_node_default_args,
84 self.full_node_default_args,
85 ["-maxreceivebuffer=20000", "-prune=550"],
86 ["-maxreceivebuffer=20000"],
87 ["-maxreceivebuffer=20000"],
88 ["-prune=550", "-blockfilterindex=1"],
89 ]
90 self.rpc_timeout = 120
91 92 def setup_network(self):
93 self.setup_nodes()
94 95 self.prunedir = os.path.join(self.nodes[2].blocks_path, '')
96 97 self.connect_nodes(0, 1)
98 self.connect_nodes(1, 2)
99 self.connect_nodes(0, 2)
100 self.connect_nodes(0, 3)
101 self.connect_nodes(0, 4)
102 self.sync_blocks(self.nodes[0:5])
103 104 def setup_nodes(self):
105 self.add_nodes(self.num_nodes, self.extra_args)
106 self.start_nodes()
107 if self.is_wallet_compiled():
108 self.import_deterministic_coinbase_privkeys()
109 110 def create_big_chain(self):
111 # Start by creating some coinbases we can spend later
112 self.generate(self.nodes[1], 200, sync_fun=lambda: self.sync_blocks(self.nodes[0:2]))
113 self.generate(self.nodes[0], 150, sync_fun=self.no_op)
114 115 # Then mine enough full blocks to create more than 550MiB of data
116 mine_large_blocks(self.nodes[0], 645)
117 118 self.sync_blocks(self.nodes[0:5])
119 120 def test_invalid_command_line_options(self):
121 self.stop_node(0)
122 self.nodes[0].assert_start_raises_init_error(
123 expected_msg='Error: Prune cannot be configured with a negative value.',
124 extra_args=['-prune=-1'],
125 )
126 self.nodes[0].assert_start_raises_init_error(
127 expected_msg='Error: Prune configured below the minimum of 550 MiB. Please use a higher number.',
128 extra_args=['-prune=549'],
129 )
130 self.nodes[0].assert_start_raises_init_error(
131 expected_msg='Error: Prune mode is incompatible with -txindex.',
132 extra_args=['-prune=550', '-txindex'],
133 )
134 self.nodes[0].assert_start_raises_init_error(
135 expected_msg='Error: Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.',
136 extra_args=['-prune=550', '-reindex-chainstate'],
137 )
138 139 def test_rescan_blockchain(self):
140 self.restart_node(0, ["-prune=550"])
141 assert_raises_rpc_error(-1, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height.", self.nodes[0].rescanblockchain)
142 143 def test_height_min(self):
144 assert os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), "blk00000.dat is missing, pruning too early"
145 self.log.info("Success")
146 self.log.info(f"Though we're already using more than 550MiB, current usage: {calc_usage(self.prunedir)}")
147 self.log.info("Mining 25 more blocks should cause the first block file to be pruned")
148 # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this
149 mine_large_blocks(self.nodes[0], 25)
150 151 # Wait for blk00000.dat to be pruned
152 self.wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30)
153 154 self.log.info("Success")
155 usage = calc_usage(self.prunedir)
156 self.log.info(f"Usage should be below target: {usage}")
157 assert_greater_than(550, usage)
158 159 def create_chain_with_staleblocks(self):
160 # Create stale blocks in manageable sized chunks
161 self.log.info("Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds")
162 163 for _ in range(12):
164 # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain
165 # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects
166 self.disconnect_nodes(0, 1)
167 self.disconnect_nodes(0, 2)
168 # Mine 24 blocks in node 1
169 mine_large_blocks(self.nodes[1], 24)
170 171 # Reorg back with 25 block chain from node 0
172 mine_large_blocks(self.nodes[0], 25)
173 174 # Create connections in the order so both nodes can see the reorg at the same time
175 self.connect_nodes(0, 1)
176 self.connect_nodes(0, 2)
177 self.sync_blocks(self.nodes[0:3])
178 179 self.log.info(f"Usage can be over target because of high stale rate: {calc_usage(self.prunedir)}")
180 181 def reorg_test(self):
182 # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip
183 # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain
184 185 height = self.nodes[1].getblockcount()
186 self.log.info(f"Current block height: {height}")
187 188 self.forkheight = height - 287
189 self.forkhash = self.nodes[1].getblockhash(self.forkheight)
190 self.log.info(f"Invalidating block {self.forkhash} at height {self.forkheight}")
191 self.nodes[1].invalidateblock(self.forkhash)
192 193 # We've now switched to our previously mined-24 block fork on node 1, but that's not what we want
194 # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago)
195 mainchainhash = self.nodes[0].getblockhash(self.forkheight - 1)
196 curhash = self.nodes[1].getblockhash(self.forkheight - 1)
197 while curhash != mainchainhash:
198 self.nodes[1].invalidateblock(curhash)
199 curhash = self.nodes[1].getblockhash(self.forkheight - 1)
200 201 assert self.nodes[1].getblockcount() == self.forkheight - 1
202 self.log.info(f"New best height: {self.nodes[1].getblockcount()}")
203 204 # Disconnect node1 and generate the new chain
205 self.disconnect_nodes(0, 1)
206 self.disconnect_nodes(1, 2)
207 208 self.log.info("Generating new longer chain of 300 more blocks")
209 self.generate(self.nodes[1], 300, sync_fun=self.no_op)
210 211 self.log.info("Reconnect nodes")
212 self.connect_nodes(0, 1)
213 self.connect_nodes(1, 2)
214 self.sync_blocks(self.nodes[0:3], timeout=120)
215 216 self.log.info(f"Verify height on node 2: {self.nodes[2].getblockcount()}")
217 self.log.info(f"Usage possibly still high because of stale blocks in block files: {calc_usage(self.prunedir)}")
218 219 self.log.info("Mine 220 more large blocks so we have requisite history")
220 221 mine_large_blocks(self.nodes[0], 220)
222 self.sync_blocks(self.nodes[0:3], timeout=120)
223 224 usage = calc_usage(self.prunedir)
225 self.log.info(f"Usage should be below target: {usage}")
226 assert_greater_than(550, usage)
227 228 def reorg_back(self):
229 # Verify that a block on the old main chain fork has been pruned away
230 assert_raises_rpc_error(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash)
231 with self.nodes[2].assert_debug_log(expected_msgs=['block verification stopping at height', '(no data)']):
232 assert not self.nodes[2].verifychain(checklevel=4, nblocks=0)
233 self.log.info(f"Will need to redownload block {self.forkheight}")
234 235 # Verify that we have enough history to reorg back to the fork point
236 # Although this is more than 288 blocks, because this chain was written more recently
237 # and only its other 299 small and 220 large blocks are in the block files after it,
238 # it is expected to still be retained
239 self.nodes[2].getblock(self.nodes[2].getblockhash(self.forkheight))
240 241 first_reorg_height = self.nodes[2].getblockcount()
242 curchainhash = self.nodes[2].getblockhash(self.mainchainheight)
243 self.nodes[2].invalidateblock(curchainhash)
244 goalbestheight = self.mainchainheight
245 goalbesthash = self.mainchainhash2
246 247 # As of 0.10 the current block download logic is not able to reorg to the original chain created in
248 # create_chain_with_stale_blocks because it doesn't know of any peer that's on that chain from which to
249 # redownload its missing blocks.
250 # Invalidate the reorg_test chain in node 0 as well, it can successfully switch to the original chain
251 # because it has all the block data.
252 # However it must mine enough blocks to have a more work chain than the reorg_test chain in order
253 # to trigger node 2's block download logic.
254 # At this point node 2 is within 288 blocks of the fork point so it will preserve its ability to reorg
255 if self.nodes[2].getblockcount() < self.mainchainheight:
256 blocks_to_mine = first_reorg_height + 1 - self.mainchainheight
257 self.log.info(f"Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: {blocks_to_mine}")
258 self.nodes[0].invalidateblock(curchainhash)
259 assert_equal(self.nodes[0].getblockcount(), self.mainchainheight)
260 assert_equal(self.nodes[0].getbestblockhash(), self.mainchainhash2)
261 goalbesthash = self.generate(self.nodes[0], blocks_to_mine, sync_fun=self.no_op)[-1]
262 goalbestheight = first_reorg_height + 1
263 264 self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload")
265 # Wait for Node 2 to reorg to proper height
266 self.wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900)
267 assert_equal(self.nodes[2].getbestblockhash(), goalbesthash)
268 # Verify we can now have the data for a block previously pruned
269 assert_equal(self.nodes[2].getblock(self.forkhash)["height"], self.forkheight)
270 271 def manual_test(self, node_number, use_timestamp):
272 # at this point, node has 995 blocks and has not yet run in prune mode
273 self.start_node(node_number)
274 node = self.nodes[node_number]
275 assert_equal(node.getblockcount(), 995)
276 assert_raises_rpc_error(-1, "Cannot prune blocks because node is not in prune mode", node.pruneblockchain, 500)
277 278 # now re-start in manual pruning mode
279 self.restart_node(node_number, extra_args=["-prune=1"])
280 node = self.nodes[node_number]
281 assert_equal(node.getblockcount(), 995)
282 283 def height(index):
284 if use_timestamp:
285 return node.getblockheader(node.getblockhash(index))["time"] + TIMESTAMP_WINDOW
286 else:
287 return index
288 289 def prune(index):
290 ret = node.pruneblockchain(height=height(index))
291 assert_equal(ret + 1, node.getblockchaininfo()['pruneheight'])
292 293 def has_block(index):
294 return os.path.isfile(os.path.join(self.nodes[node_number].blocks_path, f"blk{index:05}.dat"))
295 296 # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000)
297 assert_raises_rpc_error(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500))
298 299 # Save block transaction count before pruning, assert value
300 block1_details = node.getblock(node.getblockhash(1))
301 assert_equal(block1_details["nTx"], len(block1_details["tx"]))
302 303 # mine 6 blocks so we are at height 1001 (i.e., above PruneAfterHeight)
304 self.generate(node, 6, sync_fun=self.no_op)
305 assert_equal(node.getblockchaininfo()["blocks"], 1001)
306 307 # prune parameter in the future (block or timestamp) should raise an exception
308 future_parameter = height(1001) + 5
309 if use_timestamp:
310 assert_raises_rpc_error(-8, "Could not find block with at least the specified timestamp", node.pruneblockchain, future_parameter)
311 else:
312 assert_raises_rpc_error(-8, "Blockchain is shorter than the attempted prune height", node.pruneblockchain, future_parameter)
313 314 # Pruned block should still know the number of transactions
315 assert_equal(node.getblockheader(node.getblockhash(1))["nTx"], block1_details["nTx"])
316 317 # negative heights should raise an exception
318 assert_raises_rpc_error(-8, "Negative block height", node.pruneblockchain, -10)
319 320 # height=100 too low to prune first block file so this is a no-op
321 prune(100)
322 assert has_block(0), "blk00000.dat is missing when should still be there"
323 324 # Does nothing
325 node.pruneblockchain(height(0))
326 assert has_block(0), "blk00000.dat is missing when should still be there"
327 328 # height=500 shouldn't prune first file if there's a prune lock
329 node.setprunelock("test", {
330 "desc": "Testing",
331 "height": [2, 2],
332 })
333 assert_equal(node.listprunelocks(), {'prune_locks': [{'id': 'test', 'desc': 'Testing', 'height': [2, 2], 'temporary': False}]})
334 prune(500)
335 assert has_block(0), "blk00000.dat is missing when should still be there"
336 node.setprunelock("test", {}) # delete prune lock
337 assert_equal(node.listprunelocks(), {'prune_locks': []})
338 339 # height=500 should prune first file
340 prune(500)
341 assert not has_block(0), "blk00000.dat is still there, should be pruned by now"
342 assert has_block(1), "blk00001.dat is missing when should still be there"
343 344 # height=650 should prune second file
345 prune(650)
346 assert not has_block(1), "blk00001.dat is still there, should be pruned by now"
347 348 # height=1000 should not prune anything more, because tip-288 is in blk00002.dat.
349 prune(1000)
350 assert has_block(2), "blk00002.dat is still there, should be pruned by now"
351 352 # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat)
353 self.generate(node, MIN_BLOCKS_TO_KEEP, sync_fun=self.no_op)
354 prune(1000)
355 assert not has_block(2), "blk00002.dat is still there, should be pruned by now"
356 assert not has_block(3), "blk00003.dat is still there, should be pruned by now"
357 358 # stop node, start back up with auto-prune at 550 MiB, make sure still runs
359 self.restart_node(node_number, extra_args=["-prune=550"])
360 361 self.log.info("Success")
362 363 def test_wallet_rescan(self):
364 # check that the pruning node's wallet is still in good shape
365 self.log.info("Stop and start pruning node to trigger wallet rescan")
366 self.restart_node(2, extra_args=["-prune=550"])
367 368 wallet_info = self.nodes[2].getwalletinfo()
369 self.wait_until(lambda: wallet_info["scanning"] == False)
370 self.wait_until(lambda: wallet_info["lastprocessedblock"]["height"] == self.nodes[2].getblockcount())
371 372 # check that wallet loads successfully when restarting a pruned node after IBD.
373 # this was reported to fail in #7494.
374 self.restart_node(5, extra_args=["-prune=550", "-blockfilterindex=1"]) # restart to trigger rescan
375 376 wallet_info = self.nodes[5].getwalletinfo()
377 self.wait_until(lambda: wallet_info["scanning"] == False)
378 self.wait_until(lambda: wallet_info["lastprocessedblock"]["height"] == self.nodes[0].getblockcount())
379 380 def run_test(self):
381 self.log.info("Warning! This test requires 4GB of disk space")
382 383 self.log.info("Mining a big blockchain of 995 blocks")
384 self.create_big_chain()
385 # Chain diagram key:
386 # * blocks on main chain
387 # +,&,$,@ blocks on other forks
388 # X invalidated block
389 # N1 Node 1
390 #
391 # Start by mining a simple chain that all nodes have
392 # N0=N1=N2 **...*(995)
393 394 # stop manual-pruning node with 995 blocks
395 self.stop_node(3)
396 self.stop_node(4)
397 398 self.log.info("Check that we haven't started pruning yet because we're below PruneAfterHeight")
399 self.test_height_min()
400 # Extend this chain past the PruneAfterHeight
401 # N0=N1=N2 **...*(1020)
402 403 self.log.info("Check that we'll exceed disk space target if we have a very high stale block rate")
404 self.create_chain_with_staleblocks()
405 # Disconnect N0
406 # And mine a 24 block chain on N1 and a separate 25 block chain on N0
407 # N1=N2 **...*+...+(1044)
408 # N0 **...**...**(1045)
409 #
410 # reconnect nodes causing reorg on N1 and N2
411 # N1=N2 **...*(1020) *...**(1045)
412 # \
413 # +...+(1044)
414 #
415 # repeat this process until you have 12 stale forks hanging off the
416 # main chain on N1 and N2
417 # N0 *************************...***************************(1320)
418 #
419 # N1=N2 **...*(1020) *...**(1045) *.. ..**(1295) *...**(1320)
420 # \ \ \
421 # +...+(1044) &.. $...$(1319)
422 423 # Save some current chain state for later use
424 self.mainchainheight = self.nodes[2].getblockcount() # 1320
425 self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight)
426 427 self.log.info("Check that we can survive a 288 block reorg still")
428 self.reorg_test() # (1033, )
429 # Now create a 288 block reorg by mining a longer chain on N1
430 # First disconnect N1
431 # Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain
432 # N1 **...*(1020) **...**(1032)X..
433 # \
434 # ++...+(1031)X..
435 #
436 # Now mine 300 more blocks on N1
437 # N1 **...*(1020) **...**(1032) @@...@(1332)
438 # \ \
439 # \ X...
440 # \ \
441 # ++...+(1031)X.. ..
442 #
443 # Reconnect nodes and mine 220 more blocks on N1
444 # N1 **...*(1020) **...**(1032) @@...@@@(1552)
445 # \ \
446 # \ X...
447 # \ \
448 # ++...+(1031)X.. ..
449 #
450 # N2 **...*(1020) **...**(1032) @@...@@@(1552)
451 # \ \
452 # \ *...**(1320)
453 # \ \
454 # ++...++(1044) ..
455 #
456 # N0 ********************(1032) @@...@@@(1552)
457 # \
458 # *...**(1320)
459 460 self.log.info("Test that we can rerequest a block we previously pruned if needed for a reorg")
461 self.reorg_back()
462 # Verify that N2 still has block 1033 on current chain (@), but not on main chain (*)
463 # Invalidate 1033 on current chain (@) on N2 and we should be able to reorg to
464 # original main chain (*), but will require redownload of some blocks
465 # In order to have a peer we think we can download from, must also perform this invalidation
466 # on N0 and mine a new longest chain to trigger.
467 # Final result:
468 # N0 ********************(1032) **...****(1553)
469 # \
470 # X@...@@@(1552)
471 #
472 # N2 **...*(1020) **...**(1032) **...****(1553)
473 # \ \
474 # \ X@...@@@(1552)
475 # \
476 # +..
477 #
478 # N1 doesn't change because 1033 on main chain (*) is invalid
479 480 self.log.info("Test manual pruning with block indices")
481 self.manual_test(3, use_timestamp=False)
482 483 self.log.info("Test manual pruning with timestamps")
484 self.manual_test(4, use_timestamp=True)
485 486 self.log.info("Syncing node 5 to node 0")
487 self.connect_nodes(0, 5)
488 self.sync_blocks([self.nodes[0], self.nodes[5]], wait=5, timeout=300)
489 490 if self.is_wallet_compiled():
491 self.log.info("Test wallet re-scan")
492 self.test_wallet_rescan()
493 494 self.log.info("Test it's not possible to rescan beyond pruned data")
495 self.test_rescan_blockchain()
496 497 self.log.info("Test invalid pruning command line options")
498 self.test_invalid_command_line_options()
499 500 self.log.info("Test scanblocks can not return pruned data")
501 self.test_scanblocks_pruned()
502 503 self.log.info("Test pruneheight reflects the presence of block and undo data")
504 self.test_pruneheight_undo_presence()
505 506 self.log.info("Done")
507 508 def test_scanblocks_pruned(self):
509 node = self.nodes[5]
510 genesis_blockhash = node.getblockhash(0)
511 false_positive_spk = bytes.fromhex("001400000000000000000000000000000000000cadcb")
512 513 assert genesis_blockhash in node.scanblocks(
514 "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0)['relevant_blocks']
515 516 assert_raises_rpc_error(-1, "Block not available (pruned data)", node.scanblocks,
517 "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0, "basic", {"filter_false_positives": True})
518 519 def test_pruneheight_undo_presence(self):
520 node = self.nodes[5]
521 pruneheight = node.getblockchaininfo()["pruneheight"]
522 fetch_block = node.getblockhash(pruneheight - 1)
523 524 self.connect_nodes(1, 5)
525 peers = node.getpeerinfo()
526 node.getblockfrompeer(fetch_block, peers[0]["id"])
527 self.wait_until(lambda: not try_rpc(-1, "Block not available (pruned data)", node.getblock, fetch_block), timeout=5)
528 529 new_pruneheight = node.getblockchaininfo()["pruneheight"]
530 assert_equal(pruneheight, new_pruneheight)
531 532 if __name__ == '__main__':
533 PruneTest(__file__).main()
534