1 #!/usr/bin/env python3
2 # Copyright (c) 2014-present The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test the pruning code.
6 7 WARNING:
8 This test uses 4GB of disk space.
9 """
10 import os
11 12 from test_framework.blocktools import (
13 MIN_BLOCKS_TO_KEEP,
14 create_block,
15 create_coinbase,
16 )
17 from test_framework.script import (
18 CScript,
19 OP_NOP,
20 OP_RETURN,
21 )
22 from test_framework.test_framework import BitcoinTestFramework
23 from test_framework.util import (
24 assert_equal,
25 assert_greater_than,
26 assert_raises_rpc_error,
27 try_rpc,
28 )
29 30 # Rescans start at the earliest block up to 2 hours before a key timestamp, so
31 # the manual prune RPC avoids pruning blocks in the same window to be
32 # compatible with pruning based on key creation time.
33 TIMESTAMP_WINDOW = 2 * 60 * 60
34 35 def mine_large_blocks(node, n):
36 # Make a large scriptPubKey for the coinbase transaction. This is OP_RETURN
37 # followed by 950k of OP_NOP. This would be non-standard in a non-coinbase
38 # transaction but is consensus valid.
39 40 # Set the nTime if this is the first time this function has been called.
41 # A static variable ensures that time is monotonicly increasing and is therefore
42 # different for each block created => blockhash is unique.
43 if "nTime" not in mine_large_blocks.__dict__:
44 mine_large_blocks.nTime = 0
45 46 # Get the block parameters for the first block
47 big_script = CScript([OP_RETURN] + [OP_NOP] * 950000)
48 best_block = node.getblock(node.getbestblockhash())
49 height = int(best_block["height"]) + 1
50 mine_large_blocks.nTime = max(mine_large_blocks.nTime, int(best_block["time"])) + 1
51 previousblockhash = int(best_block["hash"], 16)
52 53 for _ in range(n):
54 block = create_block(hashprev=previousblockhash, ntime=mine_large_blocks.nTime, coinbase=create_coinbase(height, script_pubkey=big_script))
55 block.solve()
56 57 # Submit to the node
58 node.submitblock(block.serialize().hex())
59 60 previousblockhash = block.hash_int
61 height += 1
62 mine_large_blocks.nTime += 1
63 64 def calc_usage(blockdir):
65 return sum(os.path.getsize(blockdir + f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.)
66 67 class PruneTest(BitcoinTestFramework):
68 def set_test_params(self):
69 self.setup_clean_chain = True
70 self.num_nodes = 6
71 self.uses_wallet = None
72 73 # Create nodes 0 and 1 to mine.
74 # Create node 2 to test pruning.
75 self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5"]
76 # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later)
77 # Create nodes 5 to test wallet in prune mode, but do not connect
78 self.extra_args = [
79 self.full_node_default_args,
80 self.full_node_default_args,
81 ["-maxreceivebuffer=20000", "-prune=550"],
82 ["-maxreceivebuffer=20000"],
83 ["-maxreceivebuffer=20000"],
84 ["-prune=550", "-blockfilterindex=1"],
85 ]
86 self.rpc_timeout = 120
87 88 def setup_network(self):
89 self.setup_nodes()
90 91 self.prunedir = os.path.join(self.nodes[2].blocks_path, '')
92 93 self.connect_nodes(0, 1)
94 self.connect_nodes(1, 2)
95 self.connect_nodes(0, 2)
96 self.connect_nodes(0, 3)
97 self.connect_nodes(0, 4)
98 self.sync_blocks(self.nodes[0:5])
99 100 def setup_nodes(self):
101 self.add_nodes(self.num_nodes, self.extra_args)
102 self.start_nodes()
103 if self.is_wallet_compiled():
104 self.import_deterministic_coinbase_privkeys()
105 106 def create_big_chain(self):
107 # Start by creating some coinbases we can spend later
108 self.generate(self.nodes[1], 200, sync_fun=lambda: self.sync_blocks(self.nodes[0:2]))
109 self.generate(self.nodes[0], 150, sync_fun=self.no_op)
110 111 # Then mine enough full blocks to create more than 550MiB of data
112 mine_large_blocks(self.nodes[0], 645)
113 114 self.sync_blocks(self.nodes[0:5])
115 116 def test_invalid_command_line_options(self):
117 self.stop_node(0)
118 self.nodes[0].assert_start_raises_init_error(
119 expected_msg='Error: Prune cannot be configured with a negative value.',
120 extra_args=['-prune=-1'],
121 )
122 self.nodes[0].assert_start_raises_init_error(
123 expected_msg='Error: Prune configured below the minimum of 550 MiB. Please use a higher number.',
124 extra_args=['-prune=549'],
125 )
126 self.nodes[0].assert_start_raises_init_error(
127 expected_msg='Error: Prune mode is incompatible with -txindex.',
128 extra_args=['-prune=550', '-txindex'],
129 )
130 self.nodes[0].assert_start_raises_init_error(
131 expected_msg='Error: Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.',
132 extra_args=['-prune=550', '-reindex-chainstate'],
133 )
134 135 def test_rescan_blockchain(self):
136 self.restart_node(0, ["-prune=550"])
137 assert_raises_rpc_error(-1, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height.", self.nodes[0].rescanblockchain)
138 139 def test_height_min(self):
140 assert os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), "blk00000.dat is missing, pruning too early"
141 self.log.info("Success")
142 self.log.info(f"Though we're already using more than 550MiB, current usage: {calc_usage(self.prunedir)}")
143 self.log.info("Mining 25 more blocks should cause the first block file to be pruned")
144 # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this
145 mine_large_blocks(self.nodes[0], 25)
146 147 # Wait for blk00000.dat to be pruned
148 self.wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30)
149 150 self.log.info("Success")
151 usage = calc_usage(self.prunedir)
152 self.log.info(f"Usage should be below target: {usage}")
153 assert_greater_than(550, usage)
154 155 def create_chain_with_staleblocks(self):
156 # Create stale blocks in manageable sized chunks
157 self.log.info("Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds")
158 159 for _ in range(12):
160 # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain
161 # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects
162 self.disconnect_nodes(0, 1)
163 self.disconnect_nodes(0, 2)
164 # Mine 24 blocks in node 1
165 mine_large_blocks(self.nodes[1], 24)
166 167 # Reorg back with 25 block chain from node 0
168 mine_large_blocks(self.nodes[0], 25)
169 170 # Create connections in the order so both nodes can see the reorg at the same time
171 self.connect_nodes(0, 1)
172 self.connect_nodes(0, 2)
173 self.sync_blocks(self.nodes[0:3])
174 175 self.log.info(f"Usage can be over target because of high stale rate: {calc_usage(self.prunedir)}")
176 177 def reorg_test(self):
178 # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip
179 # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain
180 181 height = self.nodes[1].getblockcount()
182 self.log.info(f"Current block height: {height}")
183 184 self.forkheight = height - 287
185 self.forkhash = self.nodes[1].getblockhash(self.forkheight)
186 self.log.info(f"Invalidating block {self.forkhash} at height {self.forkheight}")
187 self.nodes[1].invalidateblock(self.forkhash)
188 189 # We've now switched to our previously mined-24 block fork on node 1, but that's not what we want
190 # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago)
191 mainchainhash = self.nodes[0].getblockhash(self.forkheight - 1)
192 curhash = self.nodes[1].getblockhash(self.forkheight - 1)
193 while curhash != mainchainhash:
194 self.nodes[1].invalidateblock(curhash)
195 curhash = self.nodes[1].getblockhash(self.forkheight - 1)
196 197 assert_equal(self.nodes[1].getblockcount(), self.forkheight - 1)
198 self.log.info(f"New best height: {self.nodes[1].getblockcount()}")
199 200 # Disconnect node1 and generate the new chain
201 self.disconnect_nodes(0, 1)
202 self.disconnect_nodes(1, 2)
203 204 self.log.info("Generating new longer chain of 300 more blocks")
205 self.generate(self.nodes[1], 300, sync_fun=self.no_op)
206 207 self.log.info("Reconnect nodes")
208 self.connect_nodes(0, 1)
209 self.connect_nodes(1, 2)
210 self.sync_blocks(self.nodes[0:3], timeout=120)
211 212 self.log.info(f"Verify height on node 2: {self.nodes[2].getblockcount()}")
213 self.log.info(f"Usage possibly still high because of stale blocks in block files: {calc_usage(self.prunedir)}")
214 215 self.log.info("Mine 220 more large blocks so we have requisite history")
216 217 mine_large_blocks(self.nodes[0], 220)
218 self.sync_blocks(self.nodes[0:3], timeout=120)
219 220 usage = calc_usage(self.prunedir)
221 self.log.info(f"Usage should be below target: {usage}")
222 assert_greater_than(550, usage)
223 224 def reorg_back(self):
225 # Verify that a block on the old main chain fork has been pruned away
226 assert_raises_rpc_error(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash)
227 with self.nodes[2].assert_debug_log(expected_msgs=["Block verification stopping at height", "(no data)"]):
228 assert not self.nodes[2].verifychain(checklevel=4, nblocks=0)
229 self.log.info(f"Will need to redownload block {self.forkheight}")
230 231 # Verify that we have enough history to reorg back to the fork point
232 # Although this is more than 288 blocks, because this chain was written more recently
233 # and only its other 299 small and 220 large blocks are in the block files after it,
234 # it is expected to still be retained
235 self.nodes[2].getblock(self.nodes[2].getblockhash(self.forkheight))
236 237 first_reorg_height = self.nodes[2].getblockcount()
238 block_hash_1295 = self.nodes[2].getblockhash(1295)
239 self.nodes[2].invalidateblock(block_hash_1295)
240 goalbestheight = self.mainchainheight
241 goalbesthash = self.mainchainhash2
242 243 # As of 0.10 the current block download logic is not able to reorg to the original chain created in
244 # create_chain_with_stale_blocks because it doesn't know of any peer that's on that chain from which to
245 # redownload its missing blocks.
246 # Invalidate the reorg_test chain in node 0 as well, it can successfully switch to the original chain
247 # because it has all the block data.
248 # However it must mine enough blocks to have a more work chain than the reorg_test chain in order
249 # to trigger node 2's block download logic.
250 # At this point node 2 is within 288 blocks of the fork point so it will preserve its ability to reorg
251 if self.nodes[2].getblockcount() < self.mainchainheight:
252 blocks_to_mine = first_reorg_height + 1 - self.mainchainheight
253 self.log.info(f"Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: {blocks_to_mine}")
254 self.nodes[0].invalidateblock(block_hash_1295)
255 assert_equal(self.nodes[0].getblockcount(), self.mainchainheight)
256 assert_equal(self.nodes[0].getbestblockhash(), self.mainchainhash2)
257 goalbesthash = self.generate(self.nodes[0], blocks_to_mine, sync_fun=self.no_op)[-1]
258 goalbestheight = first_reorg_height + 1
259 260 self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload")
261 # Wait for Node 2 to reorg to proper height
262 self.wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900)
263 assert_equal(self.nodes[2].getbestblockhash(), goalbesthash)
264 # Verify we can now have the data for a block previously pruned
265 assert_equal(self.nodes[2].getblock(self.forkhash)["height"], self.forkheight)
266 267 def manual_test(self, node_number, use_timestamp):
268 # at this point, node has 995 blocks and has not yet run in prune mode
269 self.start_node(node_number)
270 node = self.nodes[node_number]
271 assert_equal(node.getblockcount(), 995)
272 assert_raises_rpc_error(-1, "Cannot prune blocks because node is not in prune mode", node.pruneblockchain, 500)
273 274 # now re-start in manual pruning mode
275 self.restart_node(node_number, extra_args=["-prune=1"])
276 node = self.nodes[node_number]
277 assert_equal(node.getblockcount(), 995)
278 279 def height(index):
280 if use_timestamp:
281 return node.getblockheader(node.getblockhash(index))["time"] + TIMESTAMP_WINDOW
282 else:
283 return index
284 285 def prune(index):
286 ret = node.pruneblockchain(height=height(index))
287 assert_equal(ret + 1, node.getblockchaininfo()['pruneheight'])
288 289 def has_block(index):
290 return os.path.isfile(os.path.join(self.nodes[node_number].blocks_path, f"blk{index:05}.dat"))
291 292 # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000)
293 assert_raises_rpc_error(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500))
294 295 # Save block transaction count before pruning, assert value
296 block1_details = node.getblock(node.getblockhash(1))
297 assert_equal(block1_details["nTx"], len(block1_details["tx"]))
298 299 # mine 6 blocks so we are at height 1001 (i.e., above PruneAfterHeight)
300 self.generate(node, 6, sync_fun=self.no_op)
301 assert_equal(node.getblockchaininfo()["blocks"], 1001)
302 303 # prune parameter in the future (block or timestamp) should raise an exception
304 future_parameter = height(1001) + 5
305 if use_timestamp:
306 assert_raises_rpc_error(-8, "Could not find block with at least the specified timestamp", node.pruneblockchain, future_parameter)
307 else:
308 assert_raises_rpc_error(-8, "Blockchain is shorter than the attempted prune height", node.pruneblockchain, future_parameter)
309 310 # Pruned block should still know the number of transactions
311 assert_equal(node.getblockheader(node.getblockhash(1))["nTx"], block1_details["nTx"])
312 313 # negative heights should raise an exception
314 assert_raises_rpc_error(-8, "Negative block height", node.pruneblockchain, -10)
315 316 # height=100 too low to prune first block file so this is a no-op
317 prune(100)
318 assert has_block(0), "blk00000.dat is missing when should still be there"
319 320 # Does nothing
321 node.pruneblockchain(height(0))
322 assert has_block(0), "blk00000.dat is missing when should still be there"
323 324 # height=500 should prune first file
325 prune(500)
326 assert not has_block(0), "blk00000.dat is still there, should be pruned by now"
327 assert has_block(1), "blk00001.dat is missing when should still be there"
328 329 # height=650 should prune second file
330 prune(650)
331 assert not has_block(1), "blk00001.dat is still there, should be pruned by now"
332 333 # height=1000 should not prune anything more, because tip-288 is in blk00002.dat.
334 prune(1000)
335 assert has_block(2), "blk00002.dat is still there, should be pruned by now"
336 337 # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat)
338 self.generate(node, MIN_BLOCKS_TO_KEEP, sync_fun=self.no_op)
339 prune(1000)
340 assert not has_block(2), "blk00002.dat is still there, should be pruned by now"
341 assert not has_block(3), "blk00003.dat is still there, should be pruned by now"
342 343 # stop node, start back up with auto-prune at 550 MiB, make sure still runs
344 self.restart_node(node_number, extra_args=["-prune=550"])
345 346 self.log.info("Success")
347 348 def test_wallet_rescan(self):
349 # check that the pruning node's wallet is still in good shape
350 self.log.info("Stop and start pruning node to trigger wallet rescan")
351 self.restart_node(2, extra_args=["-prune=550"])
352 353 self.wait_until(lambda: self.nodes[2].getwalletinfo()["scanning"] == False)
354 self.wait_until(lambda: self.nodes[2].getwalletinfo()["lastprocessedblock"]["height"] == self.nodes[2].getblockcount())
355 356 # check that wallet loads successfully when restarting a pruned node after IBD.
357 # this was reported to fail in #7494.
358 self.restart_node(5, extra_args=["-prune=550", "-blockfilterindex=1"]) # restart to trigger rescan
359 360 self.wait_until(lambda: self.nodes[5].getwalletinfo()["scanning"] == False)
361 self.wait_until(lambda: self.nodes[5].getwalletinfo()["lastprocessedblock"]["height"] == self.nodes[0].getblockcount())
362 363 def run_test(self):
364 self.log.info("Warning! This test requires 4GB of disk space")
365 366 self.log.info("Mining a big blockchain of 995 blocks")
367 self.create_big_chain()
368 # Chain diagram key:
369 # * blocks on main chain
370 # +,&,$,@ blocks on other forks
371 # X invalidated block
372 # N1 Node 1
373 #
374 # Start by mining a simple chain that all nodes have
375 # N0=N1=N2 **...*(995)
376 377 # stop manual-pruning node with 995 blocks
378 self.stop_node(3)
379 self.stop_node(4)
380 381 self.log.info("Check that we haven't started pruning yet because we're below PruneAfterHeight")
382 self.test_height_min()
383 # Extend this chain past the PruneAfterHeight
384 # N0=N1=N2 **...*(1020)
385 386 self.log.info("Check that we'll exceed disk space target if we have a very high stale block rate")
387 self.create_chain_with_staleblocks()
388 # Disconnect N0
389 # And mine a 24 block chain on N1 and a separate 25 block chain on N0
390 # N1=N2 **...*+...+(1044)
391 # N0 **...**...**(1045)
392 #
393 # reconnect nodes causing reorg on N1 and N2
394 # N1=N2 **...*(1020) *...**(1045)
395 # \
396 # +...+(1044)
397 #
398 # repeat this process until you have 12 stale forks hanging off the
399 # main chain on N1 and N2
400 # N0 *************************...***************************(1320)
401 #
402 # N1=N2 **...*(1020) *...**(1045) *.. ..**(1295) *...**(1320)
403 # \ \ \
404 # +...+(1044) &.. $...$(1319)
405 406 # Save some current chain state for later use
407 self.mainchainheight = self.nodes[2].getblockcount() # 1320
408 self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight)
409 410 self.log.info("Check that we can survive a 288 block reorg still")
411 self.reorg_test() # (1033, )
412 # Now create a 288 block reorg by mining a longer chain on N1
413 # First disconnect N1
414 # Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain
415 # N1 **...*(1020) **...**(1032)X..
416 # \
417 # ++...+(1031)X..
418 #
419 # Now mine 300 more blocks on N1
420 # N1 **...*(1020) **...**(1032) @@...@(1332)
421 # \ \
422 # \ X...
423 # \ \
424 # ++...+(1031)X.. ..
425 #
426 # Reconnect nodes and mine 220 more blocks on N1
427 # N1 **...*(1020) **...**(1032) @@...@@@(1552)
428 # \ \
429 # \ X...
430 # \ \
431 # ++...+(1031)X.. ..
432 #
433 # N2 **...*(1020) **...**(1032) @@...@@@(1552)
434 # \ \
435 # \ *...**(1320)
436 # \ \
437 # ++...++(1044) ..
438 #
439 # N0 ********************(1032) @@...@@@(1552)
440 # \
441 # *...**(1320)
442 443 self.log.info("Test that we can rerequest a block we previously pruned if needed for a reorg")
444 self.reorg_back()
445 # Verify that N2 still has block 1033 on current chain (@), but not on main chain (*)
446 # Invalidate 1033 on current chain (@) on N2 and we should be able to reorg to
447 # original main chain (*), but will require redownload of some blocks
448 # In order to have a peer we think we can download from, must also perform this invalidation
449 # on N0 and mine a new longest chain to trigger.
450 # Final result:
451 # N0 ********************(1032) **...****(1553)
452 # \
453 # X@...@@@(1552)
454 #
455 # N2 **...*(1020) **...**(1032) **...****(1553)
456 # \ \
457 # \ X@...@@@(1552)
458 # \
459 # +..
460 #
461 # N1 doesn't change because 1033 on main chain (*) is invalid
462 463 self.log.info("Test manual pruning with block indices")
464 self.manual_test(3, use_timestamp=False)
465 466 self.log.info("Test manual pruning with timestamps")
467 self.manual_test(4, use_timestamp=True)
468 469 self.log.info("Syncing node 5 to node 0")
470 self.connect_nodes(0, 5)
471 self.sync_blocks([self.nodes[0], self.nodes[5]], wait=5, timeout=300)
472 473 if self.is_wallet_compiled():
474 self.log.info("Test wallet re-scan")
475 self.test_wallet_rescan()
476 477 self.log.info("Test it's not possible to rescan beyond pruned data")
478 self.test_rescan_blockchain()
479 480 self.log.info("Test invalid pruning command line options")
481 self.test_invalid_command_line_options()
482 483 self.log.info("Test scanblocks can not return pruned data")
484 self.test_scanblocks_pruned()
485 486 self.log.info("Test pruneheight reflects the presence of block and undo data")
487 self.test_pruneheight_undo_presence()
488 489 self.log.info("Done")
490 491 def test_scanblocks_pruned(self):
492 node = self.nodes[5]
493 genesis_blockhash = node.getblockhash(0)
494 false_positive_spk = bytes.fromhex("001400000000000000000000000000000000000cadcb")
495 496 assert genesis_blockhash in node.scanblocks(
497 "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0)['relevant_blocks']
498 499 assert_raises_rpc_error(-1, "Block not available (pruned data)", node.scanblocks,
500 "start", [{"desc": f"raw({false_positive_spk.hex()})"}], 0, 0, "basic", {"filter_false_positives": True})
501 502 def test_pruneheight_undo_presence(self):
503 node = self.nodes[5]
504 pruneheight = node.getblockchaininfo()["pruneheight"]
505 fetch_block = node.getblockhash(pruneheight - 1)
506 507 self.connect_nodes(1, 5)
508 peers = node.getpeerinfo()
509 node.getblockfrompeer(fetch_block, peers[0]["id"])
510 self.wait_until(lambda: not try_rpc(-1, "Block not available (pruned data)", node.getblock, fetch_block), timeout=5)
511 512 new_pruneheight = node.getblockchaininfo()["pruneheight"]
513 assert_equal(pruneheight, new_pruneheight)
514 515 if __name__ == '__main__':
516 PruneTest(__file__).main()
517