1 #!/usr/bin/env python3
2 # Copyright (c) 2014-present The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test behavior of headers messages to announce blocks.
6 7 Setup:
8 9 - Two nodes:
10 - node0 is the node-under-test. We create two p2p connections to it. The
11 first p2p connection is a control and should only ever receive inv's. The
12 second p2p connection tests the headers sending logic.
13 - node1 is used to create reorgs.
14 15 test_null_locators
16 ==================
17 18 Sends two getheaders requests with null locator values. First request's hashstop
19 value refers to validated block, while second request's hashstop value refers to
20 a block which hasn't been validated. Verifies only the first request returns
21 headers.
22 23 test_nonnull_locators
24 =====================
25 26 Part 1: No headers announcements before "sendheaders"
27 a. node mines a block [expect: inv]
28 send getdata for the block [expect: block]
29 b. node mines another block [expect: inv]
30 send getheaders and getdata [expect: headers, then block]
31 c. node mines another block [expect: inv]
32 peer mines a block, announces with header [expect: getdata]
33 d. node mines another block [expect: inv]
34 35 Part 2: After "sendheaders", headers announcements should generally work.
36 a. peer sends sendheaders [expect: no response]
37 peer sends getheaders with current tip [expect: no response]
38 b. node mines a block [expect: tip header]
39 c. for N in 1, ..., 10:
40 * for announce-type in {inv, header}
41 - peer mines N blocks, announces with announce-type
42 [ expect: getheaders/getdata or getdata, deliver block(s) ]
43 - node mines a block [ expect: 1 header ]
44 45 Part 3: Headers announcements stop after large reorg and resume after getheaders or inv from peer.
46 - For response-type in {inv, getheaders}
47 * node mines a 7 block reorg [ expect: headers announcement of 8 blocks ]
48 * node mines an 8-block reorg [ expect: inv at tip ]
49 * peer responds with getblocks/getdata [expect: inv, blocks ]
50 * node mines another block [ expect: inv at tip, peer sends getdata, expect: block ]
51 * node mines another block at tip [ expect: inv ]
52 * peer responds with getheaders with an old hashstop more than 8 blocks back [expect: headers]
53 * peer requests block [ expect: block ]
54 * node mines another block at tip [ expect: inv, peer sends getdata, expect: block ]
55 * peer sends response-type [expect headers if getheaders, getheaders/getdata if mining new block]
56 * node mines 1 block [expect: 1 header, peer responds with getdata]
57 58 Part 4: Test direct fetch behavior
59 a. Announce 2 old block headers.
60 Expect: no getdata requests.
61 b. Announce 3 new blocks via 1 headers message.
62 Expect: one getdata request for all 3 blocks.
63 (Send blocks.)
64 c. Announce 1 header that forks off the last two blocks.
65 Expect: no response.
66 d. Announce 1 more header that builds on that fork.
67 Expect: one getdata request for two blocks.
68 e. Announce 16 more headers that build on that fork.
69 Expect: getdata request for 14 more blocks.
70 f. Announce 1 more header that builds on that fork.
71 Expect: no response.
72 73 Part 5: Test handling of headers that don't connect.
74 a. Repeat 100 times:
75 1. Announce a header that doesn't connect.
76 Expect: getheaders message
77 2. Send headers chain.
78 Expect: getdata for the missing blocks, tip update.
79 b. Then send 99 more headers that don't connect.
80 Expect: getheaders message each time.
81 """
82 from test_framework.blocktools import create_block
83 from test_framework.messages import CInv
84 from test_framework.p2p import (
85 CBlockHeader,
86 NODE_WITNESS,
87 P2PInterface,
88 p2p_lock,
89 MSG_BLOCK,
90 msg_block,
91 msg_getblocks,
92 msg_getdata,
93 msg_getheaders,
94 msg_headers,
95 msg_inv,
96 msg_sendheaders,
97 )
98 from test_framework.test_framework import BitcoinTestFramework
99 from test_framework.util import (
100 assert_equal,
101 )
102 103 DIRECT_FETCH_RESPONSE_TIME = 0.05
104 105 class BaseNode(P2PInterface):
106 def __init__(self):
107 super().__init__()
108 109 self.block_announced = False
110 self.last_blockhash_announced = None
111 self.recent_headers_announced = []
112 113 def send_get_data(self, block_hashes):
114 """Request data for a list of block hashes."""
115 msg = msg_getdata()
116 for x in block_hashes:
117 msg.inv.append(CInv(MSG_BLOCK, x))
118 self.send_without_ping(msg)
119 120 def send_get_headers(self, locator, hashstop):
121 msg = msg_getheaders()
122 msg.locator.vHave = locator
123 msg.hashstop = hashstop
124 self.send_without_ping(msg)
125 126 def send_block_inv(self, blockhash):
127 msg = msg_inv()
128 msg.inv = [CInv(MSG_BLOCK, blockhash)]
129 self.send_without_ping(msg)
130 131 def send_header_for_blocks(self, new_blocks):
132 headers_message = msg_headers()
133 headers_message.headers = [CBlockHeader(b) for b in new_blocks]
134 self.send_without_ping(headers_message)
135 136 def send_getblocks(self, locator):
137 getblocks_message = msg_getblocks()
138 getblocks_message.locator.vHave = locator
139 self.send_without_ping(getblocks_message)
140 141 def wait_for_block_announcement(self, block_hash, timeout=60):
142 test_function = lambda: self.last_blockhash_announced == block_hash
143 self.wait_until(test_function, timeout=timeout)
144 145 def on_inv(self, message):
146 self.block_announced = True
147 self.last_blockhash_announced = message.inv[-1].hash
148 149 def on_headers(self, message):
150 if len(message.headers):
151 self.block_announced = True
152 for x in message.headers:
153 # append because headers may be announced over multiple messages.
154 self.recent_headers_announced.append(x.hash_int)
155 self.last_blockhash_announced = message.headers[-1].hash_int
156 157 def clear_block_announcements(self):
158 with p2p_lock:
159 self.block_announced = False
160 self.last_message.pop("inv", None)
161 self.last_message.pop("headers", None)
162 self.recent_headers_announced = []
163 164 165 def check_last_headers_announcement(self, headers):
166 """Test whether the last headers announcements received are right.
167 Headers may be announced across more than one message."""
168 test_function = lambda: (len(self.recent_headers_announced) >= len(headers))
169 self.wait_until(test_function)
170 with p2p_lock:
171 assert_equal(self.recent_headers_announced, headers)
172 self.block_announced = False
173 self.last_message.pop("headers", None)
174 self.recent_headers_announced = []
175 176 def check_last_inv_announcement(self, inv):
177 """Test whether the last announcement received had the right inv.
178 inv should be a list of block hashes."""
179 180 test_function = lambda: self.block_announced
181 self.wait_until(test_function)
182 183 with p2p_lock:
184 compare_inv = []
185 if "inv" in self.last_message:
186 compare_inv = [x.hash for x in self.last_message["inv"].inv]
187 assert_equal(compare_inv, inv)
188 self.block_announced = False
189 self.last_message.pop("inv", None)
190 191 class SendHeadersTest(BitcoinTestFramework):
192 def set_test_params(self):
193 self.setup_clean_chain = True
194 self.num_nodes = 2
195 196 def mine_blocks(self, count):
197 """Mine count blocks and return the new tip."""
198 199 # Clear out block announcements from each p2p listener
200 [x.clear_block_announcements() for x in self.nodes[0].p2ps]
201 self.generatetoaddress(self.nodes[0], count, self.nodes[0].get_deterministic_priv_key().address)
202 return int(self.nodes[0].getbestblockhash(), 16)
203 204 def mine_reorg(self, length):
205 """Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks).
206 207 Note: we clear the state of our p2p connections after the
208 to-be-reorged-out blocks are mined, so that we don't break later tests.
209 return the list of block hashes newly mined."""
210 211 # make sure all invalidated blocks are node0's
212 self.generatetoaddress(self.nodes[0], length, self.nodes[0].get_deterministic_priv_key().address)
213 for x in self.nodes[0].p2ps:
214 x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16))
215 x.clear_block_announcements()
216 217 tip_height = self.nodes[1].getblockcount()
218 hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1))
219 self.nodes[1].invalidateblock(hash_to_invalidate)
220 all_hashes = self.generatetoaddress(self.nodes[1], length + 1, self.nodes[1].get_deterministic_priv_key().address) # Must be longer than the orig chain
221 return [int(x, 16) for x in all_hashes]
222 223 def run_test(self):
224 # Setup the p2p connections
225 inv_node = self.nodes[0].add_p2p_connection(BaseNode())
226 # Make sure NODE_NETWORK is not set for test_node, so no block download
227 # will occur outside of direct fetching
228 test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_WITNESS)
229 230 self.test_null_locators(test_node, inv_node)
231 self.test_nonnull_locators(test_node, inv_node)
232 233 def test_null_locators(self, test_node, inv_node):
234 tip = self.nodes[0].getblockheader(self.generatetoaddress(self.nodes[0], 1, self.nodes[0].get_deterministic_priv_key().address)[0])
235 tip_hash = int(tip["hash"], 16)
236 237 inv_node.check_last_inv_announcement(inv=[tip_hash])
238 test_node.check_last_inv_announcement(inv=[tip_hash])
239 240 self.log.info("Verify getheaders with null locator and valid hashstop returns headers.")
241 test_node.clear_block_announcements()
242 test_node.send_get_headers(locator=[], hashstop=tip_hash)
243 test_node.check_last_headers_announcement(headers=[tip_hash])
244 245 self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.")
246 block = create_block(int(tip["hash"], 16), height=tip["height"] + 1, ntime=tip["mediantime"] + 1)
247 block.solve()
248 test_node.send_header_for_blocks([block])
249 test_node.clear_block_announcements()
250 test_node.send_get_headers(locator=[], hashstop=block.hash_int)
251 test_node.sync_with_ping()
252 assert_equal(test_node.block_announced, False)
253 inv_node.clear_block_announcements()
254 test_node.send_without_ping(msg_block(block))
255 inv_node.check_last_inv_announcement(inv=[block.hash_int])
256 257 def test_nonnull_locators(self, test_node, inv_node):
258 tip = int(self.nodes[0].getbestblockhash(), 16)
259 260 # PART 1
261 # 1. Mine a block; expect inv announcements each time
262 self.log.info("Part 1: headers don't start before sendheaders message...")
263 for i in range(4):
264 self.log.debug("Part 1.{}: starting...".format(i))
265 old_tip = tip
266 tip = self.mine_blocks(1)
267 inv_node.check_last_inv_announcement(inv=[tip])
268 test_node.check_last_inv_announcement(inv=[tip])
269 # Try a few different responses; none should affect next announcement
270 if i == 0:
271 # first request the block
272 test_node.send_get_data([tip])
273 test_node.wait_for_block(tip)
274 elif i == 1:
275 # next try requesting header and block
276 test_node.send_get_headers(locator=[old_tip], hashstop=tip)
277 test_node.send_get_data([tip])
278 test_node.wait_for_block(tip)
279 test_node.clear_block_announcements() # since we requested headers...
280 elif i == 2:
281 # this time announce own block via headers
282 inv_node.clear_block_announcements()
283 height = self.nodes[0].getblockcount()
284 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
285 block_time = last_time + 1
286 new_block = create_block(tip, height=height + 1, ntime=block_time)
287 new_block.solve()
288 test_node.send_header_for_blocks([new_block])
289 test_node.wait_for_getdata([new_block.hash_int])
290 test_node.send_and_ping(msg_block(new_block)) # make sure this block is processed
291 inv_node.wait_until(lambda: inv_node.block_announced)
292 inv_node.clear_block_announcements()
293 test_node.clear_block_announcements()
294 295 self.log.info("Part 1: success!")
296 self.log.info("Part 2: announce blocks with headers after sendheaders message...")
297 # PART 2
298 # 2. Send a sendheaders message and test that headers announcements
299 # commence and keep working.
300 test_node.send_without_ping(msg_sendheaders())
301 prev_tip = int(self.nodes[0].getbestblockhash(), 16)
302 test_node.send_get_headers(locator=[prev_tip], hashstop=0)
303 test_node.sync_with_ping()
304 305 # Now that we've synced headers, headers announcements should work
306 tip = self.mine_blocks(1)
307 expected_hash = tip
308 inv_node.check_last_inv_announcement(inv=[tip])
309 test_node.check_last_headers_announcement(headers=[tip])
310 311 height = self.nodes[0].getblockcount() + 1
312 block_time += 10 # Advance far enough ahead
313 for i in range(10):
314 self.log.debug("Part 2.{}: starting...".format(i))
315 # Mine i blocks, and alternate announcing either via
316 # inv (of tip) or via headers. After each, new blocks
317 # mined by the node should successfully be announced
318 # with block header, even though the blocks are never requested
319 for j in range(2):
320 self.log.debug("Part 2.{}.{}: starting...".format(i, j))
321 blocks = []
322 for _ in range(i + 1):
323 blocks.append(create_block(tip, height=height, ntime=block_time))
324 blocks[-1].solve()
325 tip = blocks[-1].hash_int
326 block_time += 1
327 height += 1
328 if j == 0:
329 # Announce via inv
330 test_node.send_block_inv(tip)
331 if i == 0:
332 test_node.wait_for_getheaders(block_hash=expected_hash)
333 else:
334 assert "getheaders" not in test_node.last_message
335 # Should have received a getheaders now
336 test_node.send_header_for_blocks(blocks)
337 # Test that duplicate inv's won't result in duplicate
338 # getdata requests, or duplicate headers announcements
339 [inv_node.send_block_inv(x.hash_int) for x in blocks]
340 test_node.wait_for_getdata([x.hash_int for x in blocks])
341 inv_node.sync_with_ping()
342 else:
343 # Announce via headers
344 test_node.send_header_for_blocks(blocks)
345 test_node.wait_for_getdata([x.hash_int for x in blocks])
346 # Test that duplicate headers won't result in duplicate
347 # getdata requests (the check is further down)
348 inv_node.send_header_for_blocks(blocks)
349 inv_node.sync_with_ping()
350 [test_node.send_without_ping(msg_block(x)) for x in blocks]
351 test_node.sync_with_ping()
352 inv_node.sync_with_ping()
353 # This block should not be announced to the inv node (since it also
354 # broadcast it)
355 assert "inv" not in inv_node.last_message
356 assert "headers" not in inv_node.last_message
357 tip = self.mine_blocks(1)
358 inv_node.check_last_inv_announcement(inv=[tip])
359 test_node.check_last_headers_announcement(headers=[tip])
360 height += 1
361 block_time += 1
362 363 self.log.info("Part 2: success!")
364 365 self.log.info("Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer...")
366 367 # PART 3. Headers announcements can stop after large reorg, and resume after
368 # getheaders or inv from peer.
369 for j in range(2):
370 self.log.debug("Part 3.{}: starting...".format(j))
371 # First try mining a reorg that can propagate with header announcement
372 new_block_hashes = self.mine_reorg(length=7)
373 tip = new_block_hashes[-1]
374 inv_node.check_last_inv_announcement(inv=[tip])
375 test_node.check_last_headers_announcement(headers=new_block_hashes)
376 377 block_time += 8
378 379 # Mine a too-large reorg, which should be announced with a single inv
380 new_block_hashes = self.mine_reorg(length=8)
381 tip = new_block_hashes[-1]
382 inv_node.check_last_inv_announcement(inv=[tip])
383 test_node.check_last_inv_announcement(inv=[tip])
384 385 block_time += 9
386 387 fork_point = self.nodes[0].getblock("%064x" % new_block_hashes[0])["previousblockhash"]
388 fork_point = int(fork_point, 16)
389 390 # Use getblocks/getdata
391 test_node.send_getblocks(locator=[fork_point])
392 test_node.check_last_inv_announcement(inv=new_block_hashes)
393 test_node.send_get_data(new_block_hashes)
394 test_node.wait_for_block(new_block_hashes[-1])
395 396 for i in range(3):
397 self.log.debug("Part 3.{}.{}: starting...".format(j, i))
398 399 # Mine another block, still should get only an inv
400 tip = self.mine_blocks(1)
401 inv_node.check_last_inv_announcement(inv=[tip])
402 test_node.check_last_inv_announcement(inv=[tip])
403 if i == 0:
404 # Just get the data -- shouldn't cause headers announcements to resume
405 test_node.send_get_data([tip])
406 test_node.wait_for_block(tip)
407 elif i == 1:
408 # Send a getheaders message that shouldn't trigger headers announcements
409 # to resume (best header sent will be too old)
410 test_node.send_get_headers(locator=[fork_point], hashstop=new_block_hashes[1])
411 test_node.send_get_data([tip])
412 test_node.wait_for_block(tip)
413 elif i == 2:
414 # This time, try sending either a getheaders to trigger resumption
415 # of headers announcements, or mine a new block and inv it, also
416 # triggering resumption of headers announcements.
417 test_node.send_get_data([tip])
418 test_node.wait_for_block(tip)
419 if j == 0:
420 test_node.send_get_headers(locator=[tip], hashstop=0)
421 test_node.sync_with_ping()
422 else:
423 test_node.send_block_inv(tip)
424 test_node.sync_with_ping()
425 # New blocks should now be announced with header
426 tip = self.mine_blocks(1)
427 inv_node.check_last_inv_announcement(inv=[tip])
428 test_node.check_last_headers_announcement(headers=[tip])
429 430 self.log.info("Part 3: success!")
431 432 self.log.info("Part 4: Testing direct fetch behavior...")
433 tip = self.mine_blocks(1)
434 height = self.nodes[0].getblockcount() + 1
435 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
436 block_time = last_time + 1
437 438 # Create 2 blocks. Send the blocks, then send the headers.
439 blocks = []
440 for _ in range(2):
441 blocks.append(create_block(tip, height=height, ntime=block_time))
442 blocks[-1].solve()
443 tip = blocks[-1].hash_int
444 block_time += 1
445 height += 1
446 inv_node.send_without_ping(msg_block(blocks[-1]))
447 448 inv_node.sync_with_ping() # Make sure blocks are processed
449 test_node.last_message.pop("getdata", None)
450 test_node.send_header_for_blocks(blocks)
451 test_node.sync_with_ping()
452 # should not have received any getdata messages
453 with p2p_lock:
454 assert "getdata" not in test_node.last_message
455 456 # This time, direct fetch should work
457 blocks = []
458 for _ in range(3):
459 blocks.append(create_block(tip, height=height, ntime=block_time))
460 blocks[-1].solve()
461 tip = blocks[-1].hash_int
462 block_time += 1
463 height += 1
464 465 test_node.send_header_for_blocks(blocks)
466 test_node.sync_with_ping()
467 test_node.wait_for_getdata([x.hash_int for x in blocks], timeout=DIRECT_FETCH_RESPONSE_TIME)
468 469 [test_node.send_without_ping(msg_block(x)) for x in blocks]
470 471 test_node.sync_with_ping()
472 473 # Now announce a header that forks the last two blocks
474 tip = blocks[0].hash_int
475 height -= 2
476 blocks = []
477 478 # Create extra blocks for later
479 for _ in range(20):
480 blocks.append(create_block(tip, height=height, ntime=block_time))
481 blocks[-1].solve()
482 tip = blocks[-1].hash_int
483 block_time += 1
484 height += 1
485 486 # Announcing one block on fork should not trigger direct fetch
487 # (less work than tip)
488 test_node.last_message.pop("getdata", None)
489 test_node.send_header_for_blocks(blocks[0:1])
490 test_node.sync_with_ping()
491 with p2p_lock:
492 assert "getdata" not in test_node.last_message
493 494 # Announcing one more block on fork should trigger direct fetch for
495 # both blocks (same work as tip)
496 test_node.send_header_for_blocks(blocks[1:2])
497 test_node.sync_with_ping()
498 test_node.wait_for_getdata([x.hash_int for x in blocks[0:2]], timeout=DIRECT_FETCH_RESPONSE_TIME)
499 500 # Announcing 16 more headers should trigger direct fetch for 14 more
501 # blocks
502 test_node.send_header_for_blocks(blocks[2:18])
503 test_node.sync_with_ping()
504 test_node.wait_for_getdata([x.hash_int for x in blocks[2:16]], timeout=DIRECT_FETCH_RESPONSE_TIME)
505 506 # Announcing 1 more header should not trigger any response
507 test_node.last_message.pop("getdata", None)
508 test_node.send_header_for_blocks(blocks[18:19])
509 test_node.sync_with_ping()
510 with p2p_lock:
511 assert "getdata" not in test_node.last_message
512 513 self.log.info("Part 4: success!")
514 515 # Now deliver all those blocks we announced.
516 [test_node.send_without_ping(msg_block(x)) for x in blocks]
517 518 self.log.info("Part 5: Testing handling of unconnecting headers")
519 # First we test that receipt of an unconnecting header doesn't prevent
520 # chain sync.
521 expected_hash = tip
522 NUM_HEADERS = 100
523 for i in range(NUM_HEADERS):
524 self.log.debug("Part 5.{}: starting...".format(i))
525 test_node.last_message.pop("getdata", None)
526 blocks = []
527 # Create two more blocks.
528 for _ in range(2):
529 blocks.append(create_block(tip, height=height, ntime=block_time))
530 blocks[-1].solve()
531 tip = blocks[-1].hash_int
532 block_time += 1
533 height += 1
534 # Send the header of the second block -> this won't connect.
535 test_node.send_header_for_blocks([blocks[1]])
536 test_node.wait_for_getheaders(block_hash=expected_hash)
537 test_node.send_header_for_blocks(blocks)
538 test_node.wait_for_getdata([x.hash_int for x in blocks])
539 [test_node.send_without_ping(msg_block(x)) for x in blocks]
540 test_node.sync_with_ping()
541 assert_equal(self.nodes[0].getbestblockhash(), blocks[1].hash_hex)
542 expected_hash = blocks[1].hash_int
543 544 blocks = []
545 # Now we test that if we repeatedly don't send connecting headers, we
546 # don't go into an infinite loop trying to get them to connect.
547 for _ in range(NUM_HEADERS + 1):
548 blocks.append(create_block(tip, height=height, ntime=block_time))
549 blocks[-1].solve()
550 tip = blocks[-1].hash_int
551 block_time += 1
552 height += 1
553 554 for i in range(1, NUM_HEADERS):
555 with p2p_lock:
556 test_node.last_message.pop("getheaders", None)
557 # Send an empty header as a failed response to the received getheaders
558 # (from the previous iteration). Otherwise, the new headers will be
559 # treated as a response instead of as an announcement.
560 test_node.send_header_for_blocks([])
561 # Send the actual unconnecting header, which should trigger a new getheaders.
562 test_node.send_header_for_blocks([blocks[i]])
563 test_node.wait_for_getheaders(block_hash=expected_hash)
564 565 # Finally, check that the inv node never received a getdata request,
566 # throughout the test
567 assert "getdata" not in inv_node.last_message
568 569 if __name__ == '__main__':
570 SendHeadersTest(__file__).main()
571