1 #!/usr/bin/env python3
2 # Copyright (c) 2014-2021 The Limenka developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test behavior of headers messages to announce blocks.
6 7 Setup:
8 9 - Two nodes:
10 - node0 is the node-under-test. We create two p2p connections to it. The
11 first p2p connection is a control and should only ever receive inv's. The
12 second p2p connection tests the headers sending logic.
13 - node1 is used to create reorgs.
14 15 test_null_locators
16 ==================
17 18 Sends two getheaders requests with null locator values. First request's hashstop
19 value refers to validated block, while second request's hashstop value refers to
20 a block which hasn't been validated. Verifies only the first request returns
21 headers.
22 23 test_nonnull_locators
24 =====================
25 26 Part 1: No headers announcements before "sendheaders"
27 a. node mines a block [expect: inv]
28 send getdata for the block [expect: block]
29 b. node mines another block [expect: inv]
30 send getheaders and getdata [expect: headers, then block]
31 c. node mines another block [expect: inv]
32 peer mines a block, announces with header [expect: getdata]
33 d. node mines another block [expect: inv]
34 35 Part 2: After "sendheaders", headers announcements should generally work.
36 a. peer sends sendheaders [expect: no response]
37 peer sends getheaders with current tip [expect: no response]
38 b. node mines a block [expect: tip header]
39 c. for N in 1, ..., 10:
40 * for announce-type in {inv, header}
41 - peer mines N blocks, announces with announce-type
42 [ expect: getheaders/getdata or getdata, deliver block(s) ]
43 - node mines a block [ expect: 1 header ]
44 45 Part 3: Headers announcements stop after large reorg and resume after getheaders or inv from peer.
46 - For response-type in {inv, getheaders}
47 * node mines a 7 block reorg [ expect: headers announcement of 8 blocks ]
48 * node mines an 8-block reorg [ expect: inv at tip ]
49 * peer responds with getblocks/getdata [expect: inv, blocks ]
50 * node mines another block [ expect: inv at tip, peer sends getdata, expect: block ]
51 * node mines another block at tip [ expect: inv ]
52 * peer responds with getheaders with an old hashstop more than 8 blocks back [expect: headers]
53 * peer requests block [ expect: block ]
54 * node mines another block at tip [ expect: inv, peer sends getdata, expect: block ]
55 * peer sends response-type [expect headers if getheaders, getheaders/getdata if mining new block]
56 * node mines 1 block [expect: 1 header, peer responds with getdata]
57 58 Part 4: Test direct fetch behavior
59 a. Announce 2 old block headers.
60 Expect: no getdata requests.
61 b. Announce 3 new blocks via 1 headers message.
62 Expect: one getdata request for all 3 blocks.
63 (Send blocks.)
64 c. Announce 1 header that forks off the last two blocks.
65 Expect: no response.
66 d. Announce 1 more header that builds on that fork.
67 Expect: one getdata request for two blocks.
68 e. Announce 16 more headers that build on that fork.
69 Expect: getdata request for 14 more blocks.
70 f. Announce 1 more header that builds on that fork.
71 Expect: no response.
72 73 Part 5: Test handling of headers that don't connect.
74 a. Repeat 100 times:
75 1. Announce a header that doesn't connect.
76 Expect: getheaders message
77 2. Send headers chain.
78 Expect: getdata for the missing blocks, tip update.
79 b. Then send 99 more headers that don't connect.
80 Expect: getheaders message each time.
81 """
82 from test_framework.blocktools import create_block, create_coinbase
83 from test_framework.messages import CInv
84 from test_framework.p2p import (
85 CBlockHeader,
86 NODE_WITNESS,
87 P2PInterface,
88 p2p_lock,
89 MSG_BLOCK,
90 msg_block,
91 msg_getblocks,
92 msg_getdata,
93 msg_getheaders,
94 msg_headers,
95 msg_inv,
96 msg_sendheaders,
97 )
98 from test_framework.test_framework import LimenkaTestFramework
99 from test_framework.util import (
100 assert_equal,
101 )
102 103 DIRECT_FETCH_RESPONSE_TIME = 0.05
104 105 class BaseNode(P2PInterface):
106 def __init__(self):
107 super().__init__()
108 109 self.block_announced = False
110 self.last_blockhash_announced = None
111 self.recent_headers_announced = []
112 113 def send_get_data(self, block_hashes):
114 """Request data for a list of block hashes."""
115 msg = msg_getdata()
116 for x in block_hashes:
117 msg.inv.append(CInv(MSG_BLOCK, x))
118 self.send_message(msg)
119 120 def send_get_headers(self, locator, hashstop):
121 msg = msg_getheaders()
122 msg.locator.vHave = locator
123 msg.hashstop = hashstop
124 self.send_message(msg)
125 126 def send_block_inv(self, blockhash):
127 msg = msg_inv()
128 msg.inv = [CInv(MSG_BLOCK, blockhash)]
129 self.send_message(msg)
130 131 def send_header_for_blocks(self, new_blocks):
132 headers_message = msg_headers()
133 headers_message.headers = [CBlockHeader(b) for b in new_blocks]
134 self.send_message(headers_message)
135 136 def send_getblocks(self, locator):
137 getblocks_message = msg_getblocks()
138 getblocks_message.locator.vHave = locator
139 self.send_message(getblocks_message)
140 141 def wait_for_block_announcement(self, block_hash, timeout=60):
142 test_function = lambda: self.last_blockhash_announced == block_hash
143 self.wait_until(test_function, timeout=timeout)
144 145 def on_inv(self, message):
146 self.block_announced = True
147 self.last_blockhash_announced = message.inv[-1].hash
148 149 def on_headers(self, message):
150 if len(message.headers):
151 self.block_announced = True
152 for x in message.headers:
153 x.calc_sha256()
154 # append because headers may be announced over multiple messages.
155 self.recent_headers_announced.append(x.sha256)
156 self.last_blockhash_announced = message.headers[-1].sha256
157 158 def clear_block_announcements(self):
159 with p2p_lock:
160 self.block_announced = False
161 self.last_message.pop("inv", None)
162 self.last_message.pop("headers", None)
163 self.recent_headers_announced = []
164 165 166 def check_last_headers_announcement(self, headers):
167 """Test whether the last headers announcements received are right.
168 Headers may be announced across more than one message."""
169 test_function = lambda: (len(self.recent_headers_announced) >= len(headers))
170 self.wait_until(test_function)
171 with p2p_lock:
172 assert_equal(self.recent_headers_announced, headers)
173 self.block_announced = False
174 self.last_message.pop("headers", None)
175 self.recent_headers_announced = []
176 177 def check_last_inv_announcement(self, inv):
178 """Test whether the last announcement received had the right inv.
179 inv should be a list of block hashes."""
180 181 test_function = lambda: self.block_announced
182 self.wait_until(test_function)
183 184 with p2p_lock:
185 compare_inv = []
186 if "inv" in self.last_message:
187 compare_inv = [x.hash for x in self.last_message["inv"].inv]
188 assert_equal(compare_inv, inv)
189 self.block_announced = False
190 self.last_message.pop("inv", None)
191 192 class SendHeadersTest(LimenkaTestFramework):
193 def set_test_params(self):
194 self.setup_clean_chain = True
195 self.num_nodes = 2
196 197 def mine_blocks(self, count):
198 """Mine count blocks and return the new tip."""
199 200 # Clear out block announcements from each p2p listener
201 [x.clear_block_announcements() for x in self.nodes[0].p2ps]
202 self.generatetoaddress(self.nodes[0], count, self.nodes[0].get_deterministic_priv_key().address)
203 return int(self.nodes[0].getbestblockhash(), 16)
204 205 def mine_reorg(self, length):
206 """Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks).
207 208 Note: we clear the state of our p2p connections after the
209 to-be-reorged-out blocks are mined, so that we don't break later tests.
210 return the list of block hashes newly mined."""
211 212 # make sure all invalidated blocks are node0's
213 self.generatetoaddress(self.nodes[0], length, self.nodes[0].get_deterministic_priv_key().address)
214 for x in self.nodes[0].p2ps:
215 x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16))
216 x.clear_block_announcements()
217 218 tip_height = self.nodes[1].getblockcount()
219 hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1))
220 self.nodes[1].invalidateblock(hash_to_invalidate)
221 all_hashes = self.generatetoaddress(self.nodes[1], length + 1, self.nodes[1].get_deterministic_priv_key().address) # Must be longer than the orig chain
222 return [int(x, 16) for x in all_hashes]
223 224 def run_test(self):
225 # Setup the p2p connections
226 inv_node = self.nodes[0].add_p2p_connection(BaseNode())
227 # Make sure NODE_NETWORK is not set for test_node, so no block download
228 # will occur outside of direct fetching
229 test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_WITNESS)
230 231 self.test_null_locators(test_node, inv_node)
232 self.test_nonnull_locators(test_node, inv_node)
233 234 def test_null_locators(self, test_node, inv_node):
235 tip = self.nodes[0].getblockheader(self.generatetoaddress(self.nodes[0], 1, self.nodes[0].get_deterministic_priv_key().address)[0])
236 tip_hash = int(tip["hash"], 16)
237 238 inv_node.check_last_inv_announcement(inv=[tip_hash])
239 test_node.check_last_inv_announcement(inv=[tip_hash])
240 241 self.log.info("Verify getheaders with null locator and valid hashstop returns headers.")
242 test_node.clear_block_announcements()
243 test_node.send_get_headers(locator=[], hashstop=tip_hash)
244 test_node.check_last_headers_announcement(headers=[tip_hash])
245 246 self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.")
247 block = create_block(int(tip["hash"], 16), create_coinbase(tip["height"] + 1), tip["mediantime"] + 1)
248 block.solve()
249 test_node.send_header_for_blocks([block])
250 test_node.clear_block_announcements()
251 test_node.send_get_headers(locator=[], hashstop=int(block.hash, 16))
252 test_node.sync_with_ping()
253 assert_equal(test_node.block_announced, False)
254 inv_node.clear_block_announcements()
255 test_node.send_message(msg_block(block))
256 inv_node.check_last_inv_announcement(inv=[int(block.hash, 16)])
257 258 def test_nonnull_locators(self, test_node, inv_node):
259 tip = int(self.nodes[0].getbestblockhash(), 16)
260 261 # PART 1
262 # 1. Mine a block; expect inv announcements each time
263 self.log.info("Part 1: headers don't start before sendheaders message...")
264 for i in range(4):
265 self.log.debug("Part 1.{}: starting...".format(i))
266 old_tip = tip
267 tip = self.mine_blocks(1)
268 inv_node.check_last_inv_announcement(inv=[tip])
269 test_node.check_last_inv_announcement(inv=[tip])
270 # Try a few different responses; none should affect next announcement
271 if i == 0:
272 # first request the block
273 test_node.send_get_data([tip])
274 test_node.wait_for_block(tip)
275 elif i == 1:
276 # next try requesting header and block
277 test_node.send_get_headers(locator=[old_tip], hashstop=tip)
278 test_node.send_get_data([tip])
279 test_node.wait_for_block(tip)
280 test_node.clear_block_announcements() # since we requested headers...
281 elif i == 2:
282 # this time announce own block via headers
283 inv_node.clear_block_announcements()
284 height = self.nodes[0].getblockcount()
285 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
286 block_time = last_time + 1
287 new_block = create_block(tip, create_coinbase(height + 1), block_time)
288 new_block.solve()
289 test_node.send_header_for_blocks([new_block])
290 test_node.wait_for_getdata([new_block.sha256])
291 test_node.send_and_ping(msg_block(new_block)) # make sure this block is processed
292 inv_node.wait_until(lambda: inv_node.block_announced)
293 inv_node.clear_block_announcements()
294 test_node.clear_block_announcements()
295 296 self.log.info("Part 1: success!")
297 self.log.info("Part 2: announce blocks with headers after sendheaders message...")
298 # PART 2
299 # 2. Send a sendheaders message and test that headers announcements
300 # commence and keep working.
301 test_node.send_message(msg_sendheaders())
302 prev_tip = int(self.nodes[0].getbestblockhash(), 16)
303 test_node.send_get_headers(locator=[prev_tip], hashstop=0)
304 test_node.sync_with_ping()
305 306 # Now that we've synced headers, headers announcements should work
307 tip = self.mine_blocks(1)
308 expected_hash = tip
309 inv_node.check_last_inv_announcement(inv=[tip])
310 test_node.check_last_headers_announcement(headers=[tip])
311 312 height = self.nodes[0].getblockcount() + 1
313 block_time += 10 # Advance far enough ahead
314 for i in range(10):
315 self.log.debug("Part 2.{}: starting...".format(i))
316 # Mine i blocks, and alternate announcing either via
317 # inv (of tip) or via headers. After each, new blocks
318 # mined by the node should successfully be announced
319 # with block header, even though the blocks are never requested
320 for j in range(2):
321 self.log.debug("Part 2.{}.{}: starting...".format(i, j))
322 blocks = []
323 for _ in range(i + 1):
324 blocks.append(create_block(tip, create_coinbase(height), block_time))
325 blocks[-1].solve()
326 tip = blocks[-1].sha256
327 block_time += 1
328 height += 1
329 if j == 0:
330 # Announce via inv
331 test_node.send_block_inv(tip)
332 if i == 0:
333 test_node.wait_for_getheaders(block_hash=expected_hash)
334 else:
335 assert "getheaders" not in test_node.last_message
336 # Should have received a getheaders now
337 test_node.send_header_for_blocks(blocks)
338 # Test that duplicate inv's won't result in duplicate
339 # getdata requests, or duplicate headers announcements
340 [inv_node.send_block_inv(x.sha256) for x in blocks]
341 test_node.wait_for_getdata([x.sha256 for x in blocks])
342 inv_node.sync_with_ping()
343 else:
344 # Announce via headers
345 test_node.send_header_for_blocks(blocks)
346 test_node.wait_for_getdata([x.sha256 for x in blocks])
347 # Test that duplicate headers won't result in duplicate
348 # getdata requests (the check is further down)
349 inv_node.send_header_for_blocks(blocks)
350 inv_node.sync_with_ping()
351 [test_node.send_message(msg_block(x)) for x in blocks]
352 test_node.sync_with_ping()
353 inv_node.sync_with_ping()
354 # This block should not be announced to the inv node (since it also
355 # broadcast it)
356 assert "inv" not in inv_node.last_message
357 assert "headers" not in inv_node.last_message
358 tip = self.mine_blocks(1)
359 inv_node.check_last_inv_announcement(inv=[tip])
360 test_node.check_last_headers_announcement(headers=[tip])
361 height += 1
362 block_time += 1
363 364 self.log.info("Part 2: success!")
365 366 self.log.info("Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer...")
367 368 # PART 3. Headers announcements can stop after large reorg, and resume after
369 # getheaders or inv from peer.
370 for j in range(2):
371 self.log.debug("Part 3.{}: starting...".format(j))
372 # First try mining a reorg that can propagate with header announcement
373 new_block_hashes = self.mine_reorg(length=7)
374 tip = new_block_hashes[-1]
375 inv_node.check_last_inv_announcement(inv=[tip])
376 test_node.check_last_headers_announcement(headers=new_block_hashes)
377 378 block_time += 8
379 380 # Mine a too-large reorg, which should be announced with a single inv
381 new_block_hashes = self.mine_reorg(length=8)
382 tip = new_block_hashes[-1]
383 inv_node.check_last_inv_announcement(inv=[tip])
384 test_node.check_last_inv_announcement(inv=[tip])
385 386 block_time += 9
387 388 fork_point = self.nodes[0].getblock("%064x" % new_block_hashes[0])["previousblockhash"]
389 fork_point = int(fork_point, 16)
390 391 # Use getblocks/getdata
392 test_node.send_getblocks(locator=[fork_point])
393 test_node.check_last_inv_announcement(inv=new_block_hashes)
394 test_node.send_get_data(new_block_hashes)
395 test_node.wait_for_block(new_block_hashes[-1])
396 397 for i in range(3):
398 self.log.debug("Part 3.{}.{}: starting...".format(j, i))
399 400 # Mine another block, still should get only an inv
401 tip = self.mine_blocks(1)
402 inv_node.check_last_inv_announcement(inv=[tip])
403 test_node.check_last_inv_announcement(inv=[tip])
404 if i == 0:
405 # Just get the data -- shouldn't cause headers announcements to resume
406 test_node.send_get_data([tip])
407 test_node.wait_for_block(tip)
408 elif i == 1:
409 # Send a getheaders message that shouldn't trigger headers announcements
410 # to resume (best header sent will be too old)
411 test_node.send_get_headers(locator=[fork_point], hashstop=new_block_hashes[1])
412 test_node.send_get_data([tip])
413 test_node.wait_for_block(tip)
414 elif i == 2:
415 # This time, try sending either a getheaders to trigger resumption
416 # of headers announcements, or mine a new block and inv it, also
417 # triggering resumption of headers announcements.
418 test_node.send_get_data([tip])
419 test_node.wait_for_block(tip)
420 if j == 0:
421 test_node.send_get_headers(locator=[tip], hashstop=0)
422 test_node.sync_with_ping()
423 else:
424 test_node.send_block_inv(tip)
425 test_node.sync_with_ping()
426 # New blocks should now be announced with header
427 tip = self.mine_blocks(1)
428 inv_node.check_last_inv_announcement(inv=[tip])
429 test_node.check_last_headers_announcement(headers=[tip])
430 431 self.log.info("Part 3: success!")
432 433 self.log.info("Part 4: Testing direct fetch behavior...")
434 tip = self.mine_blocks(1)
435 height = self.nodes[0].getblockcount() + 1
436 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
437 block_time = last_time + 1
438 439 # Create 2 blocks. Send the blocks, then send the headers.
440 blocks = []
441 for _ in range(2):
442 blocks.append(create_block(tip, create_coinbase(height), block_time))
443 blocks[-1].solve()
444 tip = blocks[-1].sha256
445 block_time += 1
446 height += 1
447 inv_node.send_message(msg_block(blocks[-1]))
448 449 inv_node.sync_with_ping() # Make sure blocks are processed
450 test_node.last_message.pop("getdata", None)
451 test_node.send_header_for_blocks(blocks)
452 test_node.sync_with_ping()
453 # should not have received any getdata messages
454 with p2p_lock:
455 assert "getdata" not in test_node.last_message
456 457 # This time, direct fetch should work
458 blocks = []
459 for _ in range(3):
460 blocks.append(create_block(tip, create_coinbase(height), block_time))
461 blocks[-1].solve()
462 tip = blocks[-1].sha256
463 block_time += 1
464 height += 1
465 466 test_node.send_header_for_blocks(blocks)
467 test_node.sync_with_ping()
468 test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=DIRECT_FETCH_RESPONSE_TIME)
469 470 [test_node.send_message(msg_block(x)) for x in blocks]
471 472 test_node.sync_with_ping()
473 474 # Now announce a header that forks the last two blocks
475 tip = blocks[0].sha256
476 height -= 2
477 blocks = []
478 479 # Create extra blocks for later
480 for _ in range(20):
481 blocks.append(create_block(tip, create_coinbase(height), block_time))
482 blocks[-1].solve()
483 tip = blocks[-1].sha256
484 block_time += 1
485 height += 1
486 487 # Announcing one block on fork should not trigger direct fetch
488 # (less work than tip)
489 test_node.last_message.pop("getdata", None)
490 test_node.send_header_for_blocks(blocks[0:1])
491 test_node.sync_with_ping()
492 with p2p_lock:
493 assert "getdata" not in test_node.last_message
494 495 # Announcing one more block on fork should trigger direct fetch for
496 # both blocks (same work as tip)
497 test_node.send_header_for_blocks(blocks[1:2])
498 test_node.sync_with_ping()
499 test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=DIRECT_FETCH_RESPONSE_TIME)
500 501 # Announcing 16 more headers should trigger direct fetch for 14 more
502 # blocks
503 test_node.send_header_for_blocks(blocks[2:18])
504 test_node.sync_with_ping()
505 test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=DIRECT_FETCH_RESPONSE_TIME)
506 507 # Announcing 1 more header should not trigger any response
508 test_node.last_message.pop("getdata", None)
509 test_node.send_header_for_blocks(blocks[18:19])
510 test_node.sync_with_ping()
511 with p2p_lock:
512 assert "getdata" not in test_node.last_message
513 514 self.log.info("Part 4: success!")
515 516 # Now deliver all those blocks we announced.
517 [test_node.send_message(msg_block(x)) for x in blocks]
518 519 self.log.info("Part 5: Testing handling of unconnecting headers")
520 # First we test that receipt of an unconnecting header doesn't prevent
521 # chain sync.
522 expected_hash = tip
523 NUM_HEADERS = 100
524 for i in range(NUM_HEADERS):
525 self.log.debug("Part 5.{}: starting...".format(i))
526 test_node.last_message.pop("getdata", None)
527 blocks = []
528 # Create two more blocks.
529 for _ in range(2):
530 blocks.append(create_block(tip, create_coinbase(height), block_time))
531 blocks[-1].solve()
532 tip = blocks[-1].sha256
533 block_time += 1
534 height += 1
535 # Send the header of the second block -> this won't connect.
536 test_node.send_header_for_blocks([blocks[1]])
537 test_node.wait_for_getheaders(block_hash=expected_hash)
538 test_node.send_header_for_blocks(blocks)
539 test_node.wait_for_getdata([x.sha256 for x in blocks])
540 [test_node.send_message(msg_block(x)) for x in blocks]
541 test_node.sync_with_ping()
542 assert_equal(int(self.nodes[0].getbestblockhash(), 16), blocks[1].sha256)
543 expected_hash = blocks[1].sha256
544 545 blocks = []
546 # Now we test that if we repeatedly don't send connecting headers, we
547 # don't go into an infinite loop trying to get them to connect.
548 for _ in range(NUM_HEADERS + 1):
549 blocks.append(create_block(tip, create_coinbase(height), block_time))
550 blocks[-1].solve()
551 tip = blocks[-1].sha256
552 block_time += 1
553 height += 1
554 555 for i in range(1, NUM_HEADERS):
556 with p2p_lock:
557 test_node.last_message.pop("getheaders", None)
558 # Send an empty header as a failed response to the received getheaders
559 # (from the previous iteration). Otherwise, the new headers will be
560 # treated as a response instead of as an announcement.
561 test_node.send_header_for_blocks([])
562 # Send the actual unconnecting header, which should trigger a new getheaders.
563 test_node.send_header_for_blocks([blocks[i]])
564 test_node.wait_for_getheaders(block_hash=expected_hash)
565 566 # Finally, check that the inv node never received a getdata request,
567 # throughout the test
568 assert "getdata" not in inv_node.last_message
569 570 if __name__ == '__main__':
571 SendHeadersTest(__file__).main()
572