p2p_compactblocks.py raw
1 #!/usr/bin/env python3
2 # Copyright (c) 2016-2022 The Limenka developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test compact blocks (BIP 152)."""
6 import random
7
8 from test_framework.blocktools import (
9 COINBASE_MATURITY,
10 NORMAL_GBT_REQUEST_PARAMS,
11 add_witness_commitment,
12 create_block,
13 )
14 from test_framework.messages import (
15 BlockTransactions,
16 BlockTransactionsRequest,
17 CBlock,
18 CBlockHeader,
19 CInv,
20 COutPoint,
21 CTransaction,
22 CTxIn,
23 CTxInWitness,
24 CTxOut,
25 from_hex,
26 HeaderAndShortIDs,
27 MSG_BLOCK,
28 MSG_CMPCT_BLOCK,
29 MSG_WITNESS_FLAG,
30 P2PHeaderAndShortIDs,
31 PrefilledTransaction,
32 calculate_shortid,
33 msg_block,
34 msg_blocktxn,
35 msg_cmpctblock,
36 msg_getblocktxn,
37 msg_getdata,
38 msg_getheaders,
39 msg_headers,
40 msg_inv,
41 msg_no_witness_block,
42 msg_no_witness_blocktxn,
43 msg_sendcmpct,
44 msg_sendheaders,
45 msg_tx,
46 ser_uint256,
47 tx_from_hex,
48 )
49 from test_framework.p2p import (
50 P2PInterface,
51 p2p_lock,
52 )
53 from test_framework.script import (
54 CScript,
55 OP_DROP,
56 OP_TRUE,
57 )
58 from test_framework.test_framework import LimenkaTestFramework
59 from test_framework.util import (
60 assert_equal,
61 softfork_active,
62 )
63 from test_framework.wallet import MiniWallet
64
65
66 # TestP2PConn: A peer we use to send messages to limenkad, and store responses.
67 class TestP2PConn(P2PInterface):
68 def __init__(self):
69 super().__init__()
70 self.last_sendcmpct = []
71 self.block_announced = False
72 # Store the hashes of blocks we've seen announced.
73 # This is for synchronizing the p2p message traffic,
74 # so we can eg wait until a particular block is announced.
75 self.announced_blockhashes = set()
76
77 def on_sendcmpct(self, message):
78 self.last_sendcmpct.append(message)
79
80 def on_cmpctblock(self, message):
81 self.block_announced = True
82 self.last_message["cmpctblock"].header_and_shortids.header.calc_sha256()
83 self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.sha256)
84
85 def on_headers(self, message):
86 self.block_announced = True
87 for x in self.last_message["headers"].headers:
88 x.calc_sha256()
89 self.announced_blockhashes.add(x.sha256)
90
91 def on_inv(self, message):
92 for x in self.last_message["inv"].inv:
93 if x.type == MSG_BLOCK:
94 self.block_announced = True
95 self.announced_blockhashes.add(x.hash)
96
97 # Requires caller to hold p2p_lock
98 def received_block_announcement(self):
99 return self.block_announced
100
101 def clear_block_announcement(self):
102 with p2p_lock:
103 self.block_announced = False
104 self.last_message.pop("inv", None)
105 self.last_message.pop("headers", None)
106 self.last_message.pop("cmpctblock", None)
107
108 def clear_getblocktxn(self):
109 with p2p_lock:
110 self.last_message.pop("getblocktxn", None)
111
112 def get_headers(self, locator, hashstop):
113 msg = msg_getheaders()
114 msg.locator.vHave = locator
115 msg.hashstop = hashstop
116 self.send_message(msg)
117
118 def send_header_for_blocks(self, new_blocks):
119 headers_message = msg_headers()
120 headers_message.headers = [CBlockHeader(b) for b in new_blocks]
121 self.send_message(headers_message)
122
123 def request_headers_and_sync(self, locator, hashstop=0):
124 self.clear_block_announcement()
125 self.get_headers(locator, hashstop)
126 self.wait_until(self.received_block_announcement, timeout=30)
127 self.clear_block_announcement()
128
129 # Block until a block announcement for a particular block hash is
130 # received.
131 def wait_for_block_announcement(self, block_hash, timeout=30):
132 def received_hash():
133 return (block_hash in self.announced_blockhashes)
134 self.wait_until(received_hash, timeout=timeout)
135
136 def send_await_disconnect(self, message, timeout=30):
137 """Sends a message to the node and wait for disconnect.
138
139 This is used when we want to send a message into the node that we expect
140 will get us disconnected, eg an invalid block."""
141 self.send_message(message)
142 self.wait_for_disconnect(timeout=timeout)
143
144 class CompactBlocksTest(LimenkaTestFramework):
145 def set_test_params(self):
146 self.setup_clean_chain = True
147 self.num_nodes = 1
148 self.extra_args = [[
149 "-acceptnonstdtxn=1",
150 ]]
151 self.utxos = []
152
153 def build_block_on_tip(self, node):
154 block = create_block(tmpl=node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS))
155 block.solve()
156 return block
157
158 # Create 10 more anyone-can-spend utxo's for testing.
159 def make_utxos(self):
160 block = self.build_block_on_tip(self.nodes[0])
161 self.segwit_node.send_and_ping(msg_no_witness_block(block))
162 assert int(self.nodes[0].getbestblockhash(), 16) == block.sha256
163 self.generate(self.wallet, COINBASE_MATURITY)
164
165 total_value = block.vtx[0].vout[0].nValue
166 out_value = total_value // 10
167 tx = CTransaction()
168 tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b''))
169 for _ in range(10):
170 tx.vout.append(CTxOut(out_value, CScript([OP_TRUE])))
171 tx.rehash()
172
173 block2 = self.build_block_on_tip(self.nodes[0])
174 block2.vtx.append(tx)
175 block2.hashMerkleRoot = block2.calc_merkle_root()
176 block2.solve()
177 self.segwit_node.send_and_ping(msg_no_witness_block(block2))
178 assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256)
179 self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)])
180
181
182 # Test "sendcmpct" (between peers preferring the same version):
183 # - No compact block announcements unless sendcmpct is sent.
184 # - If sendcmpct is sent with version = 1, the message is ignored.
185 # - If sendcmpct is sent with version > 2, the message is ignored.
186 # - If sendcmpct is sent with boolean 0, then block announcements are not
187 # made with compact blocks.
188 # - If sendcmpct is then sent with boolean 1, then new block announcements
189 # are made with compact blocks.
190 def test_sendcmpct(self, test_node):
191 node = self.nodes[0]
192
193 # Make sure we get a SENDCMPCT message from our peer
194 def received_sendcmpct():
195 return (len(test_node.last_sendcmpct) > 0)
196 test_node.wait_until(received_sendcmpct, timeout=30)
197 with p2p_lock:
198 # Check that version 2 is received.
199 assert_equal(test_node.last_sendcmpct[0].version, 2)
200 test_node.last_sendcmpct = []
201
202 tip = int(node.getbestblockhash(), 16)
203
204 def check_announcement_of_new_block(node, peer, predicate):
205 peer.clear_block_announcement()
206 block_hash = int(self.generate(node, 1)[0], 16)
207 peer.wait_for_block_announcement(block_hash, timeout=30)
208 assert peer.block_announced
209
210 with p2p_lock:
211 assert predicate(peer), (
212 "block_hash={!r}, cmpctblock={!r}, inv={!r}".format(
213 block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None)))
214
215 # We shouldn't get any block announcements via cmpctblock yet.
216 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message)
217
218 # Try one more time, this time after requesting headers.
219 test_node.request_headers_and_sync(locator=[tip])
220 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "inv" in p.last_message)
221
222 # Test a few ways of using sendcmpct that should NOT
223 # result in compact block announcements.
224 # Before each test, sync the headers chain.
225 test_node.request_headers_and_sync(locator=[tip])
226
227 # Now try a SENDCMPCT message with too-low version
228 test_node.send_and_ping(msg_sendcmpct(announce=True, version=1))
229 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message)
230
231 # Headers sync before next test.
232 test_node.request_headers_and_sync(locator=[tip])
233
234 # Now try a SENDCMPCT message with too-high version
235 test_node.send_and_ping(msg_sendcmpct(announce=True, version=3))
236 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message)
237
238 # Headers sync before next test.
239 test_node.request_headers_and_sync(locator=[tip])
240
241 # Now try a SENDCMPCT message with valid version, but announce=False
242 test_node.send_and_ping(msg_sendcmpct(announce=False, version=2))
243 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message)
244
245 # Headers sync before next test.
246 test_node.request_headers_and_sync(locator=[tip])
247
248 # Finally, try a SENDCMPCT message with announce=True
249 test_node.send_and_ping(msg_sendcmpct(announce=True, version=2))
250 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
251
252 # Try one more time (no headers sync should be needed!)
253 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
254
255 # Try one more time, after turning on sendheaders
256 test_node.send_and_ping(msg_sendheaders())
257 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
258
259 # Try one more time, after sending a version=1, announce=false message.
260 test_node.send_and_ping(msg_sendcmpct(announce=False, version=1))
261 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message)
262
263 # Now turn off announcements
264 test_node.send_and_ping(msg_sendcmpct(announce=False, version=2))
265 check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message)
266
267 # This test actually causes limenkad to (reasonably!) disconnect us, so do this last.
268 def test_invalid_cmpctblock_message(self):
269 self.generate(self.nodes[0], COINBASE_MATURITY + 1)
270 block = self.build_block_on_tip(self.nodes[0])
271
272 cmpct_block = P2PHeaderAndShortIDs()
273 cmpct_block.header = CBlockHeader(block)
274 cmpct_block.prefilled_txn_length = 1
275 # This index will be too high
276 prefilled_txn = PrefilledTransaction(1, block.vtx[0])
277 cmpct_block.prefilled_txn = [prefilled_txn]
278 self.segwit_node.send_await_disconnect(msg_cmpctblock(cmpct_block))
279 assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock)
280
281 # Compare the generated shortids to what we expect based on BIP 152, given
282 # limenkad's choice of nonce.
283 def test_compactblock_construction(self, test_node):
284 node = self.nodes[0]
285 # Generate a bunch of transactions.
286 self.generate(node, COINBASE_MATURITY + 1)
287 num_transactions = 25
288
289 segwit_tx_generated = False
290 for _ in range(num_transactions):
291 hex_tx = self.wallet.send_self_transfer(from_node=self.nodes[0])['hex']
292 tx = tx_from_hex(hex_tx)
293 if not tx.wit.is_null():
294 segwit_tx_generated = True
295
296 assert segwit_tx_generated # check that our test is not broken
297
298 # Wait until we've seen the block announcement for the resulting tip
299 tip = int(node.getbestblockhash(), 16)
300 test_node.wait_for_block_announcement(tip)
301
302 # Make sure we will receive a fast-announce compact block
303 self.request_cb_announcements(test_node)
304
305 # Now mine a block, and look at the resulting compact block.
306 test_node.clear_block_announcement()
307 block_hash = int(self.generate(node, 1)[0], 16)
308
309 # Store the raw block in our internal format.
310 block = from_hex(CBlock(), node.getblock("%064x" % block_hash, False))
311 for tx in block.vtx:
312 tx.calc_sha256()
313 block.rehash()
314
315 # Wait until the block was announced (via compact blocks)
316 test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30)
317
318 # Now fetch and check the compact block
319 header_and_shortids = None
320 with p2p_lock:
321 # Convert the on-the-wire representation to absolute indexes
322 header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids)
323 self.check_compactblock_construction_from_block(header_and_shortids, block_hash, block)
324
325 # Now fetch the compact block using a normal non-announce getdata
326 test_node.clear_block_announcement()
327 inv = CInv(MSG_CMPCT_BLOCK, block_hash)
328 test_node.send_message(msg_getdata([inv]))
329
330 test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30)
331
332 # Now fetch and check the compact block
333 header_and_shortids = None
334 with p2p_lock:
335 # Convert the on-the-wire representation to absolute indexes
336 header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids)
337 self.check_compactblock_construction_from_block(header_and_shortids, block_hash, block)
338
339 def check_compactblock_construction_from_block(self, header_and_shortids, block_hash, block):
340 # Check that we got the right block!
341 header_and_shortids.header.calc_sha256()
342 assert_equal(header_and_shortids.header.sha256, block_hash)
343
344 # Make sure the prefilled_txn appears to have included the coinbase
345 assert len(header_and_shortids.prefilled_txn) >= 1
346 assert_equal(header_and_shortids.prefilled_txn[0].index, 0)
347
348 # Check that all prefilled_txn entries match what's in the block.
349 for entry in header_and_shortids.prefilled_txn:
350 entry.tx.calc_sha256()
351 # This checks the non-witness parts of the tx agree
352 assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256)
353
354 # And this checks the witness
355 wtxid = entry.tx.calc_sha256(True)
356 assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True))
357
358 # Check that the cmpctblock message announced all the transactions.
359 assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx))
360
361 # And now check that all the shortids are as expected as well.
362 # Determine the siphash keys to use.
363 [k0, k1] = header_and_shortids.get_siphash_keys()
364
365 index = 0
366 while index < len(block.vtx):
367 if (len(header_and_shortids.prefilled_txn) > 0 and
368 header_and_shortids.prefilled_txn[0].index == index):
369 # Already checked prefilled transactions above
370 header_and_shortids.prefilled_txn.pop(0)
371 else:
372 tx_hash = block.vtx[index].calc_sha256(True)
373 shortid = calculate_shortid(k0, k1, tx_hash)
374 assert_equal(shortid, header_and_shortids.shortids[0])
375 header_and_shortids.shortids.pop(0)
376 index += 1
377
378 # Test that limenkad requests compact blocks when we announce new blocks
379 # via header or inv, and that responding to getblocktxn causes the block
380 # to be successfully reconstructed.
381 def test_compactblock_requests(self, test_node):
382 node = self.nodes[0]
383 # Try announcing a block with an inv or header, expect a compactblock
384 # request
385 for announce in ["inv", "header"]:
386 block = self.build_block_on_tip(node)
387
388 if announce == "inv":
389 test_node.send_message(msg_inv([CInv(MSG_BLOCK, block.sha256)]))
390 test_node.wait_for_getheaders(timeout=30)
391 test_node.send_header_for_blocks([block])
392 else:
393 test_node.send_header_for_blocks([block])
394 test_node.wait_for_getdata([block.sha256], timeout=30)
395 assert_equal(test_node.last_message["getdata"].inv[0].type, 4)
396
397 # Send back a compactblock message that omits the coinbase
398 comp_block = HeaderAndShortIDs()
399 comp_block.header = CBlockHeader(block)
400 comp_block.nonce = 0
401 [k0, k1] = comp_block.get_siphash_keys()
402 coinbase_hash = block.vtx[0].calc_sha256(True)
403 comp_block.shortids = [calculate_shortid(k0, k1, coinbase_hash)]
404 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
405 assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
406 # Expect a getblocktxn message.
407 with p2p_lock:
408 assert "getblocktxn" in test_node.last_message
409 absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute()
410 assert_equal(absolute_indexes, [0]) # should be a coinbase request
411
412 # Send the coinbase, and verify that the tip advances.
413 msg = msg_blocktxn()
414 msg.block_transactions.blockhash = block.sha256
415 msg.block_transactions.transactions = [block.vtx[0]]
416 test_node.send_and_ping(msg)
417 assert_equal(int(node.getbestblockhash(), 16), block.sha256)
418
419 # Create a chain of transactions from given utxo, and add to a new block.
420 def build_block_with_transactions(self, node, utxo, num_transactions):
421 block = self.build_block_on_tip(node)
422
423 for _ in range(num_transactions):
424 tx = CTransaction()
425 tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b''))
426 tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE])))
427 tx.rehash()
428 utxo = [tx.sha256, 0, tx.vout[0].nValue]
429 block.vtx.append(tx)
430
431 block.hashMerkleRoot = block.calc_merkle_root()
432 block.solve()
433 return block
434
435 # Test that we only receive getblocktxn requests for transactions that the
436 # node needs, and that responding to them causes the block to be
437 # reconstructed.
438 def test_getblocktxn_requests(self, test_node):
439 node = self.nodes[0]
440
441 def test_getblocktxn_response(compact_block, peer, expected_result):
442 msg = msg_cmpctblock(compact_block.to_p2p())
443 peer.send_and_ping(msg)
444 with p2p_lock:
445 assert "getblocktxn" in peer.last_message
446 absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute()
447 assert_equal(absolute_indexes, expected_result)
448
449 def test_tip_after_message(node, peer, msg, tip):
450 peer.send_and_ping(msg)
451 assert_equal(int(node.getbestblockhash(), 16), tip)
452
453 # First try announcing compactblocks that won't reconstruct, and verify
454 # that we receive getblocktxn messages back.
455 utxo = self.utxos.pop(0)
456
457 block = self.build_block_with_transactions(node, utxo, 5)
458 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
459 comp_block = HeaderAndShortIDs()
460 comp_block.initialize_from_block(block, use_witness=True)
461
462 test_getblocktxn_response(comp_block, test_node, [1, 2, 3, 4, 5])
463
464 msg_bt = msg_no_witness_blocktxn()
465 msg_bt = msg_blocktxn() # serialize with witnesses
466 msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[1:])
467 test_tip_after_message(node, test_node, msg_bt, block.sha256)
468
469 utxo = self.utxos.pop(0)
470 block = self.build_block_with_transactions(node, utxo, 5)
471 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
472
473 # Now try interspersing the prefilled transactions
474 comp_block.initialize_from_block(block, prefill_list=[0, 1, 5], use_witness=True)
475 test_getblocktxn_response(comp_block, test_node, [2, 3, 4])
476 msg_bt.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5])
477 test_tip_after_message(node, test_node, msg_bt, block.sha256)
478
479 # Now try giving one transaction ahead of time.
480 utxo = self.utxos.pop(0)
481 block = self.build_block_with_transactions(node, utxo, 5)
482 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
483 test_node.send_and_ping(msg_tx(block.vtx[1]))
484 assert block.vtx[1].hash in node.getrawmempool()
485
486 # Prefill 4 out of the 6 transactions, and verify that only the one
487 # that was not in the mempool is requested.
488 comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4], use_witness=True)
489 test_getblocktxn_response(comp_block, test_node, [5])
490
491 msg_bt.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]])
492 test_tip_after_message(node, test_node, msg_bt, block.sha256)
493
494 # Now provide all transactions to the node before the block is
495 # announced and verify reconstruction happens immediately.
496 utxo = self.utxos.pop(0)
497 block = self.build_block_with_transactions(node, utxo, 10)
498 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
499 for tx in block.vtx[1:]:
500 test_node.send_message(msg_tx(tx))
501 test_node.sync_with_ping()
502 # Make sure all transactions were accepted.
503 mempool = node.getrawmempool()
504 for tx in block.vtx[1:]:
505 assert tx.hash in mempool
506
507 # Clear out last request.
508 with p2p_lock:
509 test_node.last_message.pop("getblocktxn", None)
510
511 # Send compact block
512 comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True)
513 test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256)
514 with p2p_lock:
515 # Shouldn't have gotten a request for any transaction
516 assert "getblocktxn" not in test_node.last_message
517
518 # Incorrectly responding to a getblocktxn shouldn't cause the block to be
519 # permanently failed.
520 def test_incorrect_blocktxn_response(self, test_node):
521 node = self.nodes[0]
522 utxo = self.utxos.pop(0)
523
524 block = self.build_block_with_transactions(node, utxo, 10)
525 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
526 # Relay the first 5 transactions from the block in advance
527 for tx in block.vtx[1:6]:
528 test_node.send_message(msg_tx(tx))
529 test_node.sync_with_ping()
530 # Make sure all transactions were accepted.
531 mempool = node.getrawmempool()
532 for tx in block.vtx[1:6]:
533 assert tx.hash in mempool
534
535 # Send compact block
536 comp_block = HeaderAndShortIDs()
537 comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True)
538 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
539 absolute_indexes = []
540 with p2p_lock:
541 assert "getblocktxn" in test_node.last_message
542 absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute()
543 assert_equal(absolute_indexes, [6, 7, 8, 9, 10])
544
545 # Now give an incorrect response.
546 # Note that it's possible for limenkad to be smart enough to know we're
547 # lying, since it could check to see if the shortid matches what we're
548 # sending, and eg disconnect us for misbehavior. If that behavior
549 # change was made, we could just modify this test by having a
550 # different peer provide the block further down, so that we're still
551 # verifying that the block isn't marked bad permanently. This is good
552 # enough for now.
553 msg = msg_blocktxn()
554 msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:])
555 test_node.send_and_ping(msg)
556
557 # Tip should not have updated
558 assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
559
560 # We should receive a getdata request
561 test_node.wait_for_getdata([block.sha256], timeout=10)
562 assert test_node.last_message["getdata"].inv[0].type == MSG_BLOCK or \
563 test_node.last_message["getdata"].inv[0].type == MSG_BLOCK | MSG_WITNESS_FLAG
564
565 # Deliver the block
566 test_node.send_and_ping(msg_block(block))
567 assert_equal(int(node.getbestblockhash(), 16), block.sha256)
568
569 # Multiple blocktxn responses will cause a node to get disconnected.
570 def test_multiple_blocktxn_response(self, test_node):
571 node = self.nodes[0]
572 utxo = self.utxos[0]
573
574 block = self.build_block_with_transactions(node, utxo, 2)
575
576 # Send compact block
577 comp_block = HeaderAndShortIDs()
578 comp_block.initialize_from_block(block, prefill_list=[0], use_witness=True)
579 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
580 absolute_indexes = []
581 with p2p_lock:
582 assert "getblocktxn" in test_node.last_message
583 absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute()
584 assert_equal(absolute_indexes, [1, 2])
585
586 # Send a blocktxn that does not succeed in reconstruction, triggering
587 # getdata fallback.
588 msg = msg_blocktxn()
589 msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[2]] + [block.vtx[1]])
590 test_node.send_and_ping(msg)
591
592 # Tip should not have updated
593 assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock)
594
595 # We should receive a getdata request
596 test_node.wait_for_getdata([block.sha256], timeout=10)
597 assert test_node.last_message["getdata"].inv[0].type == MSG_BLOCK or \
598 test_node.last_message["getdata"].inv[0].type == MSG_BLOCK | MSG_WITNESS_FLAG
599
600 # Send the same blocktxn and assert the sender gets disconnected.
601 with node.assert_debug_log(['previous compact block reconstruction attempt failed']):
602 test_node.send_message(msg)
603 test_node.wait_for_disconnect()
604
605 def test_getblocktxn_handler(self, test_node):
606 node = self.nodes[0]
607 # limenkad will not send blocktxn responses for blocks whose height is
608 # more than 10 blocks deep.
609 MAX_GETBLOCKTXN_DEPTH = 10
610 chain_height = node.getblockcount()
611 current_height = chain_height
612 while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH):
613 block_hash = node.getblockhash(current_height)
614 block = from_hex(CBlock(), node.getblock(block_hash, False))
615
616 msg = msg_getblocktxn()
617 msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [])
618 num_to_request = random.randint(1, len(block.vtx))
619 msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request)))
620 test_node.send_message(msg)
621 test_node.wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10)
622
623 [tx.calc_sha256() for tx in block.vtx]
624 with p2p_lock:
625 assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16))
626 all_indices = msg.block_txn_request.to_absolute()
627 for index in all_indices:
628 tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0)
629 tx.calc_sha256()
630 assert_equal(tx.sha256, block.vtx[index].sha256)
631 # Check that the witness matches
632 assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True))
633 test_node.last_message.pop("blocktxn", None)
634 current_height -= 1
635
636 # Next request should send a full block response, as we're past the
637 # allowed depth for a blocktxn response.
638 block_hash = node.getblockhash(current_height)
639 msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0])
640 with p2p_lock:
641 test_node.last_message.pop("block", None)
642 test_node.last_message.pop("blocktxn", None)
643 test_node.send_and_ping(msg)
644 with p2p_lock:
645 test_node.last_message["block"].block.calc_sha256()
646 assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16))
647 assert "blocktxn" not in test_node.last_message
648
649 # Request with out-of-bounds tx index results in disconnect
650 bad_peer = self.nodes[0].add_p2p_connection(TestP2PConn())
651 block_hash = node.getblockhash(chain_height)
652 block = from_hex(CBlock(), node.getblock(block_hash, False))
653 msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [len(block.vtx)])
654 with node.assert_debug_log(['getblocktxn with out-of-bounds tx indices']):
655 bad_peer.send_message(msg)
656 bad_peer.wait_for_disconnect()
657
658 def test_low_work_compactblocks(self, test_node):
659 # A compactblock with insufficient work won't get its header included
660 node = self.nodes[0]
661 hashPrevBlock = int(node.getblockhash(node.getblockcount() - 150), 16)
662 block = self.build_block_on_tip(node)
663 block.hashPrevBlock = hashPrevBlock
664 block.solve()
665
666 comp_block = HeaderAndShortIDs()
667 comp_block.initialize_from_block(block)
668 with self.nodes[0].assert_debug_log(['[net] Ignoring low-work compact block from peer 0']):
669 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
670
671 tips = node.getchaintips()
672 found = False
673 for x in tips:
674 if x["hash"] == block.hash:
675 found = True
676 break
677 assert not found
678
679 def test_compactblocks_not_at_tip(self, test_node):
680 node = self.nodes[0]
681 # Test that requesting old compactblocks doesn't work.
682 MAX_CMPCTBLOCK_DEPTH = 5
683 new_blocks = []
684 for _ in range(MAX_CMPCTBLOCK_DEPTH + 1):
685 test_node.clear_block_announcement()
686 new_blocks.append(self.generate(node, 1)[0])
687 test_node.wait_until(test_node.received_block_announcement, timeout=30)
688
689 test_node.clear_block_announcement()
690 test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))]))
691 test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30)
692
693 test_node.clear_block_announcement()
694 self.generate(node, 1)
695 test_node.wait_until(test_node.received_block_announcement, timeout=30)
696 test_node.clear_block_announcement()
697 with p2p_lock:
698 test_node.last_message.pop("block", None)
699 test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))]))
700 test_node.wait_until(lambda: "block" in test_node.last_message, timeout=30)
701 with p2p_lock:
702 test_node.last_message["block"].block.calc_sha256()
703 assert_equal(test_node.last_message["block"].block.sha256, int(new_blocks[0], 16))
704
705 # Generate an old compactblock, and verify that it's not accepted.
706 cur_height = node.getblockcount()
707 hashPrevBlock = int(node.getblockhash(cur_height - 5), 16)
708 block = self.build_block_on_tip(node)
709 block.hashPrevBlock = hashPrevBlock
710 block.solve()
711
712 comp_block = HeaderAndShortIDs()
713 comp_block.initialize_from_block(block)
714 test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
715
716 tips = node.getchaintips()
717 found = False
718 for x in tips:
719 if x["hash"] == block.hash:
720 assert_equal(x["status"], "headers-only")
721 found = True
722 break
723 assert found
724
725 # Requesting this block via getblocktxn should silently fail
726 # (to avoid fingerprinting attacks).
727 msg = msg_getblocktxn()
728 msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0])
729 with p2p_lock:
730 test_node.last_message.pop("blocktxn", None)
731 test_node.send_and_ping(msg)
732 with p2p_lock:
733 assert "blocktxn" not in test_node.last_message
734
735 def test_end_to_end_block_relay(self, listeners):
736 node = self.nodes[0]
737 utxo = self.utxos.pop(0)
738
739 block = self.build_block_with_transactions(node, utxo, 10)
740
741 [l.clear_block_announcement() for l in listeners]
742
743 # serialize without witness (this block has no witnesses anyway).
744 # TODO: repeat this test with witness tx's to a segwit node.
745 node.submitblock(block.serialize().hex())
746
747 for l in listeners:
748 l.wait_until(lambda: "cmpctblock" in l.last_message, timeout=30)
749 with p2p_lock:
750 for l in listeners:
751 l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256()
752 assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256)
753
754 # Test that we don't get disconnected if we relay a compact block with valid header,
755 # but invalid transactions.
756 def test_invalid_tx_in_compactblock(self, test_node):
757 node = self.nodes[0]
758 assert len(self.utxos)
759 utxo = self.utxos[0]
760
761 block = self.build_block_with_transactions(node, utxo, 5)
762 del block.vtx[3]
763 block.hashMerkleRoot = block.calc_merkle_root()
764 # Drop the coinbase witness but include the witness commitment.
765 add_witness_commitment(block)
766 block.vtx[0].wit.vtxinwit = []
767 block.solve()
768
769 # Now send the compact block with all transactions prefilled, and
770 # verify that we don't get disconnected.
771 comp_block = HeaderAndShortIDs()
772 comp_block.initialize_from_block(block, prefill_list=list(range(len(block.vtx))), use_witness=True)
773 msg = msg_cmpctblock(comp_block.to_p2p())
774 test_node.send_and_ping(msg)
775
776 # Check that the tip didn't advance
777 assert int(node.getbestblockhash(), 16) != block.sha256
778 test_node.sync_with_ping()
779
780 # Helper for enabling cb announcements
781 # Send the sendcmpct request and sync headers
782 def request_cb_announcements(self, peer):
783 node = self.nodes[0]
784 tip = node.getbestblockhash()
785 peer.get_headers(locator=[int(tip, 16)], hashstop=0)
786 peer.send_and_ping(msg_sendcmpct(announce=True, version=2))
787
788 def test_compactblock_reconstruction_stalling_peer(self, stalling_peer, delivery_peer):
789 node = self.nodes[0]
790 assert len(self.utxos)
791
792 def announce_cmpct_block(node, peer):
793 utxo = self.utxos.pop(0)
794 block = self.build_block_with_transactions(node, utxo, 5)
795
796 cmpct_block = HeaderAndShortIDs()
797 cmpct_block.initialize_from_block(block)
798 msg = msg_cmpctblock(cmpct_block.to_p2p())
799 peer.send_and_ping(msg)
800 with p2p_lock:
801 assert "getblocktxn" in peer.last_message
802 return block, cmpct_block
803
804 block, cmpct_block = announce_cmpct_block(node, stalling_peer)
805
806 for tx in block.vtx[1:]:
807 delivery_peer.send_message(msg_tx(tx))
808 delivery_peer.sync_with_ping()
809 mempool = node.getrawmempool()
810 for tx in block.vtx[1:]:
811 assert tx.hash in mempool
812
813 delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
814 assert_equal(int(node.getbestblockhash(), 16), block.sha256)
815
816 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
817
818 # Now test that delivering an invalid compact block won't break relay
819
820 block, cmpct_block = announce_cmpct_block(node, stalling_peer)
821 for tx in block.vtx[1:]:
822 delivery_peer.send_message(msg_tx(tx))
823 delivery_peer.sync_with_ping()
824
825 cmpct_block.prefilled_txn[0].tx.wit.vtxinwit = [CTxInWitness()]
826 cmpct_block.prefilled_txn[0].tx.wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)]
827
828 cmpct_block.use_witness = True
829 delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
830 assert int(node.getbestblockhash(), 16) != block.sha256
831
832 msg = msg_no_witness_blocktxn()
833 msg.block_transactions.blockhash = block.sha256
834 msg.block_transactions.transactions = block.vtx[1:]
835 stalling_peer.send_and_ping(msg)
836 assert_equal(int(node.getbestblockhash(), 16), block.sha256)
837
838 def test_highbandwidth_mode_states_via_getpeerinfo(self):
839 # create new p2p connection for a fresh state w/o any prior sendcmpct messages sent
840 hb_test_node = self.nodes[0].add_p2p_connection(TestP2PConn())
841
842 # assert the RPC getpeerinfo boolean fields `bip152_hb_{to, from}`
843 # match the given parameters for the last peer of a given node
844 def assert_highbandwidth_states(node, hb_to, hb_from):
845 peerinfo = node.getpeerinfo()[-1]
846 assert_equal(peerinfo['bip152_hb_to'], hb_to)
847 assert_equal(peerinfo['bip152_hb_from'], hb_from)
848
849 # initially, neither node has selected the other peer as high-bandwidth yet
850 assert_highbandwidth_states(self.nodes[0], hb_to=False, hb_from=False)
851
852 # peer requests high-bandwidth mode by sending sendcmpct(1)
853 hb_test_node.send_and_ping(msg_sendcmpct(announce=True, version=2))
854 assert_highbandwidth_states(self.nodes[0], hb_to=False, hb_from=True)
855
856 # peer generates a block and sends it to node, which should
857 # select the peer as high-bandwidth (up to 3 peers according to BIP 152)
858 block = self.build_block_on_tip(self.nodes[0])
859 hb_test_node.send_and_ping(msg_block(block))
860 assert_highbandwidth_states(self.nodes[0], hb_to=True, hb_from=True)
861
862 # peer requests low-bandwidth mode by sending sendcmpct(0)
863 hb_test_node.send_and_ping(msg_sendcmpct(announce=False, version=2))
864 assert_highbandwidth_states(self.nodes[0], hb_to=True, hb_from=False)
865
866 def test_compactblock_reconstruction_parallel_reconstruction(self, stalling_peer, delivery_peer, inbound_peer, outbound_peer):
867 """ All p2p connections are inbound except outbound_peer. We test that ultimate parallel slot
868 can only be taken by an outbound node unless prior attempts were done by an outbound
869 """
870 node = self.nodes[0]
871 assert len(self.utxos)
872
873 def announce_cmpct_block(node, peer, txn_count):
874 utxo = self.utxos.pop(0)
875 block = self.build_block_with_transactions(node, utxo, txn_count)
876
877 cmpct_block = HeaderAndShortIDs()
878 cmpct_block.initialize_from_block(block)
879 msg = msg_cmpctblock(cmpct_block.to_p2p())
880 peer.send_and_ping(msg)
881 with p2p_lock:
882 assert "getblocktxn" in peer.last_message
883 return block, cmpct_block
884
885 for name, peer in [("delivery", delivery_peer), ("inbound", inbound_peer), ("outbound", outbound_peer)]:
886 self.log.info(f"Setting {name} as high bandwidth peer")
887 block, cmpct_block = announce_cmpct_block(node, peer, 1)
888 msg = msg_blocktxn()
889 msg.block_transactions.blockhash = block.sha256
890 msg.block_transactions.transactions = block.vtx[1:]
891 peer.send_and_ping(msg)
892 assert_equal(int(node.getbestblockhash(), 16), block.sha256)
893 peer.clear_getblocktxn()
894
895 # Test the simple parallel download case...
896 for num_missing in [1, 5, 20]:
897
898 # Remaining low-bandwidth peer is stalling_peer, who announces first
899 assert_equal([peer['bip152_hb_to'] for peer in node.getpeerinfo()], [False, True, True, True])
900
901 block, cmpct_block = announce_cmpct_block(node, stalling_peer, num_missing)
902
903 delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
904 with p2p_lock:
905 # The second peer to announce should still get a getblocktxn
906 assert "getblocktxn" in delivery_peer.last_message
907 assert int(node.getbestblockhash(), 16) != block.sha256
908
909 inbound_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
910 with p2p_lock:
911 # The third inbound peer to announce should *not* get a getblocktxn
912 assert "getblocktxn" not in inbound_peer.last_message
913 assert int(node.getbestblockhash(), 16) != block.sha256
914
915 outbound_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
916 with p2p_lock:
917 # The third peer to announce should get a getblocktxn if outbound
918 assert "getblocktxn" in outbound_peer.last_message
919 assert int(node.getbestblockhash(), 16) != block.sha256
920
921 # Second peer completes the compact block first
922 msg = msg_blocktxn()
923 msg.block_transactions.blockhash = block.sha256
924 msg.block_transactions.transactions = block.vtx[1:]
925 delivery_peer.send_and_ping(msg)
926 assert_equal(int(node.getbestblockhash(), 16), block.sha256)
927
928 # Nothing bad should happen if we get a late fill from the first peer...
929 stalling_peer.send_and_ping(msg)
930 self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue])
931
932 delivery_peer.clear_getblocktxn()
933 inbound_peer.clear_getblocktxn()
934 outbound_peer.clear_getblocktxn()
935
936
937 def run_test(self):
938 self.wallet = MiniWallet(self.nodes[0])
939
940 # Setup the p2p connections
941 self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn())
942 self.additional_segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn())
943 self.onemore_inbound_node = self.nodes[0].add_p2p_connection(TestP2PConn())
944 self.outbound_node = self.nodes[0].add_outbound_p2p_connection(TestP2PConn(), p2p_idx=3, connection_type="outbound-full-relay")
945
946 # We will need UTXOs to construct transactions in later tests.
947 self.make_utxos()
948
949 assert softfork_active(self.nodes[0], "segwit")
950
951 self.log.info("Testing SENDCMPCT p2p message... ")
952 self.test_sendcmpct(self.segwit_node)
953 self.test_sendcmpct(self.additional_segwit_node)
954 self.test_sendcmpct(self.onemore_inbound_node)
955 self.test_sendcmpct(self.outbound_node)
956
957 self.log.info("Testing compactblock construction...")
958 self.test_compactblock_construction(self.segwit_node)
959
960 self.log.info("Testing compactblock requests (segwit node)... ")
961 self.test_compactblock_requests(self.segwit_node)
962
963 self.log.info("Testing getblocktxn requests (segwit node)...")
964 self.test_getblocktxn_requests(self.segwit_node)
965
966 self.log.info("Testing getblocktxn handler (segwit node should return witnesses)...")
967 self.test_getblocktxn_handler(self.segwit_node)
968
969 self.log.info("Testing compactblock requests/announcements not at chain tip...")
970 self.test_compactblocks_not_at_tip(self.segwit_node)
971
972 self.log.info("Testing handling of low-work compact blocks...")
973 self.test_low_work_compactblocks(self.segwit_node)
974
975 self.log.info("Testing handling of incorrect blocktxn responses...")
976 self.test_incorrect_blocktxn_response(self.segwit_node)
977
978 self.log.info("Testing reconstructing compact blocks with a stalling peer...")
979 self.test_compactblock_reconstruction_stalling_peer(self.segwit_node, self.additional_segwit_node)
980
981 self.log.info("Testing reconstructing compact blocks from multiple peers...")
982 self.test_compactblock_reconstruction_parallel_reconstruction(stalling_peer=self.segwit_node, inbound_peer=self.onemore_inbound_node, delivery_peer=self.additional_segwit_node, outbound_peer=self.outbound_node)
983
984 # Test that if we submitblock to node1, we'll get a compact block
985 # announcement to all peers.
986 # (Post-segwit activation, blocks won't propagate from node0 to node1
987 # automatically, so don't bother testing a block announced to node0.)
988 self.log.info("Testing end-to-end block relay...")
989 self.request_cb_announcements(self.segwit_node)
990 self.request_cb_announcements(self.additional_segwit_node)
991 self.test_end_to_end_block_relay([self.segwit_node, self.additional_segwit_node])
992
993 self.log.info("Testing handling of invalid compact blocks...")
994 self.test_invalid_tx_in_compactblock(self.segwit_node)
995
996 self.log.info("Testing handling of multiple blocktxn responses...")
997 self.test_multiple_blocktxn_response(self.segwit_node)
998
999 # The previous test will lead to a disconnection. Reconnect before continuing.
1000 self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn())
1001
1002 self.log.info("Testing invalid index in cmpctblock message...")
1003 self.test_invalid_cmpctblock_message()
1004
1005 self.log.info("Testing high-bandwidth mode states via getpeerinfo...")
1006 self.test_highbandwidth_mode_states_via_getpeerinfo()
1007
1008
1009 if __name__ == '__main__':
1010 CompactBlocksTest(__file__).main()
1011