p2p_compactblocks_extratxs.py raw
1 #!/usr/bin/env python3
2 # Copyright (c) 2025 The Limenka developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test blockreconstructionextratxn and blockreconstructionextratxnsize options with compact blocks."""
6
7 from test_framework.blocktools import (
8 COINBASE_MATURITY,
9 NORMAL_GBT_REQUEST_PARAMS,
10 create_block,
11 add_witness_commitment,
12 )
13 from test_framework.messages import (
14 CTxOut,
15 HeaderAndShortIDs,
16 MSG_BLOCK,
17 msg_cmpctblock,
18 msg_sendcmpct,
19 msg_tx,
20 tx_from_hex,
21 )
22 from test_framework.p2p import (
23 P2PInterface,
24 p2p_lock,
25 )
26 from test_framework.script import (
27 CScript,
28 OP_DROP,
29 OP_TRUE,
30 OP_RETURN,
31 )
32 from test_framework.script_util import (
33 keys_to_multisig_script,
34 )
35 from test_framework.test_framework import LimenkaTestFramework
36 from test_framework.util import (
37 assert_equal,
38 softfork_active,
39 )
40 from decimal import Decimal
41 from test_framework.wallet import MiniWallet
42
43
44 # TestP2PConn: A peer we use to send messages to limenkad, and store responses.
45 class TestP2PConn(P2PInterface):
46 def __init__(self):
47 super().__init__()
48 self.last_sendcmpct = []
49 self.block_announced = False
50 # Store the hashes of blocks we've seen announced.
51 # This is for synchronizing the p2p message traffic,
52 # so we can eg wait until a particular block is announced.
53 self.announced_blockhashes = set()
54
55 def on_sendcmpct(self, message):
56 self.last_sendcmpct.append(message)
57
58 def on_cmpctblock(self, message):
59 self.block_announced = True
60 self.last_message["cmpctblock"].header_and_shortids.header.calc_sha256()
61 self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.sha256)
62
63 def on_headers(self, message):
64 self.block_announced = True
65 for x in self.last_message["headers"].headers:
66 x.calc_sha256()
67 self.announced_blockhashes.add(x.sha256)
68
69 def on_inv(self, message):
70 for x in self.last_message["inv"].inv:
71 if x.type == MSG_BLOCK:
72 self.block_announced = True
73 self.announced_blockhashes.add(x.hash)
74
75 # Requires caller to hold p2p_lock
76 def received_block_announcement(self):
77 return self.block_announced
78
79 def clear_block_announcement(self):
80 with p2p_lock:
81 self.block_announced = False
82 self.last_message.pop("inv", None)
83 self.last_message.pop("headers", None)
84 self.last_message.pop("cmpctblock", None)
85
86 def clear_getblocktxn(self):
87 with p2p_lock:
88 self.last_message.pop("getblocktxn", None)
89
90
91 class CompactBlocksBlockReconstructionLimitTest(LimenkaTestFramework):
92 def set_test_params(self):
93 self.setup_clean_chain = True
94 self.num_nodes = 1
95 self.extra_args = [[
96 "-acceptnonstdtxn=0",
97 "-incrementalrelayfee=0.00001",
98 "-debug=net",
99 ]]
100 self.utxos = []
101
102 def build_block_on_tip(self, node):
103 """Build a block on top of the current tip."""
104 block = create_block(tmpl=node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS))
105 block.solve()
106 return block
107
108 def make_utxos(self):
109 """Generate blocks to create UTXOs for the wallet."""
110 self.generate(self.wallet, COINBASE_MATURITY + 1000)
111
112 def restart_node_with_limit(self, *, memory_mb=None, count=None):
113 """Restart node with specific size and/or count limits."""
114 extra_args = self.extra_args[0] + [
115 "-datacarriersize=83",
116 ]
117
118 if memory_mb is not None:
119 self.log.info(f"Setting size limit: {memory_mb} MB")
120 extra_args.append(f"-blockreconstructionextratxnsize={memory_mb}")
121
122 if count is not None:
123 self.log.info(f"Setting transaction count limit: {count}")
124 extra_args.append(f"-blockreconstructionextratxn={count}")
125
126 self.log.info(f"Restarting node with args: {extra_args}")
127 self.restart_node(0, extra_args=extra_args)
128 self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn())
129 self.segwit_node.send_and_ping(msg_sendcmpct(announce=True, version=2))
130
131 def create_policy_rejected_tx(self, rejection_type="dust", target_size=None):
132 """Create a transaction that will be rejected for policy reasons but added to extra pool."""
133
134 if rejection_type == "dust":
135 tx_info = self.wallet.create_self_transfer()
136 dust_amount = 100
137 dust_script = CScript([OP_TRUE])
138 tx_info['tx'].vout.append(CTxOut(dust_amount, dust_script))
139 tx_info['tx'].vout[0].nValue -= dust_amount
140
141 elif rejection_type == "low_fee":
142 tx_info = self.wallet.create_self_transfer(fee_rate=Decimal('0.00000100'))
143
144 elif rejection_type == "op_return_size":
145 tx_info = self.wallet.create_self_transfer()
146 data = b'x' * 85
147 tx_info['tx'].vout.append(CTxOut(0, CScript([OP_RETURN, data])))
148
149 elif rejection_type == "nonstandard_script":
150 tx_info = self.wallet.create_self_transfer()
151 pubkeys = []
152 for _ in range(4):
153 pubkeys.append(bytes([0x02] + [0x00] * 32))
154 multisig_script = keys_to_multisig_script(pubkeys, k=4)
155 tx_info['tx'].vout.append(CTxOut(10000, multisig_script))
156 tx_info['tx'].vout[0].nValue -= 10000
157
158 else:
159 raise ValueError(f"Unknown rejection type: {rejection_type}")
160
161 # Add padding outputs to reach target size
162 if target_size:
163 # Estimate current transaction size
164 tx_info['tx'].rehash()
165 base_size = len(tx_info['tx'].serialize())
166
167 if base_size < target_size:
168 # Each padded output approximately 200 bytes
169 bytes_per_output = 200
170 num_outputs = (target_size - base_size) // bytes_per_output
171
172 for _ in range(num_outputs):
173 padding_data = b'x' * 190
174 script = CScript([padding_data, OP_DROP, OP_TRUE])
175 tx_info['tx'].vout.append(CTxOut(100, script))
176 tx_info['tx'].vout[0].nValue -= 100
177
178 tx_info['tx'].rehash()
179 tx_info['hex'] = tx_info['tx'].serialize().hex()
180 return tx_info
181
182 def populate_extra_pool(self, num_txs, rejection_type="dust", target_size=None):
183 """Populate the extra transaction pool using policy-rejected transactions."""
184 rejected_txs = []
185
186 for i in range(num_txs):
187 tx_info = self.create_policy_rejected_tx(rejection_type, target_size=target_size)
188 tx_obj = tx_from_hex(tx_info['hex'])
189 self.segwit_node.send_message(msg_tx(tx_obj))
190 rejected_txs.append(tx_info)
191
192 self.segwit_node.sync_with_ping()
193
194 return rejected_txs
195
196 def send_compact_block(self, transactions, indices):
197 """Send a compact block and check which transactions are requested for reconstruction."""
198 node = self.nodes[0]
199
200 # Build block
201 block = self.build_block_on_tip(node)
202
203 for i in indices:
204 tx_obj = tx_from_hex(transactions[i]['hex'])
205 block.vtx.append(tx_obj)
206
207 # Add witness commitment for blocks with witness transactions
208 add_witness_commitment(block)
209 block.solve()
210
211 # Send as compact block
212 cmpct_block = HeaderAndShortIDs()
213 cmpct_block.initialize_from_block(block, use_witness=True)
214 self.segwit_node.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p()))
215
216 # Check if node requested missing transactions
217 with p2p_lock:
218 getblocktxn = self.segwit_node.last_message.get("getblocktxn")
219
220 num_tx_requested = len(getblocktxn.block_txn_request.indexes) if getblocktxn else 0
221 self.segwit_node.clear_getblocktxn()
222
223 # Convert differential encoding to absolute indices (from BlockTransactionRequest)
224 missing_indices = []
225 if getblocktxn:
226 absolute_block_indices = getblocktxn.block_txn_request.to_absolute()
227 # Convert from block positions to transaction indices (subtract 1 for coinbase)
228 missing_indices = [idx - 1 for idx in absolute_block_indices]
229
230 return {
231 "block": block,
232 "getblocktxn": getblocktxn,
233 "num_tx_requested": num_tx_requested,
234 "missing_indices": missing_indices
235 }
236
237 # TEST: policy-rejected transactions
238
239 def test_policy_rejection_types(self):
240 """Test that each policy rejection type adds transactions to extra pool."""
241 self.log.info("Testing policy rejection types for extra pool...")
242
243 self.restart_node_with_limit(count=100)
244
245 rejection_types = ["dust", "low_fee", "op_return_size", "nonstandard_script"]
246 rejected_txs = []
247
248 for rejection_type in rejection_types:
249 self.log.info(f"Testing {rejection_type} rejection...")
250 tx_info = self.create_policy_rejected_tx(rejection_type)
251
252 tx_obj = tx_from_hex(tx_info['hex'])
253 self.segwit_node.send_message(msg_tx(tx_obj))
254
255 rejected_txs.append({
256 'type': rejection_type,
257 'tx_info': tx_info,
258 'txid': tx_info['tx'].hash,
259 'wtxid': tx_info['tx'].getwtxid()
260 })
261
262 self.segwit_node.sync_with_ping()
263
264 mempool = self.nodes[0].getrawmempool()
265 for rejected in rejected_txs:
266 assert_equal(rejected['txid'] in mempool, False)
267 self.log.info(f"✓ {rejected['type']} transaction rejected from mempool")
268
269 indices = list(range(len(rejected_txs)))
270 tx_list = [r['tx_info'] for r in rejected_txs]
271
272 result = self.send_compact_block(tx_list, indices)
273
274 assert_equal(result["missing_indices"], [])
275 self.log.info("✓ All rejected transactions are available in extra pool")
276
277 # TEST: blockreconstructionextratxn
278
279 def test_extratxnpool_disabled(self):
280 """Test that setting count to 0 disables the extra transaction pool."""
281 self.log.info("Testing disabled extra transaction pool (0 capacity)...")
282
283 self.restart_node_with_limit(count=0)
284 buffersize = 5
285 rejected_txs = self.populate_extra_pool(buffersize)
286
287 indices = list(range(buffersize))
288 result = self.send_compact_block(rejected_txs, indices)
289 assert_equal(result["missing_indices"], indices)
290 self.log.info(f"✓ All {buffersize} transactions are missing (extra txn pool disabled)")
291
292 def test_extratxnpool_capacity_and_wraparound(self):
293 """Test extra transaction pool capacity and wraparound behavior."""
294 self.log.info("Testing extra transaction pool capacity (400 transactions)...")
295
296 buffersize = 400
297 self.restart_node_with_limit(count=buffersize)
298
299 rejected_txs = self.populate_extra_pool(buffersize)
300
301 indices = list(range(buffersize))
302 result = self.send_compact_block(rejected_txs, indices)
303
304 assert_equal(result["missing_indices"], [])
305 self.log.info("✓ All rejected transactions are in the extra txn pool")
306
307 # Test that adding a 401st transaction causes eviction
308 self.log.info("Adding transaction to test eviction...")
309 self.populate_extra_pool(1)
310
311 # Check original transactions again - first one should be evicted
312 result2 = self.send_compact_block(rejected_txs, indices)
313 assert_equal(result2["missing_indices"], [0])
314 self.log.info("✓ Transaction 0 was evicted as expected (wraparound)")
315
316 def test_single_extratxnpool_capacity(self):
317 """Test edge case of single capacity extra transaction pool."""
318 self.log.info("Testing single capacity extra transaction pool...")
319
320 self.restart_node_with_limit(count=1)
321 tx_count = 5
322
323 rejected_txs = self.populate_extra_pool(tx_count)
324
325 indices = list(range(tx_count))
326 result = self.send_compact_block(rejected_txs, indices)
327
328 expected_missing = list(range(tx_count - 1))
329 assert_equal(result["missing_indices"], expected_missing)
330
331 def test_extratxn_invalid_parameters(self):
332 """Test handling of invalid blockreconstructionextratxn values."""
333 self.log.info("Testing invalid parameter values...")
334
335 # Test negative value - should be clamped to 0 (disabled)
336 self.log.info("Testing negative value (-1)...")
337 self.restart_node_with_limit(count=-1)
338
339 # Add a transaction and verify pool is disabled
340 rejected_txs = self.populate_extra_pool(1)
341 result = self.send_compact_block(rejected_txs, [0])
342 assert_equal(result["missing_indices"], [0])
343 self.log.info("✓ Negative value correctly treated as disabled")
344
345 # TEST: blockreconstructionextratxnsize
346
347 def test_extratxnsize_zero_limit(self):
348 """Test extra transaction pool zero size limit prevents extra txn pool."""
349 self.log.info("Testing extra transaction pool zero size limit prevents extra txn pool...")
350 self.restart_node_with_limit(memory_mb=0)
351
352 rejected_txs = self.populate_extra_pool(1)
353 result = self.send_compact_block(rejected_txs, [0])
354
355 # Should fail - no size limit for extra pool
356 assert result["getblocktxn"] is not None, "Node should try to request when zero size limit"
357 assert_equal(int(self.nodes[0].getbestblockhash(), 16), result["block"].hashPrevBlock)
358
359 def test_extratxnsize_eviction(self):
360 """Test extra transaction pool size limit eviction behavior."""
361 self.log.info("Testing extra transaction pool size limit eviction behavior...")
362
363 buffersize = 60
364
365 # First, test with 1MB limit - should fail
366 self.log.info(f"Step 1: Testing with 1MB limit for {buffersize} large transactions")
367 self.restart_node_with_limit(memory_mb=1, count=buffersize)
368
369 # Create 60 large transactions (~20KB each = ~1.2MB total)
370 # This exceeds the 1MB limit
371 self.log.info(f"Creating {buffersize} large transactions (~20KB each, ~1.2MB total)")
372 rejected_txs = self.populate_extra_pool(buffersize, target_size=20000)
373
374 indices = list(range(buffersize))
375 result_small = self.send_compact_block(rejected_txs, indices)
376
377 # Should have evictions - can't fit 1.2MB in 1MB limit
378 assert len(result_small["missing_indices"]) > 0, "1MB limit should cause evictions for 1.2MB of transactions"
379 evicted_count = len(result_small["missing_indices"])
380 self.log.info(f"✓ 1MB limit caused {evicted_count} evictions (can't fit ~1.2MB of transactions)")
381
382 # Now test with larger size limit to show it succeeds
383 self.log.info(f"Step 2: Testing with 2MB limit for same {buffersize} large transactions")
384 self.restart_node_with_limit(memory_mb=2, count=buffersize)
385
386 rejected_txs = self.populate_extra_pool(buffersize, target_size=20000)
387
388 result_large = self.send_compact_block(rejected_txs, indices)
389
390 # Should have NO evictions with 2MB limit
391 assert result_large["missing_indices"] == [], "2MB limit should store all transactions"
392 self.log.info(f"✓ 2MB limit successfully stores all {buffersize} large transactions (~1.2MB)")
393
394 def test_extratxnsize_boundary(self):
395 """Test extra transaction pool at exact size limit boundary."""
396 self.log.info("Testing extra transaction pool exact size limit boundary...")
397
398 limit_mb = 1
399 self.restart_node_with_limit(memory_mb=limit_mb)
400
401 test_count = 100
402 rejected_txs = self.populate_extra_pool(test_count, target_size=20000)
403
404 indices = list(range(test_count))
405 result = self.send_compact_block(rejected_txs, indices)
406
407 # Find the boundary - how many fit vs how many were evicted
408 num_evicted = len(result["missing_indices"])
409 num_fit = test_count - num_evicted
410
411 # Now restart and add exactly the number that fit
412 self.restart_node_with_limit(memory_mb=limit_mb)
413 rejected_txs = self.populate_extra_pool(num_fit, target_size=20000)
414
415 # Verify all fit
416 indices = list(range(num_fit))
417 result = self.send_compact_block(rejected_txs, indices)
418 assert result["missing_indices"] == [], f"Expected all {num_fit} transactions to fit at boundary"
419
420 # Add one more transaction - should evict exactly one
421 self.log.info("Adding one more transaction at the boundary...")
422 self.populate_extra_pool(1, target_size=20000)
423
424 # Check original transactions again
425 result2 = self.send_compact_block(rejected_txs, indices)
426 assert len(result2["missing_indices"]) == 1, "Expected exactly 1 eviction at boundary"
427 assert result2["missing_indices"] == [0], "Expected oldest transaction (0) to be evicted"
428
429 self.log.info("Size limit boundary behavior verified - one transaction evicted when limit exceeded")
430
431 def test_extratxnsize_small_limit(self):
432 """Test extra transaction pool with very small size limit (0.1 MB)."""
433 self.log.info("Testing extra transaction pool with 0.1 MB size limit...")
434
435 limit_mb = 0.1
436 self.restart_node_with_limit(memory_mb=limit_mb)
437
438 test_count = 100
439 rejected_txs = self.populate_extra_pool(test_count, target_size=500)
440 indices = list(range(test_count))
441 result = self.send_compact_block(rejected_txs, indices)
442
443 # Find the boundary - how many fit vs how many were evicted
444 num_evicted = len(result["missing_indices"])
445 num_fit = test_count - num_evicted
446
447 # fill exact capacity
448 self.restart_node_with_limit(memory_mb=limit_mb)
449 rejected_txs = self.populate_extra_pool(num_fit, target_size=500)
450
451 indices = list(range(num_fit))
452 result = self.send_compact_block(rejected_txs, indices)
453 assert result["missing_indices"] == [], f"Expected all {num_fit} transactions to fit at boundary"
454
455 # Add one more transaction - should evict exactly one
456 self.log.info("Adding one more transaction at the boundary...")
457 self.populate_extra_pool(1, target_size=500)
458
459 # Check original transactions again
460 result = self.send_compact_block(rejected_txs, indices)
461 assert len(result["missing_indices"]) == 1, "Expected exactly 1 eviction at boundary"
462 assert result["missing_indices"] == [0], "Expected oldest transaction (0) to be evicted"
463
464 self.log.info("Small size limit boundary behavior verified - one transaction evicted when limit exceeded")
465
466 def test_extratxnsize_large_transaction_exceeds_limit(self):
467 """Test that a transaction larger than the entire size limit is rejected."""
468 self.log.info("Testing large transaction that exceeds size limit...")
469
470 limit_mb = 0.5 # 0.5 MB/500KB
471 self.restart_node_with_limit(memory_mb=limit_mb)
472
473 # Create a 600KB transaction (larger than 500KB limit)
474 # This should be rejected from mempool AND not stored in extra pool
475 rejected_txs = self.populate_extra_pool(1, target_size=600000) # 600 KB
476
477 # try block reconstruction
478 result = self.send_compact_block(rejected_txs, [0])
479
480 # Verify the transaction is NOT available (was not stored in extra pool)
481 assert len(result["missing_indices"]) == 1, "Large transaction should not be stored in extra pool"
482
483 def run_test(self):
484 self.wallet = MiniWallet(self.nodes[0])
485
486 # Setup the p2p connection
487 self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn())
488
489 # Create UTXOs for testing
490 self.make_utxos()
491
492 # Ensure segwit is active
493 assert softfork_active(self.nodes[0], "segwit")
494
495 # Test policy rejection types first
496 self.test_policy_rejection_types()
497
498 # Extra Txn capacity tests
499 self.test_extratxnpool_disabled()
500 self.test_extratxnpool_capacity_and_wraparound()
501 self.test_single_extratxnpool_capacity()
502 self.test_extratxn_invalid_parameters()
503
504 # Extra Txn size tests
505 self.test_extratxnsize_zero_limit()
506 self.test_extratxnsize_eviction()
507 self.test_extratxnsize_boundary()
508 self.test_extratxnsize_small_limit()
509 self.test_extratxnsize_large_transaction_exceeds_limit()
510
511
512 if __name__ == '__main__':
513 CompactBlocksBlockReconstructionLimitTest(__file__).main()
514