1 #!/usr/bin/env python3
2 # Copyright (c) 2023-present The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 6 import time
7 8 from test_framework.blocktools import MAX_STANDARD_TX_WEIGHT
9 from test_framework.mempool_util import (
10 create_large_orphan,
11 tx_in_orphanage,
12 )
13 from test_framework.messages import (
14 CInv,
15 DEFAULT_ANCESTOR_LIMIT,
16 MSG_TX,
17 MSG_WITNESS_TX,
18 MSG_WTX,
19 malleate_tx_to_invalid_witness,
20 msg_getdata,
21 msg_inv,
22 msg_notfound,
23 msg_tx,
24 tx_from_hex,
25 )
26 from test_framework.p2p import (
27 GETDATA_TX_INTERVAL,
28 NONPREF_PEER_TX_DELAY,
29 OVERLOADED_PEER_TX_DELAY,
30 p2p_lock,
31 P2PInterface,
32 P2PTxInvStore,
33 TXID_RELAY_DELAY,
34 )
35 from test_framework.util import (
36 assert_not_equal,
37 assert_equal,
38 )
39 from test_framework.test_framework import BitcoinTestFramework
40 from test_framework.wallet import (
41 MiniWallet,
42 MiniWalletMode,
43 )
44 45 # Time to bump forward (using setmocktime) before waiting for the node to send getdata(tx) in response
46 # to an inv(tx), in seconds. This delay includes all possible delays + 1, so it should only be used
47 # when the value of the delay is not interesting. If we want to test that the node waits x seconds
48 # for one peer and y seconds for another, use specific values instead.
49 TXREQUEST_TIME_SKIP = NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY + OVERLOADED_PEER_TX_DELAY + 1
50 51 def cleanup(func):
52 def wrapper(self):
53 func(self)
54 55 # Clear mempool
56 self.generate(self.nodes[0], 1)
57 self.nodes[0].disconnect_p2ps()
58 # Check that mempool and orphanage have been cleared
59 self.wait_until(lambda: len(self.nodes[0].getorphantxs()) == 0)
60 assert_equal(0, len(self.nodes[0].getrawmempool()))
61 62 self.restart_node(0, extra_args=["-persistmempool=0"])
63 # Allow use of bumpmocktime again
64 self.nodes[0].setmocktime(int(time.time()))
65 self.wallet.rescan_utxos(include_mempool=True)
66 return wrapper
67 68 class PeerTxRelayer(P2PTxInvStore):
69 """A P2PTxInvStore that also remembers all of the getdata and tx messages it receives."""
70 def __init__(self, wtxidrelay=True):
71 super().__init__(wtxidrelay=wtxidrelay)
72 self._tx_received = []
73 self._getdata_received = []
74 75 @property
76 def tx_received(self):
77 with p2p_lock:
78 return self._tx_received
79 80 @property
81 def getdata_received(self):
82 with p2p_lock:
83 return self._getdata_received
84 85 def on_tx(self, message):
86 self._tx_received.append(message)
87 88 def on_getdata(self, message):
89 self._getdata_received.append(message)
90 91 def wait_for_parent_requests(self, txids):
92 """Wait for requests for missing parents by txid with witness data (MSG_WITNESS_TX or
93 WitnessTx). Requires that the getdata message match these txids exactly; all txids must be
94 requested and no additional requests are allowed."""
95 def test_function():
96 last_getdata = self.last_message.get('getdata')
97 if not last_getdata:
98 return False
99 return len(last_getdata.inv) == len(txids) and all([item.type == MSG_WITNESS_TX and item.hash in txids for item in last_getdata.inv])
100 self.wait_until(test_function, timeout=10)
101 102 def assert_no_immediate_response(self, message):
103 """Check that the node does not immediately respond to this message with any of getdata,
104 inv, tx. The node may respond later.
105 """
106 prev_lastmessage = self.last_message
107 self.send_and_ping(message)
108 after_lastmessage = self.last_message
109 for msgtype in ["getdata", "inv", "tx"]:
110 if msgtype not in prev_lastmessage:
111 assert msgtype not in after_lastmessage
112 else:
113 assert_equal(prev_lastmessage[msgtype], after_lastmessage[msgtype])
114 115 def assert_never_requested(self, txhash):
116 """Check that the node has never sent us a getdata for this hash (int type)"""
117 self.sync_with_ping()
118 for getdata in self.getdata_received:
119 for request in getdata.inv:
120 assert_not_equal(request.hash, txhash)
121 122 class OrphanHandlingTest(BitcoinTestFramework):
123 def set_test_params(self):
124 self.num_nodes = 1
125 self.extra_args = [[]]
126 127 def create_parent_and_child(self):
128 """Create package with 1 parent and 1 child, normal fees (no cpfp)."""
129 parent = self.wallet.create_self_transfer()
130 child = self.wallet.create_self_transfer(utxo_to_spend=parent['new_utxo'])
131 return child["tx"].wtxid_hex, child["tx"], parent["tx"]
132 133 def relay_transaction(self, peer, tx):
134 """Relay transaction using MSG_WTX"""
135 wtxid = tx.wtxid_int
136 peer.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=wtxid)]))
137 self.nodes[0].bumpmocktime(TXREQUEST_TIME_SKIP)
138 peer.wait_for_getdata([wtxid])
139 peer.send_and_ping(msg_tx(tx))
140 141 @cleanup
142 def test_arrival_timing_orphan(self):
143 self.log.info("Test missing parents that arrive during delay are not requested")
144 node = self.nodes[0]
145 tx_parent_arrives = self.wallet.create_self_transfer()
146 tx_parent_doesnt_arrive = self.wallet.create_self_transfer()
147 # Fake orphan spends nonexistent outputs of the two parents
148 tx_fake_orphan = self.wallet.create_self_transfer_multi(utxos_to_spend=[
149 {"txid": tx_parent_doesnt_arrive["txid"], "vout": 10, "value": tx_parent_doesnt_arrive["new_utxo"]["value"]},
150 {"txid": tx_parent_arrives["txid"], "vout": 10, "value": tx_parent_arrives["new_utxo"]["value"]}
151 ])
152 153 peer_spy = node.add_p2p_connection(PeerTxRelayer())
154 peer_normal = node.add_p2p_connection(PeerTxRelayer())
155 # This transaction is an orphan because it is missing inputs. It is a "fake" orphan that the
156 # spy peer has crafted to learn information about tx_parent_arrives even though it isn't
157 # able to spend a real output of it, but it could also just be a normal, real child tx.
158 # The node should not immediately respond with a request for orphan parents.
159 # Also, no request should be sent later because it will be resolved by
160 # the time the request is scheduled to be sent.
161 peer_spy.assert_no_immediate_response(msg_tx(tx_fake_orphan["tx"]))
162 163 # Node receives transaction. It attempts to obfuscate the exact timing at which this
164 # transaction entered its mempool. Send unsolicited because otherwise we need to wait for
165 # request delays.
166 peer_normal.send_and_ping(msg_tx(tx_parent_arrives["tx"]))
167 assert tx_parent_arrives["txid"] in node.getrawmempool()
168 169 # Spy peer should not be able to query the node for the parent yet, since it hasn't been
170 # announced / insufficient time has elapsed.
171 parent_inv = CInv(t=MSG_WTX, h=tx_parent_arrives["tx"].wtxid_int)
172 assert_equal(len(peer_spy.get_invs()), 0)
173 peer_spy.assert_no_immediate_response(msg_getdata([parent_inv]))
174 175 # Request would be scheduled with this delay because it is not a preferred relay peer.
176 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY)
177 peer_spy.assert_never_requested(int(tx_parent_arrives["txid"], 16))
178 peer_spy.assert_never_requested(int(tx_parent_doesnt_arrive["txid"], 16))
179 # Request would be scheduled with this delay because it is by txid.
180 self.nodes[0].bumpmocktime(TXID_RELAY_DELAY)
181 peer_spy.wait_for_parent_requests([int(tx_parent_doesnt_arrive["txid"], 16)])
182 peer_spy.assert_never_requested(int(tx_parent_arrives["txid"], 16))
183 184 @cleanup
185 def test_orphan_rejected_parents_exceptions(self):
186 node = self.nodes[0]
187 peer1 = node.add_p2p_connection(PeerTxRelayer())
188 peer2 = node.add_p2p_connection(PeerTxRelayer())
189 190 self.log.info("Test orphan handling when a nonsegwit parent is known to be invalid")
191 parent_overly_large_nonsegwit = self.wallet_nonsegwit.create_self_transfer(target_vsize=int(MAX_STANDARD_TX_WEIGHT / 4) + 1)
192 assert_equal(parent_overly_large_nonsegwit["txid"], parent_overly_large_nonsegwit["tx"].wtxid_hex)
193 parent_other = self.wallet_nonsegwit.create_self_transfer()
194 child_nonsegwit = self.wallet_nonsegwit.create_self_transfer_multi(
195 utxos_to_spend=[parent_other["new_utxo"], parent_overly_large_nonsegwit["new_utxo"]])
196 197 # Relay the parent. It should be rejected (and not reconsiderable) because it violated size limitations.
198 self.relay_transaction(peer1, parent_overly_large_nonsegwit["tx"])
199 assert parent_overly_large_nonsegwit["txid"] not in node.getrawmempool()
200 201 # Relay the child. It should not be accepted because it has missing inputs.
202 # Its parent should not be requested because its hash (txid == wtxid) has been added to the rejection filter.
203 self.relay_transaction(peer2, child_nonsegwit["tx"])
204 assert child_nonsegwit["txid"] not in node.getrawmempool()
205 assert not tx_in_orphanage(node, child_nonsegwit["tx"])
206 207 # No parents are requested.
208 self.nodes[0].bumpmocktime(GETDATA_TX_INTERVAL)
209 peer1.assert_never_requested(int(parent_other["txid"], 16))
210 peer2.assert_never_requested(int(parent_other["txid"], 16))
211 peer2.assert_never_requested(int(parent_overly_large_nonsegwit["txid"], 16))
212 213 self.log.info("Test orphan handling when a segwit parent was invalid but may be retried with another witness")
214 parent_low_fee = self.wallet.create_self_transfer(fee_rate=0)
215 child_low_fee = self.wallet.create_self_transfer(utxo_to_spend=parent_low_fee["new_utxo"])
216 217 # Relay the low fee parent. It should not be accepted.
218 self.relay_transaction(peer1, parent_low_fee["tx"])
219 assert parent_low_fee["txid"] not in node.getrawmempool()
220 221 # Relay the child. It should not be accepted because it has missing inputs.
222 self.relay_transaction(peer2, child_low_fee["tx"])
223 assert child_low_fee["txid"] not in node.getrawmempool()
224 assert tx_in_orphanage(node, child_low_fee["tx"])
225 226 # The parent should be requested because even though the txid commits to the fee, it doesn't
227 # commit to the feerate. Delayed because it's by txid and this is not a preferred relay peer.
228 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
229 peer2.wait_for_getdata([parent_low_fee["tx"].txid_int])
230 231 self.log.info("Test orphan handling when a parent was previously downloaded with witness stripped")
232 parent_normal = self.wallet.create_self_transfer()
233 parent1_witness_stripped = tx_from_hex(parent_normal["tx"].serialize_without_witness().hex())
234 child_invalid_witness = self.wallet.create_self_transfer(utxo_to_spend=parent_normal["new_utxo"])
235 236 # Relay the parent with witness stripped. It should not be accepted.
237 self.relay_transaction(peer1, parent1_witness_stripped)
238 assert_equal(parent_normal["txid"], parent1_witness_stripped.txid_hex)
239 assert parent1_witness_stripped.txid_hex not in node.getrawmempool()
240 241 # Relay the child. It should not be accepted because it has missing inputs.
242 self.relay_transaction(peer2, child_invalid_witness["tx"])
243 assert child_invalid_witness["txid"] not in node.getrawmempool()
244 assert tx_in_orphanage(node, child_invalid_witness["tx"])
245 246 # The parent should be requested since the unstripped wtxid would differ. Delayed because
247 # it's by txid and this is not a preferred relay peer.
248 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
249 peer2.wait_for_getdata([parent_normal["tx"].txid_int])
250 251 # parent_normal can be relayed again even though parent1_witness_stripped was rejected
252 self.relay_transaction(peer1, parent_normal["tx"])
253 assert_equal(set(node.getrawmempool()), set([parent_normal["txid"], child_invalid_witness["txid"]]))
254 255 @cleanup
256 def test_orphan_multiple_parents(self):
257 node = self.nodes[0]
258 peer = node.add_p2p_connection(PeerTxRelayer())
259 260 self.log.info("Test orphan parent requests with a mixture of confirmed, in-mempool and missing parents")
261 # This UTXO confirmed a long time ago.
262 utxo_conf_old = self.wallet.send_self_transfer(from_node=node)["new_utxo"]
263 txid_conf_old = utxo_conf_old["txid"]
264 self.generate(self.wallet, 10)
265 266 # Create a fake reorg to trigger BlockDisconnected, which resets the rolling bloom filter.
267 # The alternative is to mine thousands of transactions to push it out of the filter.
268 last_block = node.getbestblockhash()
269 node.invalidateblock(last_block)
270 node.preciousblock(last_block)
271 node.syncwithvalidationinterfacequeue()
272 273 # This UTXO confirmed recently.
274 utxo_conf_recent = self.wallet.send_self_transfer(from_node=node)["new_utxo"]
275 self.generate(node, 1)
276 277 # This UTXO is unconfirmed and in the mempool.
278 assert_equal(len(node.getrawmempool()), 0)
279 mempool_tx = self.wallet.send_self_transfer(from_node=node)
280 utxo_unconf_mempool = mempool_tx["new_utxo"]
281 282 # This UTXO is unconfirmed and missing.
283 missing_tx = self.wallet.create_self_transfer()
284 utxo_unconf_missing = missing_tx["new_utxo"]
285 assert missing_tx["txid"] not in node.getrawmempool()
286 287 orphan = self.wallet.create_self_transfer_multi(utxos_to_spend=[utxo_conf_old,
288 utxo_conf_recent, utxo_unconf_mempool, utxo_unconf_missing])
289 290 self.relay_transaction(peer, orphan["tx"])
291 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
292 peer.sync_with_ping()
293 assert tx_in_orphanage(node, orphan["tx"])
294 assert_equal(len(peer.last_message["getdata"].inv), 2)
295 peer.wait_for_parent_requests([int(txid_conf_old, 16), int(missing_tx["txid"], 16)])
296 297 # Even though the peer would send a notfound for the "old" confirmed transaction, the node
298 # doesn't give up on the orphan. Once all of the missing parents are received, it should be
299 # submitted to mempool.
300 peer.send_without_ping(msg_notfound(vec=[CInv(MSG_WITNESS_TX, int(txid_conf_old, 16))]))
301 # Sync with ping to ensure orphans are reconsidered
302 peer.send_and_ping(msg_tx(missing_tx["tx"]))
303 assert_equal(node.getmempoolentry(orphan["txid"])["ancestorcount"], 3)
304 305 @cleanup
306 def test_orphans_overlapping_parents(self):
307 node = self.nodes[0]
308 # In the process of relaying inflight_parent_AB
309 peer_txrequest = node.add_p2p_connection(PeerTxRelayer())
310 # Sends the orphans
311 peer_orphans = node.add_p2p_connection(PeerTxRelayer())
312 313 confirmed_utxos = [self.wallet_nonsegwit.get_utxo() for _ in range(4)]
314 assert all([utxo["confirmations"] > 0 for utxo in confirmed_utxos])
315 self.log.info("Test handling of multiple orphans with missing parents that are already being requested")
316 # Parent of child_A only
317 missing_parent_A = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=confirmed_utxos[0])
318 # Parents of child_A and child_B
319 missing_parent_AB = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=confirmed_utxos[1])
320 inflight_parent_AB = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=confirmed_utxos[2])
321 # Parent of child_B only
322 missing_parent_B = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=confirmed_utxos[3])
323 child_A = self.wallet_nonsegwit.create_self_transfer_multi(
324 utxos_to_spend=[missing_parent_A["new_utxo"], missing_parent_AB["new_utxo"], inflight_parent_AB["new_utxo"]]
325 )
326 child_B = self.wallet_nonsegwit.create_self_transfer_multi(
327 utxos_to_spend=[missing_parent_B["new_utxo"], missing_parent_AB["new_utxo"], inflight_parent_AB["new_utxo"]]
328 )
329 330 # The wtxid and txid need to be the same for the node to recognize that the missing input
331 # and in-flight request for inflight_parent_AB are the same transaction.
332 assert_equal(inflight_parent_AB["txid"], inflight_parent_AB["wtxid"])
333 334 # Announce inflight_parent_AB and wait for getdata
335 peer_txrequest.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=inflight_parent_AB["tx"].wtxid_int)]))
336 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY)
337 peer_txrequest.wait_for_getdata([inflight_parent_AB["tx"].wtxid_int])
338 339 self.log.info("Test that the node does not request a parent if it has an in-flight txrequest")
340 # Relay orphan child_A
341 self.relay_transaction(peer_orphans, child_A["tx"])
342 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
343 assert tx_in_orphanage(node, child_A["tx"])
344 # There are 3 missing parents. missing_parent_A and missing_parent_AB should be requested.
345 # But inflight_parent_AB should not, because there is already an in-flight request for it.
346 peer_orphans.wait_for_parent_requests([int(missing_parent_A["txid"], 16), int(missing_parent_AB["txid"], 16)])
347 348 self.log.info("Test that the node does not request a parent if it has an in-flight orphan parent request")
349 # Relay orphan child_B
350 self.relay_transaction(peer_orphans, child_B["tx"])
351 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
352 assert tx_in_orphanage(node, child_B["tx"])
353 # Only missing_parent_B should be requested. Not inflight_parent_AB or missing_parent_AB
354 # because they are already being requested from peer_txrequest and peer_orphans respectively.
355 peer_orphans.wait_for_parent_requests([int(missing_parent_B["txid"], 16)])
356 peer_orphans.assert_never_requested(int(inflight_parent_AB["txid"], 16))
357 358 # But inflight_parent_AB will be requested eventually if original peer doesn't respond
359 node.bumpmocktime(GETDATA_TX_INTERVAL)
360 peer_orphans.wait_for_parent_requests([int(inflight_parent_AB["txid"], 16)])
361 362 @cleanup
363 def test_orphan_of_orphan(self):
364 node = self.nodes[0]
365 peer = node.add_p2p_connection(PeerTxRelayer())
366 367 self.log.info("Test handling of an orphan with a parent who is another orphan")
368 missing_grandparent = self.wallet_nonsegwit.create_self_transfer()
369 missing_parent_orphan = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=missing_grandparent["new_utxo"])
370 missing_parent = self.wallet_nonsegwit.create_self_transfer()
371 orphan = self.wallet_nonsegwit.create_self_transfer_multi(utxos_to_spend=[missing_parent["new_utxo"], missing_parent_orphan["new_utxo"]])
372 373 # The node should put missing_parent_orphan into the orphanage and request missing_grandparent
374 self.relay_transaction(peer, missing_parent_orphan["tx"])
375 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
376 assert tx_in_orphanage(node, missing_parent_orphan["tx"])
377 peer.wait_for_parent_requests([int(missing_grandparent["txid"], 16)])
378 379 # The node should put the orphan into the orphanage and request missing_parent, skipping
380 # missing_parent_orphan because it already has it in the orphanage.
381 self.relay_transaction(peer, orphan["tx"])
382 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
383 assert tx_in_orphanage(node, orphan["tx"])
384 peer.wait_for_parent_requests([int(missing_parent["txid"], 16)])
385 386 @cleanup
387 def test_orphan_inherit_rejection(self):
388 node = self.nodes[0]
389 peer1 = node.add_p2p_connection(PeerTxRelayer())
390 peer2 = node.add_p2p_connection(PeerTxRelayer())
391 peer3 = node.add_p2p_connection(PeerTxRelayer(wtxidrelay=False))
392 393 self.log.info("Test that an orphan with rejected parents, along with any descendants, cannot be retried with an alternate witness")
394 parent_overly_large_nonsegwit = self.wallet_nonsegwit.create_self_transfer(target_vsize=int(MAX_STANDARD_TX_WEIGHT / 4) + 1)
395 assert_equal(parent_overly_large_nonsegwit["txid"], parent_overly_large_nonsegwit["tx"].wtxid_hex)
396 child = self.wallet.create_self_transfer(utxo_to_spend=parent_overly_large_nonsegwit["new_utxo"])
397 grandchild = self.wallet.create_self_transfer(utxo_to_spend=child["new_utxo"])
398 assert_not_equal(child["txid"], child["tx"].wtxid_hex)
399 assert_not_equal(grandchild["txid"], grandchild["tx"].wtxid_hex)
400 401 # Relay the parent. It should be rejected because it pays 0 fees.
402 self.relay_transaction(peer1, parent_overly_large_nonsegwit["tx"])
403 assert parent_overly_large_nonsegwit["txid"] not in node.getrawmempool()
404 405 # Relay the child. It should be rejected for having missing parents, and this rejection is
406 # cached by txid and wtxid.
407 self.relay_transaction(peer1, child["tx"])
408 assert_equal(0, len(node.getrawmempool()))
409 assert not tx_in_orphanage(node, child["tx"])
410 peer1.assert_never_requested(parent_overly_large_nonsegwit["txid"])
411 412 # Grandchild should also not be kept in orphanage because its parent has been rejected.
413 self.relay_transaction(peer2, grandchild["tx"])
414 assert_equal(0, len(node.getrawmempool()))
415 assert not tx_in_orphanage(node, grandchild["tx"])
416 peer2.assert_never_requested(child["txid"])
417 peer2.assert_never_requested(child["tx"].wtxid_hex)
418 419 # The child should never be requested, even if announced again with potentially different witness.
420 # Sync with ping to ensure orphans are reconsidered
421 peer3.send_and_ping(msg_inv([CInv(t=MSG_TX, h=int(child["txid"], 16))]))
422 self.nodes[0].bumpmocktime(TXREQUEST_TIME_SKIP)
423 peer3.assert_never_requested(child["txid"])
424 425 @cleanup
426 def test_same_txid_orphan(self):
427 self.log.info("Check what happens when orphan with same txid is already in orphanage")
428 node = self.nodes[0]
429 430 tx_parent = self.wallet.create_self_transfer()
431 432 # Create the real child
433 tx_child = self.wallet.create_self_transfer(utxo_to_spend=tx_parent["new_utxo"])
434 435 # Create a fake version of the child
436 tx_orphan_bad_wit = malleate_tx_to_invalid_witness(tx_child)
437 438 bad_peer = node.add_p2p_connection(P2PInterface())
439 honest_peer = node.add_p2p_connection(P2PInterface())
440 441 # 1. Fake orphan is received first. It is missing an input.
442 bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit))
443 assert tx_in_orphanage(node, tx_orphan_bad_wit)
444 445 # 2. Node requests the missing parent by txid.
446 parent_txid_int = int(tx_parent["txid"], 16)
447 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
448 bad_peer.wait_for_getdata([parent_txid_int])
449 450 # 3. Honest peer relays the real child, which is also missing parents and should be placed
451 # in the orphanage.
452 with node.assert_debug_log(["missingorspent"]):
453 honest_peer.send_and_ping(msg_tx(tx_child["tx"]))
454 assert tx_in_orphanage(node, tx_child["tx"])
455 456 # Time out the previous request for the parent (node will not request the same transaction
457 # from multiple nodes at the same time)
458 node.bumpmocktime(GETDATA_TX_INTERVAL)
459 460 # 4. The parent is requested. Honest peer sends it.
461 honest_peer.wait_for_getdata([parent_txid_int])
462 # Sync with ping to ensure orphans are reconsidered
463 honest_peer.send_and_ping(msg_tx(tx_parent["tx"]))
464 465 # 5. After parent is accepted, orphans should be reconsidered.
466 # The real child should be accepted and the fake one rejected.
467 node_mempool = node.getrawmempool()
468 assert tx_parent["txid"] in node_mempool
469 assert tx_child["txid"] in node_mempool
470 assert_equal(node.getmempoolentry(tx_child["txid"])["wtxid"], tx_child["wtxid"])
471 472 @cleanup
473 def test_same_txid_orphan_of_orphan(self):
474 self.log.info("Check what happens when orphan's parent with same txid is already in orphanage")
475 node = self.nodes[0]
476 477 tx_grandparent = self.wallet.create_self_transfer()
478 479 # Create middle tx (both parent and child) which will be in orphanage.
480 tx_middle = self.wallet.create_self_transfer(utxo_to_spend=tx_grandparent["new_utxo"])
481 482 # Create a fake version of the middle tx
483 tx_orphan_bad_wit = malleate_tx_to_invalid_witness(tx_middle)
484 485 # Create grandchild spending from tx_middle (and spending from tx_orphan_bad_wit since they
486 # have the same txid).
487 tx_grandchild = self.wallet.create_self_transfer(utxo_to_spend=tx_middle["new_utxo"])
488 489 bad_peer = node.add_p2p_connection(P2PInterface())
490 honest_peer = node.add_p2p_connection(P2PInterface())
491 492 # 1. Fake orphan is received first. It is missing an input.
493 bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit))
494 assert tx_in_orphanage(node, tx_orphan_bad_wit)
495 496 # 2. Node requests missing tx_grandparent by txid.
497 grandparent_txid_int = int(tx_grandparent["txid"], 16)
498 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
499 bad_peer.wait_for_getdata([grandparent_txid_int])
500 501 # 3. Honest peer relays the grandchild, which is missing a parent. The parent by txid already
502 # exists in orphanage, but should be re-requested because the node shouldn't assume that the
503 # witness data is the same. In this case, a same-txid-different-witness transaction exists!
504 honest_peer.send_and_ping(msg_tx(tx_grandchild["tx"]))
505 assert tx_in_orphanage(node, tx_grandchild["tx"])
506 middle_txid_int = int(tx_middle["txid"], 16)
507 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
508 honest_peer.wait_for_getdata([middle_txid_int])
509 510 # 4. Honest peer relays the real child, which is also missing parents and should be placed
511 # in the orphanage.
512 honest_peer.send_and_ping(msg_tx(tx_middle["tx"]))
513 assert tx_in_orphanage(node, tx_middle["tx"])
514 assert_equal(len(node.getrawmempool()), 0)
515 516 # 5. Honest peer sends tx_grandparent
517 honest_peer.send_and_ping(msg_tx(tx_grandparent["tx"]))
518 519 # 6. After parent is accepted, orphans should be reconsidered.
520 # The real child should be accepted and the fake one rejected.
521 node_mempool = node.getrawmempool()
522 assert tx_grandparent["txid"] in node_mempool
523 assert tx_middle["txid"] in node_mempool
524 assert tx_grandchild["txid"] in node_mempool
525 assert_equal(node.getmempoolentry(tx_middle["txid"])["wtxid"], tx_middle["wtxid"])
526 self.wait_until(lambda: len(node.getorphantxs()) == 0)
527 528 @cleanup
529 def test_orphan_txid_inv(self):
530 self.log.info("Check node does not ignore announcement with same txid as tx in orphanage")
531 node = self.nodes[0]
532 533 tx_parent = self.wallet.create_self_transfer()
534 535 # Create the real child and fake version
536 tx_child = self.wallet.create_self_transfer(utxo_to_spend=tx_parent["new_utxo"])
537 tx_orphan_bad_wit = malleate_tx_to_invalid_witness(tx_child)
538 539 bad_peer = node.add_p2p_connection(PeerTxRelayer())
540 # Must not send wtxidrelay because otherwise the inv(TX) will be ignored later
541 honest_peer = node.add_p2p_connection(P2PInterface(wtxidrelay=False))
542 543 # 1. Fake orphan is received first. It is missing an input.
544 bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit))
545 assert tx_in_orphanage(node, tx_orphan_bad_wit)
546 547 # 2. Node requests the missing parent by txid.
548 parent_txid_int = int(tx_parent["txid"], 16)
549 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
550 bad_peer.wait_for_getdata([parent_txid_int])
551 552 # 3. Honest peer announces the real child, by txid (this isn't common but the node should
553 # still keep track of it).
554 child_txid_int = int(tx_child["txid"], 16)
555 honest_peer.send_and_ping(msg_inv([CInv(t=MSG_TX, h=child_txid_int)]))
556 557 # 4. The child is requested. Honest peer sends it.
558 node.bumpmocktime(TXREQUEST_TIME_SKIP)
559 honest_peer.wait_for_getdata([child_txid_int])
560 honest_peer.send_and_ping(msg_tx(tx_child["tx"]))
561 assert tx_in_orphanage(node, tx_child["tx"])
562 563 # 5. After first parent request times out, the node sends another one for the missing parent
564 # of the real orphan child.
565 node.bumpmocktime(GETDATA_TX_INTERVAL)
566 honest_peer.wait_for_getdata([parent_txid_int])
567 honest_peer.send_and_ping(msg_tx(tx_parent["tx"]))
568 569 # 6. After parent is accepted, orphans should be reconsidered.
570 # The real child should be accepted and the fake one rejected. This may happen in either
571 # order since the message-processing is randomized. If tx_orphan_bad_wit is validated first,
572 # its consensus error leads to disconnection of bad_peer. If tx_child is validated first,
573 # tx_orphan_bad_wit is rejected for txn-same-nonwitness-data-in-mempool (no punishment).
574 node_mempool = node.getrawmempool()
575 assert tx_parent["txid"] in node_mempool
576 assert tx_child["txid"] in node_mempool
577 assert_equal(node.getmempoolentry(tx_child["txid"])["wtxid"], tx_child["wtxid"])
578 self.wait_until(lambda: len(node.getorphantxs()) == 0)
579 580 581 @cleanup
582 def test_orphan_handling_prefer_outbound(self):
583 self.log.info("Test that the node prefers requesting from outbound peers")
584 node = self.nodes[0]
585 orphan_wtxid, orphan_tx, parent_tx = self.create_parent_and_child()
586 orphan_inv = CInv(t=MSG_WTX, h=int(orphan_wtxid, 16))
587 588 peer_inbound = node.add_p2p_connection(PeerTxRelayer())
589 peer_outbound = node.add_outbound_p2p_connection(PeerTxRelayer(), p2p_idx=1)
590 591 # Inbound peer relays the transaction.
592 peer_inbound.send_and_ping(msg_inv([orphan_inv]))
593 self.nodes[0].bumpmocktime(TXREQUEST_TIME_SKIP)
594 peer_inbound.wait_for_getdata([int(orphan_wtxid, 16)])
595 596 # Both peers send invs for the orphan, so the node can expect both to know its ancestors.
597 peer_outbound.send_and_ping(msg_inv([orphan_inv]))
598 599 peer_inbound.send_and_ping(msg_tx(orphan_tx))
600 601 # There should be 1 orphan with 2 announcers (we don't know what their peer IDs are)
602 orphanage = node.getorphantxs(verbosity=2)
603 assert_equal(orphanage[0]["wtxid"], orphan_wtxid)
604 assert_equal(len(orphanage[0]["from"]), 2)
605 606 # The outbound peer should be preferred for getting orphan parents
607 self.nodes[0].bumpmocktime(TXID_RELAY_DELAY)
608 peer_outbound.wait_for_parent_requests([parent_tx.txid_int])
609 610 # There should be no request to the inbound peer
611 peer_inbound.assert_never_requested(parent_tx.txid_int)
612 613 self.log.info("Test that, if the preferred peer doesn't respond, the node sends another request")
614 self.nodes[0].bumpmocktime(GETDATA_TX_INTERVAL)
615 peer_inbound.sync_with_ping()
616 peer_inbound.wait_for_parent_requests([parent_tx.txid_int])
617 618 @cleanup
619 def test_maximal_package_protected(self):
620 self.log.info("Test that a node only announcing a maximally sized ancestor package is protected in orphanage")
621 self.nodes[0].setmocktime(int(time.time()))
622 node = self.nodes[0]
623 624 peer_normal = node.add_p2p_connection(P2PInterface())
625 peer_doser = node.add_p2p_connection(P2PInterface())
626 627 # Each of the num_peers peers creates a distinct set of orphans
628 large_orphans = [create_large_orphan() for _ in range(60)]
629 630 # Check to make sure these are orphans, within max standard size (to be accepted into the orphanage)
631 for large_orphan in large_orphans:
632 testres = node.testmempoolaccept([large_orphan.serialize().hex()])
633 assert not testres[0]["allowed"]
634 assert_equal(testres[0]["reject-reason"], "missing-inputs")
635 636 num_individual_dosers = 20
637 self.log.info(f"Connect {num_individual_dosers} peers and send a very large orphan from each one")
638 # This test assumes that unrequested transactions are processed (skipping inv and
639 # getdata steps because they require going through request delays)
640 # Connect 20 peers and have each of them send a large orphan.
641 for large_orphan in large_orphans[:num_individual_dosers]:
642 peer_doser_individual = node.add_p2p_connection(P2PInterface())
643 peer_doser_individual.send_and_ping(msg_tx(large_orphan))
644 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY + 1)
645 peer_doser_individual.wait_for_getdata([large_orphan.vin[0].prevout.hash])
646 647 # Make sure that these transactions are going through the orphan handling codepaths.
648 # Subsequent rounds will not wait for getdata because the time mocking will cause the
649 # normal package request to time out.
650 self.wait_until(lambda: len(node.getorphantxs()) == num_individual_dosers)
651 652 # Now honest peer will send a maximally sized ancestor package of 24 orphans chaining
653 # off of a single missing transaction, with a total vsize 404,000Wu
654 ancestor_package = self.wallet.create_self_transfer_chain(chain_length=DEFAULT_ANCESTOR_LIMIT - 1)
655 sum_ancestor_package_vsize = sum([tx["tx"].get_vsize() for tx in ancestor_package])
656 final_tx = self.wallet.create_self_transfer(utxo_to_spend=ancestor_package[-1]["new_utxo"], target_vsize=101000 - sum_ancestor_package_vsize)
657 ancestor_package.append(final_tx)
658 659 # Peer sends all but first tx to fill up orphange with their orphans
660 for orphan in ancestor_package[1:]:
661 peer_normal.send_and_ping(msg_tx(orphan["tx"]))
662 663 orphan_set = node.getorphantxs()
664 for orphan in ancestor_package[1:]:
665 assert orphan["txid"] in orphan_set
666 667 # Wait for ultimate parent request (the root ancestor transaction)
668 parent_txid_int = ancestor_package[0]["tx"].txid_int
669 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
670 671 self.wait_until(lambda: "getdata" in peer_normal.last_message and parent_txid_int in [inv.hash for inv in peer_normal.last_message.get("getdata").inv])
672 673 self.log.info("Send another round of very large orphans from a DoSy peer")
674 for large_orphan in large_orphans[num_individual_dosers:]:
675 peer_doser.send_and_ping(msg_tx(large_orphan))
676 677 self.log.info("Provide the top ancestor. The whole package should be re-evaluated after enough time.")
678 peer_normal.send_and_ping(msg_tx(ancestor_package[0]["tx"]))
679 680 # Wait until all transactions have been processed. When the last tx is accepted, it's
681 # guaranteed to have all ancestors.
682 self.wait_until(lambda: node.getmempoolentry(final_tx["txid"])["ancestorcount"] == DEFAULT_ANCESTOR_LIMIT)
683 684 @cleanup
685 def test_announcers_before_and_after(self):
686 self.log.info("Test that the node uses all peers who announced the tx prior to realizing it's an orphan")
687 node = self.nodes[0]
688 orphan_wtxid, orphan_tx, parent_tx = self.create_parent_and_child()
689 orphan_inv = CInv(t=MSG_WTX, h=int(orphan_wtxid, 16))
690 691 # Announces before tx is sent, disconnects while node is requesting parents
692 peer_early_disconnected = node.add_outbound_p2p_connection(PeerTxRelayer(), p2p_idx=3)
693 # Announces before tx is sent, doesn't respond to parent request
694 peer_early_unresponsive = node.add_p2p_connection(PeerTxRelayer())
695 696 # Announces after tx is sent
697 peer_late_announcer = node.add_p2p_connection(PeerTxRelayer())
698 699 # Both peers send invs for the orphan, so the node can expect both to know its ancestors.
700 peer_early_disconnected.send_and_ping(msg_inv([orphan_inv]))
701 self.nodes[0].bumpmocktime(TXREQUEST_TIME_SKIP)
702 peer_early_disconnected.wait_for_getdata([int(orphan_wtxid, 16)])
703 peer_early_unresponsive.send_and_ping(msg_inv([orphan_inv]))
704 peer_early_disconnected.send_and_ping(msg_tx(orphan_tx))
705 706 # There should be 1 orphan with 2 announcers (we don't know what their peer IDs are)
707 orphanage = node.getorphantxs(verbosity=2)
708 assert_equal(len(orphanage), 1)
709 assert_equal(orphanage[0]["wtxid"], orphan_wtxid)
710 assert_equal(len(orphanage[0]["from"]), 2)
711 712 # Peer disconnects before responding to request
713 self.nodes[0].bumpmocktime(TXID_RELAY_DELAY)
714 peer_early_disconnected.wait_for_parent_requests([parent_tx.txid_int])
715 peer_early_disconnected.peer_disconnect()
716 717 # The orphan should have 1 announcer left after the node finishes disconnecting peer_early_disconnected.
718 self.wait_until(lambda: len(node.getorphantxs(verbosity=2)[0]["from"]) == 1)
719 720 # The node should retry with the other peer that announced the orphan earlier.
721 # This node's request was additionally delayed because it's an inbound peer.
722 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY)
723 peer_early_unresponsive.wait_for_parent_requests([parent_tx.txid_int])
724 725 self.log.info("Test that the node uses peers who announce the tx after realizing it's an orphan")
726 peer_late_announcer.send_and_ping(msg_inv([orphan_inv]))
727 728 # The orphan should have 2 announcers now
729 orphanage = node.getorphantxs(verbosity=2)
730 assert_equal(orphanage[0]["wtxid"], orphan_wtxid)
731 assert_equal(len(orphanage[0]["from"]), 2)
732 733 self.nodes[0].bumpmocktime(GETDATA_TX_INTERVAL)
734 peer_late_announcer.wait_for_parent_requests([parent_tx.txid_int])
735 736 @cleanup
737 def test_parents_change(self):
738 self.log.info("Test that, if a parent goes missing during orphan reso, it is requested")
739 node = self.nodes[0]
740 # Orphan will have 2 parents, 1 missing and 1 already in mempool when received.
741 # Create missing parent.
742 parent_missing = self.wallet.create_self_transfer()
743 744 # Create parent that will already be in mempool, but become missing during orphan resolution.
745 # Get 3 UTXOs for replacement-cycled parent, UTXOS A, B, C
746 coin_A = self.wallet.get_utxo(confirmed_only=True)
747 coin_B = self.wallet.get_utxo(confirmed_only=True)
748 coin_C = self.wallet.get_utxo(confirmed_only=True)
749 # parent_peekaboo_AB spends A and B. It is replaced by tx_replacer_BC (conflicting UTXO B),
750 # and then replaced by tx_replacer_C (conflicting UTXO C). This replacement cycle is used to
751 # ensure that parent_peekaboo_AB can be reintroduced without requiring package RBF.
752 FEE_INCREMENT = 2400
753 parent_peekaboo_AB = self.wallet.create_self_transfer_multi(
754 utxos_to_spend=[coin_A, coin_B],
755 num_outputs=1,
756 fee_per_output=FEE_INCREMENT
757 )
758 tx_replacer_BC = self.wallet.create_self_transfer_multi(
759 utxos_to_spend=[coin_B, coin_C],
760 num_outputs=1,
761 fee_per_output=2*FEE_INCREMENT
762 )
763 tx_replacer_C = self.wallet.create_self_transfer(
764 utxo_to_spend=coin_C,
765 fee_per_output=3*FEE_INCREMENT
766 )
767 768 # parent_peekaboo_AB starts out in the mempool
769 node.sendrawtransaction(parent_peekaboo_AB["hex"])
770 771 orphan = self.wallet.create_self_transfer_multi(utxos_to_spend=[parent_peekaboo_AB["new_utxos"][0], parent_missing["new_utxo"]])
772 orphan_wtxid = orphan["wtxid"]
773 orphan_inv = CInv(t=MSG_WTX, h=int(orphan_wtxid, 16))
774 775 # peer1 sends the orphan and gets a request for the missing parent
776 peer1 = node.add_p2p_connection(PeerTxRelayer())
777 peer1.send_and_ping(msg_inv([orphan_inv]))
778 node.bumpmocktime(TXREQUEST_TIME_SKIP)
779 peer1.wait_for_getdata([int(orphan_wtxid, 16)])
780 peer1.send_and_ping(msg_tx(orphan["tx"]))
781 self.wait_until(lambda: node.getorphantxs(verbosity=0) == [orphan["txid"]])
782 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
783 peer1.wait_for_getdata([int(parent_missing["txid"], 16)])
784 785 # Replace parent_peekaboo_AB so that is a newly missing parent.
786 # Then, replace the replacement so that it can be resubmitted.
787 node.sendrawtransaction(tx_replacer_BC["hex"])
788 assert tx_replacer_BC["txid"] in node.getrawmempool()
789 node.sendrawtransaction(tx_replacer_C["hex"])
790 assert tx_replacer_BC["txid"] not in node.getrawmempool()
791 assert parent_peekaboo_AB["txid"] not in node.getrawmempool()
792 assert tx_replacer_C["txid"] in node.getrawmempool()
793 794 # Second peer is an additional announcer for this orphan, but its missing parents are different from when it was
795 # previously announced.
796 peer2 = node.add_p2p_connection(PeerTxRelayer())
797 peer2.send_and_ping(msg_inv([orphan_inv]))
798 assert_equal(len(node.getorphantxs(verbosity=2)[0]["from"]), 2)
799 800 # Disconnect peer1. peer2 should become the new candidate for orphan resolution.
801 peer1.peer_disconnect()
802 self.wait_until(lambda: node.num_test_p2p_connections() == 1)
803 peer2.sync_with_ping() # Sync with the 'net' thread which completes the disconnection fully
804 node.bumpmocktime(TXREQUEST_TIME_SKIP)
805 self.wait_until(lambda: len(node.getorphantxs(verbosity=2)[0]["from"]) == 1)
806 # Both parents should be requested, now that they are both missing.
807 peer2.wait_for_parent_requests([int(parent_peekaboo_AB["txid"], 16), int(parent_missing["txid"], 16)])
808 peer2.send_and_ping(msg_tx(parent_missing["tx"]))
809 peer2.send_and_ping(msg_tx(parent_peekaboo_AB["tx"]))
810 811 final_mempool = node.getrawmempool()
812 assert parent_missing["txid"] in final_mempool
813 assert parent_peekaboo_AB["txid"] in final_mempool
814 assert orphan["txid"] in final_mempool
815 assert tx_replacer_C["txid"] in final_mempool
816 817 def run_test(self):
818 self.nodes[0].setmocktime(int(time.time()))
819 self.wallet_nonsegwit = MiniWallet(self.nodes[0], mode=MiniWalletMode.RAW_P2PK)
820 self.generate(self.wallet_nonsegwit, 10)
821 self.wallet = MiniWallet(self.nodes[0])
822 self.generate(self.wallet, 160)
823 824 self.test_arrival_timing_orphan()
825 self.test_orphan_rejected_parents_exceptions()
826 self.test_orphan_multiple_parents()
827 self.test_orphans_overlapping_parents()
828 self.test_orphan_of_orphan()
829 self.test_orphan_inherit_rejection()
830 self.test_same_txid_orphan()
831 self.test_same_txid_orphan_of_orphan()
832 self.test_orphan_txid_inv()
833 self.test_orphan_handling_prefer_outbound()
834 self.test_announcers_before_and_after()
835 self.test_parents_change()
836 self.test_maximal_package_protected()
837 838 839 if __name__ == '__main__':
840 OrphanHandlingTest(__file__).main()
841