1 #!/usr/bin/env python3
2 # Copyright (c) 2023 The Limenka developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 6 import time
7 8 from test_framework.mempool_util import tx_in_orphanage
9 from test_framework.messages import (
10 CInv,
11 CTxInWitness,
12 MSG_TX,
13 MSG_WITNESS_TX,
14 MSG_WTX,
15 msg_getdata,
16 msg_inv,
17 msg_notfound,
18 msg_tx,
19 tx_from_hex,
20 )
21 from test_framework.p2p import (
22 GETDATA_TX_INTERVAL,
23 NONPREF_PEER_TX_DELAY,
24 OVERLOADED_PEER_TX_DELAY,
25 p2p_lock,
26 P2PInterface,
27 P2PTxInvStore,
28 TXID_RELAY_DELAY,
29 )
30 from test_framework.util import (
31 assert_equal,
32 )
33 from test_framework.test_framework import LimenkaTestFramework
34 from test_framework.wallet import (
35 MiniWallet,
36 MiniWalletMode,
37 )
38 39 # Time to bump forward (using setmocktime) before waiting for the node to send getdata(tx) in response
40 # to an inv(tx), in seconds. This delay includes all possible delays + 1, so it should only be used
41 # when the value of the delay is not interesting. If we want to test that the node waits x seconds
42 # for one peer and y seconds for another, use specific values instead.
43 TXREQUEST_TIME_SKIP = NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY + OVERLOADED_PEER_TX_DELAY + 1
44 45 DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100
46 47 def cleanup(func):
48 # Time to fastfoward (using setmocktime) in between subtests to ensure they do not interfere with
49 # one another, in seconds. Equal to 12 hours, which is enough to expire anything that may exist
50 # (though nothing should since state should be cleared) in p2p data structures.
51 LONG_TIME_SKIP = 12 * 60 * 60
52 53 def wrapper(self):
54 try:
55 func(self)
56 finally:
57 # Clear mempool
58 self.generate(self.nodes[0], 1)
59 self.nodes[0].disconnect_p2ps()
60 self.nodes[0].bumpmocktime(LONG_TIME_SKIP)
61 # Check that mempool and orphanage have been cleared
62 self.wait_until(lambda: len(self.nodes[0].getorphantxs()) == 0)
63 assert_equal(0, len(self.nodes[0].getrawmempool()))
64 self.wallet.rescan_utxos(include_mempool=True)
65 return wrapper
66 67 class PeerTxRelayer(P2PTxInvStore):
68 """A P2PTxInvStore that also remembers all of the getdata and tx messages it receives."""
69 def __init__(self, wtxidrelay=True):
70 super().__init__(wtxidrelay=wtxidrelay)
71 self._tx_received = []
72 self._getdata_received = []
73 74 @property
75 def tx_received(self):
76 with p2p_lock:
77 return self._tx_received
78 79 @property
80 def getdata_received(self):
81 with p2p_lock:
82 return self._getdata_received
83 84 def on_tx(self, message):
85 self._tx_received.append(message)
86 87 def on_getdata(self, message):
88 self._getdata_received.append(message)
89 90 def wait_for_parent_requests(self, txids):
91 """Wait for requests for missing parents by txid with witness data (MSG_WITNESS_TX or
92 WitnessTx). Requires that the getdata message match these txids exactly; all txids must be
93 requested and no additional requests are allowed."""
94 def test_function():
95 last_getdata = self.last_message.get('getdata')
96 if not last_getdata:
97 return False
98 return len(last_getdata.inv) == len(txids) and all([item.type == MSG_WITNESS_TX and item.hash in txids for item in last_getdata.inv])
99 self.wait_until(test_function, timeout=10)
100 101 def assert_no_immediate_response(self, message):
102 """Check that the node does not immediately respond to this message with any of getdata,
103 inv, tx. The node may respond later.
104 """
105 prev_lastmessage = self.last_message
106 self.send_and_ping(message)
107 after_lastmessage = self.last_message
108 for msgtype in ["getdata", "inv", "tx"]:
109 if msgtype not in prev_lastmessage:
110 assert msgtype not in after_lastmessage
111 else:
112 assert_equal(prev_lastmessage[msgtype], after_lastmessage[msgtype])
113 114 def assert_never_requested(self, txhash):
115 """Check that the node has never sent us a getdata for this hash (int type)"""
116 self.sync_with_ping()
117 for getdata in self.getdata_received:
118 for request in getdata.inv:
119 assert request.hash != txhash
120 121 class OrphanHandlingTest(LimenkaTestFramework):
122 def set_test_params(self):
123 self.num_nodes = 1
124 self.extra_args = [[]]
125 126 def create_parent_and_child(self):
127 """Create package with 1 parent and 1 child, normal fees (no cpfp)."""
128 parent = self.wallet.create_self_transfer()
129 child = self.wallet.create_self_transfer(utxo_to_spend=parent['new_utxo'])
130 return child["tx"].getwtxid(), child["tx"], parent["tx"]
131 132 def relay_transaction(self, peer, tx):
133 """Relay transaction using MSG_WTX"""
134 wtxid = int(tx.getwtxid(), 16)
135 peer.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=wtxid)]))
136 self.nodes[0].bumpmocktime(TXREQUEST_TIME_SKIP)
137 peer.wait_for_getdata([wtxid])
138 peer.send_and_ping(msg_tx(tx))
139 140 def create_malleated_version(self, tx):
141 """
142 Create a malleated version of the tx where the witness is replaced with garbage data.
143 Returns a CTransaction object.
144 """
145 tx_bad_wit = tx_from_hex(tx["hex"])
146 tx_bad_wit.wit.vtxinwit = [CTxInWitness()]
147 # Add garbage data to witness 0. We cannot simply strip the witness, as the node would
148 # classify it as a transaction in which the witness was missing rather than wrong.
149 tx_bad_wit.wit.vtxinwit[0].scriptWitness.stack = [b'garbage']
150 151 assert_equal(tx["txid"], tx_bad_wit.rehash())
152 assert tx["wtxid"] != tx_bad_wit.getwtxid()
153 154 return tx_bad_wit
155 156 @cleanup
157 def test_arrival_timing_orphan(self):
158 self.log.info("Test missing parents that arrive during delay are not requested")
159 node = self.nodes[0]
160 tx_parent_arrives = self.wallet.create_self_transfer()
161 tx_parent_doesnt_arrive = self.wallet.create_self_transfer()
162 # Fake orphan spends nonexistent outputs of the two parents
163 tx_fake_orphan = self.wallet.create_self_transfer_multi(utxos_to_spend=[
164 {"txid": tx_parent_doesnt_arrive["txid"], "vout": 10, "value": tx_parent_doesnt_arrive["new_utxo"]["value"]},
165 {"txid": tx_parent_arrives["txid"], "vout": 10, "value": tx_parent_arrives["new_utxo"]["value"]}
166 ])
167 168 peer_spy = node.add_p2p_connection(PeerTxRelayer())
169 peer_normal = node.add_p2p_connection(PeerTxRelayer())
170 # This transaction is an orphan because it is missing inputs. It is a "fake" orphan that the
171 # spy peer has crafted to learn information about tx_parent_arrives even though it isn't
172 # able to spend a real output of it, but it could also just be a normal, real child tx.
173 # The node should not immediately respond with a request for orphan parents.
174 # Also, no request should be sent later because it will be resolved by
175 # the time the request is scheduled to be sent.
176 peer_spy.assert_no_immediate_response(msg_tx(tx_fake_orphan["tx"]))
177 178 # Node receives transaction. It attempts to obfuscate the exact timing at which this
179 # transaction entered its mempool. Send unsolicited because otherwise we need to wait for
180 # request delays.
181 peer_normal.send_and_ping(msg_tx(tx_parent_arrives["tx"]))
182 assert tx_parent_arrives["txid"] in node.getrawmempool()
183 184 # Spy peer should not be able to query the node for the parent yet, since it hasn't been
185 # announced / insufficient time has elapsed.
186 parent_inv = CInv(t=MSG_WTX, h=int(tx_parent_arrives["tx"].getwtxid(), 16))
187 assert_equal(len(peer_spy.get_invs()), 0)
188 peer_spy.assert_no_immediate_response(msg_getdata([parent_inv]))
189 190 # Request would be scheduled with this delay because it is not a preferred relay peer.
191 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY)
192 peer_spy.assert_never_requested(int(tx_parent_arrives["txid"], 16))
193 peer_spy.assert_never_requested(int(tx_parent_doesnt_arrive["txid"], 16))
194 # Request would be scheduled with this delay because it is by txid.
195 self.nodes[0].bumpmocktime(TXID_RELAY_DELAY)
196 peer_spy.wait_for_parent_requests([int(tx_parent_doesnt_arrive["txid"], 16)])
197 peer_spy.assert_never_requested(int(tx_parent_arrives["txid"], 16))
198 199 @cleanup
200 def test_orphan_rejected_parents_exceptions(self):
201 node = self.nodes[0]
202 peer1 = node.add_p2p_connection(PeerTxRelayer())
203 peer2 = node.add_p2p_connection(PeerTxRelayer())
204 205 self.log.info("Test orphan handling when a nonsegwit parent is known to be invalid")
206 parent_low_fee_nonsegwit = self.wallet_nonsegwit.create_self_transfer(fee_rate=0)
207 assert_equal(parent_low_fee_nonsegwit["txid"], parent_low_fee_nonsegwit["tx"].getwtxid())
208 parent_other = self.wallet_nonsegwit.create_self_transfer()
209 child_nonsegwit = self.wallet_nonsegwit.create_self_transfer_multi(
210 utxos_to_spend=[parent_other["new_utxo"], parent_low_fee_nonsegwit["new_utxo"]])
211 212 # Relay the parent. It should be rejected because it pays 0 fees.
213 self.relay_transaction(peer1, parent_low_fee_nonsegwit["tx"])
214 assert parent_low_fee_nonsegwit["txid"] not in node.getrawmempool()
215 216 # Relay the child. It should not be accepted because it has missing inputs.
217 # Its parent should not be requested because its hash (txid == wtxid) has been added to the rejection filter.
218 self.relay_transaction(peer2, child_nonsegwit["tx"])
219 assert child_nonsegwit["txid"] not in node.getrawmempool()
220 assert not tx_in_orphanage(node, child_nonsegwit["tx"])
221 222 # No parents are requested.
223 self.nodes[0].bumpmocktime(GETDATA_TX_INTERVAL)
224 peer1.assert_never_requested(int(parent_other["txid"], 16))
225 peer2.assert_never_requested(int(parent_other["txid"], 16))
226 peer2.assert_never_requested(int(parent_low_fee_nonsegwit["txid"], 16))
227 228 self.log.info("Test orphan handling when a segwit parent was invalid but may be retried with another witness")
229 parent_low_fee = self.wallet.create_self_transfer(fee_rate=0)
230 child_low_fee = self.wallet.create_self_transfer(utxo_to_spend=parent_low_fee["new_utxo"])
231 232 # Relay the low fee parent. It should not be accepted.
233 self.relay_transaction(peer1, parent_low_fee["tx"])
234 assert parent_low_fee["txid"] not in node.getrawmempool()
235 236 # Relay the child. It should not be accepted because it has missing inputs.
237 self.relay_transaction(peer2, child_low_fee["tx"])
238 assert child_low_fee["txid"] not in node.getrawmempool()
239 assert tx_in_orphanage(node, child_low_fee["tx"])
240 241 # The parent should be requested because even though the txid commits to the fee, it doesn't
242 # commit to the feerate. Delayed because it's by txid and this is not a preferred relay peer.
243 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
244 peer2.wait_for_getdata([int(parent_low_fee["tx"].rehash(), 16)])
245 246 self.log.info("Test orphan handling when a parent was previously downloaded with witness stripped")
247 parent_normal = self.wallet.create_self_transfer()
248 parent1_witness_stripped = tx_from_hex(parent_normal["tx"].serialize_without_witness().hex())
249 child_invalid_witness = self.wallet.create_self_transfer(utxo_to_spend=parent_normal["new_utxo"])
250 251 # Relay the parent with witness stripped. It should not be accepted.
252 self.relay_transaction(peer1, parent1_witness_stripped)
253 assert_equal(parent_normal["txid"], parent1_witness_stripped.rehash())
254 assert parent1_witness_stripped.rehash() not in node.getrawmempool()
255 256 # Relay the child. It should not be accepted because it has missing inputs.
257 self.relay_transaction(peer2, child_invalid_witness["tx"])
258 assert child_invalid_witness["txid"] not in node.getrawmempool()
259 assert tx_in_orphanage(node, child_invalid_witness["tx"])
260 261 # The parent should be requested since the unstripped wtxid would differ. Delayed because
262 # it's by txid and this is not a preferred relay peer.
263 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
264 peer2.wait_for_getdata([int(parent_normal["tx"].rehash(), 16)])
265 266 # parent_normal can be relayed again even though parent1_witness_stripped was rejected
267 self.relay_transaction(peer1, parent_normal["tx"])
268 assert_equal(set(node.getrawmempool()), set([parent_normal["txid"], child_invalid_witness["txid"]]))
269 270 @cleanup
271 def test_orphan_multiple_parents(self):
272 node = self.nodes[0]
273 peer = node.add_p2p_connection(PeerTxRelayer())
274 275 self.log.info("Test orphan parent requests with a mixture of confirmed, in-mempool and missing parents")
276 # This UTXO confirmed a long time ago.
277 utxo_conf_old = self.wallet.send_self_transfer(from_node=node)["new_utxo"]
278 txid_conf_old = utxo_conf_old["txid"]
279 self.generate(self.wallet, 10)
280 281 # Create a fake reorg to trigger BlockDisconnected, which resets the rolling bloom filter.
282 # The alternative is to mine thousands of transactions to push it out of the filter.
283 last_block = node.getbestblockhash()
284 node.invalidateblock(last_block)
285 node.preciousblock(last_block)
286 node.syncwithvalidationinterfacequeue()
287 288 # This UTXO confirmed recently.
289 utxo_conf_recent = self.wallet.send_self_transfer(from_node=node)["new_utxo"]
290 self.generate(node, 1)
291 292 # This UTXO is unconfirmed and in the mempool.
293 assert_equal(len(node.getrawmempool()), 0)
294 mempool_tx = self.wallet.send_self_transfer(from_node=node)
295 utxo_unconf_mempool = mempool_tx["new_utxo"]
296 297 # This UTXO is unconfirmed and missing.
298 missing_tx = self.wallet.create_self_transfer()
299 utxo_unconf_missing = missing_tx["new_utxo"]
300 assert missing_tx["txid"] not in node.getrawmempool()
301 302 orphan = self.wallet.create_self_transfer_multi(utxos_to_spend=[utxo_conf_old,
303 utxo_conf_recent, utxo_unconf_mempool, utxo_unconf_missing])
304 305 self.relay_transaction(peer, orphan["tx"])
306 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
307 peer.sync_with_ping()
308 assert tx_in_orphanage(node, orphan["tx"])
309 assert_equal(len(peer.last_message["getdata"].inv), 2)
310 peer.wait_for_parent_requests([int(txid_conf_old, 16), int(missing_tx["txid"], 16)])
311 312 # Even though the peer would send a notfound for the "old" confirmed transaction, the node
313 # doesn't give up on the orphan. Once all of the missing parents are received, it should be
314 # submitted to mempool.
315 peer.send_message(msg_notfound(vec=[CInv(MSG_WITNESS_TX, int(txid_conf_old, 16))]))
316 # Sync with ping to ensure orphans are reconsidered
317 peer.send_and_ping(msg_tx(missing_tx["tx"]))
318 assert_equal(node.getmempoolentry(orphan["txid"])["ancestorcount"], 3)
319 320 @cleanup
321 def test_orphans_overlapping_parents(self):
322 node = self.nodes[0]
323 # In the process of relaying inflight_parent_AB
324 peer_txrequest = node.add_p2p_connection(PeerTxRelayer())
325 # Sends the orphans
326 peer_orphans = node.add_p2p_connection(PeerTxRelayer())
327 328 confirmed_utxos = [self.wallet_nonsegwit.get_utxo() for _ in range(4)]
329 assert all([utxo["confirmations"] > 0 for utxo in confirmed_utxos])
330 self.log.info("Test handling of multiple orphans with missing parents that are already being requested")
331 # Parent of child_A only
332 missing_parent_A = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=confirmed_utxos[0])
333 # Parents of child_A and child_B
334 missing_parent_AB = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=confirmed_utxos[1])
335 inflight_parent_AB = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=confirmed_utxos[2])
336 # Parent of child_B only
337 missing_parent_B = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=confirmed_utxos[3])
338 child_A = self.wallet_nonsegwit.create_self_transfer_multi(
339 utxos_to_spend=[missing_parent_A["new_utxo"], missing_parent_AB["new_utxo"], inflight_parent_AB["new_utxo"]]
340 )
341 child_B = self.wallet_nonsegwit.create_self_transfer_multi(
342 utxos_to_spend=[missing_parent_B["new_utxo"], missing_parent_AB["new_utxo"], inflight_parent_AB["new_utxo"]]
343 )
344 345 # The wtxid and txid need to be the same for the node to recognize that the missing input
346 # and in-flight request for inflight_parent_AB are the same transaction.
347 assert_equal(inflight_parent_AB["txid"], inflight_parent_AB["wtxid"])
348 349 # Announce inflight_parent_AB and wait for getdata
350 peer_txrequest.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=int(inflight_parent_AB["tx"].getwtxid(), 16))]))
351 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY)
352 peer_txrequest.wait_for_getdata([int(inflight_parent_AB["tx"].getwtxid(), 16)])
353 354 self.log.info("Test that the node does not request a parent if it has an in-flight txrequest")
355 # Relay orphan child_A
356 self.relay_transaction(peer_orphans, child_A["tx"])
357 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
358 assert tx_in_orphanage(node, child_A["tx"])
359 # There are 3 missing parents. missing_parent_A and missing_parent_AB should be requested.
360 # But inflight_parent_AB should not, because there is already an in-flight request for it.
361 peer_orphans.wait_for_parent_requests([int(missing_parent_A["txid"], 16), int(missing_parent_AB["txid"], 16)])
362 363 self.log.info("Test that the node does not request a parent if it has an in-flight orphan parent request")
364 # Relay orphan child_B
365 self.relay_transaction(peer_orphans, child_B["tx"])
366 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
367 assert tx_in_orphanage(node, child_B["tx"])
368 # Only missing_parent_B should be requested. Not inflight_parent_AB or missing_parent_AB
369 # because they are already being requested from peer_txrequest and peer_orphans respectively.
370 peer_orphans.wait_for_parent_requests([int(missing_parent_B["txid"], 16)])
371 peer_orphans.assert_never_requested(int(inflight_parent_AB["txid"], 16))
372 373 # But inflight_parent_AB will be requested eventually if original peer doesn't respond
374 node.bumpmocktime(GETDATA_TX_INTERVAL)
375 peer_orphans.wait_for_parent_requests([int(inflight_parent_AB["txid"], 16)])
376 377 @cleanup
378 def test_orphan_of_orphan(self):
379 node = self.nodes[0]
380 peer = node.add_p2p_connection(PeerTxRelayer())
381 382 self.log.info("Test handling of an orphan with a parent who is another orphan")
383 missing_grandparent = self.wallet_nonsegwit.create_self_transfer()
384 missing_parent_orphan = self.wallet_nonsegwit.create_self_transfer(utxo_to_spend=missing_grandparent["new_utxo"])
385 missing_parent = self.wallet_nonsegwit.create_self_transfer()
386 orphan = self.wallet_nonsegwit.create_self_transfer_multi(utxos_to_spend=[missing_parent["new_utxo"], missing_parent_orphan["new_utxo"]])
387 388 # The node should put missing_parent_orphan into the orphanage and request missing_grandparent
389 self.relay_transaction(peer, missing_parent_orphan["tx"])
390 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
391 assert tx_in_orphanage(node, missing_parent_orphan["tx"])
392 peer.wait_for_parent_requests([int(missing_grandparent["txid"], 16)])
393 394 # The node should put the orphan into the orphanage and request missing_parent, skipping
395 # missing_parent_orphan because it already has it in the orphanage.
396 self.relay_transaction(peer, orphan["tx"])
397 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
398 assert tx_in_orphanage(node, orphan["tx"])
399 peer.wait_for_parent_requests([int(missing_parent["txid"], 16)])
400 401 @cleanup
402 def test_orphan_inherit_rejection(self):
403 node = self.nodes[0]
404 peer1 = node.add_p2p_connection(PeerTxRelayer())
405 peer2 = node.add_p2p_connection(PeerTxRelayer())
406 peer3 = node.add_p2p_connection(PeerTxRelayer(wtxidrelay=False))
407 408 self.log.info("Test that an orphan with rejected parents, along with any descendants, cannot be retried with an alternate witness")
409 parent_low_fee_nonsegwit = self.wallet_nonsegwit.create_self_transfer(fee_rate=0)
410 assert_equal(parent_low_fee_nonsegwit["txid"], parent_low_fee_nonsegwit["tx"].getwtxid())
411 child = self.wallet.create_self_transfer(utxo_to_spend=parent_low_fee_nonsegwit["new_utxo"])
412 grandchild = self.wallet.create_self_transfer(utxo_to_spend=child["new_utxo"])
413 assert child["txid"] != child["tx"].getwtxid()
414 assert grandchild["txid"] != grandchild["tx"].getwtxid()
415 416 # Relay the parent. It should be rejected because it pays 0 fees.
417 self.relay_transaction(peer1, parent_low_fee_nonsegwit["tx"])
418 assert parent_low_fee_nonsegwit["txid"] not in node.getrawmempool()
419 420 # Relay the child. It should be rejected for having missing parents, and this rejection is
421 # cached by txid and wtxid.
422 self.relay_transaction(peer1, child["tx"])
423 assert_equal(0, len(node.getrawmempool()))
424 assert not tx_in_orphanage(node, child["tx"])
425 peer1.assert_never_requested(parent_low_fee_nonsegwit["txid"])
426 427 # Grandchild should also not be kept in orphanage because its parent has been rejected.
428 self.relay_transaction(peer2, grandchild["tx"])
429 assert_equal(0, len(node.getrawmempool()))
430 assert not tx_in_orphanage(node, grandchild["tx"])
431 peer2.assert_never_requested(child["txid"])
432 peer2.assert_never_requested(child["tx"].getwtxid())
433 434 # The child should never be requested, even if announced again with potentially different witness.
435 # Sync with ping to ensure orphans are reconsidered
436 peer3.send_and_ping(msg_inv([CInv(t=MSG_TX, h=int(child["txid"], 16))]))
437 self.nodes[0].bumpmocktime(TXREQUEST_TIME_SKIP)
438 peer3.assert_never_requested(child["txid"])
439 440 @cleanup
441 def test_same_txid_orphan(self):
442 self.log.info("Check what happens when orphan with same txid is already in orphanage")
443 node = self.nodes[0]
444 445 tx_parent = self.wallet.create_self_transfer()
446 447 # Create the real child
448 tx_child = self.wallet.create_self_transfer(utxo_to_spend=tx_parent["new_utxo"])
449 450 # Create a fake version of the child
451 tx_orphan_bad_wit = self.create_malleated_version(tx_child)
452 453 bad_peer = node.add_p2p_connection(P2PInterface())
454 honest_peer = node.add_p2p_connection(P2PInterface())
455 456 # 1. Fake orphan is received first. It is missing an input.
457 bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit))
458 assert tx_in_orphanage(node, tx_orphan_bad_wit)
459 460 # 2. Node requests the missing parent by txid.
461 parent_txid_int = int(tx_parent["txid"], 16)
462 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
463 bad_peer.wait_for_getdata([parent_txid_int])
464 465 # 3. Honest peer relays the real child, which is also missing parents and should be placed
466 # in the orphanage.
467 with node.assert_debug_log(["missingorspent"]):
468 honest_peer.send_and_ping(msg_tx(tx_child["tx"]))
469 assert tx_in_orphanage(node, tx_child["tx"])
470 471 # Time out the previous request for the parent (node will not request the same transaction
472 # from multiple nodes at the same time)
473 node.bumpmocktime(GETDATA_TX_INTERVAL)
474 475 # 4. The parent is requested. Honest peer sends it.
476 honest_peer.wait_for_getdata([parent_txid_int])
477 # Sync with ping to ensure orphans are reconsidered
478 honest_peer.send_and_ping(msg_tx(tx_parent["tx"]))
479 480 # 5. After parent is accepted, orphans should be reconsidered.
481 # The real child should be accepted and the fake one rejected.
482 node_mempool = node.getrawmempool()
483 assert tx_parent["txid"] in node_mempool
484 assert tx_child["txid"] in node_mempool
485 assert_equal(node.getmempoolentry(tx_child["txid"])["wtxid"], tx_child["wtxid"])
486 487 @cleanup
488 def test_same_txid_orphan_of_orphan(self):
489 self.log.info("Check what happens when orphan's parent with same txid is already in orphanage")
490 node = self.nodes[0]
491 492 tx_grandparent = self.wallet.create_self_transfer()
493 494 # Create middle tx (both parent and child) which will be in orphanage.
495 tx_middle = self.wallet.create_self_transfer(utxo_to_spend=tx_grandparent["new_utxo"])
496 497 # Create a fake version of the middle tx
498 tx_orphan_bad_wit = self.create_malleated_version(tx_middle)
499 500 # Create grandchild spending from tx_middle (and spending from tx_orphan_bad_wit since they
501 # have the same txid).
502 tx_grandchild = self.wallet.create_self_transfer(utxo_to_spend=tx_middle["new_utxo"])
503 504 bad_peer = node.add_p2p_connection(P2PInterface())
505 honest_peer = node.add_p2p_connection(P2PInterface())
506 507 # 1. Fake orphan is received first. It is missing an input.
508 bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit))
509 assert tx_in_orphanage(node, tx_orphan_bad_wit)
510 511 # 2. Node requests missing tx_grandparent by txid.
512 grandparent_txid_int = int(tx_grandparent["txid"], 16)
513 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
514 bad_peer.wait_for_getdata([grandparent_txid_int])
515 516 # 3. Honest peer relays the grandchild, which is missing a parent. The parent by txid already
517 # exists in orphanage, but should be re-requested because the node shouldn't assume that the
518 # witness data is the same. In this case, a same-txid-different-witness transaction exists!
519 honest_peer.send_and_ping(msg_tx(tx_grandchild["tx"]))
520 assert tx_in_orphanage(node, tx_grandchild["tx"])
521 middle_txid_int = int(tx_middle["txid"], 16)
522 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
523 honest_peer.wait_for_getdata([middle_txid_int])
524 525 # 4. Honest peer relays the real child, which is also missing parents and should be placed
526 # in the orphanage.
527 honest_peer.send_and_ping(msg_tx(tx_middle["tx"]))
528 assert tx_in_orphanage(node, tx_middle["tx"])
529 assert_equal(len(node.getrawmempool()), 0)
530 531 # 5. Honest peer sends tx_grandparent
532 honest_peer.send_and_ping(msg_tx(tx_grandparent["tx"]))
533 534 # 6. After parent is accepted, orphans should be reconsidered.
535 # The real child should be accepted and the fake one rejected.
536 node_mempool = node.getrawmempool()
537 assert tx_grandparent["txid"] in node_mempool
538 assert tx_middle["txid"] in node_mempool
539 assert tx_grandchild["txid"] in node_mempool
540 assert_equal(node.getmempoolentry(tx_middle["txid"])["wtxid"], tx_middle["wtxid"])
541 self.wait_until(lambda: len(node.getorphantxs()) == 0)
542 543 @cleanup
544 def test_orphan_txid_inv(self):
545 self.log.info("Check node does not ignore announcement with same txid as tx in orphanage")
546 node = self.nodes[0]
547 548 tx_parent = self.wallet.create_self_transfer()
549 550 # Create the real child and fake version
551 tx_child = self.wallet.create_self_transfer(utxo_to_spend=tx_parent["new_utxo"])
552 tx_orphan_bad_wit = self.create_malleated_version(tx_child)
553 554 bad_peer = node.add_p2p_connection(PeerTxRelayer())
555 # Must not send wtxidrelay because otherwise the inv(TX) will be ignored later
556 honest_peer = node.add_p2p_connection(P2PInterface(wtxidrelay=False))
557 558 # 1. Fake orphan is received first. It is missing an input.
559 bad_peer.send_and_ping(msg_tx(tx_orphan_bad_wit))
560 assert tx_in_orphanage(node, tx_orphan_bad_wit)
561 562 # 2. Node requests the missing parent by txid.
563 parent_txid_int = int(tx_parent["txid"], 16)
564 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
565 bad_peer.wait_for_getdata([parent_txid_int])
566 567 # 3. Honest peer announces the real child, by txid (this isn't common but the node should
568 # still keep track of it).
569 child_txid_int = int(tx_child["txid"], 16)
570 honest_peer.send_and_ping(msg_inv([CInv(t=MSG_TX, h=child_txid_int)]))
571 572 # 4. The child is requested. Honest peer sends it.
573 node.bumpmocktime(TXREQUEST_TIME_SKIP)
574 honest_peer.wait_for_getdata([child_txid_int])
575 honest_peer.send_and_ping(msg_tx(tx_child["tx"]))
576 assert tx_in_orphanage(node, tx_child["tx"])
577 578 # 5. After first parent request times out, the node sends another one for the missing parent
579 # of the real orphan child.
580 node.bumpmocktime(GETDATA_TX_INTERVAL)
581 honest_peer.wait_for_getdata([parent_txid_int])
582 honest_peer.send_and_ping(msg_tx(tx_parent["tx"]))
583 584 # 6. After parent is accepted, orphans should be reconsidered.
585 # The real child should be accepted and the fake one rejected. This may happen in either
586 # order since the message-processing is randomized. If tx_orphan_bad_wit is validated first,
587 # its consensus error leads to disconnection of bad_peer. If tx_child is validated first,
588 # tx_orphan_bad_wit is rejected for txn-same-nonwitness-data-in-mempool (no punishment).
589 node_mempool = node.getrawmempool()
590 assert tx_parent["txid"] in node_mempool
591 assert tx_child["txid"] in node_mempool
592 assert_equal(node.getmempoolentry(tx_child["txid"])["wtxid"], tx_child["wtxid"])
593 self.wait_until(lambda: len(node.getorphantxs()) == 0)
594 595 @cleanup
596 def test_max_orphan_amount(self):
597 self.log.info("Check that we never exceed our storage limits for orphans")
598 599 node = self.nodes[0]
600 self.generate(self.wallet, 1)
601 peer_1 = node.add_p2p_connection(P2PInterface())
602 603 self.log.info("Check that orphanage is empty on start of test")
604 assert len(node.getorphantxs()) == 0
605 606 self.log.info("Filling up orphanage with " + str(DEFAULT_MAX_ORPHAN_TRANSACTIONS) + "(DEFAULT_MAX_ORPHAN_TRANSACTIONS) orphans")
607 orphans = []
608 parent_orphans = []
609 for _ in range(DEFAULT_MAX_ORPHAN_TRANSACTIONS):
610 tx_parent_1 = self.wallet.create_self_transfer()
611 tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"])
612 parent_orphans.append(tx_parent_1["tx"])
613 orphans.append(tx_child_1["tx"])
614 peer_1.send_message(msg_tx(tx_child_1["tx"]))
615 616 peer_1.sync_with_ping()
617 orphanage = node.getorphantxs()
618 self.wait_until(lambda: len(node.getorphantxs()) == DEFAULT_MAX_ORPHAN_TRANSACTIONS)
619 620 for orphan in orphans:
621 assert tx_in_orphanage(node, orphan)
622 623 self.log.info("Check that we do not add more than the max orphan amount")
624 tx_parent_1 = self.wallet.create_self_transfer()
625 tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"])
626 peer_1.send_and_ping(msg_tx(tx_child_1["tx"]))
627 parent_orphans.append(tx_parent_1["tx"])
628 orphanage = node.getorphantxs()
629 assert_equal(len(orphanage), DEFAULT_MAX_ORPHAN_TRANSACTIONS)
630 631 self.log.info("Clearing the orphanage")
632 for index, parent_orphan in enumerate(parent_orphans):
633 peer_1.send_and_ping(msg_tx(parent_orphan))
634 self.wait_until(lambda: len(node.getorphantxs()) == 0)
635 636 @cleanup
637 def test_orphan_handling_prefer_outbound(self):
638 self.log.info("Test that the node prefers requesting from outbound peers")
639 node = self.nodes[0]
640 orphan_wtxid, orphan_tx, parent_tx = self.create_parent_and_child()
641 orphan_inv = CInv(t=MSG_WTX, h=int(orphan_wtxid, 16))
642 643 peer_inbound = node.add_p2p_connection(PeerTxRelayer())
644 peer_outbound = node.add_outbound_p2p_connection(PeerTxRelayer(), p2p_idx=1)
645 646 # Inbound peer relays the transaction.
647 peer_inbound.send_and_ping(msg_inv([orphan_inv]))
648 self.nodes[0].bumpmocktime(TXREQUEST_TIME_SKIP)
649 peer_inbound.wait_for_getdata([int(orphan_wtxid, 16)])
650 651 # Both peers send invs for the orphan, so the node can expect both to know its ancestors.
652 peer_outbound.send_and_ping(msg_inv([orphan_inv]))
653 654 peer_inbound.send_and_ping(msg_tx(orphan_tx))
655 656 # There should be 1 orphan with 2 announcers (we don't know what their peer IDs are)
657 orphanage = node.getorphantxs(verbosity=2)
658 assert_equal(orphanage[0]["wtxid"], orphan_wtxid)
659 assert_equal(len(orphanage[0]["from"]), 2)
660 661 # The outbound peer should be preferred for getting orphan parents
662 self.nodes[0].bumpmocktime(TXID_RELAY_DELAY)
663 peer_outbound.wait_for_parent_requests([int(parent_tx.rehash(), 16)])
664 665 # There should be no request to the inbound peer
666 peer_inbound.assert_never_requested(int(parent_tx.rehash(), 16))
667 668 self.log.info("Test that, if the preferred peer doesn't respond, the node sends another request")
669 self.nodes[0].bumpmocktime(GETDATA_TX_INTERVAL)
670 peer_inbound.sync_with_ping()
671 peer_inbound.wait_for_parent_requests([int(parent_tx.rehash(), 16)])
672 673 @cleanup
674 def test_announcers_before_and_after(self):
675 self.log.info("Test that the node uses all peers who announced the tx prior to realizing it's an orphan")
676 node = self.nodes[0]
677 orphan_wtxid, orphan_tx, parent_tx = self.create_parent_and_child()
678 orphan_inv = CInv(t=MSG_WTX, h=int(orphan_wtxid, 16))
679 680 # Announces before tx is sent, disconnects while node is requesting parents
681 peer_early_disconnected = node.add_outbound_p2p_connection(PeerTxRelayer(), p2p_idx=3)
682 # Announces before tx is sent, doesn't respond to parent request
683 peer_early_unresponsive = node.add_p2p_connection(PeerTxRelayer())
684 685 # Announces after tx is sent
686 peer_late_announcer = node.add_p2p_connection(PeerTxRelayer())
687 688 # Both peers send invs for the orphan, so the node can expect both to know its ancestors.
689 peer_early_disconnected.send_and_ping(msg_inv([orphan_inv]))
690 self.nodes[0].bumpmocktime(TXREQUEST_TIME_SKIP)
691 peer_early_disconnected.wait_for_getdata([int(orphan_wtxid, 16)])
692 peer_early_unresponsive.send_and_ping(msg_inv([orphan_inv]))
693 peer_early_disconnected.send_and_ping(msg_tx(orphan_tx))
694 695 # There should be 1 orphan with 2 announcers (we don't know what their peer IDs are)
696 orphanage = node.getorphantxs(verbosity=2)
697 assert_equal(len(orphanage), 1)
698 assert_equal(orphanage[0]["wtxid"], orphan_wtxid)
699 assert_equal(len(orphanage[0]["from"]), 2)
700 701 # Peer disconnects before responding to request
702 self.nodes[0].bumpmocktime(TXID_RELAY_DELAY)
703 peer_early_disconnected.wait_for_parent_requests([int(parent_tx.rehash(), 16)])
704 peer_early_disconnected.peer_disconnect()
705 706 # The orphan should have 1 announcer left after the node finishes disconnecting peer_early_disconnected.
707 self.wait_until(lambda: len(node.getorphantxs(verbosity=2)[0]["from"]) == 1)
708 709 # The node should retry with the other peer that announced the orphan earlier.
710 # This node's request was additionally delayed because it's an inbound peer.
711 self.nodes[0].bumpmocktime(NONPREF_PEER_TX_DELAY)
712 peer_early_unresponsive.wait_for_parent_requests([int(parent_tx.rehash(), 16)])
713 714 self.log.info("Test that the node uses peers who announce the tx after realizing it's an orphan")
715 peer_late_announcer.send_and_ping(msg_inv([orphan_inv]))
716 717 # The orphan should have 2 announcers now
718 orphanage = node.getorphantxs(verbosity=2)
719 assert_equal(orphanage[0]["wtxid"], orphan_wtxid)
720 assert_equal(len(orphanage[0]["from"]), 2)
721 722 self.nodes[0].bumpmocktime(GETDATA_TX_INTERVAL)
723 peer_late_announcer.wait_for_parent_requests([int(parent_tx.rehash(), 16)])
724 725 @cleanup
726 def test_parents_change(self):
727 self.log.info("Test that, if a parent goes missing during orphan reso, it is requested")
728 node = self.nodes[0]
729 # Orphan will have 2 parents, 1 missing and 1 already in mempool when received.
730 # Create missing parent.
731 parent_missing = self.wallet.create_self_transfer()
732 733 # Create parent that will already be in mempool, but become missing during orphan resolution.
734 # Get 3 UTXOs for replacement-cycled parent, UTXOS A, B, C
735 coin_A = self.wallet.get_utxo(confirmed_only=True)
736 coin_B = self.wallet.get_utxo(confirmed_only=True)
737 coin_C = self.wallet.get_utxo(confirmed_only=True)
738 # parent_peekaboo_AB spends A and B. It is replaced by tx_replacer_BC (conflicting UTXO B),
739 # and then replaced by tx_replacer_C (conflicting UTXO C). This replacement cycle is used to
740 # ensure that parent_peekaboo_AB can be reintroduced without requiring package RBF.
741 FEE_INCREMENT = 2400
742 parent_peekaboo_AB = self.wallet.create_self_transfer_multi(
743 utxos_to_spend=[coin_A, coin_B],
744 num_outputs=1,
745 fee_per_output=FEE_INCREMENT
746 )
747 tx_replacer_BC = self.wallet.create_self_transfer_multi(
748 utxos_to_spend=[coin_B, coin_C],
749 num_outputs=1,
750 fee_per_output=2*FEE_INCREMENT
751 )
752 tx_replacer_C = self.wallet.create_self_transfer(
753 utxo_to_spend=coin_C,
754 fee_per_output=3*FEE_INCREMENT
755 )
756 757 # parent_peekaboo_AB starts out in the mempool
758 node.sendrawtransaction(parent_peekaboo_AB["hex"])
759 760 orphan = self.wallet.create_self_transfer_multi(utxos_to_spend=[parent_peekaboo_AB["new_utxos"][0], parent_missing["new_utxo"]])
761 orphan_wtxid = orphan["wtxid"]
762 orphan_inv = CInv(t=MSG_WTX, h=int(orphan_wtxid, 16))
763 764 # peer1 sends the orphan and gets a request for the missing parent
765 peer1 = node.add_p2p_connection(PeerTxRelayer())
766 peer1.send_and_ping(msg_inv([orphan_inv]))
767 node.bumpmocktime(TXREQUEST_TIME_SKIP)
768 peer1.wait_for_getdata([int(orphan_wtxid, 16)])
769 peer1.send_and_ping(msg_tx(orphan["tx"]))
770 self.wait_until(lambda: node.getorphantxs(verbosity=0) == [orphan["txid"]])
771 node.bumpmocktime(NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY)
772 peer1.wait_for_getdata([int(parent_missing["txid"], 16)])
773 774 # Replace parent_peekaboo_AB so that is a newly missing parent.
775 # Then, replace the replacement so that it can be resubmitted.
776 node.sendrawtransaction(tx_replacer_BC["hex"])
777 assert tx_replacer_BC["txid"] in node.getrawmempool()
778 node.sendrawtransaction(tx_replacer_C["hex"])
779 assert tx_replacer_BC["txid"] not in node.getrawmempool()
780 assert parent_peekaboo_AB["txid"] not in node.getrawmempool()
781 assert tx_replacer_C["txid"] in node.getrawmempool()
782 783 # Second peer is an additional announcer for this orphan, but its missing parents are different from when it was
784 # previously announced.
785 peer2 = node.add_p2p_connection(PeerTxRelayer())
786 peer2.send_and_ping(msg_inv([orphan_inv]))
787 assert_equal(len(node.getorphantxs(verbosity=2)[0]["from"]), 2)
788 789 # Disconnect peer1. peer2 should become the new candidate for orphan resolution.
790 peer1.peer_disconnect()
791 self.wait_until(lambda: node.num_test_p2p_connections() == 1)
792 node.bumpmocktime(TXREQUEST_TIME_SKIP)
793 self.wait_until(lambda: len(node.getorphantxs(verbosity=2)[0]["from"]) == 1)
794 # Both parents should be requested, now that they are both missing.
795 peer2.wait_for_parent_requests([int(parent_peekaboo_AB["txid"], 16), int(parent_missing["txid"], 16)])
796 peer2.send_and_ping(msg_tx(parent_missing["tx"]))
797 peer2.send_and_ping(msg_tx(parent_peekaboo_AB["tx"]))
798 799 final_mempool = node.getrawmempool()
800 assert parent_missing["txid"] in final_mempool
801 assert parent_peekaboo_AB["txid"] in final_mempool
802 assert orphan["txid"] in final_mempool
803 assert tx_replacer_C["txid"] in final_mempool
804 805 def run_test(self):
806 self.nodes[0].setmocktime(int(time.time()))
807 self.wallet_nonsegwit = MiniWallet(self.nodes[0], mode=MiniWalletMode.RAW_P2PK)
808 self.generate(self.wallet_nonsegwit, 10)
809 self.wallet = MiniWallet(self.nodes[0])
810 self.generate(self.wallet, 160)
811 812 self.test_arrival_timing_orphan()
813 self.test_orphan_rejected_parents_exceptions()
814 self.test_orphan_multiple_parents()
815 self.test_orphans_overlapping_parents()
816 self.test_orphan_of_orphan()
817 self.test_orphan_inherit_rejection()
818 self.test_same_txid_orphan()
819 self.test_same_txid_orphan_of_orphan()
820 self.test_orphan_txid_inv()
821 self.test_max_orphan_amount()
822 self.test_orphan_handling_prefer_outbound()
823 self.test_announcers_before_and_after()
824 self.test_parents_change()
825 826 827 if __name__ == '__main__':
828 OrphanHandlingTest(__file__).main()
829