1 #!/usr/bin/env python3
2 # Copyright (c) 2014-present The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test mempool persistence.
6 7 By default, bitcoind will dump mempool on shutdown and
8 then reload it on startup. This can be overridden with
9 the -persistmempool=0 command line option.
10 11 Test is as follows:
12 13 - start node0, node1 and node2. node1 has -persistmempool=0
14 - create 5 transactions on node2 to its own address. Note that these
15 are not sent to node0 or node1 addresses because we don't want
16 them to be saved in the wallet.
17 - check that node0 and node1 have 5 transactions in their mempools
18 - shutdown all nodes.
19 - startup node0. Verify that it still has 5 transactions
20 in its mempool. Shutdown node0. This tests that by default the
21 mempool is persistent.
22 - startup node1. Verify that its mempool is empty. Shutdown node1.
23 This tests that with -persistmempool=0, the mempool is not
24 dumped to disk when the node is shut down.
25 - Restart node0 with -persistmempool=0. Verify that its mempool is
26 empty. Shutdown node0. This tests that with -persistmempool=0,
27 the mempool is not loaded from disk on start up.
28 - Restart node0 with -persistmempool. Verify that it has 5
29 transactions in its mempool. This tests that -persistmempool=0
30 does not overwrite a previously valid mempool stored on disk.
31 - Remove node0 mempool.dat and verify savemempool RPC recreates it
32 and verify that node1 can load it and has 5 transactions in its
33 mempool.
34 - Verify that savemempool throws when the RPC is called if
35 node1 can't write to disk.
36 37 """
38 from decimal import Decimal
39 import os
40 import time
41 42 from test_framework.p2p import P2PTxInvStore
43 from test_framework.test_framework import BitcoinTestFramework
44 from test_framework.util import (
45 assert_equal,
46 assert_greater_than_or_equal,
47 assert_raises_rpc_error,
48 )
49 from test_framework.wallet import MiniWallet, COIN
50 51 52 class MempoolPersistTest(BitcoinTestFramework):
53 def set_test_params(self):
54 self.num_nodes = 3
55 self.extra_args = [[], ["-persistmempool=0"], []]
56 self.uses_wallet = None
57 58 def run_test(self):
59 self.mini_wallet = MiniWallet(self.nodes[2])
60 if self.is_wallet_compiled():
61 self.nodes[2].createwallet(
62 wallet_name="watch",
63 disable_private_keys=True,
64 load_on_startup=False,
65 )
66 wallet_watch = self.nodes[2].get_wallet_rpc("watch")
67 assert_equal([{'success': True}], wallet_watch.importdescriptors([{'desc': self.mini_wallet.get_descriptor(), 'timestamp': 0}]))
68 69 self.log.debug("Send 5 transactions from node2 (to its own address)")
70 tx_creation_time_lower = int(time.time())
71 for _ in range(5):
72 last_txid = self.mini_wallet.send_self_transfer(from_node=self.nodes[2])["txid"]
73 if self.is_wallet_compiled():
74 self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet
75 node2_balance = wallet_watch.getbalance()
76 self.sync_all()
77 tx_creation_time_higher = int(time.time())
78 79 self.log.debug("Verify that node0 and node1 have 5 transactions in their mempools")
80 assert_equal(len(self.nodes[0].getrawmempool()), 5)
81 assert_equal(len(self.nodes[1].getrawmempool()), 5)
82 83 total_fee_old = self.nodes[0].getmempoolinfo()['total_fee']
84 85 self.log.debug("Prioritize a transaction on node0")
86 fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees']
87 assert_equal(fees['base'], fees['modified'])
88 self.nodes[0].prioritisetransaction(txid=last_txid, fee_delta=1000)
89 fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees']
90 assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified'])
91 92 self.log.info('Check the total base fee is unchanged after prioritisetransaction')
93 assert_equal(total_fee_old, self.nodes[0].getmempoolinfo()['total_fee'])
94 assert_equal(total_fee_old, sum(v['fees']['base'] for k, v in self.nodes[0].getrawmempool(verbose=True).items()))
95 96 last_entry = self.nodes[0].getmempoolentry(txid=last_txid)
97 tx_creation_time = last_entry['time']
98 assert_greater_than_or_equal(tx_creation_time, tx_creation_time_lower)
99 assert_greater_than_or_equal(tx_creation_time_higher, tx_creation_time)
100 101 # disconnect nodes & make a txn that remains in the unbroadcast set.
102 self.disconnect_nodes(0, 1)
103 assert_equal(len(self.nodes[0].getpeerinfo()), 0)
104 assert_equal(len(self.nodes[0].p2ps), 0)
105 self.mini_wallet.send_self_transfer(from_node=self.nodes[0])
106 107 # Test persistence of prioritisation for transactions not in the mempool.
108 # Create a tx and prioritise but don't submit until after the restart.
109 tx_prioritised_not_submitted = self.mini_wallet.create_self_transfer()
110 self.nodes[0].prioritisetransaction(txid=tx_prioritised_not_submitted['txid'], fee_delta=9999)
111 112 self.log.debug("Stop-start the nodes. Verify that node0 has the transactions in its mempool and node1 does not. Verify that node2 calculates its balance correctly after loading wallet transactions.")
113 self.stop_nodes()
114 # Give this node a head-start, so we can be "extra-sure" that it didn't load anything later
115 # Also don't store the mempool, to keep the datadir clean
116 self.start_node(1, extra_args=["-persistmempool=0"])
117 self.start_node(0)
118 self.start_node(2)
119 assert self.nodes[0].getmempoolinfo()["loaded"] # start_node is blocking on the mempool being loaded
120 assert self.nodes[2].getmempoolinfo()["loaded"]
121 assert_equal(len(self.nodes[0].getrawmempool()), 6)
122 assert_equal(len(self.nodes[2].getrawmempool()), 5)
123 # The others have loaded their mempool. If node_1 loaded anything, we'd probably notice by now:
124 assert_equal(len(self.nodes[1].getrawmempool()), 0)
125 126 self.log.debug('Verify prioritization is loaded correctly')
127 fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees']
128 assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified'])
129 130 self.log.debug('Verify all fields are loaded correctly')
131 new_entry = self.nodes[0].getmempoolentry(txid=last_txid)
132 assert_equal({**last_entry, "clusterid": None}, {**new_entry, "clusterid": None})
133 self.nodes[0].sendrawtransaction(tx_prioritised_not_submitted['hex'])
134 entry_prioritised_before_restart = self.nodes[0].getmempoolentry(txid=tx_prioritised_not_submitted['txid'])
135 assert_equal(entry_prioritised_before_restart['fees']['base'] + Decimal('0.00009999'), entry_prioritised_before_restart['fees']['modified'])
136 137 # Verify accounting of mempool transactions after restart is correct
138 if self.is_wallet_compiled():
139 self.nodes[2].loadwallet("watch")
140 wallet_watch = self.nodes[2].get_wallet_rpc("watch")
141 self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet
142 assert_equal(node2_balance, wallet_watch.getbalance())
143 144 mempooldat0 = os.path.join(self.nodes[0].chain_path, 'mempool.dat')
145 mempooldat1 = os.path.join(self.nodes[1].chain_path, 'mempool.dat')
146 147 self.log.debug("Force -persistmempool=0 node1 to savemempool to disk via RPC")
148 assert not os.path.exists(mempooldat1)
149 result1 = self.nodes[1].savemempool()
150 assert os.path.isfile(mempooldat1)
151 assert_equal(result1['filename'], mempooldat1)
152 os.remove(mempooldat1)
153 154 self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.")
155 self.stop_nodes()
156 self.start_node(0, extra_args=["-persistmempool=0"])
157 assert self.nodes[0].getmempoolinfo()["loaded"]
158 assert_equal(len(self.nodes[0].getrawmempool()), 0)
159 160 self.log.debug("Import mempool at runtime to node0.")
161 assert_equal({}, self.nodes[0].importmempool(mempooldat0))
162 assert_equal(len(self.nodes[0].getrawmempool()), 7)
163 fees = self.nodes[0].getmempoolentry(txid=last_txid)["fees"]
164 assert_equal(fees["base"], fees["modified"])
165 assert_equal({}, self.nodes[0].importmempool(mempooldat0, {"apply_fee_delta_priority": True, "apply_unbroadcast_set": True}))
166 assert_equal(2, self.nodes[0].getmempoolinfo()["unbroadcastcount"])
167 fees = self.nodes[0].getmempoolentry(txid=last_txid)["fees"]
168 assert_equal(fees["base"] + Decimal("0.00001000"), fees["modified"])
169 170 self.log.debug("Stop-start node0. Verify that it has the transactions in its mempool.")
171 self.stop_nodes()
172 self.start_node(0)
173 assert self.nodes[0].getmempoolinfo()["loaded"]
174 assert_equal(len(self.nodes[0].getrawmempool()), 7)
175 176 self.log.debug("Remove the mempool.dat file. Verify that savemempool to disk via RPC re-creates it")
177 os.remove(mempooldat0)
178 result0 = self.nodes[0].savemempool()
179 assert os.path.isfile(mempooldat0)
180 assert_equal(result0['filename'], mempooldat0)
181 182 self.log.debug("Stop nodes, make node1 use mempool.dat from node0. Verify it has 7 transactions")
183 os.rename(mempooldat0, mempooldat1)
184 self.stop_nodes()
185 self.start_node(1, extra_args=["-persistmempool"])
186 assert self.nodes[1].getmempoolinfo()["loaded"]
187 assert_equal(len(self.nodes[1].getrawmempool()), 7)
188 189 self.log.debug("Prevent bitcoind from writing mempool.dat to disk. Verify that `savemempool` fails")
190 # to test the exception we are creating a tmp folder called mempool.dat.new
191 # which is an implementation detail that could change and break this test
192 mempooldotnew1 = mempooldat1 + '.new'
193 os.mkdir(mempooldotnew1)
194 assert_raises_rpc_error(-1, "Unable to dump mempool to disk", self.nodes[1].savemempool)
195 os.rmdir(mempooldotnew1)
196 197 self.test_importmempool_union()
198 self.test_persist_unbroadcast()
199 200 def test_persist_unbroadcast(self):
201 node0 = self.nodes[0]
202 self.start_node(0)
203 self.start_node(2)
204 205 # clear out mempool
206 self.generate(node0, 1, sync_fun=self.no_op)
207 208 # ensure node0 doesn't have any connections
209 # make a transaction that will remain in the unbroadcast set
210 assert_equal(len(node0.getpeerinfo()), 0)
211 assert_equal(len(node0.p2ps), 0)
212 self.mini_wallet.send_self_transfer(from_node=node0)
213 214 # shutdown, then startup with wallet disabled
215 self.restart_node(0, extra_args=["-disablewallet"])
216 217 # check that txn gets broadcast due to unbroadcast logic
218 conn = node0.add_p2p_connection(P2PTxInvStore())
219 node0.mockscheduler(16 * 60) # 15 min + 1 for buffer
220 self.wait_until(lambda: len(conn.get_invs()) == 1)
221 222 def test_importmempool_union(self):
223 self.log.debug("Submit different transactions to node0 and node1's mempools")
224 self.start_node(0)
225 self.start_node(2)
226 tx_node0 = self.mini_wallet.send_self_transfer(from_node=self.nodes[0])
227 tx_node1 = self.mini_wallet.send_self_transfer(from_node=self.nodes[1])
228 tx_node01 = self.mini_wallet.create_self_transfer()
229 tx_node01_secret = self.mini_wallet.create_self_transfer()
230 self.nodes[0].prioritisetransaction(tx_node01["txid"], 0, COIN)
231 self.nodes[0].prioritisetransaction(tx_node01_secret["txid"], 0, 2 * COIN)
232 self.nodes[1].prioritisetransaction(tx_node01_secret["txid"], 0, 3 * COIN)
233 self.nodes[0].sendrawtransaction(tx_node01["hex"])
234 self.nodes[1].sendrawtransaction(tx_node01["hex"])
235 assert tx_node0["txid"] in self.nodes[0].getrawmempool()
236 assert tx_node0["txid"] not in self.nodes[1].getrawmempool()
237 assert tx_node1["txid"] not in self.nodes[0].getrawmempool()
238 assert tx_node1["txid"] in self.nodes[1].getrawmempool()
239 assert tx_node01["txid"] in self.nodes[0].getrawmempool()
240 assert tx_node01["txid"] in self.nodes[1].getrawmempool()
241 assert tx_node01_secret["txid"] not in self.nodes[0].getrawmempool()
242 assert tx_node01_secret["txid"] not in self.nodes[1].getrawmempool()
243 244 self.log.debug("Check that importmempool can add txns without replacing the entire mempool")
245 mempooldat0 = str(self.nodes[0].chain_path / "mempool.dat")
246 result0 = self.nodes[0].savemempool()
247 assert_equal(mempooldat0, result0["filename"])
248 assert_equal({}, self.nodes[1].importmempool(mempooldat0, {"apply_fee_delta_priority": True}))
249 # All transactions should be in node1's mempool now.
250 assert tx_node0["txid"] in self.nodes[1].getrawmempool()
251 assert tx_node1["txid"] in self.nodes[1].getrawmempool()
252 assert tx_node1["txid"] not in self.nodes[0].getrawmempool()
253 # For transactions that already existed, priority should be changed
254 entry_node01 = self.nodes[1].getmempoolentry(tx_node01["txid"])
255 assert_equal(entry_node01["fees"]["base"] + 1, entry_node01["fees"]["modified"])
256 # Deltas for not-yet-submitted transactions should be applied as well (prioritisation is stackable).
257 self.nodes[1].sendrawtransaction(tx_node01_secret["hex"])
258 entry_node01_secret = self.nodes[1].getmempoolentry(tx_node01_secret["txid"])
259 assert_equal(entry_node01_secret["fees"]["base"] + 5, entry_node01_secret["fees"]["modified"])
260 self.stop_nodes()
261 262 263 if __name__ == "__main__":
264 MempoolPersistTest(__file__).main()
265