1 #!/usr/bin/env python3
2 # Copyright (c) 2014-2022 The Limenka developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test mempool persistence.
6 7 By default, limenkad will dump mempool on shutdown and
8 then reload it on startup. This can be overridden with
9 the -persistmempool=0 command line option.
10 11 Test is as follows:
12 13 - start node0, node1 and node2. node1 has -persistmempool=0
14 - create 5 transactions on node2 to its own address. Note that these
15 are not sent to node0 or node1 addresses because we don't want
16 them to be saved in the wallet.
17 - check that node0 and node1 have 5 transactions in their mempools
18 - shutdown all nodes.
19 - startup node0. Verify that it still has 5 transactions
20 in its mempool. Shutdown node0. This tests that by default the
21 mempool is persistent.
22 - startup node1. Verify that its mempool is empty. Shutdown node1.
23 This tests that with -persistmempool=0, the mempool is not
24 dumped to disk when the node is shut down.
25 - Restart node0 with -persistmempool=0. Verify that its mempool is
26 empty. Shutdown node0. This tests that with -persistmempool=0,
27 the mempool is not loaded from disk on start up.
28 - Restart node0 with -persistmempool. Verify that it has 5
29 transactions in its mempool. This tests that -persistmempool=0
30 does not overwrite a previously valid mempool stored on disk.
31 - Remove node0 mempool.dat and verify savemempool RPC recreates it
32 and verify that node1 can load it and has 5 transactions in its
33 mempool.
34 - Verify that savemempool throws when the RPC is called if
35 node1 can't write to disk.
36 37 """
38 from decimal import Decimal
39 import os
40 import time
41 42 from test_framework.p2p import P2PTxInvStore
43 from test_framework.test_framework import LimenkaTestFramework
44 from test_framework.util import (
45 assert_equal,
46 assert_greater_than_or_equal,
47 assert_raises_rpc_error,
48 )
49 from test_framework.wallet import MiniWallet, COIN
50 51 52 class MempoolPersistTest(LimenkaTestFramework):
53 def add_options(self, parser):
54 self.add_wallet_options(parser, legacy=False)
55 56 def set_test_params(self):
57 self.num_nodes = 3
58 self.extra_args = [[], ["-persistmempool=0"], []]
59 60 def run_test(self):
61 self.mini_wallet = MiniWallet(self.nodes[2])
62 if self.is_sqlite_compiled():
63 self.nodes[2].createwallet(
64 wallet_name="watch",
65 descriptors=True,
66 disable_private_keys=True,
67 load_on_startup=False,
68 )
69 wallet_watch = self.nodes[2].get_wallet_rpc("watch")
70 assert_equal([{'success': True}], wallet_watch.importdescriptors([{'desc': self.mini_wallet.get_descriptor(), 'timestamp': 0}]))
71 72 self.log.debug("Send 5 transactions from node2 (to its own address)")
73 tx_creation_time_lower = int(time.time())
74 for _ in range(5):
75 last_txid = self.mini_wallet.send_self_transfer(from_node=self.nodes[2])["txid"]
76 if self.is_sqlite_compiled():
77 self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet
78 node2_balance = wallet_watch.getbalance()
79 self.sync_all()
80 tx_creation_time_higher = int(time.time())
81 82 self.log.debug("Verify that node0 and node1 have 5 transactions in their mempools")
83 assert_equal(len(self.nodes[0].getrawmempool()), 5)
84 assert_equal(len(self.nodes[1].getrawmempool()), 5)
85 86 total_fee_old = self.nodes[0].getmempoolinfo()['total_fee']
87 88 self.log.debug("Prioritize a transaction on node0")
89 fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees']
90 assert_equal(fees['base'], fees['modified'])
91 self.nodes[0].prioritisetransaction(txid=last_txid, fee_delta=1000)
92 fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees']
93 assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified'])
94 95 self.log.info('Check the total base fee is unchanged after prioritisetransaction')
96 assert_equal(total_fee_old, self.nodes[0].getmempoolinfo()['total_fee'])
97 assert_equal(total_fee_old, sum(v['fees']['base'] for k, v in self.nodes[0].getrawmempool(verbose=True).items()))
98 99 last_entry = self.nodes[0].getmempoolentry(txid=last_txid)
100 tx_creation_time = last_entry['time']
101 assert_greater_than_or_equal(tx_creation_time, tx_creation_time_lower)
102 assert_greater_than_or_equal(tx_creation_time_higher, tx_creation_time)
103 104 # disconnect nodes & make a txn that remains in the unbroadcast set.
105 self.disconnect_nodes(0, 1)
106 assert_equal(len(self.nodes[0].getpeerinfo()), 0)
107 assert_equal(len(self.nodes[0].p2ps), 0)
108 self.mini_wallet.send_self_transfer(from_node=self.nodes[0])
109 110 # Test persistence of prioritisation for transactions not in the mempool.
111 # Create a tx and prioritise but don't submit until after the restart.
112 tx_prioritised_not_submitted = self.mini_wallet.create_self_transfer()
113 self.nodes[0].prioritisetransaction(txid=tx_prioritised_not_submitted['txid'], fee_delta=9999)
114 115 self.log.debug("Stop-start the nodes. Verify that node0 has the transactions in its mempool and node1 does not. Verify that node2 calculates its balance correctly after loading wallet transactions.")
116 self.stop_nodes()
117 # Give this node a head-start, so we can be "extra-sure" that it didn't load anything later
118 # Also don't store the mempool, to keep the datadir clean
119 self.start_node(1, extra_args=["-persistmempool=0"])
120 self.start_node(0)
121 self.start_node(2)
122 assert self.nodes[0].getmempoolinfo()["loaded"] # start_node is blocking on the mempool being loaded
123 assert self.nodes[2].getmempoolinfo()["loaded"]
124 assert_equal(len(self.nodes[0].getrawmempool()), 6)
125 assert_equal(len(self.nodes[2].getrawmempool()), 5)
126 # The others have loaded their mempool. If node_1 loaded anything, we'd probably notice by now:
127 assert_equal(len(self.nodes[1].getrawmempool()), 0)
128 129 self.log.debug('Verify prioritization is loaded correctly')
130 fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees']
131 assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified'])
132 133 self.log.debug('Verify all fields are loaded correctly')
134 assert_equal(last_entry, self.nodes[0].getmempoolentry(txid=last_txid))
135 self.nodes[0].sendrawtransaction(tx_prioritised_not_submitted['hex'])
136 entry_prioritised_before_restart = self.nodes[0].getmempoolentry(txid=tx_prioritised_not_submitted['txid'])
137 assert_equal(entry_prioritised_before_restart['fees']['base'] + Decimal('0.00009999'), entry_prioritised_before_restart['fees']['modified'])
138 139 # Verify accounting of mempool transactions after restart is correct
140 if self.is_sqlite_compiled():
141 self.nodes[2].loadwallet("watch")
142 wallet_watch = self.nodes[2].get_wallet_rpc("watch")
143 self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet
144 assert_equal(node2_balance, wallet_watch.getbalance())
145 146 mempooldat0 = os.path.join(self.nodes[0].chain_path, 'mempool.dat')
147 mempooldat1 = os.path.join(self.nodes[1].chain_path, 'mempool.dat')
148 149 self.log.debug("Force -persistmempool=0 node1 to savemempool to disk via RPC")
150 assert not os.path.exists(mempooldat1)
151 result1 = self.nodes[1].savemempool()
152 assert os.path.isfile(mempooldat1)
153 assert_equal(result1['filename'], mempooldat1)
154 os.remove(mempooldat1)
155 156 self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.")
157 self.stop_nodes()
158 self.start_node(0, extra_args=["-persistmempool=0"])
159 assert self.nodes[0].getmempoolinfo()["loaded"]
160 assert_equal(len(self.nodes[0].getrawmempool()), 0)
161 162 self.log.debug("Import mempool at runtime to node0.")
163 assert_equal({}, self.nodes[0].importmempool(mempooldat0))
164 assert_equal(len(self.nodes[0].getrawmempool()), 7)
165 fees = self.nodes[0].getmempoolentry(txid=last_txid)["fees"]
166 assert_equal(fees["base"], fees["modified"])
167 assert_equal({}, self.nodes[0].importmempool(mempooldat0, {"apply_fee_delta_priority": True, "apply_unbroadcast_set": True}))
168 assert_equal(2, self.nodes[0].getmempoolinfo()["unbroadcastcount"])
169 fees = self.nodes[0].getmempoolentry(txid=last_txid)["fees"]
170 assert_equal(fees["base"] + Decimal("0.00001000"), fees["modified"])
171 172 self.log.debug("Stop-start node0. Verify that it has the transactions in its mempool.")
173 self.stop_nodes()
174 self.start_node(0)
175 assert self.nodes[0].getmempoolinfo()["loaded"]
176 assert_equal(len(self.nodes[0].getrawmempool()), 7)
177 178 self.log.debug("Remove the mempool.dat file. Verify that savemempool to disk via RPC re-creates it")
179 os.remove(mempooldat0)
180 result0 = self.nodes[0].savemempool()
181 assert os.path.isfile(mempooldat0)
182 assert_equal(result0['filename'], mempooldat0)
183 184 self.log.debug("Stop nodes, make node1 use mempool.dat from node0. Verify it has 7 transactions")
185 os.rename(mempooldat0, mempooldat1)
186 self.stop_nodes()
187 self.start_node(1, extra_args=["-persistmempool"])
188 assert self.nodes[1].getmempoolinfo()["loaded"]
189 assert_equal(len(self.nodes[1].getrawmempool()), 7)
190 191 self.log.debug("Prevent limenkad from writing mempool.dat to disk. Verify that `savemempool` fails")
192 # to test the exception we are creating a tmp folder called mempool.dat.new
193 # which is an implementation detail that could change and break this test
194 mempooldotnew1 = mempooldat1 + '.new'
195 os.mkdir(mempooldotnew1)
196 assert_raises_rpc_error(-1, "Unable to dump mempool to disk", self.nodes[1].savemempool)
197 os.rmdir(mempooldotnew1)
198 199 self.test_importmempool_union()
200 self.test_persist_unbroadcast()
201 202 def test_persist_unbroadcast(self):
203 node0 = self.nodes[0]
204 self.start_node(0)
205 self.start_node(2)
206 207 # clear out mempool
208 self.generate(node0, 1, sync_fun=self.no_op)
209 210 # ensure node0 doesn't have any connections
211 # make a transaction that will remain in the unbroadcast set
212 assert_equal(len(node0.getpeerinfo()), 0)
213 assert_equal(len(node0.p2ps), 0)
214 self.mini_wallet.send_self_transfer(from_node=node0)
215 216 # shutdown, then startup with wallet disabled
217 self.restart_node(0, extra_args=["-disablewallet"])
218 219 # check that txn gets broadcast due to unbroadcast logic
220 conn = node0.add_p2p_connection(P2PTxInvStore())
221 node0.mockscheduler(16 * 60) # 15 min + 1 for buffer
222 self.wait_until(lambda: len(conn.get_invs()) == 1)
223 224 def test_importmempool_union(self):
225 self.log.debug("Submit different transactions to node0 and node1's mempools")
226 self.start_node(0)
227 self.start_node(2)
228 tx_node0 = self.mini_wallet.send_self_transfer(from_node=self.nodes[0])
229 tx_node1 = self.mini_wallet.send_self_transfer(from_node=self.nodes[1])
230 tx_node01 = self.mini_wallet.create_self_transfer()
231 tx_node01_secret = self.mini_wallet.create_self_transfer()
232 self.nodes[0].prioritisetransaction(tx_node01["txid"], 0, COIN)
233 self.nodes[0].prioritisetransaction(tx_node01_secret["txid"], 0, 2 * COIN)
234 self.nodes[1].prioritisetransaction(tx_node01_secret["txid"], 0, 3 * COIN)
235 self.nodes[0].sendrawtransaction(tx_node01["hex"])
236 self.nodes[1].sendrawtransaction(tx_node01["hex"])
237 assert tx_node0["txid"] in self.nodes[0].getrawmempool()
238 assert not tx_node0["txid"] in self.nodes[1].getrawmempool()
239 assert not tx_node1["txid"] in self.nodes[0].getrawmempool()
240 assert tx_node1["txid"] in self.nodes[1].getrawmempool()
241 assert tx_node01["txid"] in self.nodes[0].getrawmempool()
242 assert tx_node01["txid"] in self.nodes[1].getrawmempool()
243 assert not tx_node01_secret["txid"] in self.nodes[0].getrawmempool()
244 assert not tx_node01_secret["txid"] in self.nodes[1].getrawmempool()
245 246 self.log.debug("Check that importmempool can add txns without replacing the entire mempool")
247 mempooldat0 = str(self.nodes[0].chain_path / "mempool.dat")
248 result0 = self.nodes[0].savemempool()
249 assert_equal(mempooldat0, result0["filename"])
250 assert_equal({}, self.nodes[1].importmempool(mempooldat0, {"apply_fee_delta_priority": True}))
251 # All transactions should be in node1's mempool now.
252 assert tx_node0["txid"] in self.nodes[1].getrawmempool()
253 assert tx_node1["txid"] in self.nodes[1].getrawmempool()
254 assert not tx_node1["txid"] in self.nodes[0].getrawmempool()
255 # For transactions that already existed, priority should be changed
256 entry_node01 = self.nodes[1].getmempoolentry(tx_node01["txid"])
257 assert_equal(entry_node01["fees"]["base"] + 1, entry_node01["fees"]["modified"])
258 # Deltas for not-yet-submitted transactions should be applied as well (prioritisation is stackable).
259 self.nodes[1].sendrawtransaction(tx_node01_secret["hex"])
260 entry_node01_secret = self.nodes[1].getmempoolentry(tx_node01_secret["txid"])
261 assert_equal(entry_node01_secret["fees"]["base"] + 5, entry_node01_secret["fees"]["modified"])
262 self.stop_nodes()
263 264 265 if __name__ == "__main__":
266 MempoolPersistTest(__file__).main()
267