wallet_upgradewallet.py raw
1 #!/usr/bin/env python3
2 # Copyright (c) 2018-2022 The Limenka developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """upgradewallet RPC functional test
6
7 Test upgradewallet RPC. Download node binaries:
8
9 Requires previous releases binaries, see test/README.md.
10 Only v0.15.2 and v0.16.3 are required by this test.
11 """
12
13 import os
14 import shutil
15 import struct
16
17 from io import BytesIO
18
19 from test_framework.blocktools import COINBASE_MATURITY
20 from test_framework.bdb import dump_bdb_kv
21 from test_framework.messages import deser_compact_size, deser_string
22 from test_framework.test_framework import LimenkaTestFramework
23 from test_framework.util import (
24 assert_equal,
25 assert_is_hex_string,
26 sha256sum_file,
27 )
28
29
30 UPGRADED_KEYMETA_VERSION = 12
31
32 def deser_keymeta(f):
33 ver, create_time = struct.unpack('<Iq', f.read(12))
34 kp_str = deser_string(f)
35 seed_id = f.read(20)
36 fpr = f.read(4)
37 path_len = 0
38 path = []
39 has_key_orig = False
40 if ver == UPGRADED_KEYMETA_VERSION:
41 path_len = deser_compact_size(f)
42 for i in range(0, path_len):
43 path.append(struct.unpack('<I', f.read(4))[0])
44 has_key_orig = bool(f.read(1))
45 return ver, create_time, kp_str, seed_id, fpr, path_len, path, has_key_orig
46
47 class UpgradeWalletTest(LimenkaTestFramework):
48 def add_options(self, parser):
49 self.add_wallet_options(parser, descriptors=False)
50
51 def set_test_params(self):
52 self.setup_clean_chain = True
53 self.num_nodes = 3
54 self.extra_args = [
55 ["-addresstype=bech32", "-keypool=2"], # current wallet version
56 ["-usehd=1", "-keypool=2"], # v0.16.3 wallet
57 ["-usehd=0", "-keypool=2"] # v0.15.2 wallet
58 ]
59 self.wallet_names = [self.default_wallet_name, None, None]
60
61 def skip_test_if_missing_module(self):
62 self.skip_if_no_wallet()
63 self.skip_if_no_bdb()
64 self.skip_if_no_previous_releases()
65
66 def setup_network(self):
67 self.setup_nodes()
68
69 def setup_nodes(self):
70 self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[
71 None,
72 160300,
73 150200,
74 ])
75 self.start_nodes()
76 self.import_deterministic_coinbase_privkeys()
77
78 def dumb_sync_blocks(self):
79 """
80 Little helper to sync older wallets.
81 Notice that v0.15.2's regtest is hardforked, so there is
82 no sync for it.
83 v0.15.2 is only being used to test for version upgrade
84 and master hash key presence.
85 v0.16.3 is being used to test for version upgrade and balances.
86 Further info: https://github.com/limenka/limenka/pull/18774#discussion_r416967844
87 """
88 node_from = self.nodes[0]
89 v16_3_node = self.nodes[1]
90 to_height = node_from.getblockcount()
91 height = self.nodes[1].getblockcount()
92 for i in range(height, to_height+1):
93 b = node_from.getblock(blockhash=node_from.getblockhash(i), verbose=0)
94 v16_3_node.submitblock(b)
95 assert_equal(v16_3_node.getblockcount(), to_height)
96
97 def test_upgradewallet(self, wallet, previous_version, requested_version=None, expected_version=None):
98 unchanged = expected_version == previous_version
99 new_version = previous_version if unchanged else expected_version if expected_version else requested_version
100 old_wallet_info = wallet.getwalletinfo()
101 assert_equal(old_wallet_info["walletversion"], previous_version)
102 assert_equal(wallet.upgradewallet(requested_version),
103 {
104 "wallet_name": old_wallet_info["walletname"],
105 "previous_version": previous_version,
106 "current_version": new_version,
107 "result": "Already at latest version. Wallet version unchanged." if unchanged else "Wallet upgraded successfully from version {} to version {}.".format(previous_version, new_version),
108 }
109 )
110 assert_equal(wallet.getwalletinfo()["walletversion"], new_version)
111
112 def test_upgradewallet_error(self, wallet, previous_version, requested_version, msg):
113 assert_equal(wallet.getwalletinfo()["walletversion"], previous_version)
114 assert_equal(wallet.upgradewallet(requested_version),
115 {
116 "wallet_name": "",
117 "previous_version": previous_version,
118 "current_version": previous_version,
119 "error": msg,
120 }
121 )
122 assert_equal(wallet.getwalletinfo()["walletversion"], previous_version)
123
124 def run_test(self):
125 self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, self.nodes[0].getnewaddress(), sync_fun=lambda: self.dumb_sync_blocks())
126 # # Sanity check the test framework:
127 res = self.nodes[0].getblockchaininfo()
128 assert_equal(res['blocks'], COINBASE_MATURITY + 1)
129 node_master = self.nodes[0]
130 v16_3_node = self.nodes[1]
131 v15_2_node = self.nodes[2]
132
133 # Send coins to old wallets for later conversion checks.
134 v16_3_wallet = v16_3_node.get_wallet_rpc('wallet.dat')
135 v16_3_address = v16_3_wallet.getnewaddress()
136 self.generatetoaddress(node_master, COINBASE_MATURITY + 1, v16_3_address, sync_fun=lambda: self.dumb_sync_blocks())
137 v16_3_balance = v16_3_wallet.getbalance()
138
139 self.log.info("Test upgradewallet RPC...")
140 # Prepare for copying of the older wallet
141 node_master_wallet_dir = node_master.wallets_path / self.default_wallet_name
142 node_master_wallet = node_master_wallet_dir / self.default_wallet_name / self.wallet_data_filename
143 v16_3_wallet = v16_3_node.wallets_path / "wallet.dat"
144 v15_2_wallet = v15_2_node.chain_path / "wallet.dat"
145 split_hd_wallet = v15_2_node.chain_path / "splithd"
146 self.stop_nodes()
147
148 # Make split hd wallet
149 self.start_node(2, ['-usehd=1', '-keypool=2', '-wallet=splithd'])
150 self.stop_node(2)
151
152 def copy_v16():
153 node_master.get_wallet_rpc(self.default_wallet_name).unloadwallet()
154 # Copy the 0.16.3 wallet to the last Limenka version and open it:
155 shutil.rmtree(node_master_wallet_dir)
156 os.mkdir(node_master_wallet_dir)
157 shutil.copy(
158 v16_3_wallet,
159 node_master_wallet_dir
160 )
161 node_master.loadwallet(self.default_wallet_name)
162
163 def copy_non_hd():
164 node_master.get_wallet_rpc(self.default_wallet_name).unloadwallet()
165 # Copy the 0.15.2 non hd wallet to the last Limenka version and open it:
166 shutil.rmtree(node_master_wallet_dir)
167 os.mkdir(node_master_wallet_dir)
168 shutil.copy(
169 v15_2_wallet,
170 node_master_wallet_dir
171 )
172 node_master.loadwallet(self.default_wallet_name)
173
174 def copy_split_hd():
175 node_master.get_wallet_rpc(self.default_wallet_name).unloadwallet()
176 # Copy the 0.15.2 split hd wallet to the last Limenka version and open it:
177 shutil.rmtree(node_master_wallet_dir)
178 os.mkdir(node_master_wallet_dir)
179 shutil.copy(
180 split_hd_wallet,
181 os.path.join(node_master_wallet_dir, 'wallet.dat')
182 )
183 node_master.loadwallet(self.default_wallet_name)
184
185 self.restart_node(0)
186 copy_v16()
187 wallet = node_master.get_wallet_rpc(self.default_wallet_name)
188 assert_equal(wallet.getbalance(), v16_3_balance)
189 self.log.info("Test upgradewallet without a version argument")
190 self.test_upgradewallet(wallet, previous_version=159900, expected_version=169900)
191 # wallet should still contain the same balance
192 assert_equal(wallet.getbalance(), v16_3_balance)
193
194 copy_non_hd()
195 wallet = node_master.get_wallet_rpc(self.default_wallet_name)
196 # should have no master key hash before conversion
197 assert_equal('hdseedid' in wallet.getwalletinfo(), False)
198 self.log.info("Test upgradewallet with explicit version number")
199 self.test_upgradewallet(wallet, previous_version=60000, requested_version=169900)
200 # after conversion master key hash should be present
201 assert_is_hex_string(wallet.getwalletinfo()['hdseedid'])
202
203 self.log.info("Intermediary versions don't effect anything")
204 copy_non_hd()
205 # Wallet starts with 60000
206 assert_equal(60000, wallet.getwalletinfo()['walletversion'])
207 wallet.unloadwallet()
208 before_checksum = sha256sum_file(node_master_wallet)
209 node_master.loadwallet('')
210 # Test an "upgrade" from 60000 to 129999 has no effect, as the next version is 130000
211 self.test_upgradewallet(wallet, previous_version=60000, requested_version=129999, expected_version=60000)
212 wallet.unloadwallet()
213 assert_equal(before_checksum, sha256sum_file(node_master_wallet))
214 node_master.loadwallet('')
215
216 self.log.info('Wallets cannot be downgraded')
217 copy_non_hd()
218 self.test_upgradewallet_error(wallet, previous_version=60000, requested_version=40000,
219 msg="Cannot downgrade wallet from version 60000 to version 40000. Wallet version unchanged.")
220 wallet.unloadwallet()
221 assert_equal(before_checksum, sha256sum_file(node_master_wallet))
222 node_master.loadwallet('')
223
224 self.log.info('Can upgrade to HD')
225 # Inspect the old wallet and make sure there is no hdchain
226 orig_kvs = dump_bdb_kv(node_master_wallet)
227 assert b'\x07hdchain' not in orig_kvs
228 # Upgrade to HD, no split
229 self.test_upgradewallet(wallet, previous_version=60000, requested_version=130000)
230 # Check that there is now a hd chain and it is version 1, no internal chain counter
231 new_kvs = dump_bdb_kv(node_master_wallet)
232 assert b'\x07hdchain' in new_kvs
233 hd_chain = new_kvs[b'\x07hdchain']
234 assert_equal(28, len(hd_chain))
235 hd_chain_version, _external_counter, seed_id = struct.unpack('<iI20s', hd_chain)
236 assert_equal(1, hd_chain_version)
237 seed_id = bytearray(seed_id)
238 seed_id.reverse()
239
240 # New keys (including change) should be HD (the two old keys have been flushed)
241 info = wallet.getaddressinfo(wallet.getnewaddress())
242 assert_equal(seed_id.hex(), info['hdseedid'])
243 assert_equal('m/0\'/0\'/0\'', info['hdkeypath'])
244 prev_seed_id = info['hdseedid']
245 # Change key should be HD and from the same keypool
246 info = wallet.getaddressinfo(wallet.getrawchangeaddress())
247 assert_equal(prev_seed_id, info['hdseedid'])
248 assert_equal('m/0\'/0\'/1\'', info['hdkeypath'])
249
250 self.log.info('Cannot upgrade to HD Split, needs Pre Split Keypool')
251 for version in [139900, 159900, 169899]:
252 self.test_upgradewallet_error(wallet, previous_version=130000, requested_version=version,
253 msg="Cannot upgrade a non HD split wallet from version {} to version {} without upgrading to "
254 "support pre-split keypool. Please use version 169900 or no version specified.".format(130000, version))
255
256 self.log.info('Upgrade HD to HD chain split')
257 self.test_upgradewallet(wallet, previous_version=130000, requested_version=169900)
258 # Check that the hdchain updated correctly
259 new_kvs = dump_bdb_kv(node_master_wallet)
260 hd_chain = new_kvs[b'\x07hdchain']
261 assert_equal(32, len(hd_chain))
262 hd_chain_version, _external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain)
263 assert_equal(2, hd_chain_version)
264 assert_equal(0, internal_counter)
265 seed_id = bytearray(seed_id)
266 seed_id.reverse()
267 assert_equal(seed_id.hex(), prev_seed_id)
268 # Next change address is the same keypool
269 info = wallet.getaddressinfo(wallet.getrawchangeaddress())
270 assert_equal(prev_seed_id, info['hdseedid'])
271 assert_equal('m/0\'/0\'/2\'', info['hdkeypath'])
272 # Next change address is the new keypool
273 info = wallet.getaddressinfo(wallet.getrawchangeaddress())
274 assert_equal(prev_seed_id, info['hdseedid'])
275 assert_equal('m/0\'/1\'/0\'', info['hdkeypath'])
276 # External addresses use the same keypool
277 info = wallet.getaddressinfo(wallet.getnewaddress())
278 assert_equal(prev_seed_id, info['hdseedid'])
279 assert_equal('m/0\'/0\'/3\'', info['hdkeypath'])
280
281 self.log.info('Upgrade non-HD to HD chain split')
282 copy_non_hd()
283 self.test_upgradewallet(wallet, previous_version=60000, requested_version=169900)
284 # Check that the hdchain updated correctly
285 new_kvs = dump_bdb_kv(node_master_wallet)
286 hd_chain = new_kvs[b'\x07hdchain']
287 assert_equal(32, len(hd_chain))
288 hd_chain_version, _external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain)
289 assert_equal(2, hd_chain_version)
290 assert_equal(2, internal_counter)
291 # The next addresses are HD and should be on different HD chains (the one remaining key in each pool should have been flushed)
292 info = wallet.getaddressinfo(wallet.getnewaddress())
293 ext_id = info['hdseedid']
294 assert_equal('m/0\'/0\'/0\'', info['hdkeypath'])
295 info = wallet.getaddressinfo(wallet.getrawchangeaddress())
296 assert_equal(ext_id, info['hdseedid'])
297 assert_equal('m/0\'/1\'/0\'', info['hdkeypath'])
298
299 self.log.info('KeyMetadata should upgrade when loading into master')
300 copy_v16()
301 old_kvs = dump_bdb_kv(v16_3_wallet)
302 new_kvs = dump_bdb_kv(node_master_wallet)
303 for k, old_v in old_kvs.items():
304 if k.startswith(b'\x07keymeta'):
305 new_ver, new_create_time, new_kp_str, new_seed_id, _new_fpr, new_path_len, new_path, new_has_key_orig = deser_keymeta(BytesIO(new_kvs[k]))
306 old_ver, old_create_time, old_kp_str, old_seed_id, _old_fpr, old_path_len, old_path, old_has_key_orig = deser_keymeta(BytesIO(old_v))
307 assert_equal(10, old_ver)
308 if old_kp_str == b"": # imported things that don't have keymeta (i.e. imported coinbase privkeys) won't be upgraded
309 assert_equal(new_kvs[k], old_v)
310 continue
311 assert_equal(12, new_ver)
312 assert_equal(new_create_time, old_create_time)
313 assert_equal(new_kp_str, old_kp_str)
314 assert_equal(new_seed_id, old_seed_id)
315 assert_equal(0, old_path_len)
316 assert_equal(new_path_len, len(new_path))
317 assert_equal([], old_path)
318 assert_equal(False, old_has_key_orig)
319 assert_equal(True, new_has_key_orig)
320
321 # Check that the path is right
322 built_path = []
323 for s in new_kp_str.decode().split('/')[1:]:
324 h = 0
325 if s[-1] == '\'':
326 s = s[:-1]
327 h = 0x80000000
328 p = int(s) | h
329 built_path.append(p)
330 assert_equal(new_path, built_path)
331
332 self.log.info('Upgrading to NO_DEFAULT_KEY should not remove the defaultkey')
333 copy_split_hd()
334 # Check the wallet has a default key initially
335 old_kvs = dump_bdb_kv(node_master_wallet)
336 defaultkey = old_kvs[b'\x0adefaultkey']
337 self.log.info("Upgrade the wallet. Should still have the same default key.")
338 self.test_upgradewallet(wallet, previous_version=139900, requested_version=159900)
339 new_kvs = dump_bdb_kv(node_master_wallet)
340 up_defaultkey = new_kvs[b'\x0adefaultkey']
341 assert_equal(defaultkey, up_defaultkey)
342 # 0.16.3 doesn't have a default key
343 v16_3_kvs = dump_bdb_kv(v16_3_wallet)
344 assert b'\x0adefaultkey' not in v16_3_kvs
345
346 if self.is_sqlite_compiled():
347 self.log.info("Checking that descriptor wallets do nothing, successfully")
348 self.nodes[0].createwallet(wallet_name="desc_upgrade", descriptors=True)
349 desc_wallet = self.nodes[0].get_wallet_rpc("desc_upgrade")
350 self.test_upgradewallet(desc_wallet, previous_version=169900, expected_version=169900)
351
352 self.log.info("Checking that descriptor wallets without privkeys do nothing, successfully")
353 self.nodes[0].createwallet(wallet_name="desc_upgrade_nopriv", descriptors=True, disable_private_keys=True)
354 desc_wallet = self.nodes[0].get_wallet_rpc("desc_upgrade_nopriv")
355 self.test_upgradewallet(desc_wallet, previous_version=169900, expected_version=169900)
356
357 if self.is_bdb_compiled():
358 self.log.info("Upgrading a wallet with private keys disabled")
359 self.nodes[0].createwallet(wallet_name="privkeys_disabled_upgrade", disable_private_keys=True, descriptors=False)
360 disabled_wallet = self.nodes[0].get_wallet_rpc("privkeys_disabled_upgrade")
361 self.test_upgradewallet(disabled_wallet, previous_version=169900, expected_version=169900)
362
363 if __name__ == '__main__':
364 UpgradeWalletTest(__file__).main()
365