1 #!/usr/bin/env python3
2 # Copyright (c) 2018-present The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Backwards compatibility functional test
6 7 Test various backwards compatibility scenarios. Requires previous releases binaries,
8 see test/README.md.
9 10 Due to RPC changes introduced in various versions the below tests
11 won't work for older versions without some patches or workarounds.
12 13 Use only the latest patch version of each release, unless a test specifically
14 needs an older patch version.
15 """
16 17 import json
18 import os
19 import shutil
20 21 from test_framework.blocktools import COINBASE_MATURITY
22 from test_framework.test_framework import BitcoinTestFramework
23 from test_framework.descriptors import descsum_create
24 from test_framework.messages import ser_string
25 26 from test_framework.util import (
27 assert_equal,
28 assert_greater_than,
29 assert_raises_rpc_error,
30 )
31 32 LAST_KEYPOOL_INDEX = 9 # Index of the last derived address with the keypool size of 10
33 34 class BackwardsCompatibilityTest(BitcoinTestFramework):
35 def set_test_params(self):
36 self.setup_clean_chain = True
37 self.num_nodes = 8
38 # Add new version after each release:
39 self.extra_args = [
40 ["-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to mine blocks. noban for immediate tx relay
41 ["-nowallet", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # Pre-release: use to receive coins, swap wallets, etc
42 ["-nowallet", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v25.0
43 ["-nowallet", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v24.0.1
44 ["-nowallet", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v23.0
45 ["-nowallet", "-addresstype=bech32", "-whitelist=noban@127.0.0.1", f"-keypool={LAST_KEYPOOL_INDEX + 1}"], # v22.0
46 ["-nowallet", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.21.0
47 ["-nowallet", "-addresstype=bech32", "-whitelist=noban@127.0.0.1"], # v0.20.1
48 ]
49 self.wallet_names = [self.default_wallet_name]
50 51 def skip_test_if_missing_module(self):
52 self.skip_if_no_wallet()
53 self.skip_if_no_previous_releases()
54 55 def setup_nodes(self):
56 self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[
57 None,
58 None,
59 250000,
60 240001,
61 230000,
62 220000,
63 210000,
64 200100,
65 ])
66 67 self.start_nodes()
68 self.import_deterministic_coinbase_privkeys()
69 70 def split_version(self, node):
71 major = node.version // 10000
72 minor = (node.version % 10000) // 100
73 patch = (node.version % 100)
74 return (major, minor, patch)
75 76 def major_version_equals(self, node, major):
77 node_major, _, _ = self.split_version(node)
78 return node_major == major
79 80 def major_version_less_than(self, node, major):
81 node_major, _, _ = self.split_version(node)
82 return node_major < major
83 84 def major_version_at_least(self, node, major):
85 node_major, _, _ = self.split_version(node)
86 return node_major >= major
87 88 def test_v22_inactivehdchain_path(self):
89 self.log.info("Testing inactive hd chain bad derivation path cleanup")
90 # 0.21.x and 22.x would both produce bad derivation paths when topping up an inactive hd chain
91 # Make sure that this is being automatically cleaned up by migration
92 node_master = self.nodes[1]
93 node_v22 = self.nodes[self.num_nodes - 3]
94 wallet_name = "bad_deriv_path"
95 node_v22.createwallet(wallet_name=wallet_name, descriptors=False)
96 bad_deriv_wallet = node_v22.get_wallet_rpc(wallet_name)
97 98 # Make a dump of the wallet to get an unused address
99 dump_path = node_v22.wallets_path / f"{wallet_name}.dump"
100 bad_deriv_wallet.dumpwallet(dump_path)
101 addr = None
102 seed = None
103 with open(dump_path) as f:
104 for line in f:
105 if f"hdkeypath=m/0'/0'/{LAST_KEYPOOL_INDEX}'" in line:
106 addr = line.split(" ")[4].split("=")[1]
107 elif " hdseed=1 " in line:
108 seed = line.split(" ")[0]
109 assert addr is not None
110 assert seed is not None
111 # Rotate seed and unload
112 bad_deriv_wallet.sethdseed()
113 bad_deriv_wallet.unloadwallet()
114 # Receive at addr to trigger inactive chain topup on next load
115 self.nodes[0].sendtoaddress(addr, 1)
116 self.generate(self.nodes[0], 1, sync_fun=self.no_op)
117 self.sync_all(nodes=[self.nodes[0], node_master, node_v22])
118 node_v22.loadwallet(wallet_name)
119 120 # Dump again to find bad hd keypath
121 bad_deriv_path = f"m/0'/0'/{LAST_KEYPOOL_INDEX}'/0'/0'/{LAST_KEYPOOL_INDEX + 1}'"
122 good_deriv_path = f"m/0h/0h/{LAST_KEYPOOL_INDEX + 1}h"
123 os.unlink(dump_path)
124 bad_deriv_wallet.dumpwallet(dump_path)
125 bad_path_addr = None
126 with open(dump_path) as f:
127 for line in f:
128 if f"hdkeypath={bad_deriv_path}" in line:
129 bad_path_addr = line.split(" ")[4].split("=")[1]
130 assert bad_path_addr is not None
131 assert_equal(bad_deriv_wallet.getaddressinfo(bad_path_addr)["hdkeypath"], bad_deriv_path)
132 133 # Verify that this bad derivation path addr is actually at m/0'/0'/10' by making a new wallet with the same seed but larger keypool
134 node_v22.createwallet(wallet_name="path_verify", descriptors=False, blank=True)
135 verify_wallet = node_v22.get_wallet_rpc("path_verify")
136 verify_wallet.sethdseed(True, seed)
137 # Bad addr is after keypool, so need to generate it by refilling
138 verify_wallet.keypoolrefill(LAST_KEYPOOL_INDEX + 2)
139 assert_equal(verify_wallet.getaddressinfo(bad_path_addr)["hdkeypath"], good_deriv_path.replace("h", "'"))
140 141 # Migrate with master
142 # Since all keymeta records are now deleted after migration, the derivation path
143 # should now be correct as it is derived on-the-fly from the inactive hd chain's descriptor
144 backup_path = node_v22.wallets_path / f"{wallet_name}.bak"
145 bad_deriv_wallet.backupwallet(backup_path)
146 wallet_dir_master = os.path.join(node_master.wallets_path, wallet_name)
147 os.makedirs(wallet_dir_master, exist_ok=True)
148 shutil.copy(backup_path, os.path.join(wallet_dir_master, "wallet.dat"))
149 node_master.migratewallet(wallet_name)
150 bad_deriv_wallet_master = node_master.get_wallet_rpc(wallet_name)
151 assert_equal(bad_deriv_wallet_master.getaddressinfo(bad_path_addr)["hdkeypath"], good_deriv_path)
152 bad_deriv_wallet_master.unloadwallet()
153 154 def check_keymeta(conn):
155 # Retrieve all records that have the "keymeta" prefix. The remaining key data varies for each record.
156 keymeta_rec = conn.execute(f"SELECT value FROM main where key >= x'{ser_string(b'keymeta').hex()}' AND key < x'{ser_string(b'keymetb').hex()}'").fetchone()
157 assert_equal(keymeta_rec, None)
158 159 wallet_db = node_master.wallets_path / wallet_name / "wallet.dat"
160 self.inspect_sqlite_db(wallet_db, check_keymeta)
161 162 def test_ignore_legacy_during_startup(self, legacy_nodes, node_master):
163 self.log.info("Test that legacy wallets are ignored during startup on v29+")
164 165 legacy_node = legacy_nodes[0]
166 wallet_name = f"legacy_up_{legacy_node.version}"
167 legacy_node.loadwallet(wallet_name)
168 legacy_wallet = legacy_node.get_wallet_rpc(wallet_name)
169 170 # Move legacy wallet to latest node
171 wallet_path = node_master.wallets_path / wallet_name
172 wallet_path.mkdir()
173 legacy_wallet.backupwallet(wallet_path / "wallet.dat")
174 legacy_wallet.unloadwallet()
175 176 # Write wallet so it is automatically loaded during init
177 settings_path = node_master.chain_path / "settings.json"
178 with settings_path.open("w") as fp:
179 json.dump({"wallet": [wallet_name]}, fp)
180 181 # Restart latest node and verify that the legacy wallet load is skipped without exiting early during init.
182 self.restart_node(node_master.index, extra_args=[])
183 # Ensure we receive the warning message and clear the stderr pipe.
184 node_master.stderr.seek(0)
185 warning_msg = node_master.stderr.read().decode('utf-8').strip()
186 assert "The wallet appears to be a Legacy wallet, please use the wallet migration tool (migratewallet RPC or the GUI option)" in warning_msg
187 node_master.stderr.truncate(0), node_master.stderr.seek(0) # reset buffer
188 189 # Verify the node is still running (no shutdown occurred during startup)
190 node_master.getblockcount()
191 # Reset settings for any subsequent test
192 os.remove(settings_path)
193 194 def run_test(self):
195 node_miner = self.nodes[0]
196 node_master = self.nodes[1]
197 node_v21 = self.nodes[self.num_nodes - 2]
198 node_v20 = self.nodes[self.num_nodes - 1] # bdb only
199 200 legacy_nodes = self.nodes[2:] # Nodes that support legacy wallets
201 descriptors_nodes = self.nodes[2:-1] # Nodes that support descriptor wallets
202 203 self.generatetoaddress(node_miner, COINBASE_MATURITY + 1, node_miner.getnewaddress())
204 205 # Sanity check the test framework:
206 assert_equal(node_v20.getblockchaininfo()["blocks"], COINBASE_MATURITY + 1)
207 208 self.log.info("Test wallet backwards compatibility...")
209 # Create a number of wallets and open them in older versions:
210 211 # w1: regular wallet, created on master: update this test when default
212 # wallets can no longer be opened by older versions.
213 node_master.createwallet(wallet_name="w1")
214 wallet = node_master.get_wallet_rpc("w1")
215 info = wallet.getwalletinfo()
216 assert info['private_keys_enabled']
217 assert info['keypoolsize'] > 0
218 # Create a confirmed transaction, receiving coins
219 address = wallet.getnewaddress()
220 node_miner.sendtoaddress(address, 10)
221 self.sync_mempools()
222 self.generate(node_miner, 1)
223 # Create a conflicting transaction using RBF
224 return_address = node_miner.getnewaddress()
225 tx1_id = node_master.sendtoaddress(return_address, 1)
226 tx2_id = node_master.bumpfee(tx1_id)["txid"]
227 # Confirm the transaction
228 self.sync_mempools()
229 self.generate(node_miner, 1)
230 # Create another conflicting transaction using RBF
231 tx3_id = node_master.sendtoaddress(return_address, 1)
232 tx4_id = node_master.bumpfee(tx3_id)["txid"]
233 self.sync_mempools()
234 # Abandon transaction, but don't confirm
235 node_master.abandontransaction(tx3_id)
236 237 # w2: wallet with private keys disabled, created on master: update this
238 # test when default wallets private keys disabled can no longer be
239 # opened by older versions.
240 node_master.createwallet(wallet_name="w2", disable_private_keys=True)
241 wallet = node_master.get_wallet_rpc("w2")
242 info = wallet.getwalletinfo()
243 assert_equal(info['private_keys_enabled'], False)
244 assert_equal(info['keypoolsize'], 0)
245 246 # w3: blank wallet, created on master: update this
247 # test when default blank wallets can no longer be opened by older versions.
248 node_master.createwallet(wallet_name="w3", blank=True)
249 wallet = node_master.get_wallet_rpc("w3")
250 info = wallet.getwalletinfo()
251 assert info['private_keys_enabled']
252 assert_equal(info['keypoolsize'], 0)
253 254 # Unload wallets and copy to older nodes:
255 node_master_wallets_dir = node_master.wallets_path
256 node_master.unloadwallet("w1")
257 node_master.unloadwallet("w2")
258 node_master.unloadwallet("w3")
259 260 for node in legacy_nodes:
261 # Copy wallets to previous version
262 for wallet in os.listdir(node_master_wallets_dir):
263 dest = node.wallets_path / wallet
264 source = node_master_wallets_dir / wallet
265 shutil.copytree(source, dest)
266 267 self.log.info("Test that a wallet made on master can be opened on:")
268 # This test only works on the nodes that support descriptor wallets
269 # since we can no longer create legacy wallets.
270 for node in descriptors_nodes:
271 self.log.info(f"- {node.version}")
272 for wallet_name in ["w1", "w2", "w3"]:
273 if self.major_version_less_than(node, 22) and wallet_name == "w1":
274 # Descriptor wallets created after 0.21 have taproot descriptors which 0.21 does not support, tested below
275 continue
276 # Also try to reopen on master after opening on old
277 for n in [node, node_master]:
278 n.loadwallet(wallet_name)
279 wallet = n.get_wallet_rpc(wallet_name)
280 info = wallet.getwalletinfo()
281 if wallet_name == "w1":
282 assert_equal(info['private_keys_enabled'], True)
283 assert info['keypoolsize'] > 0
284 txs = wallet.listtransactions()
285 assert_equal(len(txs), 5)
286 assert_equal(txs[1]["txid"], tx1_id)
287 assert_equal(txs[2]["walletconflicts"], [tx1_id])
288 assert_equal(txs[1]["replaced_by_txid"], tx2_id)
289 assert not txs[1]["abandoned"]
290 assert_equal(txs[1]["confirmations"], -1)
291 assert_equal(txs[2]["blockindex"], 1)
292 assert txs[3]["abandoned"]
293 assert_equal(txs[4]["walletconflicts"], [tx3_id])
294 assert_equal(txs[3]["replaced_by_txid"], tx4_id)
295 assert not hasattr(txs[3], "blockindex")
296 elif wallet_name == "w2":
297 assert_equal(info['private_keys_enabled'], False)
298 assert_equal(info['keypoolsize'], 0)
299 else:
300 assert_equal(info['private_keys_enabled'], True)
301 assert_equal(info['keypoolsize'], 0)
302 303 # Copy back to master
304 wallet.unloadwallet()
305 if n == node:
306 (node_master.wallets_path / wallet_name / "wallet.dat").unlink()
307 (node_master.wallets_path / wallet_name).rmdir()
308 shutil.copytree(
309 n.wallets_path / wallet_name,
310 node_master.wallets_path / wallet_name,
311 )
312 313 # Check that descriptor wallets don't work on legacy only nodes
314 self.log.info("Test descriptor wallet incompatibility on v0.20")
315 # Descriptor wallets appear to be corrupted wallets to old software
316 assert self.major_version_equals(node_v20, 20)
317 for wallet_name in ["w1", "w2", "w3"]:
318 assert_raises_rpc_error(-4, "Wallet file verification failed: wallet.dat corrupt, salvage failed", node_v20.loadwallet, wallet_name)
319 320 # w1 cannot be opened by 0.21 since it contains a taproot descriptor
321 self.log.info("Test that 0.21 cannot open wallet containing tr() descriptors")
322 assert_raises_rpc_error(-1, "map::at", node_v21.loadwallet, "w1")
323 324 self.log.info("Test that a wallet can upgrade to and downgrade from master, from:")
325 for node in descriptors_nodes:
326 self.log.info(f"- {node.version}")
327 wallet_name = f"up_{node.version}"
328 node.createwallet(wallet_name=wallet_name, descriptors=True)
329 wallet_prev = node.get_wallet_rpc(wallet_name)
330 address = wallet_prev.getnewaddress('', "bech32")
331 addr_info = wallet_prev.getaddressinfo(address)
332 333 hdkeypath = addr_info["hdkeypath"].replace("'", "h")
334 pubkey = addr_info["pubkey"]
335 336 # Make a backup of the wallet file
337 backup_path = os.path.join(self.options.tmpdir, f"{wallet_name}.dat")
338 wallet_prev.backupwallet(backup_path)
339 340 # Remove the wallet from old node
341 wallet_prev.unloadwallet()
342 343 # Open backup with sqlite and get flags
344 def get_flags(conn):
345 flags_rec = conn.execute(f"SELECT value FROM main WHERE key = x'{ser_string(b'flags').hex()}'").fetchone()
346 return int.from_bytes(flags_rec[0], byteorder="little")
347 348 old_flags = self.inspect_sqlite_db(backup_path, get_flags)
349 350 # Restore the wallet to master
351 load_res = node_master.restorewallet(wallet_name, backup_path)
352 # There should be no warnings
353 assert "warnings" not in load_res
354 355 wallet = node_master.get_wallet_rpc(wallet_name)
356 info = wallet.getaddressinfo(address)
357 descriptor = f"wpkh([{info['hdmasterfingerprint']}{hdkeypath[1:]}]{pubkey})"
358 assert_equal(info["desc"], descsum_create(descriptor))
359 360 # Make backup so the wallet can be copied back to old node
361 down_wallet_name = f"re_down_{node.version}"
362 down_backup_path = os.path.join(self.options.tmpdir, f"{down_wallet_name}.dat")
363 wallet.backupwallet(down_backup_path)
364 365 # Check that taproot descriptors can be added to 0.21 wallets
366 # This must be done after the backup is created so that 0.21 can still load
367 # the backup
368 if self.major_version_equals(node, 21):
369 assert_raises_rpc_error(-12, "No bech32m addresses available", wallet.getnewaddress, address_type="bech32m")
370 xpubs = wallet.gethdkeys(active_only=True)
371 assert_equal(len(xpubs), 1)
372 assert_equal(len(xpubs[0]["descriptors"]), 6)
373 wallet.createwalletdescriptor("bech32m")
374 xpubs = wallet.gethdkeys(active_only=True)
375 assert_equal(len(xpubs), 1)
376 assert_equal(len(xpubs[0]["descriptors"]), 8)
377 tr_descs = [desc["desc"] for desc in xpubs[0]["descriptors"] if desc["desc"].startswith("tr(")]
378 assert_equal(len(tr_descs), 2)
379 for desc in tr_descs:
380 assert info["hdmasterfingerprint"] in desc
381 wallet.getnewaddress(address_type="bech32m")
382 383 wallet.unloadwallet()
384 385 # Open the wallet with sqlite and inspect the flags and records
386 def check_upgraded_records(conn, old_flags):
387 flags_rec = conn.execute(f"SELECT value FROM main WHERE key = x'{ser_string(b'flags').hex()}'").fetchone()
388 new_flags = int.from_bytes(flags_rec[0], byteorder="little")
389 diff_flags = new_flags & ~old_flags
390 391 # Check for last hardened xpubs if the flag is newly set
392 if diff_flags & (1 << 2):
393 self.log.debug("Checking descriptor cache was upgraded")
394 # Fetch all records with the walletdescriptorlhcache prefix
395 lh_cache_recs = conn.execute(f"SELECT value FROM main where key >= x'{ser_string(b'walletdescriptorlhcache').hex()}' AND key < x'{ser_string(b'walletdescriptorlhcachf').hex()}'").fetchall()
396 assert_greater_than(len(lh_cache_recs), 0)
397 398 self.inspect_sqlite_db(down_backup_path, check_upgraded_records, old_flags)
399 400 # Check that no automatic upgrade broke downgrading the wallet
401 target_dir = node.wallets_path / down_wallet_name
402 os.makedirs(target_dir, exist_ok=True)
403 shutil.copyfile(
404 down_backup_path,
405 target_dir / "wallet.dat"
406 )
407 node.loadwallet(down_wallet_name)
408 wallet_res = node.get_wallet_rpc(down_wallet_name)
409 info = wallet_res.getaddressinfo(address)
410 assert_equal(info, addr_info)
411 412 self.log.info("Test that a wallet from a legacy only node must be migrated, from:")
413 for node in legacy_nodes:
414 self.log.info(f"- {node.version}")
415 wallet_name = f"legacy_up_{node.version}"
416 if self.major_version_at_least(node, 21):
417 node.createwallet(wallet_name=wallet_name, descriptors=False)
418 else:
419 node.createwallet(wallet_name=wallet_name)
420 wallet_prev = node.get_wallet_rpc(wallet_name)
421 address = wallet_prev.getnewaddress('', "bech32")
422 addr_info = wallet_prev.getaddressinfo(address)
423 424 # Make a backup of the wallet file
425 backup_path = os.path.join(self.options.tmpdir, f"{wallet_name}.dat")
426 wallet_prev.backupwallet(backup_path)
427 428 # Remove the wallet from old node
429 wallet_prev.unloadwallet()
430 431 # Restore the wallet to master
432 # Legacy wallets are no longer supported. Trying to load these should result in an error
433 assert_raises_rpc_error(-18, "The wallet appears to be a Legacy wallet, please use the wallet migration tool (migratewallet RPC or the GUI option)", node_master.restorewallet, wallet_name, backup_path)
434 435 self.test_v22_inactivehdchain_path()
436 self.test_ignore_legacy_during_startup(legacy_nodes, node_master)
437 438 if __name__ == '__main__':
439 BackwardsCompatibilityTest(__file__).main()
440