test_runner.py raw
1 #!/usr/bin/env python3
2 # Copyright (c) 2014-present The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Run regression test suite.
6
7 This module calls down into individual test cases via subprocess. It will
8 forward all unrecognized arguments onto the individual test scripts.
9
10 For a description of arguments recognized by test scripts, see
11 `test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`.
12
13 """
14
15 import argparse
16 from collections import deque
17 from concurrent import futures
18 import configparser
19 import csv
20 import datetime
21 import os
22 import pathlib
23 import platform
24 import time
25 import shutil
26 import signal
27 import subprocess
28 import sys
29 import tempfile
30 import re
31 import logging
32 from test_framework.util import (
33 Binaries,
34 export_env_build_path,
35 get_binary_paths,
36 )
37
38 # Minimum amount of space to run the tests.
39 MIN_FREE_SPACE = 1.1 * 1024 * 1024 * 1024
40 # Additional space to run an extra job.
41 ADDITIONAL_SPACE_PER_JOB = 100 * 1024 * 1024
42 # Minimum amount of space required for --nocleanup
43 MIN_NO_CLEANUP_SPACE = 12 * 1024 * 1024 * 1024
44
45 # Formatting. Default colors to empty strings.
46 DEFAULT, BOLD, GREEN, RED = ("", ""), ("", ""), ("", ""), ("", "")
47 try:
48 # Make sure python thinks it can write unicode to its stdout
49 "\u2713".encode("utf_8").decode(sys.stdout.encoding)
50 TICK = "✓ "
51 CROSS = "✖ "
52 CIRCLE = "○ "
53 except UnicodeDecodeError:
54 TICK = "P "
55 CROSS = "x "
56 CIRCLE = "o "
57
58 if platform.system() == 'Windows':
59 import ctypes
60 kernel32 = ctypes.windll.kernel32 # type: ignore
61 ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4
62 STD_OUTPUT_HANDLE = -11
63 STD_ERROR_HANDLE = -12
64 # Enable ascii color control to stdout
65 stdout = kernel32.GetStdHandle(STD_OUTPUT_HANDLE)
66 stdout_mode = ctypes.c_int32()
67 kernel32.GetConsoleMode(stdout, ctypes.byref(stdout_mode))
68 kernel32.SetConsoleMode(stdout, stdout_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
69 # Enable ascii color control to stderr
70 stderr = kernel32.GetStdHandle(STD_ERROR_HANDLE)
71 stderr_mode = ctypes.c_int32()
72 kernel32.GetConsoleMode(stderr, ctypes.byref(stderr_mode))
73 kernel32.SetConsoleMode(stderr, stderr_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
74 else:
75 # primitive formatting on supported
76 # terminal via ANSI escape sequences:
77 DEFAULT = ('\033[0m', '\033[0m')
78 BOLD = ('\033[0m', '\033[1m')
79 GREEN = ('\033[0m', '\033[0;32m')
80 RED = ('\033[0m', '\033[0;31m')
81
82 TEST_EXIT_PASSED = 0
83 TEST_EXIT_SKIPPED = 77
84
85 TEST_FRAMEWORK_UNIT_TESTS = 'feature_framework_unit_tests.py'
86
87 EXTENDED_SCRIPTS = [
88 # These tests are not run by default.
89 # Longest test should go first, to favor running tests in parallel
90 'feature_pruning.py',
91 'feature_dbcrash.py',
92 'feature_index_prune.py',
93 ]
94
95 # Special script to run each bench sanity check
96 TOOL_BENCH_SANITY_CHECK = "tool_bench_sanity_check.py"
97
98 BASE_SCRIPTS = [
99 # Special scripts that are "expanded" later
100 TOOL_BENCH_SANITY_CHECK,
101 # Scripts that are run by default.
102 # Longest test should go first, to favor running tests in parallel
103 # vv Tests less than 5m vv
104 'feature_fee_estimation.py',
105 'feature_taproot.py',
106 'feature_block.py',
107 'mempool_ephemeral_dust.py',
108 'wallet_conflicts.py',
109 'p2p_opportunistic_1p1c.py',
110 'p2p_node_network_limited.py --v1transport',
111 'p2p_node_network_limited.py --v2transport',
112 # vv Tests less than 2m vv
113 'mining_getblocktemplate_longpoll.py',
114 'p2p_segwit.py',
115 'feature_maxuploadtarget.py',
116 'feature_assumeutxo.py',
117 'mempool_updatefromblock.py',
118 'mempool_persist.py',
119 # vv Tests less than 60s vv
120 'rpc_psbt.py',
121 'wallet_fundrawtransaction.py',
122 'wallet_bumpfee.py',
123 'wallet_v3_txs.py',
124 'wallet_backup.py',
125 'feature_segwit.py --v2transport',
126 'feature_segwit.py --v1transport',
127 'p2p_tx_download.py',
128 'wallet_avoidreuse.py',
129 'feature_abortnode.py',
130 'wallet_address_types.py',
131 'p2p_orphan_handling.py',
132 'wallet_basic.py',
133 'feature_maxtipage.py',
134 'wallet_multiwallet.py',
135 'wallet_multiwallet.py --usecli',
136 'p2p_dns_seeds.py',
137 'wallet_groups.py',
138 'p2p_blockfilters.py',
139 'feature_assumevalid.py',
140 'wallet_taproot.py',
141 'feature_bip68_sequence.py',
142 'rpc_packages.py',
143 'rpc_bind.py --ipv4',
144 'rpc_bind.py --ipv6',
145 'rpc_bind.py --nonloopback',
146 'p2p_headers_sync_with_minchainwork.py',
147 'p2p_feefilter.py',
148 'feature_csv_activation.py',
149 'p2p_sendheaders.py',
150 'feature_config_args.py',
151 'wallet_listtransactions.py',
152 'wallet_miniscript.py',
153 # vv Tests less than 30s vv
154 'wallet_deprecated_rbf.py',
155 'p2p_invalid_messages.py',
156 'rpc_createmultisig.py',
157 'p2p_timeouts.py --v1transport',
158 'p2p_timeouts.py --v2transport',
159 'rpc_signer.py',
160 'wallet_signer.py',
161 'mempool_limit.py',
162 'rpc_txoutproof.py',
163 'rpc_orphans.py',
164 'wallet_listreceivedby.py',
165 'wallet_abandonconflict.py',
166 'wallet_anchor.py',
167 'feature_reindex.py',
168 'feature_reindex_readonly.py',
169 'wallet_labels.py',
170 'p2p_compactblocks.py',
171 'p2p_compactblocks_blocksonly.py',
172 'wallet_hd.py',
173 'wallet_blank.py',
174 'wallet_keypool_topup.py',
175 'wallet_fast_rescan.py',
176 'wallet_gethdkeys.py',
177 'wallet_createwalletdescriptor.py',
178 'wallet_exported_watchonly.py',
179 'interface_zmq.py',
180 'rpc_invalid_address_message.py',
181 'rpc_validateaddress.py',
182 'interface_bitcoin_cli.py',
183 'feature_bind_extra.py',
184 'mempool_resurrect.py',
185 'wallet_txn_doublespend.py --mineblock',
186 'tool_bitcoin_chainstate.py',
187 'tool_wallet.py',
188 'tool_utils.py',
189 'tool_signet_miner.py',
190 'wallet_txn_clone.py',
191 'wallet_txn_clone.py --segwit',
192 'rpc_getchaintips.py',
193 'rpc_misc.py',
194 'p2p_1p1c_network.py',
195 'interface_rest.py',
196 'mempool_spend_coinbase.py',
197 'wallet_avoid_mixing_output_types.py',
198 'mempool_reorg.py',
199 'p2p_block_sync.py --v1transport',
200 'p2p_block_sync.py --v2transport',
201 'wallet_createwallet.py --usecli',
202 'wallet_createwallet.py',
203 'wallet_reindex.py',
204 'wallet_reorgsrestore.py',
205 'interface_http.py',
206 'interface_rpc.py',
207 'interface_usdt_coinselection.py',
208 'interface_usdt_mempool.py',
209 'interface_usdt_net.py',
210 'interface_usdt_utxocache.py',
211 'interface_usdt_validation.py',
212 'rpc_users.py',
213 'rpc_whitelist.py',
214 'feature_proxy.py',
215 'wallet_signrawtransactionwithwallet.py',
216 'rpc_signrawtransactionwithkey.py',
217 'rpc_rawtransaction.py',
218 'wallet_transactiontime_rescan.py',
219 'p2p_addrv2_relay.py',
220 'p2p_compactblocks_hb.py --v1transport',
221 'p2p_compactblocks_hb.py --v2transport',
222 'p2p_disconnect_ban.py --v1transport',
223 'p2p_disconnect_ban.py --v2transport',
224 'feature_posix_fs_permissions.py',
225 'rpc_decodescript.py',
226 'rpc_blockchain.py --v1transport',
227 'rpc_blockchain.py --v2transport',
228 'mining_template_verification.py',
229 'rpc_deprecated.py',
230 'wallet_disable.py',
231 'wallet_change_address.py',
232 'p2p_addr_relay.py',
233 'p2p_getaddr_caching.py',
234 'p2p_getdata.py',
235 'p2p_addrfetch.py',
236 'rpc_net.py --v1transport',
237 'rpc_net.py --v2transport',
238 'wallet_keypool.py',
239 'wallet_descriptor.py',
240 'p2p_nobloomfilter_messages.py',
241 TEST_FRAMEWORK_UNIT_TESTS,
242 'p2p_filter.py',
243 'rpc_setban.py --v1transport',
244 'rpc_setban.py --v2transport',
245 'p2p_blocksonly.py',
246 'mining_prioritisetransaction.py',
247 'p2p_invalid_locator.py',
248 'p2p_invalid_block.py --v1transport',
249 'p2p_invalid_block.py --v2transport',
250 'p2p_invalid_tx.py --v1transport',
251 'p2p_invalid_tx.py --v2transport',
252 'p2p_v2_transport.py',
253 'p2p_v2_encrypted.py',
254 'p2p_v2_misbehaving.py',
255 'example_test.py',
256 'mempool_truc.py',
257 'wallet_multisig_descriptor_psbt.py',
258 'wallet_miniscript_decaying_multisig_descriptor_psbt.py',
259 'wallet_txn_doublespend.py',
260 'wallet_backwards_compatibility.py',
261 'wallet_txn_clone.py --mineblock',
262 'feature_notifications.py',
263 'rpc_getblockfilter.py',
264 'rpc_getblockfrompeer.py',
265 'rpc_invalidateblock.py',
266 'feature_utxo_set_hash.py',
267 'feature_rbf.py',
268 'mempool_packages.py',
269 'mempool_package_limits.py',
270 'mempool_package_rbf.py',
271 'tool_utxo_to_sqlite.py',
272 'feature_versionbits_warning.py',
273 'feature_blocksxor.py',
274 'rpc_preciousblock.py',
275 'wallet_importprunedfunds.py',
276 'p2p_leak_tx.py --v1transport',
277 'p2p_leak_tx.py --v2transport',
278 'p2p_eviction.py',
279 'p2p_outbound_eviction.py',
280 'p2p_ibd_stalling.py --v1transport',
281 'p2p_ibd_stalling.py --v2transport',
282 'p2p_net_deadlock.py --v1transport',
283 'p2p_net_deadlock.py --v2transport',
284 'wallet_signmessagewithaddress.py',
285 'rpc_signmessagewithprivkey.py',
286 'rpc_generate.py',
287 'wallet_balance.py',
288 'p2p_initial_headers_sync.py',
289 'feature_nulldummy.py',
290 'mempool_accept.py',
291 'p2p_addr_selfannouncement.py',
292 'mempool_expiry.py',
293 'wallet_importdescriptors.py',
294 'wallet_crosschain.py',
295 'mining_basic.py',
296 'mining_mainnet.py',
297 'feature_signet.py',
298 'p2p_mutated_blocks.py',
299 'rpc_named_arguments.py',
300 'feature_startupnotify.py',
301 'wallet_simulaterawtx.py',
302 'wallet_listsinceblock.py',
303 'wallet_listdescriptors.py',
304 'p2p_leak.py',
305 'p2p_bip434_feature.py',
306 'p2p_bip434_feature.py --v2transport',
307 'wallet_encryption.py',
308 'feature_dersig.py',
309 'feature_reindex_init.py',
310 'feature_cltv.py',
311 'rpc_uptime.py',
312 'feature_discover.py',
313 'wallet_resendwallettransactions.py',
314 'wallet_fallbackfee.py',
315 'rpc_dumptxoutset.py',
316 'feature_minchainwork.py',
317 'rpc_estimatefee.py',
318 'p2p_private_broadcast.py',
319 'p2p_private_broadcast_cap.py',
320 'rpc_getblockstats.py',
321 'feature_port.py',
322 'feature_bind_port_externalip.py',
323 'wallet_create_tx.py',
324 'wallet_send.py',
325 'wallet_sendall.py',
326 'wallet_sendmany.py',
327 'wallet_spend_unconfirmed.py',
328 'wallet_rescan_unconfirmed.py',
329 'p2p_fingerprint.py',
330 'feature_uacomment.py',
331 'feature_init.py',
332 'wallet_coinbase_category.py',
333 'feature_filelock.py',
334 'feature_loadblock.py',
335 'wallet_assumeutxo.py',
336 'p2p_add_connections.py',
337 'feature_bind_port_discover.py',
338 'p2p_unrequested_blocks.py',
339 'p2p_message_capture.py',
340 'feature_includeconf.py',
341 'feature_addrman.py',
342 'feature_asmap.py',
343 'feature_chain_tiebreaks.py',
344 'feature_fastprune.py',
345 'feature_framework_miniwallet.py',
346 'mempool_unbroadcast.py',
347 'mempool_compatibility.py',
348 'mempool_accept_wtxid.py',
349 'mempool_dust.py',
350 'mempool_sigoplimit.py',
351 'rpc_deriveaddresses.py',
352 'rpc_deriveaddresses.py --usecli',
353 'p2p_ping.py',
354 'p2p_tx_privacy.py',
355 'rpc_getdescriptoractivity.py',
356 'rpc_scanblocks.py',
357 'tool_bitcoin.py',
358 'p2p_sendtxrcncl.py',
359 'rpc_scantxoutset.py',
360 'feature_torcontrol.py',
361 'feature_unsupported_utxo_db.py',
362 'mempool_cluster.py',
363 'feature_logging.py',
364 'interface_ipc.py',
365 'interface_ipc_mining.py',
366 'feature_anchors.py',
367 'mempool_datacarrier.py',
368 'feature_coinstatsindex.py',
369 'feature_coinstatsindex_compatibility.py',
370 'wallet_orphanedreward.py',
371 'wallet_musig.py',
372 'wallet_timelock.py',
373 'p2p_permissions.py',
374 'feature_blocksdir.py',
375 'wallet_startup.py',
376 'p2p_private_broadcast_retry_v1.py',
377 'feature_remove_pruned_files_on_startup.py',
378 'feature_prune_stale_fork.py',
379 'p2p_i2p_ports.py',
380 'p2p_i2p_sessions.py',
381 'feature_presegwit_node_upgrade.py',
382 'feature_settings.py',
383 'rpc_getdescriptorinfo.py',
384 'rpc_gettxspendingprevout.py',
385 'rpc_help.py',
386 'feature_framework_testshell.py',
387 'tool_rpcauth.py',
388 'p2p_handshake.py',
389 'p2p_handshake.py --v2transport',
390 'interface_ipc_cli.py',
391 'feature_dirsymlinks.py',
392 'feature_help.py',
393 'feature_framework_startup_failures.py',
394 'feature_shutdown.py',
395 'wallet_migration.py',
396 'p2p_ibd_txrelay.py',
397 'p2p_seednode.py',
398 # Don't append tests at the end to avoid merge conflicts
399 # Put them in a random line within the section that fits their approximate run-time
400 ]
401
402 # Place EXTENDED_SCRIPTS first since it has the 3 longest running tests
403 ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS
404
405 NON_SCRIPTS = [
406 # These are python files that live in the functional tests directory, but are not test scripts.
407 "combine_logs.py",
408 "create_cache.py",
409 "test_runner.py",
410 ]
411
412 def main():
413 # Parse arguments and pass through unrecognised args
414 parser = argparse.ArgumentParser(add_help=False,
415 usage='%(prog)s [test_runner.py options] [script options] [scripts]',
416 description=__doc__,
417 epilog='''
418 Help text and arguments for individual test script:''',
419 formatter_class=argparse.RawTextHelpFormatter)
420 parser.add_argument('--ansi', action='store_true', default=sys.stdout.isatty(), help="Use ANSI colors and dots in output (enabled by default when standard output is a TTY)")
421 parser.add_argument('--combinedlogslen', '-c', type=int, default=0, metavar='n', help='On failure, print a log (of length n lines) to the console, combined from the test framework and all test nodes.')
422 parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface')
423 parser.add_argument('--exclude', '-x', action='append', help='specify a script to exclude. Can be specified multiple times. The .py extension is optional.')
424 parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
425 parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
426 parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
427 parser.add_argument('--quiet', '-q', action='store_true', help='only print dots, results summary and failure logs')
428 parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs")
429 parser.add_argument('--failfast', '-F', action='store_true', help='stop execution after the first test failure')
430 parser.add_argument('--filter', help='filter scripts to run by regular expression')
431 parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true",
432 help="Leave bitcoinds and test.* datadir on exit or error")
433 parser.add_argument('--resultsfile', '-r', help='store test results (as CSV) to the provided file')
434
435 args, unknown_args = parser.parse_known_args()
436 # Fail on self-check warnings before running the tests.
437 fail_on_warn = True
438 if not args.ansi:
439 global DEFAULT, BOLD, GREEN, RED
440 DEFAULT = ("", "")
441 BOLD = ("", "")
442 GREEN = ("", "")
443 RED = ("", "")
444
445 # args to be passed on always start with two dashes; tests are the remaining unknown args
446 tests = [arg for arg in unknown_args if arg[:2] != "--"]
447 passon_args = [arg for arg in unknown_args if arg[:2] == "--"]
448
449 # Read config generated by configure.
450 config = configparser.ConfigParser()
451 configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
452 config.read_file(open(configfile))
453
454 passon_args.append("--configfile=%s" % configfile)
455
456 # Set up logging
457 logging_level = logging.INFO if args.quiet else logging.DEBUG
458 logging.basicConfig(format='%(message)s', level=logging_level)
459
460 # Create base test directory
461 tmpdir = "%s/test_runner_₿_🏃_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
462
463 os.makedirs(tmpdir)
464
465 logging.debug("Temporary test directory at %s" % tmpdir)
466
467 results_filepath = None
468 if args.resultsfile:
469 results_filepath = pathlib.Path(args.resultsfile)
470 # Stop early if the parent directory doesn't exist
471 assert results_filepath.parent.exists(), "Results file parent directory does not exist"
472 logging.debug("Test results will be written to " + str(results_filepath))
473
474 enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND")
475
476 if not enable_bitcoind:
477 print("No functional tests to run.")
478 print("Re-compile with the -DBUILD_DAEMON=ON build option")
479 sys.exit(1)
480
481 export_env_build_path(config)
482
483 # Build tests
484 test_list = deque()
485 if tests:
486 # Individual tests have been specified. Run specified tests that exist
487 # in the ALL_SCRIPTS list. Accept names with or without a .py extension.
488 # Specified tests can contain wildcards, but in that case the supplied
489 # paths should be coherent, e.g. the same path as that provided to call
490 # test_runner.py. Examples:
491 # `test/functional/test_runner.py test/functional/wallet*`
492 # `test/functional/test_runner.py ./test/functional/wallet*`
493 # `test_runner.py wallet*`
494 # but not:
495 # `test/functional/test_runner.py wallet*`
496 # Multiple wildcards can be passed:
497 # `test_runner.py tool* mempool*`
498 for test in tests:
499 script = test.split("/")[-1]
500 script = script + ".py" if ".py" not in script else script
501 matching_scripts = [s for s in ALL_SCRIPTS if s.startswith(script)]
502 if matching_scripts:
503 test_list += matching_scripts
504 else:
505 print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], test))
506 elif args.extended:
507 # Include extended tests
508 test_list += ALL_SCRIPTS
509 else:
510 # Run base tests only
511 test_list += BASE_SCRIPTS
512
513 # Remove the test cases that the user has explicitly asked to exclude.
514 # The user can specify a test case with or without the .py extension.
515 if args.exclude:
516
517 def print_warning_missing_test(test_name):
518 print("{}WARNING!{} Test '{}' not found in current test list. Check the --exclude options.".format(BOLD[1], BOLD[0], test_name))
519 if fail_on_warn:
520 sys.exit(1)
521
522 def remove_tests(exclude_list):
523 if not exclude_list:
524 print_warning_missing_test(exclude_test)
525 for exclude_item in exclude_list:
526 print("Excluding %s" % exclude_item)
527 test_list.remove(exclude_item)
528
529 for exclude_test in args.exclude:
530 if ',' in exclude_test:
531 print("{}WARNING!{} --exclude '{}' contains a comma. Use --exclude for each test.".format(BOLD[1], BOLD[0], exclude_test))
532 if fail_on_warn:
533 sys.exit(1)
534
535 for exclude_test in args.exclude:
536 # A space in the name indicates it has arguments such as "rpc_bind.py --ipv4"
537 if ' ' in exclude_test:
538 remove_tests([test for test in test_list if test.replace('.py', '') == exclude_test.replace('.py', '')])
539 else:
540 # Exclude all variants of a test
541 remove_tests([test for test in test_list if test.split('.py')[0] == exclude_test.split('.py')[0]])
542
543 if config["components"].getboolean("BUILD_BENCH") and TOOL_BENCH_SANITY_CHECK in test_list:
544 # Remove it, and expand it for each bench in the list
545 test_list.remove(TOOL_BENCH_SANITY_CHECK)
546 bench_cmd = Binaries(get_binary_paths(config), bin_dir=None).bench_argv() + ["-list"]
547 bench_list = subprocess.check_output(bench_cmd, text=True).splitlines()
548 bench_list = [f"{TOOL_BENCH_SANITY_CHECK} --bench={b}" for b in bench_list]
549 # Start with special scripts (variable, unknown runtime)
550 test_list.extendleft(reversed(bench_list))
551
552 if args.filter:
553 test_list = deque(filter(re.compile(args.filter).search, test_list))
554
555 if not test_list:
556 print("No valid test scripts specified. Check that your test is in one "
557 "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests")
558 sys.exit(1)
559
560 if args.help:
561 # Print help for test_runner.py, then print help of the first script (with args removed) and exit.
562 parser.print_help()
563 subprocess.check_call([sys.executable, os.path.join(config["environment"]["SRCDIR"], 'test', 'functional', test_list[0].split()[0]), '-h'])
564 sys.exit(0)
565
566 # Warn if there is not enough space on tmpdir to run the tests with --nocleanup
567 if args.nocleanup:
568 if shutil.disk_usage(tmpdir).free < MIN_NO_CLEANUP_SPACE:
569 print(f"{BOLD[1]}WARNING!{BOLD[0]} There may be insufficient free space in {tmpdir} to run the functional test suite with --nocleanup. "
570 f"A minimum of {MIN_NO_CLEANUP_SPACE // (1024 * 1024 * 1024)} GB of free space is required.")
571 passon_args.append("--nocleanup")
572
573 check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=fail_on_warn)
574 check_script_prefixes()
575
576 run_tests(
577 test_list=test_list,
578 build_dir=config["environment"]["BUILDDIR"],
579 tmpdir=tmpdir,
580 jobs=args.jobs,
581 enable_coverage=args.coverage,
582 args=passon_args,
583 combined_logs_len=args.combinedlogslen,
584 failfast=args.failfast,
585 use_term_control=args.ansi,
586 results_filepath=results_filepath,
587 )
588
589 def run_tests(*, test_list, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control, results_filepath=None):
590 args = args or []
591
592 # Some optional Python dependencies (e.g. pycapnp) may emit warnings or fail under
593 # CPython free-threaded builds when the GIL is disabled. Force it on for all
594 # functional tests so every child process inherits PYTHON_GIL=1.
595 os.environ["PYTHON_GIL"] = "1"
596
597 # Warn if bitcoind is already running
598 try:
599 # pgrep exits with code zero when one or more matching processes found
600 if subprocess.run(["pgrep", "-x", "bitcoind"], stdout=subprocess.DEVNULL).returncode == 0:
601 print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0]))
602 except OSError:
603 # pgrep not supported
604 pass
605
606 # Warn if there is not enough space on the testing dir
607 min_space = MIN_FREE_SPACE + (jobs - 1) * ADDITIONAL_SPACE_PER_JOB
608 if shutil.disk_usage(tmpdir).free < min_space:
609 print(f"{BOLD[1]}WARNING!{BOLD[0]} There may be insufficient free space in {tmpdir} to run the Bitcoin functional test suite. "
610 f"Running the test suite with fewer than {min_space // (1024 * 1024)} MB of free space might cause tests to fail.")
611
612 tests_dir = f"{build_dir}/test/functional/"
613 # This allows `test_runner.py` to work from an out-of-source build directory using a symlink,
614 # a hard link or a copy on any platform. See https://github.com/bitcoin/bitcoin/pull/27561.
615 sys.path.append(tests_dir)
616
617 cache_tmp_dir = tempfile.TemporaryDirectory(prefix="functional_test_cache")
618 flags = [f"--cachedir={cache_tmp_dir.name}"] + args
619
620 if enable_coverage:
621 coverage = RPCCoverage()
622 flags.append(coverage.flag)
623 logging.debug("Initializing coverage directory at %s" % coverage.dir)
624 else:
625 coverage = None
626
627 if len(test_list) > 1 and jobs > 1:
628 # Populate cache
629 try:
630 subprocess.check_output([sys.executable, tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir])
631 except subprocess.CalledProcessError as e:
632 sys.stdout.buffer.write(e.output)
633 raise
634
635 # Run Tests
636 job_queue = TestHandler(
637 num_tests_parallel=jobs,
638 tests_dir=tests_dir,
639 tmpdir=tmpdir,
640 test_list=test_list,
641 flags=flags,
642 use_term_control=use_term_control,
643 )
644 start_time = time.time()
645 test_results = []
646
647 max_len_name = len(max(test_list, key=len))
648 test_count = len(test_list)
649 all_passed = True
650 while not job_queue.done():
651 if failfast and not all_passed:
652 break
653 for test_result, testdir, stdout, stderr, skip_reason in job_queue.get_next():
654 test_results.append(test_result)
655 done_str = f"{len(test_results)}/{test_count} - {BOLD[1]}{test_result.name}{BOLD[0]}"
656 if test_result.status == "Passed":
657 logging.debug("%s passed, Duration: %s s" % (done_str, test_result.time))
658 elif test_result.status == "Skipped":
659 logging.debug(f"{done_str} skipped ({skip_reason})")
660 else:
661 all_passed = False
662 print("%s failed, Duration: %s s\n" % (done_str, test_result.time))
663 print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n')
664 print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n')
665 if combined_logs_len and os.path.isdir(testdir):
666 # Print the final `combinedlogslen` lines of the combined logs
667 print('{}Combine the logs and print the last {} lines ...{}'.format(BOLD[1], combined_logs_len, BOLD[0]))
668 print('\n============')
669 print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0]))
670 print('============\n')
671 combined_logs_args = [sys.executable, os.path.join(tests_dir, 'combine_logs.py'), testdir]
672 if BOLD[0]:
673 combined_logs_args += ['--color']
674 combined_logs, _ = subprocess.Popen(combined_logs_args, text=True, stdout=subprocess.PIPE).communicate()
675 print("\n".join(deque(combined_logs.splitlines(), combined_logs_len)))
676
677 if failfast:
678 logging.debug("Early exiting after test failure")
679 break
680
681 if "[Errno 28] No space left on device" in stdout:
682 sys.exit(f"Early exiting after test failure due to insufficient free space in {tmpdir}\n"
683 f"Test execution data left in {tmpdir}.\n"
684 f"Additional storage is needed to execute testing.")
685
686 runtime = int(time.time() - start_time)
687 print_results(test_results, max_len_name, runtime)
688 if results_filepath:
689 write_results(test_results, results_filepath, runtime)
690
691 if coverage:
692 coverage_passed = coverage.report_rpc_coverage()
693 else:
694 coverage_passed = True
695
696 # Clear up the temp directory if all subdirectories are gone
697 if not os.listdir(tmpdir):
698 os.rmdir(tmpdir)
699
700 all_passed = all_passed and coverage_passed
701
702 # Clean up dangling processes if any. This may only happen with --failfast option.
703 # Killing the process group will also terminate the current process but that is
704 # not an issue
705 if not os.getenv("CI_FAILFAST_TEST_LEAVE_DANGLING") and len(job_queue.jobs):
706 os.killpg(os.getpgid(0), signal.SIGKILL)
707
708 sys.exit(not all_passed)
709
710
711 def print_results(test_results, max_len_name, runtime):
712 results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0]
713
714 test_results.sort(key=TestResult.sort_key)
715 all_passed = True
716 time_sum = 0
717
718 for test_result in test_results:
719 all_passed = all_passed and test_result.was_successful
720 time_sum += test_result.time
721 test_result.padding = max_len_name
722 results += str(test_result)
723
724 status = TICK + "Passed" if all_passed else CROSS + "Failed"
725 if not all_passed:
726 results += RED[1]
727 results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ("ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0]
728 if not all_passed:
729 results += RED[0]
730 results += "Runtime: %s s\n" % (runtime)
731 print(results)
732
733
734 def write_results(test_results, filepath, total_runtime):
735 with open(filepath, mode="w") as results_file:
736 results_writer = csv.writer(results_file)
737 results_writer.writerow(['test', 'status', 'duration(seconds)'])
738 all_passed = True
739 for test_result in test_results:
740 all_passed = all_passed and test_result.was_successful
741 results_writer.writerow([test_result.name, test_result.status, str(test_result.time)])
742 results_writer.writerow(['ALL', ("Passed" if all_passed else "Failed"), str(total_runtime)])
743
744 class TestHandler:
745 """
746 Trigger the test scripts passed in via the list.
747 """
748 def __init__(self, *, num_tests_parallel, tests_dir, tmpdir, test_list, flags, use_term_control):
749 assert num_tests_parallel >= 1
750 self.executor = futures.ThreadPoolExecutor(max_workers=num_tests_parallel)
751 self.num_jobs = num_tests_parallel
752 self.tests_dir = tests_dir
753 self.tmpdir = tmpdir
754 self.test_list = test_list
755 self.flags = flags
756 self.jobs = {}
757 self.use_term_control = use_term_control
758
759 def done(self):
760 return not (self.jobs or self.test_list)
761
762 def get_next(self):
763 while len(self.jobs) < self.num_jobs and self.test_list:
764 # Add tests
765 test = self.test_list.popleft()
766 portseed = len(self.test_list)
767 portseed_arg = ["--portseed={}".format(portseed)]
768 log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
769 log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
770 test_argv = test.split()
771 testdir = "{}/{}_{}".format(self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)
772 tmpdir_arg = ["--tmpdir={}".format(testdir)]
773
774 def proc_wait(task):
775 task[2].wait()
776 return task
777
778 task = [
779 test,
780 time.time(),
781 subprocess.Popen(
782 [sys.executable, self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir_arg,
783 text=True,
784 stdout=log_stdout,
785 stderr=log_stderr,
786 ),
787 testdir,
788 log_stdout,
789 log_stderr,
790 ]
791 fut = self.executor.submit(proc_wait, task)
792 self.jobs[fut] = test
793 assert self.jobs # Must not be empty here
794
795 # Print remaining running jobs when all jobs have been started.
796 if not self.test_list:
797 print("Remaining jobs: [{}]".format(", ".join(sorted(self.jobs.values()))))
798
799 dot_count = 0
800 while True:
801 # Return all procs that have finished, if any. Otherwise sleep until there is one.
802 procs = futures.wait(self.jobs.keys(), timeout=.5, return_when=futures.FIRST_COMPLETED)
803 self.jobs = {fut: self.jobs[fut] for fut in procs.not_done}
804 ret = []
805 for job in procs.done:
806 (name, start_time, proc, testdir, log_out, log_err) = job.result()
807
808 log_out.seek(0), log_err.seek(0)
809 [stdout, stderr] = [log_file.read().decode('utf-8') for log_file in (log_out, log_err)]
810 log_out.close(), log_err.close()
811 skip_reason = None
812 if proc.returncode == TEST_EXIT_PASSED and stderr == "":
813 status = "Passed"
814 elif proc.returncode == TEST_EXIT_SKIPPED:
815 status = "Skipped"
816 skip_reason = re.search(r"Test Skipped: (.*)", stdout).group(1).strip()
817 else:
818 status = "Failed"
819
820 if self.use_term_control:
821 clearline = '\r' + (' ' * dot_count) + '\r'
822 print(clearline, end='', flush=True)
823 dot_count = 0
824 ret.append((TestResult(name, status, int(time.time() - start_time)), testdir, stdout, stderr, skip_reason))
825 if ret:
826 return ret
827 if self.use_term_control:
828 print('.', end='', flush=True)
829 dot_count += 1
830
831
832 class TestResult():
833 def __init__(self, name, status, time):
834 self.name = name
835 self.status = status
836 self.time = time
837 self.padding = 0
838
839 def sort_key(self):
840 if self.status == "Passed":
841 return 0, self.name.lower()
842 elif self.status == "Failed":
843 return 2, self.name.lower()
844 elif self.status == "Skipped":
845 return 1, self.name.lower()
846
847 def __repr__(self):
848 if self.status == "Passed":
849 color = GREEN
850 glyph = TICK
851 elif self.status == "Failed":
852 color = RED
853 glyph = CROSS
854 elif self.status == "Skipped":
855 color = DEFAULT
856 glyph = CIRCLE
857
858 return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0]
859
860 @property
861 def was_successful(self):
862 return self.status != "Failed"
863
864
865 def check_script_prefixes():
866 """Check that test scripts start with one of the allowed name prefixes."""
867
868 good_prefixes_re = re.compile("^(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool)_")
869 bad_script_names = [script for script in ALL_SCRIPTS if good_prefixes_re.match(script) is None]
870
871 if bad_script_names:
872 print("%sERROR:%s %d tests not meeting naming conventions:" % (BOLD[1], BOLD[0], len(bad_script_names)))
873 print(" %s" % ("\n ".join(sorted(bad_script_names))))
874 raise AssertionError("Some tests are not following naming convention!")
875
876
877 def check_script_list(*, src_dir, fail_on_warn):
878 """Check scripts directory.
879
880 Check that all python files in this directory are categorized
881 as a test script or meta script."""
882 script_dir = src_dir + '/test/functional/'
883 python_files = set([test_file for test_file in os.listdir(script_dir) if test_file.endswith(".py")])
884 missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS)))
885 if len(missed_tests) != 0:
886 print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests)))
887 if fail_on_warn:
888 sys.exit(1)
889
890
891 class RPCCoverage():
892 """
893 Coverage reporting utilities for test_runner.
894
895 Coverage calculation works by having each test script subprocess write
896 coverage files into a particular directory. These files contain the RPC
897 commands invoked during testing, as well as a complete listing of RPC
898 commands per `bitcoin-cli help` (`rpc_interface.txt`).
899
900 After all tests complete, the commands run are combined and diff'd against
901 the complete list to calculate uncovered RPC commands.
902
903 See also: test/functional/test_framework/coverage.py
904
905 """
906 def __init__(self):
907 self.temp_dir = tempfile.TemporaryDirectory(prefix="coverage")
908 self.dir = self.temp_dir.name
909 self.flag = '--coveragedir=%s' % self.dir
910
911 def report_rpc_coverage(self):
912 """
913 Print out RPC commands that were unexercised by tests.
914
915 """
916 uncovered = self._get_uncovered_rpc_commands()
917
918 if uncovered:
919 print("Uncovered RPC commands:")
920 print("".join((" - %s\n" % command) for command in sorted(uncovered)))
921 return False
922 else:
923 print("All RPC commands covered.")
924 return True
925
926 def _get_uncovered_rpc_commands(self):
927 """
928 Return a set of currently untested RPC commands.
929
930 """
931 # This is shared from `test/functional/test_framework/coverage.py`
932 reference_filename = 'rpc_interface.txt'
933 coverage_file_prefix = 'coverage.'
934
935 coverage_ref_filename = os.path.join(self.dir, reference_filename)
936 coverage_filenames = set()
937 all_cmds = set()
938 # Consider RPC generate covered, because it is overloaded in
939 # test_framework/test_node.py and not seen by the coverage check.
940 covered_cmds = set({'generate'})
941
942 if not os.path.isfile(coverage_ref_filename):
943 raise RuntimeError("No coverage reference found")
944
945 with open(coverage_ref_filename, 'r') as coverage_ref_file:
946 all_cmds.update([line.strip() for line in coverage_ref_file.readlines()])
947
948 for root, _, files in os.walk(self.dir):
949 for filename in files:
950 if filename.startswith(coverage_file_prefix):
951 coverage_filenames.add(os.path.join(root, filename))
952
953 for filename in coverage_filenames:
954 with open(filename, 'r') as coverage_file:
955 covered_cmds.update([line.strip() for line in coverage_file.readlines()])
956
957 return all_cmds - covered_cmds
958
959
960 if __name__ == '__main__':
961 main()
962