test_runner.py raw
1 #!/usr/bin/env python3
2 # Copyright (c) 2014-2022 The Limenka developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Run regression test suite.
6
7 This module calls down into individual test cases via subprocess. It will
8 forward all unrecognized arguments onto the individual test scripts.
9
10 For a description of arguments recognized by test scripts, see
11 `test/functional/test_framework/test_framework.py:LimenkaTestFramework.main`.
12
13 """
14
15 import argparse
16 from collections import deque
17 import configparser
18 import csv
19 import datetime
20 import os
21 import pathlib
22 import platform
23 import time
24 import shutil
25 import signal
26 import subprocess
27 import sys
28 import tempfile
29 import re
30 import logging
31
32 os.environ["REQUIRE_WALLET_TYPE_SET"] = "1"
33
34 # Minimum amount of space to run the tests.
35 MIN_FREE_SPACE = 1.1 * 1024 * 1024 * 1024
36 # Additional space to run an extra job.
37 ADDITIONAL_SPACE_PER_JOB = 100 * 1024 * 1024
38 # Minimum amount of space required for --nocleanup
39 MIN_NO_CLEANUP_SPACE = 12 * 1024 * 1024 * 1024
40
41 # Formatting. Default colors to empty strings.
42 DEFAULT, BOLD, GREEN, RED = ("", ""), ("", ""), ("", ""), ("", "")
43 try:
44 # Make sure python thinks it can write unicode to its stdout
45 "\u2713".encode("utf_8").decode(sys.stdout.encoding)
46 TICK = "✓ "
47 CROSS = "✖ "
48 CIRCLE = "○ "
49 except UnicodeDecodeError:
50 TICK = "P "
51 CROSS = "x "
52 CIRCLE = "o "
53
54 if platform.system() == 'Windows':
55 import ctypes
56 kernel32 = ctypes.windll.kernel32 # type: ignore
57 ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4
58 STD_OUTPUT_HANDLE = -11
59 STD_ERROR_HANDLE = -12
60 # Enable ascii color control to stdout
61 stdout = kernel32.GetStdHandle(STD_OUTPUT_HANDLE)
62 stdout_mode = ctypes.c_int32()
63 kernel32.GetConsoleMode(stdout, ctypes.byref(stdout_mode))
64 kernel32.SetConsoleMode(stdout, stdout_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
65 # Enable ascii color control to stderr
66 stderr = kernel32.GetStdHandle(STD_ERROR_HANDLE)
67 stderr_mode = ctypes.c_int32()
68 kernel32.GetConsoleMode(stderr, ctypes.byref(stderr_mode))
69 kernel32.SetConsoleMode(stderr, stderr_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
70 else:
71 # primitive formatting on supported
72 # terminal via ANSI escape sequences:
73 DEFAULT = ('\033[0m', '\033[0m')
74 BOLD = ('\033[0m', '\033[1m')
75 GREEN = ('\033[0m', '\033[0;32m')
76 RED = ('\033[0m', '\033[0;31m')
77
78 TEST_EXIT_PASSED = 0
79 TEST_EXIT_SKIPPED = 77
80
81 TEST_FRAMEWORK_UNIT_TESTS = 'feature_framework_unit_tests.py'
82
83 EXTENDED_SCRIPTS = [
84 # These tests are not run by default.
85 # Longest test should go first, to favor running tests in parallel
86 'feature_pruning.py',
87 'feature_dbcrash.py',
88 'feature_index_prune.py',
89 'wallet_pruning.py --legacy-wallet',
90 ]
91
92 BASE_SCRIPTS = [
93 # Scripts that are run by default.
94 # Longest test should go first, to favor running tests in parallel
95 # vv Tests less than 5m vv
96 'feature_fee_estimation.py',
97 'feature_taproot.py',
98 'feature_reduced_data_temporary_deployment.py',
99 'feature_bip9_max_activation_height.py',
100 'feature_rdts.py',
101 'feature_block.py',
102 'mempool_ephemeral_dust.py',
103 'wallet_conflicts.py --legacy-wallet',
104 'wallet_conflicts.py --descriptors',
105 'p2p_opportunistic_1p1c.py',
106 'p2p_node_network_limited.py --v1transport',
107 'p2p_node_network_limited.py --v2transport',
108 # vv Tests less than 2m vv
109 'mining_getblocktemplate_longpoll.py',
110 'p2p_segwit.py',
111 'feature_maxuploadtarget.py',
112 'feature_assumeutxo.py',
113 'mempool_updatefromblock.py',
114 'mempool_persist.py --descriptors',
115 # vv Tests less than 60s vv
116 'rpc_psbt.py --legacy-wallet',
117 'rpc_psbt.py --descriptors',
118 'wallet_fundrawtransaction.py --legacy-wallet',
119 'wallet_fundrawtransaction.py --descriptors',
120 'wallet_bumpfee.py --legacy-wallet',
121 'wallet_bumpfee.py --descriptors',
122 'wallet_import_rescan.py --legacy-wallet',
123 'wallet_backup.py --legacy-wallet',
124 'wallet_backup.py --descriptors',
125 'feature_segwit.py --legacy-wallet',
126 'feature_segwit.py --descriptors --v1transport',
127 'feature_segwit.py --descriptors --v2transport',
128 'p2p_tx_download.py',
129 'wallet_avoidreuse.py --legacy-wallet',
130 'wallet_avoidreuse.py --descriptors',
131 'feature_abortnode.py',
132 'wallet_address_types.py --legacy-wallet',
133 'wallet_address_types.py --descriptors',
134 'p2p_orphan_handling.py',
135 'wallet_basic.py --legacy-wallet',
136 'wallet_basic.py --descriptors',
137 'feature_maxtipage.py',
138 'wallet_multiwallet.py --legacy-wallet',
139 'wallet_multiwallet.py --descriptors',
140 'wallet_multiwallet.py --usecli',
141 'p2p_dns_seeds.py',
142 'wallet_groups.py --legacy-wallet',
143 'wallet_groups.py --descriptors',
144 'p2p_blockfilters.py',
145 'feature_assumevalid.py',
146 'wallet_taproot.py --descriptors',
147 'feature_bip68_sequence.py',
148 'rpc_packages.py',
149 'rpc_bind.py --ipv4',
150 'rpc_bind.py --ipv6',
151 'rpc_bind.py --nonloopback',
152 'p2p_headers_sync_with_minchainwork.py',
153 'p2p_feefilter.py',
154 'feature_csv_activation.py',
155 'feature_reduced_data_utxo_height.py',
156 'p2p_sendheaders.py',
157 'feature_config_args.py',
158 'wallet_listtransactions.py --legacy-wallet',
159 'wallet_listtransactions.py --descriptors',
160 'wallet_miniscript.py --descriptors',
161 # vv Tests less than 30s vv
162 'p2p_invalid_messages.py',
163 'rpc_createmultisig.py',
164 'p2p_timeouts.py --v1transport',
165 'p2p_timeouts.py --v2transport',
166 'wallet_dump.py --legacy-wallet',
167 'rpc_signer.py',
168 'wallet_signer.py --descriptors',
169 'wallet_importmulti.py --legacy-wallet',
170 'mempool_limit.py',
171 'rpc_txoutproof.py',
172 'rpc_orphans.py',
173 'wallet_listreceivedby.py --legacy-wallet',
174 'wallet_listreceivedby.py --descriptors',
175 'wallet_abandonconflict.py --legacy-wallet',
176 'wallet_abandonconflict.py --descriptors',
177 'wallet_anchor.py --legacy-wallet',
178 'wallet_anchor.py --descriptors',
179 'feature_reindex.py',
180 'feature_reindex_readonly.py',
181 'wallet_labels.py --legacy-wallet',
182 'wallet_labels.py --descriptors',
183 'p2p_compactblocks.py',
184 'p2p_compactblocks_blocksonly.py',
185 'p2p_compactblocks_extratxs.py',
186 'wallet_hd.py --legacy-wallet',
187 'wallet_hd.py --descriptors',
188 'wallet_blank.py --legacy-wallet',
189 'wallet_blank.py --descriptors',
190 'wallet_keypool_topup.py --legacy-wallet',
191 'wallet_keypool_topup.py --descriptors',
192 'wallet_fast_rescan.py --descriptors',
193 'wallet_gethdkeys.py --descriptors',
194 'wallet_createwalletdescriptor.py --descriptors',
195 'feature_fee_estimates_persist.py',
196 'interface_zmq.py',
197 'rpc_invalid_address_message.py',
198 'rpc_validateaddress.py',
199 'interface_limenka_cli.py --legacy-wallet',
200 'interface_limenka_cli.py --descriptors',
201 'feature_bind_extra.py',
202 'mempool_resurrect.py',
203 'wallet_sweepprivkeys.py',
204 'wallet_txn_doublespend.py --mineblock',
205 'tool_cli_completion.py',
206 'tool_wallet.py --legacy-wallet',
207 'tool_wallet.py --legacy-wallet --bdbro',
208 'tool_wallet.py --legacy-wallet --bdbro --swap-bdb-endian',
209 'tool_wallet.py --descriptors',
210 'tool_signet_miner.py --legacy-wallet',
211 'tool_signet_miner.py --descriptors',
212 'wallet_txn_clone.py',
213 'wallet_txn_clone.py --segwit',
214 'mining_coin_age_priority.py',
215 'rpc_getchaintips.py',
216 'rpc_misc.py',
217 'p2p_1p1c_network.py',
218 'interface_rest.py',
219 'mempool_spend_coinbase.py',
220 'wallet_avoid_mixing_output_types.py --descriptors',
221 'mempool_reorg.py',
222 'p2p_block_sync.py --v1transport',
223 'p2p_block_sync.py --v2transport',
224 'p2p_block_times.py',
225 'wallet_createwallet.py --legacy-wallet',
226 'wallet_createwallet.py --usecli',
227 'wallet_createwallet.py --descriptors',
228 'wallet_watchonly.py --legacy-wallet',
229 'wallet_watchonly.py --usecli --legacy-wallet',
230 'wallet_reindex.py --legacy-wallet',
231 'wallet_reindex.py --descriptors',
232 'wallet_reorgsrestore.py',
233 'interface_http.py',
234 'interface_rpc.py',
235 'interface_usdt_coinselection.py',
236 'interface_usdt_mempool.py',
237 'interface_usdt_net.py',
238 'interface_usdt_utxocache.py',
239 'interface_usdt_validation.py',
240 'rpc_users.py',
241 'rpc_whitelist.py',
242 'rpc_getrpcwhitelist.py',
243 'feature_proxy.py',
244 'wallet_signrawtransactionwithwallet.py --legacy-wallet',
245 'wallet_signrawtransactionwithwallet.py --descriptors',
246 'rpc_signrawtransactionwithkey.py',
247 'rpc_rawtransaction.py --legacy-wallet',
248 'wallet_transactiontime_rescan.py --descriptors',
249 'wallet_transactiontime_rescan.py --legacy-wallet',
250 'p2p_addrv2_relay.py',
251 'p2p_compactblocks_hb.py --v1transport',
252 'p2p_compactblocks_hb.py --v2transport',
253 'p2p_disconnect_ban.py --v1transport',
254 'p2p_disconnect_ban.py --v2transport',
255 'feature_posix_fs_permissions.py',
256 'rpc_decodescript.py',
257 'rpc_blockchain.py --v1transport',
258 'rpc_blockchain.py --v2transport',
259 'rpc_deprecated.py',
260 'wallet_disable.py',
261 'wallet_change_address.py --legacy-wallet',
262 'wallet_change_address.py --descriptors',
263 'p2p_addr_relay.py',
264 'p2p_getaddr_caching.py',
265 'p2p_getdata.py',
266 'p2p_addrfetch.py',
267 'rpc_net.py --v1transport',
268 'rpc_net.py --v2transport',
269 'wallet_keypool.py --legacy-wallet',
270 'wallet_keypool.py --descriptors',
271 'wallet_descriptor.py --descriptors',
272 'p2p_nobloomfilter_messages.py',
273 TEST_FRAMEWORK_UNIT_TESTS,
274 'p2p_filter.py',
275 'rpc_setban.py --v1transport',
276 'rpc_setban.py --v2transport',
277 'p2p_blocksonly.py',
278 'mining_prioritisetransaction.py',
279 'p2p_invalid_locator.py',
280 'p2p_invalid_block.py --v1transport',
281 'p2p_invalid_block.py --v2transport',
282 'p2p_invalid_tx.py --v1transport',
283 'p2p_invalid_tx.py --v2transport',
284 'p2p_v2_transport.py',
285 'p2p_v2_encrypted.py',
286 'p2p_v2_misbehaving.py',
287 'example_test.py',
288 'mempool_truc.py',
289 'wallet_txn_doublespend.py --legacy-wallet',
290 'wallet_multisig_descriptor_psbt.py --descriptors',
291 'wallet_miniscript_decaying_multisig_descriptor_psbt.py --descriptors',
292 'wallet_txn_doublespend.py --descriptors',
293 'wallet_backwards_compatibility.py --legacy-wallet',
294 'wallet_backwards_compatibility.py --descriptors',
295 'wallet_txn_clone.py --mineblock',
296 'feature_notifications.py',
297 'rpc_getblockfilter.py',
298 'rpc_getblockfrompeer.py',
299 'rpc_invalidateblock.py',
300 'feature_utxo_set_hash.py',
301 'feature_rbf.py',
302 'mempool_packages.py',
303 'mempool_package_onemore.py',
304 'mempool_package_limits.py',
305 'mempool_package_rbf.py',
306 'tool_utxo_to_sqlite.py',
307 'feature_versionbits_warning.py',
308 'feature_blocksxor.py',
309 'rpc_preciousblock.py',
310 'wallet_importprunedfunds.py --legacy-wallet',
311 'wallet_importprunedfunds.py --descriptors',
312 'p2p_leak_tx.py --v1transport',
313 'p2p_leak_tx.py --v2transport',
314 'p2p_eviction.py',
315 'p2p_outbound_eviction.py',
316 'p2p_ibd_stalling.py --v1transport',
317 'p2p_ibd_stalling.py --v2transport',
318 'p2p_net_deadlock.py --v1transport',
319 'p2p_net_deadlock.py --v2transport',
320 'wallet_signmessagewithaddress.py',
321 'rpc_signmessagewithprivkey.py',
322 'rpc_generate.py',
323 'wallet_balance.py --legacy-wallet',
324 'wallet_balance.py --descriptors',
325 'p2p_initial_headers_sync.py',
326 'feature_nulldummy.py',
327 'mempool_accept.py',
328 'mempool_fee_histogram.py',
329 'mempool_expiry.py',
330 'rpc_sort_multisig.py',
331 'wallet_import_with_label.py --legacy-wallet',
332 'wallet_importdescriptors.py --descriptors',
333 'wallet_importseed.py --descriptors',
334 'wallet_upgradewallet.py --legacy-wallet',
335 'wallet_crosschain.py',
336 'mining_basic.py',
337 'mining_mainnet.py',
338 'feature_signet.py',
339 'p2p_mutated_blocks.py',
340 'wallet_implicitsegwit.py --legacy-wallet',
341 'rpc_named_arguments.py',
342 'feature_startupnotify.py',
343 'wallet_simulaterawtx.py --legacy-wallet',
344 'wallet_simulaterawtx.py --descriptors',
345 'wallet_listsinceblock.py --legacy-wallet',
346 'wallet_listsinceblock.py --descriptors',
347 'wallet_listdescriptors.py --descriptors',
348 'p2p_leak.py',
349 'wallet_encryption.py --legacy-wallet',
350 'wallet_encryption.py --descriptors',
351 'feature_dersig.py',
352 'feature_reindex_init.py',
353 'feature_cltv.py',
354 'rpc_uptime.py',
355 'feature_discover.py',
356 'wallet_resendwallettransactions.py --legacy-wallet',
357 'wallet_resendwallettransactions.py --descriptors',
358 'wallet_fallbackfee.py --legacy-wallet',
359 'wallet_fallbackfee.py --descriptors',
360 'rpc_dumptxoutset.py',
361 'rpc_getblocklocations.py',
362 'feature_minchainwork.py',
363 'rpc_estimatefee.py',
364 'rpc_getblockstats.py',
365 'feature_port.py',
366 'feature_bind_port_externalip.py',
367 'wallet_create_tx.py --legacy-wallet',
368 'wallet_send.py --legacy-wallet',
369 'wallet_send.py --descriptors',
370 'wallet_sendall.py --legacy-wallet',
371 'wallet_sendall.py --descriptors',
372 'wallet_sendmany.py --descriptors',
373 'wallet_sendmany.py --legacy-wallet',
374 'wallet_create_tx.py --descriptors',
375 'wallet_inactive_hdchains.py --legacy-wallet',
376 'wallet_spend_unconfirmed.py',
377 'wallet_rescan_unconfirmed.py --descriptors',
378 'p2p_fingerprint.py',
379 'feature_uacomment.py',
380 'feature_init.py',
381 'wallet_coinbase_category.py --legacy-wallet',
382 'wallet_coinbase_category.py --descriptors',
383 'feature_filelock.py',
384 'feature_loadblock.py',
385 'wallet_assumeutxo.py --descriptors',
386 'p2p_dos_header_tree.py',
387 'p2p_add_connections.py',
388 'feature_bind_port_discover.py',
389 'p2p_unrequested_blocks.py',
390 'p2p_message_capture.py',
391 'feature_includeconf.py',
392 'feature_addrman.py',
393 'feature_asmap.py',
394 'feature_chain_tiebreaks.py',
395 'feature_fastprune.py',
396 'feature_framework_miniwallet.py',
397 'feature_sync_coins_tip_after_chain_sync.py',
398 'mempool_unbroadcast.py',
399 'mempool_compatibility.py',
400 'mempool_accept_wtxid.py',
401 'mempool_dust.py',
402 'mempool_subdust_fee_penalty.py',
403 'mempool_sigoplimit.py',
404 'rpc_deriveaddresses.py',
405 'rpc_deriveaddresses.py --usecli',
406 'p2p_ping.py',
407 'p2p_tx_privacy.py',
408 'rpc_getdescriptoractivity.py',
409 'rpc_scanblocks.py',
410 'p2p_sendtxrcncl.py',
411 'rpc_scantxoutset.py',
412 'feature_torcontrol.py',
413 'feature_unsupported_utxo_db.py',
414 'feature_logging.py',
415 'feature_anchors.py',
416 'mempool_datacarrier.py',
417 'feature_coinstatsindex.py',
418 'wallet_orphanedreward.py',
419 'wallet_timelock.py',
420 'p2p_permissions.py',
421 'feature_blocksdir.py',
422 'wallet_startup.py',
423 'feature_remove_pruned_files_on_startup.py',
424 'p2p_i2p_ports.py',
425 'p2p_i2p_sessions.py',
426 'feature_presegwit_node_upgrade.py',
427 'feature_p2spkh.py',
428 'feature_settings.py',
429 'rpc_getdescriptorinfo.py',
430 'rpc_mempool_info.py',
431 'rpc_getgeneralinfo.py',
432 'rpc_help.py',
433 'p2p_handshake.py',
434 'p2p_handshake.py --v2transport',
435 'feature_dirsymlinks.py',
436 'feature_help.py',
437 'feature_shutdown.py',
438 'wallet_migration.py',
439 'p2p_ibd_txrelay.py',
440 'p2p_seednode.py',
441 # Don't append tests at the end to avoid merge conflicts
442 # Put them in a random line within the section that fits their approximate run-time
443 ]
444
445 # Place EXTENDED_SCRIPTS first since it has the 3 longest running tests
446 ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS
447
448 NON_SCRIPTS = [
449 # These are python files that live in the functional tests directory, but are not test scripts.
450 "combine_logs.py",
451 "create_cache.py",
452 "test_runner.py",
453 ]
454
455 def main():
456 # Parse arguments and pass through unrecognised args
457 parser = argparse.ArgumentParser(add_help=False,
458 usage='%(prog)s [test_runner.py options] [script options] [scripts]',
459 description=__doc__,
460 epilog='''
461 Help text and arguments for individual test script:''',
462 formatter_class=argparse.RawTextHelpFormatter)
463 parser.add_argument('--ansi', action='store_true', default=sys.stdout.isatty(), help="Use ANSI colors and dots in output (enabled by default when standard output is a TTY)")
464 parser.add_argument('--combinedlogslen', '-c', type=int, default=0, metavar='n', help='On failure, print a log (of length n lines) to the console, combined from the test framework and all test nodes.')
465 parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface')
466 parser.add_argument('--ci', action='store_true', help='Run checks and code that are usually only enabled in a continuous integration environment')
467 parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.')
468 parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
469 parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
470 parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
471 parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.')
472 parser.add_argument('--quiet', '-q', action='store_true', help='only print dots, results summary and failure logs')
473 parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs")
474 parser.add_argument('--failfast', '-F', action='store_true', help='stop execution after the first test failure')
475 parser.add_argument('--filter', help='filter scripts to run by regular expression')
476 parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true",
477 help="Leave limenkads and test.* datadir on exit or error")
478 parser.add_argument('--resultsfile', '-r', help='store test results (as CSV) to the provided file')
479
480 args, unknown_args = parser.parse_known_args()
481 fail_on_warn = args.ci
482 if not args.ansi:
483 global DEFAULT, BOLD, GREEN, RED
484 DEFAULT = ("", "")
485 BOLD = ("", "")
486 GREEN = ("", "")
487 RED = ("", "")
488
489 # args to be passed on always start with two dashes; tests are the remaining unknown args
490 tests = [arg for arg in unknown_args if arg[:2] != "--"]
491 passon_args = [arg for arg in unknown_args if arg[:2] == "--"]
492
493 # Read config generated by configure.
494 config = configparser.ConfigParser()
495 configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
496 config.read_file(open(configfile, encoding="utf8"))
497
498 passon_args.append("--configfile=%s" % configfile)
499
500 # Set up logging
501 logging_level = logging.INFO if args.quiet else logging.DEBUG
502 logging.basicConfig(format='%(message)s', level=logging_level)
503
504 # Create base test directory
505 tmpdir = "%s/test_runner_₿_🏃_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
506
507 os.makedirs(tmpdir)
508
509 logging.debug("Temporary test directory at %s" % tmpdir)
510
511 results_filepath = None
512 if args.resultsfile:
513 results_filepath = pathlib.Path(args.resultsfile)
514 # Stop early if the parent directory doesn't exist
515 assert results_filepath.parent.exists(), "Results file parent directory does not exist"
516 logging.debug("Test results will be written to " + str(results_filepath))
517
518 enable_limenkad = config["components"].getboolean("ENABLE_LIMENKAD")
519
520 if not enable_limenkad:
521 print("No functional tests to run.")
522 print("Re-compile with the -DBUILD_DAEMON=ON build option")
523 sys.exit(1)
524
525 # Build list of tests
526 test_list = []
527 if tests:
528 # Individual tests have been specified. Run specified tests that exist
529 # in the ALL_SCRIPTS list. Accept names with or without a .py extension.
530 # Specified tests can contain wildcards, but in that case the supplied
531 # paths should be coherent, e.g. the same path as that provided to call
532 # test_runner.py. Examples:
533 # `test/functional/test_runner.py test/functional/wallet*`
534 # `test/functional/test_runner.py ./test/functional/wallet*`
535 # `test_runner.py wallet*`
536 # but not:
537 # `test/functional/test_runner.py wallet*`
538 # Multiple wildcards can be passed:
539 # `test_runner.py tool* mempool*`
540 for test in tests:
541 script = test.split("/")[-1]
542 script = script + ".py" if ".py" not in script else script
543 matching_scripts = [s for s in ALL_SCRIPTS if s.startswith(script)]
544 if matching_scripts:
545 test_list.extend(matching_scripts)
546 else:
547 print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], test))
548 elif args.extended:
549 # Include extended tests
550 test_list += ALL_SCRIPTS
551 else:
552 # Run base tests only
553 test_list += BASE_SCRIPTS
554
555 # Remove the test cases that the user has explicitly asked to exclude.
556 # The user can specify a test case with or without the .py extension.
557 if args.exclude:
558
559 def print_warning_missing_test(test_name):
560 print("{}WARNING!{} Test '{}' not found in current test list. Check the --exclude list.".format(BOLD[1], BOLD[0], test_name))
561 if fail_on_warn:
562 sys.exit(1)
563
564 def remove_tests(exclude_list):
565 if not exclude_list:
566 print_warning_missing_test(exclude_test)
567 for exclude_item in exclude_list:
568 test_list.remove(exclude_item)
569
570 exclude_tests = [test.strip() for test in args.exclude.split(",")]
571 for exclude_test in exclude_tests:
572 # A space in the name indicates it has arguments such as "wallet_basic.py --descriptors"
573 if ' ' in exclude_test:
574 remove_tests([test for test in test_list if test.replace('.py', '') == exclude_test.replace('.py', '')])
575 else:
576 # Exclude all variants of a test
577 remove_tests([test for test in test_list if test.split('.py')[0] == exclude_test.split('.py')[0]])
578
579 if args.filter:
580 test_list = list(filter(re.compile(args.filter).search, test_list))
581
582 if not test_list:
583 print("No valid test scripts specified. Check that your test is in one "
584 "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests")
585 sys.exit(1)
586
587 if args.help:
588 # Print help for test_runner.py, then print help of the first script (with args removed) and exit.
589 parser.print_help()
590 subprocess.check_call([sys.executable, os.path.join(config["environment"]["SRCDIR"], 'test', 'functional', test_list[0].split()[0]), '-h'])
591 sys.exit(0)
592
593 # Warn if there is not enough space on tmpdir to run the tests with --nocleanup
594 if args.nocleanup:
595 if shutil.disk_usage(tmpdir).free < MIN_NO_CLEANUP_SPACE:
596 print(f"{BOLD[1]}WARNING!{BOLD[0]} There may be insufficient free space in {tmpdir} to run the functional test suite with --nocleanup. "
597 f"A minimum of {MIN_NO_CLEANUP_SPACE // (1024 * 1024 * 1024)} GB of free space is required.")
598 passon_args.append("--nocleanup")
599
600 check_script_list(src_dir=config["environment"]["SRCDIR"], fail_on_warn=fail_on_warn)
601 check_script_prefixes()
602
603 if not args.keepcache:
604 shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True)
605
606 run_tests(
607 test_list=test_list,
608 build_dir=config["environment"]["BUILDDIR"],
609 tmpdir=tmpdir,
610 jobs=args.jobs,
611 enable_coverage=args.coverage,
612 args=passon_args,
613 combined_logs_len=args.combinedlogslen,
614 failfast=args.failfast,
615 use_term_control=args.ansi,
616 results_filepath=results_filepath,
617 )
618
619 def run_tests(*, test_list, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control, results_filepath=None):
620 args = args or []
621
622 # Warn if limenkad is already running
623 try:
624 # pgrep exits with code zero when one or more matching processes found
625 if subprocess.run(["pgrep", "-x", "limenkad"], stdout=subprocess.DEVNULL).returncode == 0:
626 print("%sWARNING!%s There is already a limenkad process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0]))
627 except OSError:
628 # pgrep not supported
629 pass
630
631 # Warn if there is a cache directory
632 cache_dir = "%s/test/cache" % build_dir
633 if os.path.isdir(cache_dir):
634 print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir))
635
636 # Warn if there is not enough space on the testing dir
637 min_space = MIN_FREE_SPACE + (jobs - 1) * ADDITIONAL_SPACE_PER_JOB
638 if shutil.disk_usage(tmpdir).free < min_space:
639 print(f"{BOLD[1]}WARNING!{BOLD[0]} There may be insufficient free space in {tmpdir} to run the Limenka functional test suite. "
640 f"Running the test suite with fewer than {min_space // (1024 * 1024)} MB of free space might cause tests to fail.")
641
642 tests_dir = f"{build_dir}/test/functional/"
643 # This allows `test_runner.py` to work from an out-of-source build directory using a symlink,
644 # a hard link or a copy on any platform. See https://github.com/limenka/limenka/pull/27561.
645 sys.path.append(tests_dir)
646
647 flags = ['--cachedir={}'.format(cache_dir)] + args
648
649 if enable_coverage:
650 coverage = RPCCoverage()
651 flags.append(coverage.flag)
652 logging.debug("Initializing coverage directory at %s" % coverage.dir)
653 else:
654 coverage = None
655
656 if len(test_list) > 1 and jobs > 1:
657 # Populate cache
658 try:
659 subprocess.check_output([sys.executable, tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir])
660 except subprocess.CalledProcessError as e:
661 sys.stdout.buffer.write(e.output)
662 raise
663
664 #Run Tests
665 job_queue = TestHandler(
666 num_tests_parallel=jobs,
667 tests_dir=tests_dir,
668 tmpdir=tmpdir,
669 test_list=test_list,
670 flags=flags,
671 use_term_control=use_term_control,
672 )
673 start_time = time.time()
674 test_results = []
675
676 max_len_name = len(max(test_list, key=len))
677 test_count = len(test_list)
678 all_passed = True
679 while not job_queue.done():
680 if failfast and not all_passed:
681 break
682 for test_result, testdir, stdout, stderr, skip_reason in job_queue.get_next():
683 test_results.append(test_result)
684 done_str = f"{len(test_results)}/{test_count} - {BOLD[1]}{test_result.name}{BOLD[0]}"
685 if test_result.status == "Passed":
686 logging.debug("%s passed, Duration: %s s" % (done_str, test_result.time))
687 elif test_result.status == "Skipped":
688 logging.debug(f"{done_str} skipped ({skip_reason})")
689 else:
690 all_passed = False
691 print("%s failed, Duration: %s s\n" % (done_str, test_result.time))
692 print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n')
693 print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n')
694 if combined_logs_len and os.path.isdir(testdir):
695 # Print the final `combinedlogslen` lines of the combined logs
696 print('{}Combine the logs and print the last {} lines ...{}'.format(BOLD[1], combined_logs_len, BOLD[0]))
697 print('\n============')
698 print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0]))
699 print('============\n')
700 combined_logs_args = [sys.executable, os.path.join(tests_dir, 'combine_logs.py'), testdir]
701 if BOLD[0]:
702 combined_logs_args += ['--color']
703 combined_logs, _ = subprocess.Popen(combined_logs_args, text=True, stdout=subprocess.PIPE).communicate()
704 print("\n".join(deque(combined_logs.splitlines(), combined_logs_len)))
705
706 if failfast:
707 logging.debug("Early exiting after test failure")
708 break
709
710 if "[Errno 28] No space left on device" in stdout:
711 sys.exit(f"Early exiting after test failure due to insufficient free space in {tmpdir}\n"
712 f"Test execution data left in {tmpdir}.\n"
713 f"Additional storage is needed to execute testing.")
714
715 runtime = int(time.time() - start_time)
716 print_results(test_results, max_len_name, runtime)
717 if results_filepath:
718 write_results(test_results, results_filepath, runtime)
719
720 if coverage:
721 coverage_passed = coverage.report_rpc_coverage()
722
723 logging.debug("Cleaning up coverage data")
724 coverage.cleanup()
725 else:
726 coverage_passed = True
727
728 # Clear up the temp directory if all subdirectories are gone
729 if not os.listdir(tmpdir):
730 os.rmdir(tmpdir)
731
732
733 # Clean up dangling processes if any. This may only happen with --failfast option.
734 # Killing the process group will also terminate the current process but that is
735 # not an issue
736 if not os.getenv("CI_FAILFAST_TEST_LEAVE_DANGLING") and len(job_queue.jobs):
737 os.killpg(os.getpgid(0), signal.SIGKILL)
738
739 sys.exit(not all_passed)
740
741
742 def print_results(test_results, max_len_name, runtime):
743 results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS ", "DURATION") + BOLD[0]
744
745 test_results.sort(key=TestResult.sort_key)
746 all_passed = True
747 time_sum = 0
748
749 for test_result in test_results:
750 all_passed = all_passed and test_result.was_successful
751 time_sum += test_result.time
752 test_result.padding = max_len_name
753 results += str(test_result)
754
755 status = TICK + "Passed" if all_passed else CROSS + "Failed"
756 if not all_passed:
757 results += RED[1]
758 results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ("ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0]
759 if not all_passed:
760 results += RED[0]
761 results += "Runtime: %s s\n" % (runtime)
762 print(results)
763
764
765 def write_results(test_results, filepath, total_runtime):
766 with open(filepath, mode="w", encoding="utf8") as results_file:
767 results_writer = csv.writer(results_file)
768 results_writer.writerow(['test', 'status', 'duration(seconds)'])
769 all_passed = True
770 for test_result in test_results:
771 all_passed = all_passed and test_result.was_successful
772 results_writer.writerow([test_result.name, test_result.status, str(test_result.time)])
773 results_writer.writerow(['ALL', ("Passed" if all_passed else "Failed"), str(total_runtime)])
774
775 class TestHandler:
776 """
777 Trigger the test scripts passed in via the list.
778 """
779
780 def __init__(self, *, num_tests_parallel, tests_dir, tmpdir, test_list, flags, use_term_control):
781 assert num_tests_parallel >= 1
782 self.num_jobs = num_tests_parallel
783 self.tests_dir = tests_dir
784 self.tmpdir = tmpdir
785 self.test_list = test_list
786 self.flags = flags
787 self.jobs = []
788 self.use_term_control = use_term_control
789
790 def done(self):
791 return not (self.jobs or self.test_list)
792
793 def get_next(self):
794 while len(self.jobs) < self.num_jobs and self.test_list:
795 # Add tests
796 test = self.test_list.pop(0)
797 portseed = len(self.test_list)
798 portseed_arg = ["--portseed={}".format(portseed)]
799 log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
800 log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
801 test_argv = test.split()
802 testdir = "{}/{}_{}".format(self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)
803 tmpdir_arg = ["--tmpdir={}".format(testdir)]
804 self.jobs.append((test,
805 time.time(),
806 subprocess.Popen([sys.executable, self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir_arg,
807 text=True,
808 stdout=log_stdout,
809 stderr=log_stderr),
810 testdir,
811 log_stdout,
812 log_stderr))
813 if not self.jobs:
814 raise IndexError('pop from empty list')
815
816 # Print remaining running jobs when all jobs have been started.
817 if not self.test_list:
818 print("Remaining jobs: [{}]".format(", ".join(j[0] for j in self.jobs)))
819
820 dot_count = 0
821 while True:
822 # Return all procs that have finished, if any. Otherwise sleep until there is one.
823 time.sleep(.5)
824 ret = []
825 for job in self.jobs:
826 (name, start_time, proc, testdir, log_out, log_err) = job
827 if proc.poll() is not None:
828 log_out.seek(0), log_err.seek(0)
829 [stdout, stderr] = [log_file.read().decode('utf-8') for log_file in (log_out, log_err)]
830 log_out.close(), log_err.close()
831 skip_reason = None
832 if proc.returncode == TEST_EXIT_PASSED and stderr == "":
833 status = "Passed"
834 elif proc.returncode == TEST_EXIT_SKIPPED:
835 status = "Skipped"
836 skip_reason = re.search(r"Test Skipped: (.*)", stdout).group(1).strip()
837 else:
838 status = "Failed"
839 self.jobs.remove(job)
840 if self.use_term_control:
841 clearline = '\r' + (' ' * dot_count) + '\r'
842 print(clearline, end='', flush=True)
843 dot_count = 0
844 ret.append((TestResult(name, status, int(time.time() - start_time)), testdir, stdout, stderr, skip_reason))
845 if ret:
846 return ret
847 if self.use_term_control:
848 print('.', end='', flush=True)
849 dot_count += 1
850
851
852 class TestResult():
853 def __init__(self, name, status, time):
854 self.name = name
855 self.status = status
856 self.time = time
857 self.padding = 0
858
859 def sort_key(self):
860 if self.status == "Passed":
861 return 0, self.name.lower()
862 elif self.status == "Failed":
863 return 2, self.name.lower()
864 elif self.status == "Skipped":
865 return 1, self.name.lower()
866
867 def __repr__(self):
868 if self.status == "Passed":
869 color = GREEN
870 glyph = TICK
871 elif self.status == "Failed":
872 color = RED
873 glyph = CROSS
874 elif self.status == "Skipped":
875 color = DEFAULT
876 glyph = CIRCLE
877
878 return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0]
879
880 @property
881 def was_successful(self):
882 return self.status != "Failed"
883
884
885 def check_script_prefixes():
886 """Check that test scripts start with one of the allowed name prefixes."""
887
888 good_prefixes_re = re.compile("^(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool)_")
889 bad_script_names = [script for script in ALL_SCRIPTS if good_prefixes_re.match(script) is None]
890
891 if bad_script_names:
892 print("%sERROR:%s %d tests not meeting naming conventions:" % (BOLD[1], BOLD[0], len(bad_script_names)))
893 print(" %s" % ("\n ".join(sorted(bad_script_names))))
894 raise AssertionError("Some tests are not following naming convention!")
895
896
897 def check_script_list(*, src_dir, fail_on_warn):
898 """Check scripts directory.
899
900 Check that all python files in this directory are categorized
901 as a test script or meta script."""
902 script_dir = src_dir + '/test/functional/'
903 python_files = set([test_file for test_file in os.listdir(script_dir) if test_file.endswith(".py")])
904 missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS)))
905 if len(missed_tests) != 0:
906 print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests)))
907 if fail_on_warn:
908 sys.exit(1)
909
910
911 class RPCCoverage():
912 """
913 Coverage reporting utilities for test_runner.
914
915 Coverage calculation works by having each test script subprocess write
916 coverage files into a particular directory. These files contain the RPC
917 commands invoked during testing, as well as a complete listing of RPC
918 commands per `limenka-cli help` (`rpc_interface.txt`).
919
920 After all tests complete, the commands run are combined and diff'd against
921 the complete list to calculate uncovered RPC commands.
922
923 See also: test/functional/test_framework/coverage.py
924
925 """
926 def __init__(self):
927 self.dir = tempfile.mkdtemp(prefix="coverage")
928 self.flag = '--coveragedir=%s' % self.dir
929
930 def report_rpc_coverage(self):
931 """
932 Print out RPC commands that were unexercised by tests.
933
934 """
935 uncovered = self._get_uncovered_rpc_commands()
936
937 if uncovered:
938 print("Uncovered RPC commands:")
939 print("".join((" - %s\n" % command) for command in sorted(uncovered)))
940 return False
941 else:
942 print("All RPC commands covered.")
943 return True
944
945 def cleanup(self):
946 return shutil.rmtree(self.dir)
947
948 def _get_uncovered_rpc_commands(self):
949 """
950 Return a set of currently untested RPC commands.
951
952 """
953 # This is shared from `test/functional/test_framework/coverage.py`
954 reference_filename = 'rpc_interface.txt'
955 coverage_file_prefix = 'coverage.'
956
957 coverage_ref_filename = os.path.join(self.dir, reference_filename)
958 coverage_filenames = set()
959 all_cmds = set()
960 # Consider RPC generate covered, because it is overloaded in
961 # test_framework/test_node.py and not seen by the coverage check.
962 covered_cmds = set({'generate'})
963
964 if not os.path.isfile(coverage_ref_filename):
965 raise RuntimeError("No coverage reference found")
966
967 with open(coverage_ref_filename, 'r', encoding="utf8") as coverage_ref_file:
968 all_cmds.update([line.strip() for line in coverage_ref_file.readlines()])
969
970 for root, _, files in os.walk(self.dir):
971 for filename in files:
972 if filename.startswith(coverage_file_prefix):
973 coverage_filenames.add(os.path.join(root, filename))
974
975 for filename in coverage_filenames:
976 with open(filename, 'r', encoding="utf8") as coverage_file:
977 covered_cmds.update([line.strip() for line in coverage_file.readlines()])
978
979 return all_cmds - covered_cmds
980
981
982 if __name__ == '__main__':
983 main()
984