1 #!/usr/bin/env bash
2 #
3 # Copyright (c) 2019-2020 The Limenka developers
4 # Distributed under the MIT software license, see the accompanying
5 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
6 #
7 # Test for deterministic coverage across unit test runs.
8 9 export LC_ALL=C
10 11 # Use GCOV_EXECUTABLE="gcov" if compiling with gcc.
12 # Use GCOV_EXECUTABLE="llvm-cov gcov" if compiling with clang.
13 GCOV_EXECUTABLE="gcov"
14 15 # Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism.
16 NON_DETERMINISTIC_TESTS=(
17 "blockfilter_index_tests/blockfilter_index_initial_sync" # src/checkqueue.h: In CCheckQueue::Loop(): while (queue.empty()) { ... }
18 "coinselector_tests/knapsack_solver_test" # coinselector_tests.cpp: if (equal_sets(setCoinsRet, setCoinsRet2))
19 "fs_tests/fsbridge_fstream" # deterministic test failure?
20 "miner_tests/CreateNewBlock_validity" # validation.cpp: if (signals.CallbacksPending() > 10)
21 "scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue()
22 "scheduler_tests/singlethreadedscheduler_ordered" # scheduler.cpp: CScheduler::serviceQueue()
23 "txvalidationcache_tests/checkinputs_test" # validation.cpp: if (signals.CallbacksPending() > 10)
24 "txvalidationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (signals.CallbacksPending() > 10)
25 "txindex_tests/txindex_initial_sync" # validation.cpp: if (signals.CallbacksPending() > 10)
26 "txvalidation_tests/tx_mempool_reject_coinbase" # validation.cpp: if (signals.CallbacksPending() > 10)
27 "validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (signals.CallbacksPending() > 10)
28 "wallet_tests/coin_mark_dirty_immature_credit" # validation.cpp: if (signals.CallbacksPending() > 10)
29 "wallet_tests/dummy_input_size_test" # validation.cpp: if (signals.CallbacksPending() > 10)
30 "wallet_tests/importmulti_rescan" # validation.cpp: if (signals.CallbacksPending() > 10)
31 "wallet_tests/importwallet_rescan" # validation.cpp: if (signals.CallbacksPending() > 10)
32 "wallet_tests/ListCoins" # validation.cpp: if (signals.CallbacksPending() > 10)
33 "wallet_tests/scan_for_wallet_transactions" # validation.cpp: if (signals.CallbacksPending() > 10)
34 "wallet_tests/wallet_disableprivkeys" # validation.cpp: if (signals.CallbacksPending() > 10)
35 )
36 37 TEST_LIMENKA_BINARY="src/test/test_limenka"
38 39 print_usage() {
40 echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]"
41 }
42 43 N_TEST_RUNS=2
44 BOOST_TEST_RUN_FILTERS=""
45 if [[ $# != 0 ]]; then
46 if [[ $1 == "--help" ]]; then
47 print_usage
48 exit
49 fi
50 PARSED_ARGUMENTS=0
51 if [[ $1 =~ [a-z] ]]; then
52 BOOST_TEST_RUN_FILTERS=$1
53 PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1))
54 shift
55 fi
56 if [[ $1 =~ ^[0-9]+$ ]]; then
57 N_TEST_RUNS=$1
58 PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1))
59 shift
60 fi
61 if [[ ${PARSED_ARGUMENTS} == 0 || $# -gt 2 || ${N_TEST_RUNS} -lt 2 ]]; then
62 print_usage
63 exit
64 fi
65 fi
66 if [[ ${BOOST_TEST_RUN_FILTERS} == "" ]]; then
67 BOOST_TEST_RUN_FILTERS="$(IFS=":"; echo "!${NON_DETERMINISTIC_TESTS[*]}" | sed 's/:/:!/g')"
68 else
69 echo "Using Boost test filter: ${BOOST_TEST_RUN_FILTERS}"
70 echo
71 fi
72 73 if ! command -v gcov > /dev/null; then
74 echo "Error: gcov not installed. Exiting."
75 exit 1
76 fi
77 78 if ! command -v gcovr > /dev/null; then
79 echo "Error: gcovr not installed. Exiting."
80 exit 1
81 fi
82 83 if [[ ! -e ${TEST_LIMENKA_BINARY} ]]; then
84 echo "Error: Executable ${TEST_LIMENKA_BINARY} not found. Run \"cmake -B build -DCMAKE_BUILD_TYPE=Coverage\" and compile."
85 exit 1
86 fi
87 88 get_file_suffix_count() {
89 find src/ -type f -name "*.$1" | wc -l
90 }
91 92 if [[ $(get_file_suffix_count gcno) == 0 ]]; then
93 echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"cmake -B build -DCMAKE_BUILD_TYPE=Coverage\" and re-compile."
94 exit 1
95 fi
96 97 get_covr_filename() {
98 echo "gcovr.run-$1.txt"
99 }
100 101 TEST_RUN_ID=0
102 while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do
103 TEST_RUN_ID=$((TEST_RUN_ID + 1))
104 echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #${TEST_RUN_ID} of ${N_TEST_RUNS}"
105 find src/ -type f -name "*.gcda" -exec rm {} \;
106 if [[ $(get_file_suffix_count gcda) != 0 ]]; then
107 echo "Error: Stale *.gcda files found. Exiting."
108 exit 1
109 fi
110 TEST_OUTPUT_TEMPFILE=$(mktemp)
111 if ! BOOST_TEST_RUN_FILTERS="${BOOST_TEST_RUN_FILTERS}" ${TEST_LIMENKA_BINARY} > "${TEST_OUTPUT_TEMPFILE}" 2>&1; then
112 cat "${TEST_OUTPUT_TEMPFILE}"
113 rm "${TEST_OUTPUT_TEMPFILE}"
114 exit 1
115 fi
116 rm "${TEST_OUTPUT_TEMPFILE}"
117 if [[ $(get_file_suffix_count gcda) == 0 ]]; then
118 echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"cmake -B build -DCMAKE_BUILD_TYPE=Coverage\" and re-compile."
119 exit 1
120 fi
121 GCOVR_TEMPFILE=$(mktemp)
122 if ! gcovr --gcov-executable "${GCOV_EXECUTABLE}" -r src/ > "${GCOVR_TEMPFILE}"; then
123 echo "Error: gcovr failed. Output written to ${GCOVR_TEMPFILE}. Exiting."
124 exit 1
125 fi
126 GCOVR_FILENAME=$(get_covr_filename ${TEST_RUN_ID})
127 mv "${GCOVR_TEMPFILE}" "${GCOVR_FILENAME}"
128 if grep -E "^TOTAL *0 *0 " "${GCOVR_FILENAME}"; then
129 echo "Error: Spurious gcovr output. Make sure the correct GCOV_EXECUTABLE variable is set in $0 (\"gcov\" for gcc, \"llvm-cov gcov\" for clang)."
130 exit 1
131 fi
132 if [[ ${TEST_RUN_ID} != 1 ]]; then
133 COVERAGE_DIFF=$(diff -u "$(get_covr_filename 1)" "${GCOVR_FILENAME}")
134 if [[ ${COVERAGE_DIFF} != "" ]]; then
135 echo
136 echo "The line coverage is non-deterministic between runs. Exiting."
137 echo
138 echo "The test suite must be deterministic in the sense that the set of lines executed at least"
139 echo "once must be identical between runs. This is a necessary condition for meaningful"
140 echo "coverage measuring."
141 echo
142 echo "${COVERAGE_DIFF}"
143 exit 1
144 fi
145 rm "${GCOVR_FILENAME}"
146 fi
147 done
148 149 echo
150 echo "Coverage test passed: Deterministic coverage across ${N_TEST_RUNS} runs."
151 exit
152