test_runner.py raw
1 #!/usr/bin/env python3
2 # Copyright 2014 BitPay Inc.
3 # Copyright 2016-2017 The Limenka developers
4 # Distributed under the MIT software license, see the accompanying
5 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
6 """Test framework for limenka utils.
7
8 Runs automatically during `ctest --test-dir build/`.
9
10 Can also be run manually."""
11
12 import argparse
13 import configparser
14 import difflib
15 import json
16 import logging
17 import os
18 import pprint
19 import re
20 import subprocess
21 import sys
22
23 def main():
24 config = configparser.ConfigParser()
25 config.optionxform = str
26 with open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8") as f:
27 config.read_file(f)
28 env_conf = dict(config.items('environment'))
29
30 parser = argparse.ArgumentParser(description=__doc__)
31 parser.add_argument('-v', '--verbose', action='store_true')
32 args = parser.parse_args()
33 verbose = args.verbose
34
35 if verbose:
36 level = logging.DEBUG
37 else:
38 level = logging.WARNING
39 formatter = '%(asctime)s - %(levelname)s - %(message)s'
40 # Add the format/level to the logger
41 logging.basicConfig(format=formatter, level=level)
42
43 bctester(os.path.join(env_conf["SRCDIR"], "test", "util", "data"), "limenka-util-test.json", env_conf, config['components'])
44
45 def bctester(testDir, input_basename, buildenv, component_conf):
46 """ Loads and parses the input file, runs all tests and reports results"""
47 input_filename = os.path.join(testDir, input_basename)
48 with open(input_filename, encoding="utf8") as f:
49 raw_data = f.read()
50 input_data = json.loads(raw_data)
51
52 failed_testcases = []
53 skipped_testcases = []
54 skipped_testcase_deps = set()
55
56 for testObj in input_data:
57 m = re.match(r'^\.\/limenka-(\w+)$', testObj['exec'])
58 if not component_conf.getboolean(f'ENABLE_UTIL_{m.group(1).upper()}'):
59 logging.info("SKIPPED: " + testObj["description"])
60 skipped_testcases.append(testObj['description'])
61 skipped_testcase_deps.add(testObj['exec'])
62 continue
63
64 try:
65 bctest(testDir, testObj, buildenv)
66 logging.info("PASSED: " + testObj["description"])
67 except Exception:
68 logging.info("FAILED: " + testObj["description"])
69 failed_testcases.append(testObj["description"])
70
71 if failed_testcases:
72 error_message = "FAILED_TESTCASES:\n"
73 error_message += pprint.pformat(failed_testcases, width=400)
74 logging.error(error_message)
75 sys.exit(1)
76 else:
77 if skipped_testcases:
78 logging.warning(f'{len(skipped_testcases)} tests skipped because {skipped_testcase_deps} is not built')
79 sys.exit(0)
80
81 def bctest(testDir, testObj, buildenv):
82 """Runs a single test, comparing output and RC to expected output and RC.
83
84 Raises an error if input can't be read, executable fails, or output/RC
85 are not as expected. Error is caught by bctester() and reported.
86 """
87 # Get the exec names and arguments
88 execprog = os.path.join(buildenv["BUILDDIR"], "bin", testObj["exec"] + buildenv["EXEEXT"])
89 if testObj["exec"] == "./limenka-util":
90 execprog = os.getenv("LIMENKAUTIL", default=execprog)
91 elif testObj["exec"] == "./limenka-tx":
92 execprog = os.getenv("LIMENKATX", default=execprog)
93
94 execargs = testObj['args']
95 execrun = [execprog] + execargs
96
97 # Read the input data (if there is any)
98 inputData = None
99 if "input" in testObj:
100 filename = os.path.join(testDir, testObj["input"])
101 with open(filename, encoding="utf8") as f:
102 inputData = f.read()
103
104 # Read the expected output data (if there is any)
105 outputFn = None
106 outputData = None
107 outputType = None
108 if "output_cmp" in testObj:
109 outputFn = testObj['output_cmp']
110 outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare)
111 try:
112 with open(os.path.join(testDir, outputFn), encoding="utf8") as f:
113 outputData = f.read()
114 except Exception:
115 logging.error("Output file " + outputFn + " cannot be opened")
116 raise
117 if not outputData:
118 logging.error("Output data missing for " + outputFn)
119 raise Exception
120 if not outputType:
121 logging.error("Output file %s does not have a file extension" % outputFn)
122 raise Exception
123
124 # Run the test
125 try:
126 res = subprocess.run(execrun, capture_output=True, text=True, input=inputData)
127 except OSError:
128 logging.error("OSError, Failed to execute " + execprog)
129 raise
130
131 if outputData:
132 data_mismatch, formatting_mismatch = False, False
133 # Parse command output and expected output
134 try:
135 a_parsed = parse_output(res.stdout, outputType)
136 except Exception as e:
137 logging.error(f"Error parsing command output as {outputType}: '{str(e)}'; res: {str(res)}")
138 raise
139 try:
140 b_parsed = parse_output(outputData, outputType)
141 except Exception as e:
142 logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e))
143 raise
144 # Compare data
145 if a_parsed != b_parsed:
146 logging.error(f"Output data mismatch for {outputFn} (format {outputType}); res: {str(res)}")
147 data_mismatch = True
148 # Compare formatting
149 if res.stdout != outputData:
150 error_message = f"Output formatting mismatch for {outputFn}:\nres: {str(res)}\n"
151 error_message += "".join(difflib.context_diff(outputData.splitlines(True),
152 res.stdout.splitlines(True),
153 fromfile=outputFn,
154 tofile="returned"))
155 logging.error(error_message)
156 formatting_mismatch = True
157
158 assert not data_mismatch and not formatting_mismatch
159
160 # Compare the return code to the expected return code
161 wantRC = 0
162 if "return_code" in testObj:
163 wantRC = testObj['return_code']
164 if res.returncode != wantRC:
165 logging.error(f"Return code mismatch for {outputFn}; res: {str(res)}")
166 raise Exception
167
168 if "error_txt" in testObj:
169 want_error = testObj["error_txt"]
170 # Compare error text
171 # TODO: ideally, we'd compare the strings exactly and also assert
172 # That stderr is empty if no errors are expected. However, limenka-tx
173 # emits DISPLAY errors when running as a windows application on
174 # linux through wine. Just assert that the expected error text appears
175 # somewhere in stderr.
176 if want_error not in res.stderr:
177 logging.error(f"Error mismatch:\nExpected: {want_error}\nReceived: {res.stderr.rstrip()}\nres: {str(res)}")
178 raise Exception
179
180 def parse_output(a, fmt):
181 """Parse the output according to specified format.
182
183 Raise an error if the output can't be parsed."""
184 if fmt == 'json': # json: compare parsed data
185 return json.loads(a)
186 elif fmt == 'hex': # hex: parse and compare binary data
187 return bytes.fromhex(a.strip())
188 else:
189 raise NotImplementedError("Don't know how to compare %s" % fmt)
190
191 if __name__ == '__main__':
192 main()
193