1 #!/usr/bin/env bash
2
3
4 stderr() {
5 echo "$@" 1>&2
6 }
7
8 usage() {
9 b=$(basename "$0")
10 echo $b: ERROR: "$@" 1>&2
11
12 cat 1>&2 <<EOF
13
14 DESCRIPTION
15
16 $(basename "$0") is the script to run continuous integration commands for
17 go-toml on unix.
18
19 Requires Go and Git to be available in the PATH. Expects to be ran from the
20 root of go-toml's Git repository.
21
22 USAGE
23
24 $b COMMAND [OPTIONS...]
25
26 COMMANDS
27
28 benchmark [OPTIONS...] [BRANCH]
29
30 Run benchmarks.
31
32 ARGUMENTS
33
34 BRANCH Optional. Defines which Git branch to use when running
35 benchmarks.
36
37 OPTIONS
38
39 -d Compare benchmarks of HEAD with BRANCH using benchstats. In
40 this form the BRANCH argument is required.
41
42 -a Compare benchmarks of HEAD against go-toml v1 and
43 BurntSushi/toml.
44
45 -html When used with -a, emits the output as HTML, ready to be
46 embedded in the README.
47
48 coverage [OPTIONS...] [BRANCH]
49
50 Generates code coverage.
51
52 ARGUMENTS
53
54 BRANCH Optional. Defines which Git branch to use when reporting
55 coverage. Defaults to HEAD.
56
57 OPTIONS
58
59 -d Compare coverage of HEAD with the one of BRANCH. In this form,
60 the BRANCH argument is required. Exit code is non-zero when
61 coverage percentage decreased.
62 EOF
63 exit 1
64 }
65
66 cover() {
67 branch="${1}"
68 dir="$(mktemp -d)"
69
70 stderr "Executing coverage for ${branch} at ${dir}"
71
72 if [ "${branch}" = "HEAD" ]; then
73 cp -r . "${dir}/"
74 else
75 git worktree add "$dir" "$branch"
76 fi
77
78 pushd "$dir"
79 go test -covermode=atomic -coverpkg=./... -coverprofile=coverage.out.tmp ./...
80 cat coverage.out.tmp | grep -v fuzz | grep -v testsuite | grep -v tomltestgen | grep -v gotoml-test-decoder > coverage.out
81 go tool cover -func=coverage.out
82 echo "Coverage profile for ${branch}: ${dir}/coverage.out" >&2
83 popd
84
85 if [ "${branch}" != "HEAD" ]; then
86 git worktree remove --force "$dir"
87 fi
88 }
89
90 coverage() {
91 case "$1" in
92 -d)
93 shift
94 target="${1?Need to provide a target branch argument}"
95
96 output_dir="$(mktemp -d)"
97 target_out="${output_dir}/target.txt"
98 head_out="${output_dir}/head.txt"
99
100 cover "${target}" > "${target_out}"
101 cover "HEAD" > "${head_out}"
102
103 cat "${target_out}"
104 cat "${head_out}"
105
106 echo ""
107
108 target_pct="$(tail -n2 ${target_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%.*/\1/')"
109 head_pct="$(tail -n2 ${head_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%/\1/')"
110 echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%"
111
112 delta_pct=$(echo "$head_pct - $target_pct" | bc -l)
113 echo "Delta: ${delta_pct}"
114
115 if [[ $delta_pct = \-* ]]; then
116 echo "Regression!";
117
118 target_diff="${output_dir}/target.diff.txt"
119 head_diff="${output_dir}/head.diff.txt"
120 cat "${target_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${target_diff}"
121 cat "${head_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${head_diff}"
122
123 diff --side-by-side --suppress-common-lines "${target_diff}" "${head_diff}"
124 return 1
125 fi
126 return 0
127 ;;
128 esac
129
130 cover "${1-HEAD}"
131 }
132
133 bench() {
134 branch="${1}"
135 out="${2}"
136 replace="${3}"
137 dir="$(mktemp -d)"
138
139 stderr "Executing benchmark for ${branch} at ${dir}"
140
141 if [ "${branch}" = "HEAD" ]; then
142 cp -r . "${dir}/"
143 else
144 git worktree add "$dir" "$branch"
145 fi
146
147 pushd "$dir"
148
149 if [ "${replace}" != "" ]; then
150 find ./benchmark/ -iname '*.go' -exec sed -i -E "s|github.com/pelletier/go-toml/v2|${replace}|g" {} \;
151 go get "${replace}"
152 fi
153
154 export GOMAXPROCS=2
155 nice -n -19 taskset --cpu-list 0,1 go test '-bench=^Benchmark(Un)?[mM]arshal' -count=5 -run=Nothing ./... | tee "${out}"
156 popd
157
158 if [ "${branch}" != "HEAD" ]; then
159 git worktree remove --force "$dir"
160 fi
161 }
162
163 fmktemp() {
164 if mktemp --version|grep GNU >/dev/null; then
165 mktemp --suffix=-$1;
166 else
167 mktemp -t $1;
168 fi
169 }
170
171 benchstathtml() {
172 python3 - $1 <<'EOF'
173 import sys
174
175 lines = []
176 stop = False
177
178 with open(sys.argv[1]) as f:
179 for line in f.readlines():
180 line = line.strip()
181 if line == "":
182 stop = True
183 if not stop:
184 lines.append(line.split(','))
185
186 results = []
187 for line in reversed(lines[1:]):
188 v2 = float(line[1])
189 results.append([
190 line[0].replace("-32", ""),
191 "%.1fx" % (float(line[3])/v2), # v1
192 "%.1fx" % (float(line[5])/v2), # bs
193 ])
194 # move geomean to the end
195 results.append(results[0])
196 del results[0]
197
198
199 def printtable(data):
200 print("""
201 <table>
202 <thead>
203 <tr><th>Benchmark</th><th>go-toml v1</th><th>BurntSushi/toml</th></tr>
204 </thead>
205 <tbody>""")
206
207 for r in data:
208 print(" <tr><td>{}</td><td>{}</td><td>{}</td></tr>".format(*r))
209
210 print(""" </tbody>
211 </table>""")
212
213
214 def match(x):
215 return "ReferenceFile" in x[0] or "HugoFrontMatter" in x[0]
216
217 above = [x for x in results if match(x)]
218 below = [x for x in results if not match(x)]
219
220 printtable(above)
221 print("<details><summary>See more</summary>")
222 print("""<p>The table above has the results of the most common use-cases. The table below
223 contains the results of all benchmarks, including unrealistic ones. It is
224 provided for completeness.</p>""")
225 printtable(below)
226 print('<p>This table can be generated with <code>./ci.sh benchmark -a -html</code>.</p>')
227 print("</details>")
228
229 EOF
230 }
231
232 benchmark() {
233 case "$1" in
234 -d)
235 shift
236 target="${1?Need to provide a target branch argument}"
237
238 old=`fmktemp ${target}`
239 bench "${target}" "${old}"
240
241 new=`fmktemp HEAD`
242 bench HEAD "${new}"
243
244 benchstat "${old}" "${new}"
245 return 0
246 ;;
247 -a)
248 shift
249
250 v2stats=`fmktemp go-toml-v2`
251 bench HEAD "${v2stats}" "github.com/pelletier/go-toml/v2"
252 v1stats=`fmktemp go-toml-v1`
253 bench HEAD "${v1stats}" "github.com/pelletier/go-toml"
254 bsstats=`fmktemp bs-toml`
255 bench HEAD "${bsstats}" "github.com/BurntSushi/toml"
256
257 cp "${v2stats}" go-toml-v2.txt
258 cp "${v1stats}" go-toml-v1.txt
259 cp "${bsstats}" bs-toml.txt
260
261 if [ "$1" = "-html" ]; then
262 tmpcsv=`fmktemp csv`
263 benchstat -csv -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt > $tmpcsv
264 benchstathtml $tmpcsv
265 else
266 benchstat -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt
267 fi
268
269 rm -f go-toml-v2.txt go-toml-v1.txt bs-toml.txt
270 return $?
271 esac
272
273 bench "${1-HEAD}" `mktemp`
274 }
275
276 case "$1" in
277 coverage) shift; coverage $@;;
278 benchmark) shift; benchmark $@;;
279 *) usage "bad argument $1";;
280 esac
281