e2e_test.go raw
1 // SPDX-License-Identifier: Unlicense OR MIT
2
3 package main_test
4
5 import (
6 "bufio"
7 "errors"
8 "flag"
9 "fmt"
10 "image"
11 "image/color"
12 "io"
13 "io/ioutil"
14 "os"
15 "os/exec"
16 "strings"
17 "testing"
18 "time"
19 )
20
21 var raceEnabled = false
22
23 var headless = flag.Bool("headless", true, "run end-to-end tests in headless mode")
24
25 const appid = "localhost.gogio.endtoend"
26
27 // TestDriver is implemented by each of the platforms we can run end-to-end
28 // tests on. None of its methods return any errors, as the errors are directly
29 // reported to testing.T via methods like Fatal.
30 type TestDriver interface {
31 initBase(t *testing.T, width, height int)
32
33 // Start opens the Gio app found at path. The driver should attempt to
34 // run the app with the base driver's width and height, and the
35 // platform's background should be white.
36 //
37 // When the function returns, the gio app must be ready to use on the
38 // platform, with its initial frame fully drawn.
39 Start(path string)
40
41 // Screenshot takes a screenshot of the Gio app on the platform.
42 Screenshot() image.Image
43
44 // Click performs a pointer click at the specified coordinates,
45 // including both press and release. It returns when the next frame is
46 // fully drawn.
47 Click(x, y int)
48 }
49
50 type driverBase struct {
51 *testing.T
52
53 width, height int
54
55 output io.Reader
56 frameNotifs chan bool
57 }
58
59 func (d *driverBase) initBase(t *testing.T, width, height int) {
60 d.T = t
61 d.width, d.height = width, height
62 }
63
64 func TestEndToEnd(t *testing.T) {
65 if testing.Short() {
66 t.Skipf("end-to-end tests tend to be slow")
67 }
68
69 t.Parallel()
70
71 const (
72 testdataWithGoImportPkgPath = "github.com/p9c/p9/pkg/gel/gio/cmd/gogio/testdata"
73 testdataWithRelativePkgPath = "testdata/testdata.go"
74 )
75 // Keep this list local, to not reuse TestDriver objects.
76 subtests := []struct {
77 name string
78 driver TestDriver
79 pkgPath string
80 }{
81 {"X11 using go import path", &X11TestDriver{}, testdataWithGoImportPkgPath},
82 {"X11", &X11TestDriver{}, testdataWithRelativePkgPath},
83 {"Wayland", &WaylandTestDriver{}, testdataWithRelativePkgPath},
84 {"JS", &JSTestDriver{}, testdataWithRelativePkgPath},
85 {"Android", &AndroidTestDriver{}, testdataWithRelativePkgPath},
86 {"Windows", &WineTestDriver{}, testdataWithRelativePkgPath},
87 }
88
89 for _, subtest := range subtests {
90 t.Run(subtest.name, func(t *testing.T) {
91 subtest := subtest // copy the changing loop variable
92 t.Parallel()
93 runEndToEndTest(t, subtest.driver, subtest.pkgPath)
94 })
95 }
96 }
97
98 func runEndToEndTest(t *testing.T, driver TestDriver, pkgPath string) {
99 size := image.Point{X: 800, Y: 600}
100 driver.initBase(t, size.X, size.Y)
101
102 t.Log("starting driver and gio app")
103 driver.Start(pkgPath)
104
105 beef := color.NRGBA{R: 0xde, G: 0xad, B: 0xbe, A: 0xff}
106 white := color.NRGBA{R: 0xff, G: 0xff, B: 0xff, A: 0xff}
107 black := color.NRGBA{R: 0x00, G: 0x00, B: 0x00, A: 0xff}
108 gray := color.NRGBA{R: 0xbb, G: 0xbb, B: 0xbb, A: 0xff}
109 red := color.NRGBA{R: 0xff, G: 0x00, B: 0x00, A: 0xff}
110
111 // These are the four colors at the beginning.
112 t.Log("taking initial screenshot")
113 withRetries(t, 4*time.Second, func() error {
114 img := driver.Screenshot()
115 size = img.Bounds().Size() // override the default size
116 return checkImageCorners(img, beef, white, black, gray)
117 })
118
119 // TODO(mvdan): implement this properly in the Wayland driver; swaymsg
120 // almost works to automate clicks, but the button presses end up in the
121 // wrong coordinates.
122 if _, ok := driver.(*WaylandTestDriver); ok {
123 return
124 }
125
126 // Click the first and last sections to turn them red.
127 t.Log("clicking twice and taking another screenshot")
128 driver.Click(1*(size.X/4), 1*(size.Y/4))
129 driver.Click(3*(size.X/4), 3*(size.Y/4))
130 withRetries(t, 4*time.Second, func() error {
131 img := driver.Screenshot()
132 return checkImageCorners(img, red, white, black, red)
133 })
134 }
135
136 // withRetries keeps retrying fn until it succeeds, or until the timeout is hit.
137 // It uses a rudimentary kind of backoff, which starts with 100ms delays. As
138 // such, timeout should generally be in the order of seconds.
139 func withRetries(t *testing.T, timeout time.Duration, fn func() error) {
140 t.Helper()
141
142 timeoutTimer := time.NewTimer(timeout)
143 defer timeoutTimer.Stop()
144 backoff := 100 * time.Millisecond
145
146 tries := 0
147 var lastErr error
148 for {
149 if lastErr = fn(); lastErr == nil {
150 return
151 }
152 tries++
153 t.Logf("retrying after %s", backoff)
154
155 // Use a timer instead of a sleep, so that the timeout can stop
156 // the backoff early. Don't reuse this timer, since we're not in
157 // a hot loop, and we don't want tricky code.
158 backoffTimer := time.NewTimer(backoff)
159 defer backoffTimer.Stop()
160
161 select {
162 case <-timeoutTimer.C:
163 t.Errorf("last error: %v", lastErr)
164 t.Fatalf("hit timeout of %s after %d tries", timeout, tries)
165 case <-backoffTimer.C:
166 }
167
168 // Keep doubling it until a maximum. With the start at 100ms,
169 // we'll do: 100ms, 200ms, 400ms, 800ms, 1.6s, and 2s forever.
170 backoff *= 2
171 if max := 2 * time.Second; backoff > max {
172 backoff = max
173 }
174 }
175 }
176
177 type colorMismatch struct {
178 x, y int
179 wantRGB, gotRGB [3]uint32
180 }
181
182 func (m colorMismatch) String() string {
183 return fmt.Sprintf("%3d,%-3d got 0x%04x%04x%04x, want 0x%04x%04x%04x",
184 m.x, m.y,
185 m.gotRGB[0], m.gotRGB[1], m.gotRGB[2],
186 m.wantRGB[0], m.wantRGB[1], m.wantRGB[2],
187 )
188 }
189
190 func checkImageCorners(img image.Image, topLeft, topRight, botLeft, botRight color.Color) error {
191 // The colors are split in four rectangular sections. Check the corners
192 // of each of the sections. We check the corners left to right, top to
193 // bottom, like when reading left-to-right text.
194
195 size := img.Bounds().Size()
196 var mismatches []colorMismatch
197
198 checkColor := func(x, y int, want color.Color) {
199 r, g, b, _ := want.RGBA()
200 got := img.At(x, y)
201 r_, g_, b_, _ := got.RGBA()
202 if r_ != r || g_ != g || b_ != b {
203 mismatches = append(mismatches, colorMismatch{
204 x: x,
205 y: y,
206 wantRGB: [3]uint32{r, g, b},
207 gotRGB: [3]uint32{r_, g_, b_},
208 })
209 }
210 }
211
212 {
213 minX, minY := 5, 5
214 maxX, maxY := (size.X/2)-5, (size.Y/2)-5
215 checkColor(minX, minY, topLeft)
216 checkColor(maxX, minY, topLeft)
217 checkColor(minX, maxY, topLeft)
218 checkColor(maxX, maxY, topLeft)
219 }
220 {
221 minX, minY := (size.X/2)+5, 5
222 maxX, maxY := size.X-5, (size.Y/2)-5
223 checkColor(minX, minY, topRight)
224 checkColor(maxX, minY, topRight)
225 checkColor(minX, maxY, topRight)
226 checkColor(maxX, maxY, topRight)
227 }
228 {
229 minX, minY := 5, (size.Y/2)+5
230 maxX, maxY := (size.X/2)-5, size.Y-5
231 checkColor(minX, minY, botLeft)
232 checkColor(maxX, minY, botLeft)
233 checkColor(minX, maxY, botLeft)
234 checkColor(maxX, maxY, botLeft)
235 }
236 {
237 minX, minY := (size.X/2)+5, (size.Y/2)+5
238 maxX, maxY := size.X-5, size.Y-5
239 checkColor(minX, minY, botRight)
240 checkColor(maxX, minY, botRight)
241 checkColor(minX, maxY, botRight)
242 checkColor(maxX, maxY, botRight)
243 }
244 if n := len(mismatches); n > 0 {
245 b := new(strings.Builder)
246 fmt.Fprintf(b, "encountered %d color mismatches:\n", n)
247 for _, m := range mismatches {
248 fmt.Fprintf(b, "%s\n", m)
249 }
250 return errors.New(b.String())
251 }
252 return nil
253 }
254
255 func (d *driverBase) waitForFrame() {
256 d.Helper()
257
258 if d.frameNotifs == nil {
259 // Start the goroutine that reads output lines and notifies of
260 // new frames via frameNotifs. The test doesn't wait for this
261 // goroutine to finish; it will naturally end when the output
262 // reader reaches an error like EOF.
263 d.frameNotifs = make(chan bool, 1)
264 if d.output == nil {
265 d.Fatal("need an output reader to be notified of frames")
266 }
267 go func() {
268 scanner := bufio.NewScanner(d.output)
269 for scanner.Scan() {
270 line := scanner.Text()
271 d.Log(line)
272 if strings.Contains(line, "gio frame ready") {
273 d.frameNotifs <- true
274 }
275 }
276 // Since we're only interested in the output while the
277 // app runs, and we don't know when it finishes here,
278 // ignore "already closed" pipe errors.
279 if err := scanner.Err(); err != nil && !errors.Is(err, os.ErrClosed) {
280 d.Errorf("reading app output: %v", err)
281 }
282 }()
283 }
284
285 // Unfortunately, there isn't a way to select on a test failing, since
286 // testing.T doesn't have anything like a context or a "done" channel.
287 //
288 // We can't let selects block forever, since the default -test.timeout
289 // is ten minutes - far too long for tests that take seconds.
290 //
291 // For now, a static short timeout is better than nothing. 5s is plenty
292 // for our simple test app to render on any device.
293 select {
294 case <-d.frameNotifs:
295 case <-time.After(5 * time.Second):
296 d.Fatalf("timed out waiting for a frame to be ready")
297 }
298 }
299
300 func (d *driverBase) needPrograms(names ...string) {
301 d.Helper()
302 for _, name := range names {
303 if _, err := exec.LookPath(name); err != nil {
304 d.Skipf("%s needed to run", name)
305 }
306 }
307 }
308
309 func (d *driverBase) tempDir(name string) string {
310 d.Helper()
311 dir, err := ioutil.TempDir("", name)
312 if err != nil {
313 d.Fatal(err)
314 }
315 d.Cleanup(func() { os.RemoveAll(dir) })
316 return dir
317 }
318
319 func (d *driverBase) gogio(args ...string) {
320 d.Helper()
321 prog, err := os.Executable()
322 if err != nil {
323 d.Fatal(err)
324 }
325 cmd := exec.Command(prog, args...)
326 cmd.Env = append(os.Environ(), "RUN_GOGIO=1")
327 if out, err := cmd.CombinedOutput(); err != nil {
328 d.Fatalf("gogio error: %s:\n%s", err, out)
329 }
330 }
331