generate_test.go raw
1 // Copyright 2023 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 // This file implements a custom generator to create various go/types
6 // source files from the corresponding types2 files.
7
8 package types_test
9
10 import (
11 "bytes"
12 "flag"
13 "fmt"
14 "go/ast"
15 "go/format"
16 "go/parser"
17 "go/token"
18 "internal/diff"
19 "os"
20 "path/filepath"
21 "runtime"
22 "strings"
23 "testing"
24 )
25
26 var filesToWrite = flag.String("write", "", `go/types files to generate, or "all" for all files`)
27
28 const (
29 srcDir = "/src/cmd/compile/internal/types2/"
30 dstDir = "/src/go/types/"
31 )
32
33 // TestGenerate verifies that generated files in go/types match their types2
34 // counterpart. If -write is set, this test actually writes the expected
35 // content to go/types; otherwise, it just compares with the existing content.
36 func TestGenerate(t *testing.T) {
37 // If filesToWrite is set, write the generated content to disk.
38 // In the special case of "all", write all files in filemap.
39 write := *filesToWrite != ""
40 var files []string // files to process
41 if *filesToWrite != "" && *filesToWrite != "all" {
42 files = strings.Split(*filesToWrite, ",")
43 } else {
44 for file := range filemap {
45 files = append(files, file)
46 }
47 }
48
49 for _, filename := range files {
50 generate(t, filename, write)
51 }
52 }
53
54 func generate(t *testing.T, filename string, write bool) {
55 // parse src (cmd/compile/internal/types2)
56 srcFilename := filepath.FromSlash(runtime.GOROOT() + srcDir + filename)
57 file, err := parser.ParseFile(fset, srcFilename, nil, parser.ParseComments)
58 if err != nil {
59 t.Fatal(err)
60 }
61
62 // fix package name
63 file.Name.Name = strings.ReplaceAll(file.Name.Name, "types2", "types")
64
65 // rewrite AST as needed
66 if action := filemap[filename]; action != nil {
67 action(file)
68 }
69
70 // format AST
71 var buf bytes.Buffer
72 rel, _ := filepath.Rel(dstDir, srcDir)
73 fmt.Fprintf(&buf, "// Code generated by \"go test -run=Generate -write=all\"; DO NOT EDIT.\n")
74 fmt.Fprintf(&buf, "// Source: %s/%s\n\n", filepath.ToSlash(rel), filename)
75 if err := format.Node(&buf, fset, file); err != nil {
76 t.Fatal(err)
77 }
78 generatedContent := buf.Bytes()
79
80 // read dst (go/types)
81 dstFilename := filepath.FromSlash(runtime.GOROOT() + dstDir + filename)
82 onDiskContent, err := os.ReadFile(dstFilename)
83 if err != nil {
84 t.Fatalf("reading %q: %v", filename, err)
85 }
86
87 // compare on-disk dst with buffer generated from src.
88 if d := diff.Diff(filename+" (on disk in "+dstDir+")", onDiskContent, filename+" (generated from "+srcDir+")", generatedContent); d != nil {
89 if write {
90 t.Logf("applying change:\n%s", d)
91 if err := os.WriteFile(dstFilename, generatedContent, 0o644); err != nil {
92 t.Fatalf("writing %q: %v", filename, err)
93 }
94 } else {
95 t.Errorf("file on disk in %s is stale:\n%s", dstDir, d)
96 }
97 }
98 }
99
100 type action func(in *ast.File)
101
102 var filemap = map[string]action{
103 "alias.go": fixTokenPos,
104 "assignments.go": func(f *ast.File) {
105 renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
106 renameSelectorExprs(f, "syntax.Name->ast.Ident", "ident.Value->ident.Name", "ast.Pos->token.Pos") // must happen before renaming identifiers
107 renameIdents(f, "syntax->ast", "poser->positioner", "nopos->noposn")
108 },
109 "array.go": nil,
110 "api_predicates.go": nil,
111 "basic.go": nil,
112 "builtins.go": func(f *ast.File) {
113 renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
114 renameIdents(f, "syntax->ast")
115 renameSelectors(f, "ArgList->Args")
116 fixSelValue(f)
117 fixAtPosCall(f)
118 },
119 "builtins_test.go": func(f *ast.File) {
120 renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`, `"cmd/compile/internal/types2"->"go/types"`)
121 renameSelectorExprs(f, "syntax.Name->ast.Ident", "p.Value->p.Name") // must happen before renaming identifiers
122 renameIdents(f, "syntax->ast")
123 },
124 "chan.go": nil,
125 "const.go": fixTokenPos,
126 "context.go": nil,
127 "context_test.go": nil,
128 "conversions.go": nil,
129 "errors_test.go": func(f *ast.File) { renameIdents(f, "nopos->noposn") },
130 "errsupport.go": nil,
131 "gccgosizes.go": nil,
132 "gcsizes.go": func(f *ast.File) { renameIdents(f, "IsSyncAtomicAlign64->_IsSyncAtomicAlign64") },
133 "hilbert_test.go": func(f *ast.File) { renameImportPath(f, `"cmd/compile/internal/types2"->"go/types"`) },
134 "infer.go": func(f *ast.File) { fixTokenPos(f); fixInferSig(f) },
135 "initorder.go": nil,
136 // "initorder.go": fixErrErrorfCall, // disabled for now due to unresolved error_ use implications for gopls
137 "instantiate.go": func(f *ast.File) { fixTokenPos(f); fixCheckErrorfCall(f) },
138 "instantiate_test.go": func(f *ast.File) { renameImportPath(f, `"cmd/compile/internal/types2"->"go/types"`) },
139 "literals.go": func(f *ast.File) {
140 insertImportPath(f, `"go/token"`)
141 renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
142 renameSelectorExprs(f,
143 "syntax.IntLit->token.INT", "syntax.FloatLit->token.FLOAT", "syntax.ImagLit->token.IMAG",
144 "syntax.Name->ast.Ident", "key.Value->key.Name", "atyp.Elem->atyp.Elt") // must happen before renaming identifiers
145 renameIdents(f, "syntax->ast")
146 renameSelectors(f, "ElemList->Elts")
147 },
148 "lookup.go": func(f *ast.File) { fixTokenPos(f) },
149 "main_test.go": nil,
150 "map.go": nil,
151 "mono.go": func(f *ast.File) {
152 fixTokenPos(f)
153 insertImportPath(f, `"go/ast"`)
154 renameSelectorExprs(f, "syntax.Expr->ast.Expr")
155 },
156 "named.go": func(f *ast.File) { fixTokenPos(f); renameSelectors(f, "Trace->_Trace") },
157 "object.go": func(f *ast.File) { fixTokenPos(f); renameIdents(f, "NewTypeNameLazy->_NewTypeNameLazy") },
158 // TODO(gri) needs adjustments for TestObjectString - disabled for now
159 // "object_test.go": func(f *ast.File) { renameImportPath(f, `"cmd/compile/internal/types2"->"go/types"`) },
160 "objset.go": nil,
161 "operand.go": func(f *ast.File) {
162 insertImportPath(f, `"go/token"`)
163 renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
164 renameSelectorExprs(f,
165 "syntax.Pos->token.Pos", "syntax.LitKind->token.Token",
166 "syntax.IntLit->token.INT", "syntax.FloatLit->token.FLOAT",
167 "syntax.ImagLit->token.IMAG", "syntax.RuneLit->token.CHAR",
168 "syntax.StringLit->token.STRING") // must happen before renaming identifiers
169 renameIdents(f, "syntax->ast")
170 },
171 "package.go": nil,
172 "pointer.go": nil,
173 "predicates.go": nil,
174 "range.go": func(f *ast.File) {
175 renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
176 renameSelectorExprs(f, "syntax.Name->ast.Ident", "syntax.ForStmt->ast.RangeStmt", "ident.Value->ident.Name") // must happen before renaming identifiers
177 renameIdents(f, "syntax->ast", "poser->positioner")
178 },
179 "recording.go": func(f *ast.File) {
180 renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
181 renameSelectorExprs(f, "syntax.Name->ast.Ident") // must happen before renaming identifiers
182 renameIdents(f, "syntax->ast")
183 fixAtPosCall(f)
184 },
185 "scope.go": func(f *ast.File) { fixTokenPos(f); renameIdents(f, "InsertLazy->_InsertLazy") },
186 "selection.go": nil,
187 "sizes.go": func(f *ast.File) { renameIdents(f, "IsSyncAtomicAlign64->_IsSyncAtomicAlign64") },
188 "slice.go": nil,
189 "subst.go": func(f *ast.File) { fixTokenPos(f); renameSelectors(f, "Trace->_Trace") },
190 "termlist.go": nil,
191 "termlist_test.go": nil,
192 "tuple.go": nil,
193 "typelists.go": nil,
194 "typeset.go": func(f *ast.File) { fixTokenPos(f); renameSelectors(f, "Trace->_Trace") },
195 "typeparam.go": nil,
196 "typeterm_test.go": nil,
197 "typeterm.go": nil,
198 "typestring.go": nil,
199 "under.go": nil,
200 "unify.go": fixSprintf,
201 "universe.go": fixGlobalTypVarDecl,
202 "util_test.go": fixTokenPos,
203 "validtype.go": func(f *ast.File) { fixTokenPos(f); renameSelectors(f, "Trace->_Trace") },
204 }
205
206 // TODO(gri) We should be able to make these rewriters more configurable/composable.
207 // For now this is a good starting point.
208
209 // A renameMap maps old strings to new strings.
210 type renameMap map[string]string
211
212 // makeRenameMap returns a renameMap populates from renames entries of the form "from->to".
213 func makeRenameMap(renames ...string) renameMap {
214 m := make(renameMap)
215 for _, r := range renames {
216 s := strings.Split(r, "->")
217 if len(s) != 2 {
218 panic("invalid rename entry: " + r)
219 }
220 m[s[0]] = s[1]
221 }
222 return m
223 }
224
225 // rename renames the given string s if a corresponding rename exists in m.
226 func (m renameMap) rename(s *string) {
227 if r, ok := m[*s]; ok {
228 *s = r
229 }
230 }
231
232 // renameSel renames a selector expression of the form a.x to b.x (where a, b are identifiers)
233 // if m contains the ("a.x" : "b.y") key-value pair.
234 func (m renameMap) renameSel(n *ast.SelectorExpr) {
235 if a, _ := n.X.(*ast.Ident); a != nil {
236 a_x := a.Name + "." + n.Sel.Name
237 if r, ok := m[a_x]; ok {
238 b_y := strings.Split(r, ".")
239 if len(b_y) != 2 {
240 panic("invalid selector expression: " + r)
241 }
242 a.Name = b_y[0]
243 n.Sel.Name = b_y[1]
244 }
245 }
246 }
247
248 // renameIdents renames identifiers: each renames entry is of the form "from->to".
249 // Note: This doesn't change the use of the identifiers in comments.
250 func renameIdents(f *ast.File, renames ...string) {
251 m := makeRenameMap(renames...)
252 ast.Inspect(f, func(n ast.Node) bool {
253 switch n := n.(type) {
254 case *ast.Ident:
255 m.rename(&n.Name)
256 return false
257 }
258 return true
259 })
260 }
261
262 // renameSelectors is like renameIdents but only looks at selectors.
263 func renameSelectors(f *ast.File, renames ...string) {
264 m := makeRenameMap(renames...)
265 ast.Inspect(f, func(n ast.Node) bool {
266 switch n := n.(type) {
267 case *ast.SelectorExpr:
268 m.rename(&n.Sel.Name)
269 return false
270 }
271 return true
272 })
273
274 }
275
276 // renameSelectorExprs is like renameIdents but only looks at selector expressions.
277 // Each renames entry must be of the form "x.a->y.b".
278 func renameSelectorExprs(f *ast.File, renames ...string) {
279 m := makeRenameMap(renames...)
280 ast.Inspect(f, func(n ast.Node) bool {
281 switch n := n.(type) {
282 case *ast.SelectorExpr:
283 m.renameSel(n)
284 return false
285 }
286 return true
287 })
288 }
289
290 // renameImportPath is like renameIdents but renames import paths.
291 func renameImportPath(f *ast.File, renames ...string) {
292 m := makeRenameMap(renames...)
293 ast.Inspect(f, func(n ast.Node) bool {
294 switch n := n.(type) {
295 case *ast.ImportSpec:
296 if n.Path.Kind != token.STRING {
297 panic("invalid import path")
298 }
299 m.rename(&n.Path.Value)
300 return false
301 }
302 return true
303 })
304 }
305
306 // insertImportPath inserts the given import path.
307 // There must be at least one import declaration present already.
308 func insertImportPath(f *ast.File, path string) {
309 for _, d := range f.Decls {
310 if g, _ := d.(*ast.GenDecl); g != nil && g.Tok == token.IMPORT {
311 g.Specs = append(g.Specs, &ast.ImportSpec{Path: &ast.BasicLit{ValuePos: g.End(), Kind: token.STRING, Value: path}})
312 return
313 }
314 }
315 panic("no import declaration present")
316 }
317
318 // fixTokenPos changes imports of "cmd/compile/internal/syntax" to "go/token",
319 // uses of syntax.Pos to token.Pos, and calls to x.IsKnown() to x.IsValid().
320 func fixTokenPos(f *ast.File) {
321 m := makeRenameMap(`"cmd/compile/internal/syntax"->"go/token"`, "syntax.Pos->token.Pos", "IsKnown->IsValid")
322 ast.Inspect(f, func(n ast.Node) bool {
323 switch n := n.(type) {
324 case *ast.ImportSpec:
325 // rewrite import path "cmd/compile/internal/syntax" to "go/token"
326 if n.Path.Kind != token.STRING {
327 panic("invalid import path")
328 }
329 m.rename(&n.Path.Value)
330 return false
331 case *ast.SelectorExpr:
332 // rewrite syntax.Pos to token.Pos
333 m.renameSel(n)
334 case *ast.CallExpr:
335 // rewrite x.IsKnown() to x.IsValid()
336 if fun, _ := n.Fun.(*ast.SelectorExpr); fun != nil && len(n.Args) == 0 {
337 m.rename(&fun.Sel.Name)
338 return false
339 }
340 }
341 return true
342 })
343 }
344
345 // fixSelValue updates the selector x.Sel.Value to x.Sel.Name.
346 func fixSelValue(f *ast.File) {
347 ast.Inspect(f, func(n ast.Node) bool {
348 switch n := n.(type) {
349 case *ast.SelectorExpr:
350 if n.Sel.Name == "Value" {
351 if selx, _ := n.X.(*ast.SelectorExpr); selx != nil && selx.Sel.Name == "Sel" {
352 n.Sel.Name = "Name"
353 return false
354 }
355 }
356 }
357 return true
358 })
359 }
360
361 // fixInferSig updates the Checker.infer signature to use a positioner instead of a token.Position
362 // as first argument, renames the argument from "pos" to "posn", and updates a few internal uses of
363 // "pos" to "posn" and "posn.Pos()" respectively.
364 func fixInferSig(f *ast.File) {
365 ast.Inspect(f, func(n ast.Node) bool {
366 switch n := n.(type) {
367 case *ast.FuncDecl:
368 if n.Name.Name == "infer" {
369 // rewrite (pos token.Pos, ...) to (posn positioner, ...)
370 par := n.Type.Params.List[0]
371 if len(par.Names) == 1 && par.Names[0].Name == "pos" {
372 par.Names[0] = newIdent(par.Names[0].Pos(), "posn")
373 par.Type = newIdent(par.Type.Pos(), "positioner")
374 return true
375 }
376 }
377 case *ast.CallExpr:
378 if selx, _ := n.Fun.(*ast.SelectorExpr); selx != nil {
379 switch selx.Sel.Name {
380 case "renameTParams":
381 // rewrite check.renameTParams(pos, ... ) to check.renameTParams(posn.Pos(), ... )
382 if isIdent(n.Args[0], "pos") {
383 pos := n.Args[0].Pos()
384 fun := &ast.SelectorExpr{X: newIdent(pos, "posn"), Sel: newIdent(pos, "Pos")}
385 arg := &ast.CallExpr{Fun: fun, Lparen: pos, Args: nil, Ellipsis: token.NoPos, Rparen: pos}
386 n.Args[0] = arg
387 return false
388 }
389 case "addf":
390 // rewrite err.addf(pos, ...) to err.addf(posn, ...)
391 if isIdent(n.Args[0], "pos") {
392 pos := n.Args[0].Pos()
393 arg := newIdent(pos, "posn")
394 n.Args[0] = arg
395 return false
396 }
397 case "allowVersion":
398 // rewrite check.allowVersion(pos, ...) to check.allowVersion(posn, ...)
399 if isIdent(n.Args[0], "pos") {
400 pos := n.Args[0].Pos()
401 arg := newIdent(pos, "posn")
402 n.Args[0] = arg
403 return false
404 }
405 }
406 }
407 }
408 return true
409 })
410 }
411
412 // fixAtPosCall updates calls of the form atPos(x) to x.Pos() in argument lists of (check).dump calls.
413 // TODO(gri) can we avoid this and just use atPos consistently in go/types and types2?
414 func fixAtPosCall(f *ast.File) {
415 ast.Inspect(f, func(n ast.Node) bool {
416 switch n := n.(type) {
417 case *ast.CallExpr:
418 if selx, _ := n.Fun.(*ast.SelectorExpr); selx != nil && selx.Sel.Name == "dump" {
419 for i, arg := range n.Args {
420 if call, _ := arg.(*ast.CallExpr); call != nil {
421 // rewrite xxx.dump(..., atPos(x), ...) to xxx.dump(..., x.Pos(), ...)
422 if isIdent(call.Fun, "atPos") {
423 pos := call.Args[0].Pos()
424 fun := &ast.SelectorExpr{X: call.Args[0], Sel: newIdent(pos, "Pos")}
425 n.Args[i] = &ast.CallExpr{Fun: fun, Lparen: pos, Rparen: pos}
426 return false
427 }
428 }
429 }
430 }
431 }
432 return true
433 })
434 }
435
436 // fixErrErrorfCall updates calls of the form err.addf(obj, ...) to err.addf(obj.Pos(), ...).
437 func fixErrErrorfCall(f *ast.File) {
438 ast.Inspect(f, func(n ast.Node) bool {
439 switch n := n.(type) {
440 case *ast.CallExpr:
441 if selx, _ := n.Fun.(*ast.SelectorExpr); selx != nil {
442 if isIdent(selx.X, "err") {
443 switch selx.Sel.Name {
444 case "errorf":
445 // rewrite err.addf(obj, ... ) to err.addf(obj.Pos(), ... )
446 if ident, _ := n.Args[0].(*ast.Ident); ident != nil && ident.Name == "obj" {
447 pos := n.Args[0].Pos()
448 fun := &ast.SelectorExpr{X: ident, Sel: newIdent(pos, "Pos")}
449 n.Args[0] = &ast.CallExpr{Fun: fun, Lparen: pos, Rparen: pos}
450 return false
451 }
452 }
453 }
454 }
455 }
456 return true
457 })
458 }
459
460 // fixCheckErrorfCall updates calls of the form check.errorf(pos, ...) to check.errorf(atPos(pos), ...).
461 func fixCheckErrorfCall(f *ast.File) {
462 ast.Inspect(f, func(n ast.Node) bool {
463 switch n := n.(type) {
464 case *ast.CallExpr:
465 if selx, _ := n.Fun.(*ast.SelectorExpr); selx != nil {
466 if isIdent(selx.X, "check") {
467 switch selx.Sel.Name {
468 case "errorf":
469 // rewrite check.errorf(pos, ... ) to check.errorf(atPos(pos), ... )
470 if ident := asIdent(n.Args[0], "pos"); ident != nil {
471 pos := n.Args[0].Pos()
472 fun := newIdent(pos, "atPos")
473 n.Args[0] = &ast.CallExpr{Fun: fun, Lparen: pos, Args: []ast.Expr{ident}, Rparen: pos}
474 return false
475 }
476 }
477 }
478 }
479 }
480 return true
481 })
482 }
483
484 // fixGlobalTypVarDecl changes the global Typ variable from an array to a slice
485 // (in types2 we use an array for efficiency, in go/types it's a slice and we
486 // cannot change that).
487 func fixGlobalTypVarDecl(f *ast.File) {
488 ast.Inspect(f, func(n ast.Node) bool {
489 switch n := n.(type) {
490 case *ast.ValueSpec:
491 // rewrite type Typ = [...]Type{...} to type Typ = []Type{...}
492 if len(n.Names) == 1 && n.Names[0].Name == "Typ" && len(n.Values) == 1 {
493 n.Values[0].(*ast.CompositeLit).Type.(*ast.ArrayType).Len = nil
494 return false
495 }
496 }
497 return true
498 })
499 }
500
501 // fixSprintf adds an extra nil argument for the *token.FileSet parameter in sprintf calls.
502 func fixSprintf(f *ast.File) {
503 ast.Inspect(f, func(n ast.Node) bool {
504 switch n := n.(type) {
505 case *ast.CallExpr:
506 if isIdent(n.Fun, "sprintf") && len(n.Args) >= 4 /* ... args */ {
507 n.Args = insert(n.Args, 1, newIdent(n.Args[1].Pos(), "nil"))
508 return false
509 }
510 }
511 return true
512 })
513 }
514
515 // asIdent returns x as *ast.Ident if it is an identifier with the given name.
516 func asIdent(x ast.Node, name string) *ast.Ident {
517 if ident, _ := x.(*ast.Ident); ident != nil && ident.Name == name {
518 return ident
519 }
520 return nil
521 }
522
523 // isIdent reports whether x is an identifier with the given name.
524 func isIdent(x ast.Node, name string) bool {
525 return asIdent(x, name) != nil
526 }
527
528 // newIdent returns a new identifier with the given position and name.
529 func newIdent(pos token.Pos, name string) *ast.Ident {
530 id := ast.NewIdent(name)
531 id.NamePos = pos
532 return id
533 }
534
535 // insert inserts x at list[at] and moves the remaining elements up.
536 func insert(list []ast.Expr, at int, x ast.Expr) []ast.Expr {
537 list = append(list, nil)
538 copy(list[at+1:], list[at:])
539 list[at] = x
540 return list
541 }
542