1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4 5 package packages
6 7 import (
8 "bytes"
9 "context"
10 "encoding/json"
11 "fmt"
12 "log"
13 "os"
14 "os/exec"
15 "path"
16 "path/filepath"
17 "reflect"
18 "sort"
19 "strconv"
20 "strings"
21 "sync"
22 "unicode"
23 24 "golang.org/x/tools/internal/gocommand"
25 "golang.org/x/tools/internal/packagesinternal"
26 )
27 28 // debug controls verbose logging.
29 var debug, _ = strconv.ParseBool(os.Getenv("GOPACKAGESDEBUG"))
30 31 // A goTooOldError reports that the go command
32 // found by exec.LookPath is too old to use the new go list behavior.
33 type goTooOldError struct {
34 error
35 }
36 37 // responseDeduper wraps a DriverResponse, deduplicating its contents.
38 type responseDeduper struct {
39 seenRoots map[string]bool
40 seenPackages map[string]*Package
41 dr *DriverResponse
42 }
43 44 func newDeduper() *responseDeduper {
45 return &responseDeduper{
46 dr: &DriverResponse{},
47 seenRoots: map[string]bool{},
48 seenPackages: map[string]*Package{},
49 }
50 }
51 52 // addAll fills in r with a DriverResponse.
53 func (r *responseDeduper) addAll(dr *DriverResponse) {
54 for _, pkg := range dr.Packages {
55 r.addPackage(pkg)
56 }
57 for _, root := range dr.Roots {
58 r.addRoot(root)
59 }
60 r.dr.GoVersion = dr.GoVersion
61 }
62 63 func (r *responseDeduper) addPackage(p *Package) {
64 if r.seenPackages[p.ID] != nil {
65 return
66 }
67 r.seenPackages[p.ID] = p
68 r.dr.Packages = append(r.dr.Packages, p)
69 }
70 71 func (r *responseDeduper) addRoot(id string) {
72 if r.seenRoots[id] {
73 return
74 }
75 r.seenRoots[id] = true
76 r.dr.Roots = append(r.dr.Roots, id)
77 }
78 79 type golistState struct {
80 cfg *Config
81 ctx context.Context
82 83 runner *gocommand.Runner
84 85 // overlay is the JSON file that encodes the Config.Overlay
86 // mapping, used by 'go list -overlay=...'.
87 overlay string
88 89 envOnce sync.Once
90 goEnvError error
91 goEnv map[string]string
92 93 rootsOnce sync.Once
94 rootDirsError error
95 rootDirs map[string]string
96 97 goVersionOnce sync.Once
98 goVersionError error
99 goVersion int // The X in Go 1.X.
100 101 // vendorDirs caches the (non)existence of vendor directories.
102 vendorDirs map[string]bool
103 }
104 105 // getEnv returns Go environment variables. Only specific variables are
106 // populated -- computing all of them is slow.
107 func (state *golistState) getEnv() (map[string]string, error) {
108 state.envOnce.Do(func() {
109 var b *bytes.Buffer
110 b, state.goEnvError = state.invokeGo("env", "-json", "GOMOD", "GOPATH")
111 if state.goEnvError != nil {
112 return
113 }
114 115 state.goEnv = make(map[string]string)
116 decoder := json.NewDecoder(b)
117 if state.goEnvError = decoder.Decode(&state.goEnv); state.goEnvError != nil {
118 return
119 }
120 })
121 return state.goEnv, state.goEnvError
122 }
123 124 // mustGetEnv is a convenience function that can be used if getEnv has already succeeded.
125 func (state *golistState) mustGetEnv() map[string]string {
126 env, err := state.getEnv()
127 if err != nil {
128 panic(fmt.Sprintf("mustGetEnv: %v", err))
129 }
130 return env
131 }
132 133 // goListDriver uses the go list command to interpret the patterns and produce
134 // the build system package structure.
135 // See driver for more details.
136 //
137 // overlay is the JSON file that encodes the cfg.Overlay
138 // mapping, used by 'go list -overlay=...'
139 func goListDriver(cfg *Config, runner *gocommand.Runner, overlay string, patterns []string) (_ *DriverResponse, err error) {
140 // Make sure that any asynchronous go commands are killed when we return.
141 parentCtx := cfg.Context
142 if parentCtx == nil {
143 parentCtx = context.Background()
144 }
145 ctx, cancel := context.WithCancel(parentCtx)
146 defer cancel()
147 148 response := newDeduper()
149 150 state := &golistState{
151 cfg: cfg,
152 ctx: ctx,
153 vendorDirs: map[string]bool{},
154 overlay: overlay,
155 runner: runner,
156 }
157 158 // Fill in response.Sizes asynchronously if necessary.
159 if cfg.Mode&NeedTypesSizes != 0 || cfg.Mode&(NeedTypes|NeedTypesInfo) != 0 {
160 errCh := make(chan error)
161 go func() {
162 compiler, arch, err := getSizesForArgs(ctx, state.cfgInvocation(), runner)
163 response.dr.Compiler = compiler
164 response.dr.Arch = arch
165 errCh <- err
166 }()
167 defer func() {
168 if sizesErr := <-errCh; sizesErr != nil {
169 err = sizesErr
170 }
171 }()
172 }
173 174 // Determine files requested in contains patterns
175 var containFiles []string
176 restPatterns := make([]string, 0, len(patterns))
177 // Extract file= and other [querytype]= patterns. Report an error if querytype
178 // doesn't exist.
179 extractQueries:
180 for _, pattern := range patterns {
181 eqidx := strings.Index(pattern, "=")
182 if eqidx < 0 {
183 restPatterns = append(restPatterns, pattern)
184 } else {
185 query, value := pattern[:eqidx], pattern[eqidx+len("="):]
186 switch query {
187 case "file":
188 containFiles = append(containFiles, value)
189 case "pattern":
190 restPatterns = append(restPatterns, value)
191 case "": // not a reserved query
192 restPatterns = append(restPatterns, pattern)
193 default:
194 for _, rune := range query {
195 if rune < 'a' || rune > 'z' { // not a reserved query
196 restPatterns = append(restPatterns, pattern)
197 continue extractQueries
198 }
199 }
200 // Reject all other patterns containing "="
201 return nil, fmt.Errorf("invalid query type %q in query pattern %q", query, pattern)
202 }
203 }
204 }
205 206 // See if we have any patterns to pass through to go list. Zero initial
207 // patterns also requires a go list call, since it's the equivalent of
208 // ".".
209 if len(restPatterns) > 0 || len(patterns) == 0 {
210 dr, err := state.createDriverResponse(restPatterns...)
211 if err != nil {
212 return nil, err
213 }
214 response.addAll(dr)
215 }
216 217 if len(containFiles) != 0 {
218 if err := state.runContainsQueries(response, containFiles); err != nil {
219 return nil, err
220 }
221 }
222 223 // (We may yet return an error due to defer.)
224 return response.dr, nil
225 }
226 227 // abs returns an absolute representation of path, based on cfg.Dir.
228 func (cfg *Config) abs(path string) (string, error) {
229 if filepath.IsAbs(path) {
230 return path, nil
231 }
232 // In case cfg.Dir is relative, pass it to filepath.Abs.
233 return filepath.Abs(filepath.Join(cfg.Dir, path))
234 }
235 236 func (state *golistState) runContainsQueries(response *responseDeduper, queries []string) error {
237 for _, query := range queries {
238 // TODO(matloob): Do only one query per directory.
239 fdir := filepath.Dir(query)
240 // Pass absolute path of directory to go list so that it knows to treat it as a directory,
241 // not a package path.
242 pattern, err := state.cfg.abs(fdir)
243 if err != nil {
244 return fmt.Errorf("could not determine absolute path of file= query path %q: %v", query, err)
245 }
246 dirResponse, err := state.createDriverResponse(pattern)
247 248 // If there was an error loading the package, or no packages are returned,
249 // or the package is returned with errors, try to load the file as an
250 // ad-hoc package.
251 // Usually the error will appear in a returned package, but may not if we're
252 // in module mode and the ad-hoc is located outside a module.
253 if err != nil || len(dirResponse.Packages) == 0 || len(dirResponse.Packages) == 1 && len(dirResponse.Packages[0].GoFiles) == 0 &&
254 len(dirResponse.Packages[0].Errors) == 1 {
255 var queryErr error
256 if dirResponse, queryErr = state.adhocPackage(pattern, query); queryErr != nil {
257 return err // return the original error
258 }
259 }
260 isRoot := make(map[string]bool, len(dirResponse.Roots))
261 for _, root := range dirResponse.Roots {
262 isRoot[root] = true
263 }
264 for _, pkg := range dirResponse.Packages {
265 // Add any new packages to the main set
266 // We don't bother to filter packages that will be dropped by the changes of roots,
267 // that will happen anyway during graph construction outside this function.
268 // Over-reporting packages is not a problem.
269 response.addPackage(pkg)
270 // if the package was not a root one, it cannot have the file
271 if !isRoot[pkg.ID] {
272 continue
273 }
274 for _, pkgFile := range pkg.GoFiles {
275 if filepath.Base(query) == filepath.Base(pkgFile) {
276 response.addRoot(pkg.ID)
277 break
278 }
279 }
280 }
281 }
282 return nil
283 }
284 285 // adhocPackage attempts to load or construct an ad-hoc package for a given
286 // query, if the original call to the driver produced inadequate results.
287 func (state *golistState) adhocPackage(pattern, query string) (*DriverResponse, error) {
288 response, err := state.createDriverResponse(query)
289 if err != nil {
290 return nil, err
291 }
292 // If we get nothing back from `go list`,
293 // try to make this file into its own ad-hoc package.
294 // TODO(rstambler): Should this check against the original response?
295 if len(response.Packages) == 0 {
296 response.Packages = append(response.Packages, &Package{
297 ID: "command-line-arguments",
298 PkgPath: query,
299 GoFiles: []string{query},
300 CompiledGoFiles: []string{query},
301 Imports: make(map[string]*Package),
302 })
303 response.Roots = append(response.Roots, "command-line-arguments")
304 }
305 // Handle special cases.
306 if len(response.Packages) == 1 {
307 // golang/go#33482: If this is a file= query for ad-hoc packages where
308 // the file only exists on an overlay, and exists outside of a module,
309 // add the file to the package and remove the errors.
310 if response.Packages[0].ID == "command-line-arguments" ||
311 filepath.ToSlash(response.Packages[0].PkgPath) == filepath.ToSlash(query) {
312 if len(response.Packages[0].GoFiles) == 0 {
313 filename := filepath.Join(pattern, filepath.Base(query)) // avoid recomputing abspath
314 // TODO(matloob): check if the file is outside of a root dir?
315 for path := range state.cfg.Overlay {
316 if path == filename {
317 response.Packages[0].Errors = nil
318 response.Packages[0].GoFiles = []string{path}
319 response.Packages[0].CompiledGoFiles = []string{path}
320 }
321 }
322 }
323 }
324 }
325 return response, nil
326 }
327 328 // Fields must match go list;
329 // see $GOROOT/src/cmd/go/internal/load/pkg.go.
330 type jsonPackage struct {
331 ImportPath string
332 Dir string
333 Name string
334 Target string
335 Export string
336 GoFiles []string
337 CompiledGoFiles []string
338 IgnoredGoFiles []string
339 IgnoredOtherFiles []string
340 EmbedPatterns []string
341 EmbedFiles []string
342 CFiles []string
343 CgoFiles []string
344 CXXFiles []string
345 MFiles []string
346 HFiles []string
347 FFiles []string
348 SFiles []string
349 SwigFiles []string
350 SwigCXXFiles []string
351 SysoFiles []string
352 Imports []string
353 ImportMap map[string]string
354 Deps []string
355 Module *Module
356 TestGoFiles []string
357 TestImports []string
358 XTestGoFiles []string
359 XTestImports []string
360 ForTest string // q in a "p [q.test]" package, else ""
361 DepOnly bool
362 363 Error *packagesinternal.PackageError
364 DepsErrors []*packagesinternal.PackageError
365 }
366 367 func otherFiles(p *jsonPackage) [][]string {
368 return [][]string{p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.SwigFiles, p.SwigCXXFiles, p.SysoFiles}
369 }
370 371 // createDriverResponse uses the "go list" command to expand the pattern
372 // words and return a response for the specified packages.
373 func (state *golistState) createDriverResponse(words ...string) (*DriverResponse, error) {
374 // go list uses the following identifiers in ImportPath and Imports:
375 //
376 // "p" -- importable package or main (command)
377 // "q.test" -- q's test executable
378 // "p [q.test]" -- variant of p as built for q's test executable
379 // "q_test [q.test]" -- q's external test package
380 //
381 // The packages p that are built differently for a test q.test
382 // are q itself, plus any helpers used by the external test q_test,
383 // typically including "testing" and all its dependencies.
384 385 // Run "go list" for complete
386 // information on the specified packages.
387 goVersion, err := state.getGoVersion()
388 if err != nil {
389 return nil, err
390 }
391 buf, err := state.invokeGo("list", golistargs(state.cfg, words, goVersion)...)
392 if err != nil {
393 return nil, err
394 }
395 396 seen := make(map[string]*jsonPackage)
397 pkgs := make(map[string]*Package)
398 additionalErrors := make(map[string][]Error)
399 // Decode the JSON and convert it to Package form.
400 response := &DriverResponse{
401 GoVersion: goVersion,
402 }
403 for dec := json.NewDecoder(buf); dec.More(); {
404 p := new(jsonPackage)
405 if err := dec.Decode(p); err != nil {
406 return nil, fmt.Errorf("JSON decoding failed: %v", err)
407 }
408 409 if p.ImportPath == "" {
410 // The documentation for go list says that “[e]rroneous packages will have
411 // a non-empty ImportPath”. If for some reason it comes back empty, we
412 // prefer to error out rather than silently discarding data or handing
413 // back a package without any way to refer to it.
414 if p.Error != nil {
415 return nil, Error{
416 Pos: p.Error.Pos,
417 Msg: p.Error.Err,
418 }
419 }
420 return nil, fmt.Errorf("package missing import path: %+v", p)
421 }
422 423 // Work around https://golang.org/issue/33157:
424 // go list -e, when given an absolute path, will find the package contained at
425 // that directory. But when no package exists there, it will return a fake package
426 // with an error and the ImportPath set to the absolute path provided to go list.
427 // Try to convert that absolute path to what its package path would be if it's
428 // contained in a known module or GOPATH entry. This will allow the package to be
429 // properly "reclaimed" when overlays are processed.
430 if filepath.IsAbs(p.ImportPath) && p.Error != nil {
431 pkgPath, ok, err := state.getPkgPath(p.ImportPath)
432 if err != nil {
433 return nil, err
434 }
435 if ok {
436 p.ImportPath = pkgPath
437 }
438 }
439 440 if old, found := seen[p.ImportPath]; found {
441 // If one version of the package has an error, and the other doesn't, assume
442 // that this is a case where go list is reporting a fake dependency variant
443 // of the imported package: When a package tries to invalidly import another
444 // package, go list emits a variant of the imported package (with the same
445 // import path, but with an error on it, and the package will have a
446 // DepError set on it). An example of when this can happen is for imports of
447 // main packages: main packages can not be imported, but they may be
448 // separately matched and listed by another pattern.
449 // See golang.org/issue/36188 for more details.
450 451 // The plan is that eventually, hopefully in Go 1.15, the error will be
452 // reported on the importing package rather than the duplicate "fake"
453 // version of the imported package. Once all supported versions of Go
454 // have the new behavior this logic can be deleted.
455 // TODO(matloob): delete the workaround logic once all supported versions of
456 // Go return the errors on the proper package.
457 458 // There should be exactly one version of a package that doesn't have an
459 // error.
460 if old.Error == nil && p.Error == nil {
461 if !reflect.DeepEqual(p, old) {
462 return nil, fmt.Errorf("internal error: go list gives conflicting information for package %v", p.ImportPath)
463 }
464 continue
465 }
466 467 // Determine if this package's error needs to be bubbled up.
468 // This is a hack, and we expect for go list to eventually set the error
469 // on the package.
470 if old.Error != nil {
471 var errkind string
472 if strings.Contains(old.Error.Err, "not an importable package") {
473 errkind = "not an importable package"
474 } else if strings.Contains(old.Error.Err, "use of internal package") && strings.Contains(old.Error.Err, "not allowed") {
475 errkind = "use of internal package not allowed"
476 }
477 if errkind != "" {
478 if len(old.Error.ImportStack) < 1 {
479 return nil, fmt.Errorf(`internal error: go list gave a %q error with empty import stack`, errkind)
480 }
481 importingPkg := old.Error.ImportStack[len(old.Error.ImportStack)-1]
482 if importingPkg == old.ImportPath {
483 // Using an older version of Go which put this package itself on top of import
484 // stack, instead of the importer. Look for importer in second from top
485 // position.
486 if len(old.Error.ImportStack) < 2 {
487 return nil, fmt.Errorf(`internal error: go list gave a %q error with an import stack without importing package`, errkind)
488 }
489 importingPkg = old.Error.ImportStack[len(old.Error.ImportStack)-2]
490 }
491 additionalErrors[importingPkg] = append(additionalErrors[importingPkg], Error{
492 Pos: old.Error.Pos,
493 Msg: old.Error.Err,
494 Kind: ListError,
495 })
496 }
497 }
498 499 // Make sure that if there's a version of the package without an error,
500 // that's the one reported to the user.
501 if old.Error == nil {
502 continue
503 }
504 505 // This package will replace the old one at the end of the loop.
506 }
507 seen[p.ImportPath] = p
508 509 pkg := &Package{
510 Name: p.Name,
511 ID: p.ImportPath,
512 Dir: p.Dir,
513 Target: p.Target,
514 GoFiles: absJoin(p.Dir, p.GoFiles, p.CgoFiles),
515 CompiledGoFiles: absJoin(p.Dir, p.CompiledGoFiles),
516 OtherFiles: absJoin(p.Dir, otherFiles(p)...),
517 EmbedFiles: absJoin(p.Dir, p.EmbedFiles),
518 EmbedPatterns: absJoin(p.Dir, p.EmbedPatterns),
519 IgnoredFiles: absJoin(p.Dir, p.IgnoredGoFiles, p.IgnoredOtherFiles),
520 ForTest: p.ForTest,
521 depsErrors: p.DepsErrors,
522 Module: p.Module,
523 }
524 525 if (state.cfg.Mode&typecheckCgo) != 0 && len(p.CgoFiles) != 0 {
526 if len(p.CompiledGoFiles) > len(p.GoFiles) {
527 // We need the cgo definitions, which are in the first
528 // CompiledGoFile after the non-cgo ones. This is a hack but there
529 // isn't currently a better way to find it. We also need the pure
530 // Go files and unprocessed cgo files, all of which are already
531 // in pkg.GoFiles.
532 cgoTypes := p.CompiledGoFiles[len(p.GoFiles)]
533 pkg.CompiledGoFiles = append([]string{cgoTypes}, pkg.GoFiles...)
534 } else {
535 // golang/go#38990: go list silently fails to do cgo processing
536 pkg.CompiledGoFiles = nil
537 pkg.Errors = append(pkg.Errors, Error{
538 Msg: "go list failed to return CompiledGoFiles. This may indicate failure to perform cgo processing; try building at the command line. See https://golang.org/issue/38990.",
539 Kind: ListError,
540 })
541 }
542 }
543 544 // Work around https://golang.org/issue/28749:
545 // cmd/go puts assembly, C, and C++ files in CompiledGoFiles.
546 // Remove files from CompiledGoFiles that are non-go files
547 // (or are not files that look like they are from the cache).
548 if len(pkg.CompiledGoFiles) > 0 {
549 out := pkg.CompiledGoFiles[:0]
550 for _, f := range pkg.CompiledGoFiles {
551 if ext := filepath.Ext(f); ext != ".go" && ext != "" { // ext == "" means the file is from the cache, so probably cgo-processed file
552 continue
553 }
554 out = append(out, f)
555 }
556 pkg.CompiledGoFiles = out
557 }
558 559 // Extract the PkgPath from the package's ID.
560 if i := strings.IndexByte(pkg.ID, ' '); i >= 0 {
561 pkg.PkgPath = pkg.ID[:i]
562 } else {
563 pkg.PkgPath = pkg.ID
564 }
565 566 if pkg.PkgPath == "unsafe" {
567 pkg.CompiledGoFiles = nil // ignore fake unsafe.go file (#59929)
568 } else if len(pkg.CompiledGoFiles) == 0 {
569 // Work around for pre-go.1.11 versions of go list.
570 // TODO(matloob): they should be handled by the fallback.
571 // Can we delete this?
572 pkg.CompiledGoFiles = pkg.GoFiles
573 }
574 575 // Assume go list emits only absolute paths for Dir.
576 if p.Dir != "" && !filepath.IsAbs(p.Dir) {
577 log.Fatalf("internal error: go list returned non-absolute Package.Dir: %s", p.Dir)
578 }
579 580 if p.Export != "" && !filepath.IsAbs(p.Export) {
581 pkg.ExportFile = filepath.Join(p.Dir, p.Export)
582 } else {
583 pkg.ExportFile = p.Export
584 }
585 586 // imports
587 //
588 // Imports contains the IDs of all imported packages.
589 // ImportsMap records (path, ID) only where they differ.
590 ids := make(map[string]bool)
591 for _, id := range p.Imports {
592 ids[id] = true
593 }
594 pkg.Imports = make(map[string]*Package)
595 for path, id := range p.ImportMap {
596 pkg.Imports[path] = &Package{ID: id} // non-identity import
597 delete(ids, id)
598 }
599 for id := range ids {
600 if id == "C" {
601 continue
602 }
603 604 pkg.Imports[id] = &Package{ID: id} // identity import
605 }
606 if !p.DepOnly {
607 response.Roots = append(response.Roots, pkg.ID)
608 }
609 610 // Temporary work-around for golang/go#39986. Parse filenames out of
611 // error messages. This happens if there are unrecoverable syntax
612 // errors in the source, so we can't match on a specific error message.
613 //
614 // TODO(rfindley): remove this heuristic, in favor of considering
615 // InvalidGoFiles from the list driver.
616 if err := p.Error; err != nil && state.shouldAddFilenameFromError(p) {
617 addFilenameFromPos := func(pos string) bool {
618 split := strings.Split(pos, ":")
619 if len(split) < 1 {
620 return false
621 }
622 filename := strings.TrimSpace(split[0])
623 if filename == "" {
624 return false
625 }
626 if !filepath.IsAbs(filename) {
627 filename = filepath.Join(state.cfg.Dir, filename)
628 }
629 info, _ := os.Stat(filename)
630 if info == nil {
631 return false
632 }
633 pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, filename)
634 pkg.GoFiles = append(pkg.GoFiles, filename)
635 return true
636 }
637 found := addFilenameFromPos(err.Pos)
638 // In some cases, go list only reports the error position in the
639 // error text, not the error position. One such case is when the
640 // file's package name is a keyword (see golang.org/issue/39763).
641 if !found {
642 addFilenameFromPos(err.Err)
643 }
644 }
645 646 if p.Error != nil {
647 msg := strings.TrimSpace(p.Error.Err) // Trim to work around golang.org/issue/32363.
648 // Address golang.org/issue/35964 by appending import stack to error message.
649 if msg == "import cycle not allowed" && len(p.Error.ImportStack) != 0 {
650 msg += fmt.Sprintf(": import stack: %v", p.Error.ImportStack)
651 }
652 pkg.Errors = append(pkg.Errors, Error{
653 Pos: p.Error.Pos,
654 Msg: msg,
655 Kind: ListError,
656 })
657 }
658 659 pkgs[pkg.ID] = pkg
660 }
661 662 for id, errs := range additionalErrors {
663 if p, ok := pkgs[id]; ok {
664 p.Errors = append(p.Errors, errs...)
665 }
666 }
667 for _, pkg := range pkgs {
668 response.Packages = append(response.Packages, pkg)
669 }
670 sort.Slice(response.Packages, func(i, j int) bool { return response.Packages[i].ID < response.Packages[j].ID })
671 672 return response, nil
673 }
674 675 func (state *golistState) shouldAddFilenameFromError(p *jsonPackage) bool {
676 if len(p.GoFiles) > 0 || len(p.CompiledGoFiles) > 0 {
677 return false
678 }
679 680 goV, err := state.getGoVersion()
681 if err != nil {
682 return false
683 }
684 685 // On Go 1.14 and earlier, only add filenames from errors if the import stack is empty.
686 // The import stack behaves differently for these versions than newer Go versions.
687 if goV < 15 {
688 return len(p.Error.ImportStack) == 0
689 }
690 691 // On Go 1.15 and later, only parse filenames out of error if there's no import stack,
692 // or the current package is at the top of the import stack. This is not guaranteed
693 // to work perfectly, but should avoid some cases where files in errors don't belong to this
694 // package.
695 return len(p.Error.ImportStack) == 0 || p.Error.ImportStack[len(p.Error.ImportStack)-1] == p.ImportPath
696 }
697 698 // getGoVersion returns the effective minor version of the go command.
699 func (state *golistState) getGoVersion() (int, error) {
700 state.goVersionOnce.Do(func() {
701 state.goVersion, state.goVersionError = gocommand.GoVersion(state.ctx, state.cfgInvocation(), state.runner)
702 })
703 return state.goVersion, state.goVersionError
704 }
705 706 // getPkgPath finds the package path of a directory if it's relative to a root
707 // directory.
708 func (state *golistState) getPkgPath(dir string) (string, bool, error) {
709 if !filepath.IsAbs(dir) {
710 panic("non-absolute dir passed to getPkgPath")
711 }
712 roots, err := state.determineRootDirs()
713 if err != nil {
714 return "", false, err
715 }
716 717 for rdir, rpath := range roots {
718 // Make sure that the directory is in the module,
719 // to avoid creating a path relative to another module.
720 if !strings.HasPrefix(dir, rdir) {
721 continue
722 }
723 // TODO(matloob): This doesn't properly handle symlinks.
724 r, err := filepath.Rel(rdir, dir)
725 if err != nil {
726 continue
727 }
728 if rpath != "" {
729 // We choose only one root even though the directory even it can belong in multiple modules
730 // or GOPATH entries. This is okay because we only need to work with absolute dirs when a
731 // file is missing from disk, for instance when gopls calls go/packages in an overlay.
732 // Once the file is saved, gopls, or the next invocation of the tool will get the correct
733 // result straight from golist.
734 // TODO(matloob): Implement module tiebreaking?
735 return path.Join(rpath, filepath.ToSlash(r)), true, nil
736 }
737 return filepath.ToSlash(r), true, nil
738 }
739 return "", false, nil
740 }
741 742 // absJoin absolutizes and flattens the lists of files.
743 func absJoin(dir string, fileses ...[]string) (res []string) {
744 for _, files := range fileses {
745 for _, file := range files {
746 if !filepath.IsAbs(file) {
747 file = filepath.Join(dir, file)
748 }
749 res = append(res, file)
750 }
751 }
752 return res
753 }
754 755 func jsonFlag(cfg *Config, goVersion int) string {
756 if goVersion < 19 {
757 return "-json"
758 }
759 var fields []string
760 added := make(map[string]bool)
761 addFields := func(fs ...string) {
762 for _, f := range fs {
763 if !added[f] {
764 added[f] = true
765 fields = append(fields, f)
766 }
767 }
768 }
769 addFields("Name", "ImportPath", "Error") // These fields are always needed
770 if cfg.Mode&NeedFiles != 0 || cfg.Mode&(NeedTypes|NeedTypesInfo) != 0 {
771 addFields("Dir", "GoFiles", "IgnoredGoFiles", "IgnoredOtherFiles", "CFiles",
772 "CgoFiles", "CXXFiles", "MFiles", "HFiles", "FFiles", "SFiles",
773 "SwigFiles", "SwigCXXFiles", "SysoFiles")
774 if cfg.Tests {
775 addFields("TestGoFiles", "XTestGoFiles")
776 }
777 }
778 if cfg.Mode&(NeedTypes|NeedTypesInfo) != 0 {
779 // CompiledGoFiles seems to be required for the test case TestCgoNoSyntax,
780 // even when -compiled isn't passed in.
781 // TODO(#52435): Should we make the test ask for -compiled, or automatically
782 // request CompiledGoFiles in certain circumstances?
783 addFields("Dir", "CompiledGoFiles")
784 }
785 if cfg.Mode&NeedCompiledGoFiles != 0 {
786 addFields("Dir", "CompiledGoFiles", "Export")
787 }
788 if cfg.Mode&NeedImports != 0 {
789 // When imports are requested, DepOnly is used to distinguish between packages
790 // explicitly requested and transitive imports of those packages.
791 addFields("DepOnly", "Imports", "ImportMap")
792 if cfg.Tests {
793 addFields("TestImports", "XTestImports")
794 }
795 }
796 if cfg.Mode&NeedDeps != 0 {
797 addFields("DepOnly")
798 }
799 if usesExportData(cfg) {
800 // Request Dir in the unlikely case Export is not absolute.
801 addFields("Dir", "Export")
802 }
803 if cfg.Mode&NeedForTest != 0 {
804 addFields("ForTest")
805 }
806 if cfg.Mode&needInternalDepsErrors != 0 {
807 addFields("DepsErrors")
808 }
809 if cfg.Mode&NeedModule != 0 {
810 addFields("Module")
811 }
812 if cfg.Mode&NeedEmbedFiles != 0 {
813 addFields("EmbedFiles")
814 }
815 if cfg.Mode&NeedEmbedPatterns != 0 {
816 addFields("EmbedPatterns")
817 }
818 if cfg.Mode&NeedTarget != 0 {
819 addFields("Target")
820 }
821 return "-json=" + strings.Join(fields, ",")
822 }
823 824 func golistargs(cfg *Config, words []string, goVersion int) []string {
825 const findFlags = NeedImports | NeedTypes | NeedSyntax | NeedTypesInfo
826 fullargs := []string{
827 "-e", jsonFlag(cfg, goVersion),
828 fmt.Sprintf("-compiled=%t", cfg.Mode&(NeedCompiledGoFiles|NeedSyntax|NeedTypes|NeedTypesInfo|NeedTypesSizes) != 0),
829 fmt.Sprintf("-test=%t", cfg.Tests),
830 fmt.Sprintf("-export=%t", usesExportData(cfg)),
831 fmt.Sprintf("-deps=%t", cfg.Mode&NeedImports != 0),
832 // go list doesn't let you pass -test and -find together,
833 // probably because you'd just get the TestMain.
834 fmt.Sprintf("-find=%t", !cfg.Tests && cfg.Mode&findFlags == 0 && !usesExportData(cfg)),
835 }
836 837 // golang/go#60456: with go1.21 and later, go list serves pgo variants, which
838 // can be costly to compute and may result in redundant processing for the
839 // caller. Disable these variants. If someone wants to add e.g. a NeedPGO
840 // mode flag, that should be a separate proposal.
841 if goVersion >= 21 {
842 fullargs = append(fullargs, "-pgo=off")
843 }
844 845 fullargs = append(fullargs, cfg.BuildFlags...)
846 fullargs = append(fullargs, "--")
847 fullargs = append(fullargs, words...)
848 return fullargs
849 }
850 851 // cfgInvocation returns an Invocation that reflects cfg's settings.
852 func (state *golistState) cfgInvocation() gocommand.Invocation {
853 cfg := state.cfg
854 return gocommand.Invocation{
855 BuildFlags: cfg.BuildFlags,
856 CleanEnv: cfg.Env != nil,
857 Env: cfg.Env,
858 Logf: cfg.Logf,
859 WorkingDir: cfg.Dir,
860 Overlay: state.overlay,
861 }
862 }
863 864 // invokeGo returns the stdout of a go command invocation.
865 func (state *golistState) invokeGo(verb string, args ...string) (*bytes.Buffer, error) {
866 cfg := state.cfg
867 868 inv := state.cfgInvocation()
869 inv.Verb = verb
870 inv.Args = args
871 872 stdout, stderr, friendlyErr, err := state.runner.RunRaw(cfg.Context, inv)
873 if err != nil {
874 // Check for 'go' executable not being found.
875 if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound {
876 return nil, fmt.Errorf("'go list' driver requires 'go', but %s", exec.ErrNotFound)
877 }
878 879 exitErr, ok := err.(*exec.ExitError)
880 if !ok {
881 // Catastrophic error:
882 // - context cancellation
883 return nil, fmt.Errorf("couldn't run 'go': %w", err)
884 }
885 886 // Old go version?
887 if strings.Contains(stderr.String(), "flag provided but not defined") {
888 return nil, goTooOldError{fmt.Errorf("unsupported version of go: %s: %s", exitErr, stderr)}
889 }
890 891 // Related to #24854
892 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "unexpected directory layout") {
893 return nil, friendlyErr
894 }
895 896 // Return an error if 'go list' failed due to missing tools in
897 // $GOROOT/pkg/tool/$GOOS_$GOARCH (#69606).
898 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), `go: no such tool`) {
899 return nil, friendlyErr
900 }
901 902 // Is there an error running the C compiler in cgo? This will be reported in the "Error" field
903 // and should be suppressed by go list -e.
904 //
905 // This condition is not perfect yet because the error message can include other error messages than runtime/cgo.
906 isPkgPathRune := func(r rune) bool {
907 // From https://golang.org/ref/spec#Import_declarations:
908 // Implementation restriction: A compiler may restrict ImportPaths to non-empty strings
909 // using only characters belonging to Unicode's L, M, N, P, and S general categories
910 // (the Graphic characters without spaces) and may also exclude the
911 // characters !"#$%&'()*,:;<=>?[\]^`{|} and the Unicode replacement character U+FFFD.
912 return unicode.IsOneOf([]*unicode.RangeTable{unicode.L, unicode.M, unicode.N, unicode.P, unicode.S}, r) &&
913 !strings.ContainsRune("!\"#$%&'()*,:;<=>?[\\]^`{|}\uFFFD", r)
914 }
915 // golang/go#36770: Handle case where cmd/go prints module download messages before the error.
916 msg := stderr.String()
917 for strings.HasPrefix(msg, "go: downloading") {
918 msg = msg[strings.IndexRune(msg, '\n')+1:]
919 }
920 if len(stderr.String()) > 0 && strings.HasPrefix(stderr.String(), "# ") {
921 msg := msg[len("# "):]
922 if strings.HasPrefix(strings.TrimLeftFunc(msg, isPkgPathRune), "\n") {
923 return stdout, nil
924 }
925 // Treat pkg-config errors as a special case (golang.org/issue/36770).
926 if strings.HasPrefix(msg, "pkg-config") {
927 return stdout, nil
928 }
929 }
930 931 // This error only appears in stderr. See golang.org/cl/166398 for a fix in go list to show
932 // the error in the Err section of stdout in case -e option is provided.
933 // This fix is provided for backwards compatibility.
934 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "named files must be .go files") {
935 output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
936 strings.Trim(stderr.String(), "\n"))
937 return bytes.NewBufferString(output), nil
938 }
939 940 // Similar to the previous error, but currently lacks a fix in Go.
941 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "named files must all be in one directory") {
942 output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
943 strings.Trim(stderr.String(), "\n"))
944 return bytes.NewBufferString(output), nil
945 }
946 947 // Backwards compatibility for Go 1.11 because 1.12 and 1.13 put the directory in the ImportPath.
948 // If the package doesn't exist, put the absolute path of the directory into the error message,
949 // as Go 1.13 list does.
950 const noSuchDirectory = "no such directory"
951 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), noSuchDirectory) {
952 errstr := stderr.String()
953 abspath := strings.TrimSpace(errstr[strings.Index(errstr, noSuchDirectory)+len(noSuchDirectory):])
954 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
955 abspath, strings.Trim(stderr.String(), "\n"))
956 return bytes.NewBufferString(output), nil
957 }
958 959 // Workaround for #29280: go list -e has incorrect behavior when an ad-hoc package doesn't exist.
960 // Note that the error message we look for in this case is different that the one looked for above.
961 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no such file or directory") {
962 output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
963 strings.Trim(stderr.String(), "\n"))
964 return bytes.NewBufferString(output), nil
965 }
966 967 // Workaround for #34273. go list -e with GO111MODULE=on has incorrect behavior when listing a
968 // directory outside any module.
969 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside available modules") {
970 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
971 // TODO(matloob): command-line-arguments isn't correct here.
972 "command-line-arguments", strings.Trim(stderr.String(), "\n"))
973 return bytes.NewBufferString(output), nil
974 }
975 976 // Another variation of the previous error
977 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside module root") {
978 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
979 // TODO(matloob): command-line-arguments isn't correct here.
980 "command-line-arguments", strings.Trim(stderr.String(), "\n"))
981 return bytes.NewBufferString(output), nil
982 }
983 984 // Workaround for an instance of golang.org/issue/26755: go list -e will return a non-zero exit
985 // status if there's a dependency on a package that doesn't exist. But it should return
986 // a zero exit status and set an error on that package.
987 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no Go files in") {
988 // Don't clobber stdout if `go list` actually returned something.
989 if len(stdout.String()) > 0 {
990 return stdout, nil
991 }
992 // try to extract package name from string
993 stderrStr := stderr.String()
994 var importPath string
995 colon := strings.Index(stderrStr, ":")
996 if colon > 0 && strings.HasPrefix(stderrStr, "go build ") {
997 importPath = stderrStr[len("go build "):colon]
998 }
999 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
1000 importPath, strings.Trim(stderrStr, "\n"))
1001 return bytes.NewBufferString(output), nil
1002 }
1003 1004 // Export mode entails a build.
1005 // If that build fails, errors appear on stderr
1006 // (despite the -e flag) and the Export field is blank.
1007 // Do not fail in that case.
1008 // The same is true if an ad-hoc package given to go list doesn't exist.
1009 // TODO(matloob): Remove these once we can depend on go list to exit with a zero status with -e even when
1010 // packages don't exist or a build fails.
1011 if !usesExportData(cfg) && !containsGoFile(args) {
1012 return nil, friendlyErr
1013 }
1014 }
1015 return stdout, nil
1016 }
1017 1018 func containsGoFile(s []string) bool {
1019 for _, f := range s {
1020 if strings.HasSuffix(f, ".go") {
1021 return true
1022 }
1023 }
1024 return false
1025 }
1026 1027 func cmdDebugStr(cmd *exec.Cmd) string {
1028 env := make(map[string]string)
1029 for _, kv := range cmd.Env {
1030 split := strings.SplitN(kv, "=", 2)
1031 k, v := split[0], split[1]
1032 env[k] = v
1033 }
1034 1035 var args []string
1036 for _, arg := range cmd.Args {
1037 quoted := strconv.Quote(arg)
1038 if quoted[1:len(quoted)-1] != arg || strings.Contains(arg, " ") {
1039 args = append(args, quoted)
1040 } else {
1041 args = append(args, arg)
1042 }
1043 }
1044 return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v GOPROXY=%v PWD=%v %v", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["GOPROXY"], env["PWD"], strings.Join(args, " "))
1045 }
1046 1047 // getSizesForArgs queries 'go list' for the appropriate
1048 // Compiler and GOARCH arguments to pass to [types.SizesFor].
1049 func getSizesForArgs(ctx context.Context, inv gocommand.Invocation, gocmdRunner *gocommand.Runner) (string, string, error) {
1050 inv.Verb = "list"
1051 inv.Args = []string{"-f", "{{context.GOARCH}} {{context.Compiler}}", "--", "unsafe"}
1052 stdout, stderr, friendlyErr, rawErr := gocmdRunner.RunRaw(ctx, inv)
1053 var goarch, compiler string
1054 if rawErr != nil {
1055 rawErrMsg := rawErr.Error()
1056 if strings.Contains(rawErrMsg, "cannot find main module") ||
1057 strings.Contains(rawErrMsg, "go.mod file not found") {
1058 // User's running outside of a module.
1059 // All bets are off. Get GOARCH and guess compiler is gc.
1060 // TODO(matloob): Is this a problem in practice?
1061 inv.Verb = "env"
1062 inv.Args = []string{"GOARCH"}
1063 envout, enverr := gocmdRunner.Run(ctx, inv)
1064 if enverr != nil {
1065 return "", "", enverr
1066 }
1067 goarch = strings.TrimSpace(envout.String())
1068 compiler = "gc"
1069 } else if friendlyErr != nil {
1070 return "", "", friendlyErr
1071 } else {
1072 // This should be unreachable, but be defensive
1073 // in case RunRaw's error results are inconsistent.
1074 return "", "", rawErr
1075 }
1076 } else {
1077 fields := strings.Fields(stdout.String())
1078 if len(fields) < 2 {
1079 return "", "", fmt.Errorf("could not parse GOARCH and Go compiler in format \"<GOARCH> <compiler>\":\nstdout: <<%s>>\nstderr: <<%s>>",
1080 stdout.String(), stderr.String())
1081 }
1082 goarch = fields[0]
1083 compiler = fields[1]
1084 }
1085 return compiler, goarch, nil
1086 }
1087