1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4 5 package packages
6 7 // See doc.go for package documentation and implementation notes.
8 9 import (
10 "context"
11 "encoding/json"
12 "errors"
13 "fmt"
14 "go/ast"
15 "go/parser"
16 "go/scanner"
17 "go/token"
18 "go/types"
19 "log"
20 "os"
21 "path/filepath"
22 "runtime"
23 "strings"
24 "sync"
25 "sync/atomic"
26 "time"
27 28 "golang.org/x/sync/errgroup"
29 30 "golang.org/x/tools/go/gcexportdata"
31 "golang.org/x/tools/internal/gocommand"
32 "golang.org/x/tools/internal/packagesinternal"
33 "golang.org/x/tools/internal/typesinternal"
34 )
35 36 // A LoadMode controls the amount of detail to return when loading.
37 // The bits below can be combined to specify which fields should be
38 // filled in the result packages.
39 //
40 // The zero value is a special case, equivalent to combining
41 // the NeedName, NeedFiles, and NeedCompiledGoFiles bits.
42 //
43 // ID and Errors (if present) will always be filled.
44 // [Load] may return more information than requested.
45 //
46 // The Mode flag is a union of several bits named NeedName,
47 // NeedFiles, and so on, each of which determines whether
48 // a given field of Package (Name, Files, etc) should be
49 // populated.
50 //
51 // For convenience, we provide named constants for the most
52 // common combinations of Need flags:
53 //
54 // [LoadFiles] lists of files in each package
55 // [LoadImports] ... plus imports
56 // [LoadTypes] ... plus type information
57 // [LoadSyntax] ... plus type-annotated syntax
58 // [LoadAllSyntax] ... for all dependencies
59 //
60 // Unfortunately there are a number of open bugs related to
61 // interactions among the LoadMode bits:
62 // - https://go.dev/issue/56633
63 // - https://go.dev/issue/56677
64 // - https://go.dev/issue/58726
65 // - https://go.dev/issue/63517
66 type LoadMode int
67 68 const (
69 // NeedName adds Name and PkgPath.
70 NeedName LoadMode = 1 << iota
71 72 // NeedFiles adds Dir, GoFiles, OtherFiles, and IgnoredFiles
73 NeedFiles
74 75 // NeedCompiledGoFiles adds CompiledGoFiles.
76 NeedCompiledGoFiles
77 78 // NeedImports adds Imports. If NeedDeps is not set, the Imports field will contain
79 // "placeholder" Packages with only the ID set.
80 NeedImports
81 82 // NeedDeps adds the fields requested by the LoadMode in the packages in Imports.
83 NeedDeps
84 85 // NeedExportFile adds ExportFile.
86 NeedExportFile
87 88 // NeedTypes adds Types, Fset, and IllTyped.
89 NeedTypes
90 91 // NeedSyntax adds Syntax and Fset.
92 NeedSyntax
93 94 // NeedTypesInfo adds TypesInfo and Fset.
95 NeedTypesInfo
96 97 // NeedTypesSizes adds TypesSizes.
98 NeedTypesSizes
99 100 // needInternalDepsErrors adds the internal deps errors field for use by gopls.
101 needInternalDepsErrors
102 103 // NeedForTest adds ForTest.
104 //
105 // Tests must also be set on the context for this field to be populated.
106 NeedForTest
107 108 // typecheckCgo enables full support for type checking cgo. Requires Go 1.15+.
109 // Modifies CompiledGoFiles and Types, and has no effect on its own.
110 typecheckCgo
111 112 // NeedModule adds Module.
113 NeedModule
114 115 // NeedEmbedFiles adds EmbedFiles.
116 NeedEmbedFiles
117 118 // NeedEmbedPatterns adds EmbedPatterns.
119 NeedEmbedPatterns
120 121 // NeedTarget adds Target.
122 NeedTarget
123 124 // Be sure to update loadmode_string.go when adding new items!
125 )
126 127 const (
128 // LoadFiles loads the name and file names for the initial packages.
129 LoadFiles = NeedName | NeedFiles | NeedCompiledGoFiles
130 131 // LoadImports loads the name, file names, and import mapping for the initial packages.
132 LoadImports = LoadFiles | NeedImports
133 134 // LoadTypes loads exported type information for the initial packages.
135 LoadTypes = LoadImports | NeedTypes | NeedTypesSizes
136 137 // LoadSyntax loads typed syntax for the initial packages.
138 LoadSyntax = LoadTypes | NeedSyntax | NeedTypesInfo
139 140 // LoadAllSyntax loads typed syntax for the initial packages and all dependencies.
141 LoadAllSyntax = LoadSyntax | NeedDeps
142 143 // Deprecated: NeedExportsFile is a historical misspelling of NeedExportFile.
144 //
145 //go:fix inline
146 NeedExportsFile = NeedExportFile
147 )
148 149 // A Config specifies details about how packages should be loaded.
150 // The zero value is a valid configuration.
151 //
152 // Calls to [Load] do not modify this struct.
153 type Config struct {
154 // Mode controls the level of information returned for each package.
155 Mode LoadMode
156 157 // Context specifies the context for the load operation.
158 // Cancelling the context may cause [Load] to abort and
159 // return an error.
160 Context context.Context
161 162 // Logf is the logger for the config.
163 // If the user provides a logger, debug logging is enabled.
164 // If the GOPACKAGESDEBUG environment variable is set to true,
165 // but the logger is nil, default to log.Printf.
166 Logf func(format string, args ...any)
167 168 // Dir is the directory in which to run the build system's query tool
169 // that provides information about the packages.
170 // If Dir is empty, the tool is run in the current directory.
171 Dir string
172 173 // Env is the environment to use when invoking the build system's query tool.
174 // If Env is nil, the current environment is used.
175 // As in os/exec's Cmd, only the last value in the slice for
176 // each environment key is used. To specify the setting of only
177 // a few variables, append to the current environment, as in:
178 //
179 // opt.Env = append(os.Environ(), "GOOS=plan9", "GOARCH=386")
180 //
181 Env []string
182 183 // BuildFlags is a list of command-line flags to be passed through to
184 // the build system's query tool.
185 BuildFlags []string
186 187 // Fset provides source position information for syntax trees and types.
188 // If Fset is nil, Load will use a new fileset, but preserve Fset's value.
189 Fset *token.FileSet
190 191 // ParseFile is called to read and parse each file
192 // when preparing a package's type-checked syntax tree.
193 // It must be safe to call ParseFile simultaneously from multiple goroutines.
194 // If ParseFile is nil, the loader will uses parser.ParseFile.
195 //
196 // ParseFile should parse the source from src and use filename only for
197 // recording position information.
198 //
199 // An application may supply a custom implementation of ParseFile
200 // to change the effective file contents or the behavior of the parser,
201 // or to modify the syntax tree. For example, selectively eliminating
202 // unwanted function bodies can significantly accelerate type checking.
203 ParseFile func(fset *token.FileSet, filename string, src []byte) (*ast.File, error)
204 205 // If Tests is set, the loader includes not just the packages
206 // matching a particular pattern but also any related test packages,
207 // including test-only variants of the package and the test executable.
208 //
209 // For example, when using the go command, loading "fmt" with Tests=true
210 // returns four packages, with IDs "fmt" (the standard package),
211 // "fmt [fmt.test]" (the package as compiled for the test),
212 // "fmt_test" (the test functions from source files in package fmt_test),
213 // and "fmt.test" (the test binary).
214 //
215 // In build systems with explicit names for tests,
216 // setting Tests may have no effect.
217 Tests bool
218 219 // Overlay is a mapping from absolute file paths to file contents.
220 //
221 // For each map entry, [Load] uses the alternative file
222 // contents provided by the overlay mapping instead of reading
223 // from the file system. This mechanism can be used to enable
224 // editor-integrated tools to correctly analyze the contents
225 // of modified but unsaved buffers, for example.
226 //
227 // The overlay mapping is passed to the build system's driver
228 // (see "The driver protocol") so that it too can report
229 // consistent package metadata about unsaved files. However,
230 // drivers may vary in their level of support for overlays.
231 Overlay map[string][]byte
232 }
233 234 // Load loads and returns the Go packages named by the given patterns.
235 //
236 // The cfg parameter specifies loading options; nil behaves the same as an empty [Config].
237 //
238 // The [Config.Mode] field is a set of bits that determine what kinds
239 // of information should be computed and returned. Modes that require
240 // more information tend to be slower. See [LoadMode] for details
241 // and important caveats. Its zero value is equivalent to
242 // [NeedName] | [NeedFiles] | [NeedCompiledGoFiles].
243 //
244 // Each call to Load returns a new set of [Package] instances.
245 // The Packages and their Imports form a directed acyclic graph.
246 //
247 // If the [NeedTypes] mode flag was set, each call to Load uses a new
248 // [types.Importer], so [types.Object] and [types.Type] values from
249 // different calls to Load must not be mixed as they will have
250 // inconsistent notions of type identity.
251 //
252 // If any of the patterns was invalid as defined by the
253 // underlying build system, Load returns an error.
254 // It may return an empty list of packages without an error,
255 // for instance for an empty expansion of a valid wildcard.
256 // Errors associated with a particular package are recorded in the
257 // corresponding Package's Errors list, and do not cause Load to
258 // return an error. Clients may need to handle such errors before
259 // proceeding with further analysis. The [PrintErrors] function is
260 // provided for convenient display of all errors.
261 func Load(cfg *Config, patterns ...string) ([]*Package, error) {
262 ld := newLoader(cfg)
263 response, external, err := defaultDriver(&ld.Config, patterns...)
264 if err != nil {
265 return nil, err
266 }
267 268 ld.sizes = types.SizesFor(response.Compiler, response.Arch)
269 if ld.sizes == nil && ld.Config.Mode&(NeedTypes|NeedTypesSizes|NeedTypesInfo) != 0 {
270 // Type size information is needed but unavailable.
271 if external {
272 // An external driver may fail to populate the Compiler/GOARCH fields,
273 // especially since they are relatively new (see #63700).
274 // Provide a sensible fallback in this case.
275 ld.sizes = types.SizesFor("gc", runtime.GOARCH)
276 if ld.sizes == nil { // gccgo-only arch
277 ld.sizes = types.SizesFor("gc", "amd64")
278 }
279 } else {
280 // Go list should never fail to deliver accurate size information.
281 // Reject the whole Load since the error is the same for every package.
282 return nil, fmt.Errorf("can't determine type sizes for compiler %q on GOARCH %q",
283 response.Compiler, response.Arch)
284 }
285 }
286 287 return ld.refine(response)
288 }
289 290 // defaultDriver is a driver that implements go/packages' fallback behavior.
291 // It will try to request to an external driver, if one exists. If there's
292 // no external driver, or the driver returns a response with NotHandled set,
293 // defaultDriver will fall back to the go list driver.
294 // The boolean result indicates that an external driver handled the request.
295 func defaultDriver(cfg *Config, patterns ...string) (*DriverResponse, bool, error) {
296 const (
297 // windowsArgMax specifies the maximum command line length for
298 // the Windows' CreateProcess function.
299 windowsArgMax = 32767
300 // maxEnvSize is a very rough estimation of the maximum environment
301 // size of a user.
302 maxEnvSize = 16384
303 // safeArgMax specifies the maximum safe command line length to use
304 // by the underlying driver excl. the environment. We choose the Windows'
305 // ARG_MAX as the starting point because it's one of the lowest ARG_MAX
306 // constants out of the different supported platforms,
307 // e.g., https://www.in-ulm.de/~mascheck/various/argmax/#results.
308 safeArgMax = windowsArgMax - maxEnvSize
309 )
310 chunks, err := splitIntoChunks(patterns, safeArgMax)
311 if err != nil {
312 return nil, false, err
313 }
314 315 if driver := findExternalDriver(cfg); driver != nil {
316 response, err := callDriverOnChunks(driver, cfg, chunks)
317 if err != nil {
318 return nil, false, err
319 } else if !response.NotHandled {
320 return response, true, nil
321 }
322 // not handled: fall through
323 }
324 325 // go list fallback
326 327 // Write overlays once, as there are many calls
328 // to 'go list' (one per chunk plus others too).
329 overlayFile, cleanupOverlay, err := gocommand.WriteOverlays(cfg.Overlay)
330 if err != nil {
331 return nil, false, err
332 }
333 defer cleanupOverlay()
334 335 var runner gocommand.Runner // (shared across many 'go list' calls)
336 driver := func(cfg *Config, patterns []string) (*DriverResponse, error) {
337 return goListDriver(cfg, &runner, overlayFile, patterns)
338 }
339 response, err := callDriverOnChunks(driver, cfg, chunks)
340 if err != nil {
341 return nil, false, err
342 }
343 return response, false, err
344 }
345 346 // splitIntoChunks chunks the slice so that the total number of characters
347 // in a chunk is no longer than argMax.
348 func splitIntoChunks(patterns []string, argMax int) ([][]string, error) {
349 if argMax <= 0 {
350 return nil, errors.New("failed to split patterns into chunks, negative safe argMax value")
351 }
352 var chunks [][]string
353 charsInChunk := 0
354 nextChunkStart := 0
355 for i, v := range patterns {
356 vChars := len(v)
357 if vChars > argMax {
358 // a single pattern is longer than the maximum safe ARG_MAX, hardly should happen
359 return nil, errors.New("failed to split patterns into chunks, a pattern is too long")
360 }
361 charsInChunk += vChars + 1 // +1 is for a whitespace between patterns that has to be counted too
362 if charsInChunk > argMax {
363 chunks = append(chunks, patterns[nextChunkStart:i])
364 nextChunkStart = i
365 charsInChunk = vChars
366 }
367 }
368 // add the last chunk
369 if nextChunkStart < len(patterns) {
370 chunks = append(chunks, patterns[nextChunkStart:])
371 }
372 return chunks, nil
373 }
374 375 func callDriverOnChunks(driver driver, cfg *Config, chunks [][]string) (*DriverResponse, error) {
376 if len(chunks) == 0 {
377 return driver(cfg, nil)
378 }
379 responses := make([]*DriverResponse, len(chunks))
380 errNotHandled := errors.New("driver returned NotHandled")
381 var g errgroup.Group
382 for i, chunk := range chunks {
383 g.Go(func() (err error) {
384 responses[i], err = driver(cfg, chunk)
385 if responses[i] != nil && responses[i].NotHandled {
386 err = errNotHandled
387 }
388 return err
389 })
390 }
391 if err := g.Wait(); err != nil {
392 if errors.Is(err, errNotHandled) {
393 return &DriverResponse{NotHandled: true}, nil
394 }
395 return nil, err
396 }
397 return mergeResponses(responses...), nil
398 }
399 400 func mergeResponses(responses ...*DriverResponse) *DriverResponse {
401 if len(responses) == 0 {
402 return nil
403 }
404 response := newDeduper()
405 response.dr.NotHandled = false
406 response.dr.Compiler = responses[0].Compiler
407 response.dr.Arch = responses[0].Arch
408 response.dr.GoVersion = responses[0].GoVersion
409 for _, v := range responses {
410 response.addAll(v)
411 }
412 return response.dr
413 }
414 415 // A Package describes a loaded Go package.
416 //
417 // It also defines part of the JSON schema of [DriverResponse].
418 // See the package documentation for an overview.
419 type Package struct {
420 // ID is a unique identifier for a package,
421 // in a syntax provided by the underlying build system.
422 //
423 // Because the syntax varies based on the build system,
424 // clients should treat IDs as opaque and not attempt to
425 // interpret them.
426 ID string
427 428 // Name is the package name as it appears in the package source code.
429 Name string
430 431 // PkgPath is the package path as used by the go/types package.
432 PkgPath string
433 434 // Dir is the directory associated with the package, if it exists.
435 //
436 // For packages listed by the go command, this is the directory containing
437 // the package files.
438 Dir string
439 440 // Errors contains any errors encountered querying the metadata
441 // of the package, or while parsing or type-checking its files.
442 Errors []Error
443 444 // TypeErrors contains the subset of errors produced during type checking.
445 TypeErrors []types.Error
446 447 // GoFiles lists the absolute file paths of the package's Go source files.
448 // It may include files that should not be compiled, for example because
449 // they contain non-matching build tags, are documentary pseudo-files such as
450 // unsafe/unsafe.go or builtin/builtin.go, or are subject to cgo preprocessing.
451 GoFiles []string
452 453 // CompiledGoFiles lists the absolute file paths of the package's source
454 // files that are suitable for type checking.
455 // This may differ from GoFiles if files are processed before compilation.
456 CompiledGoFiles []string
457 458 // OtherFiles lists the absolute file paths of the package's non-Go source files,
459 // including assembly, C, C++, Fortran, Objective-C, SWIG, and so on.
460 OtherFiles []string
461 462 // EmbedFiles lists the absolute file paths of the package's files
463 // embedded with go:embed.
464 EmbedFiles []string
465 466 // EmbedPatterns lists the absolute file patterns of the package's
467 // files embedded with go:embed.
468 EmbedPatterns []string
469 470 // IgnoredFiles lists source files that are not part of the package
471 // using the current build configuration but that might be part of
472 // the package using other build configurations.
473 IgnoredFiles []string
474 475 // ExportFile is the absolute path to a file containing type
476 // information for the package as provided by the build system.
477 ExportFile string
478 479 // Target is the absolute install path of the .a file, for libraries,
480 // and of the executable file, for binaries.
481 Target string
482 483 // Imports maps import paths appearing in the package's Go source files
484 // to corresponding loaded Packages.
485 Imports map[string]*Package
486 487 // Module is the module information for the package if it exists.
488 //
489 // Note: it may be missing for std and cmd; see Go issue #65816.
490 Module *Module
491 492 // -- The following fields are not part of the driver JSON schema. --
493 494 // Types provides type information for the package.
495 // The NeedTypes LoadMode bit sets this field for packages matching the
496 // patterns; type information for dependencies may be missing or incomplete,
497 // unless NeedDeps and NeedImports are also set.
498 //
499 // Each call to [Load] returns a consistent set of type
500 // symbols, as defined by the comment at [types.Identical].
501 // Avoid mixing type information from two or more calls to [Load].
502 Types *types.Package `json:"-"`
503 504 // Fset provides position information for Types, TypesInfo, and Syntax.
505 // It is set only when Types is set.
506 Fset *token.FileSet `json:"-"`
507 508 // IllTyped indicates whether the package or any dependency contains errors.
509 // It is set only when Types is set.
510 IllTyped bool `json:"-"`
511 512 // Syntax is the package's syntax trees, for the files listed in CompiledGoFiles.
513 //
514 // The NeedSyntax LoadMode bit populates this field for packages matching the patterns.
515 // If NeedDeps and NeedImports are also set, this field will also be populated
516 // for dependencies.
517 //
518 // Syntax is kept in the same order as CompiledGoFiles, with the caveat that nils are
519 // removed. If parsing returned nil, Syntax may be shorter than CompiledGoFiles.
520 Syntax []*ast.File `json:"-"`
521 522 // TypesInfo provides type information about the package's syntax trees.
523 // It is set only when Syntax is set.
524 TypesInfo *types.Info `json:"-"`
525 526 // TypesSizes provides the effective size function for types in TypesInfo.
527 TypesSizes types.Sizes `json:"-"`
528 529 // -- internal --
530 531 // ForTest is the package under test, if any.
532 ForTest string
533 534 // depsErrors is the DepsErrors field from the go list response, if any.
535 depsErrors []*packagesinternal.PackageError
536 }
537 538 // Module provides module information for a package.
539 //
540 // It also defines part of the JSON schema of [DriverResponse].
541 // See the package documentation for an overview.
542 type Module struct {
543 Path string // module path
544 Version string // module version
545 Replace *Module // replaced by this module
546 Time *time.Time // time version was created
547 Main bool // is this the main module?
548 Indirect bool // is this module only an indirect dependency of main module?
549 Dir string // directory holding files for this module, if any
550 GoMod string // path to go.mod file used when loading this module, if any
551 GoVersion string // go version used in module
552 Error *ModuleError // error loading module
553 }
554 555 // ModuleError holds errors loading a module.
556 type ModuleError struct {
557 Err string // the error itself
558 }
559 560 func init() {
561 packagesinternal.GetDepsErrors = func(p any) []*packagesinternal.PackageError {
562 return p.(*Package).depsErrors
563 }
564 packagesinternal.TypecheckCgo = int(typecheckCgo)
565 packagesinternal.DepsErrors = int(needInternalDepsErrors)
566 }
567 568 // An Error describes a problem with a package's metadata, syntax, or types.
569 type Error struct {
570 Pos string // "file:line:col" or "file:line" or "" or "-"
571 Msg string
572 Kind ErrorKind
573 }
574 575 // ErrorKind describes the source of the error, allowing the user to
576 // differentiate between errors generated by the driver, the parser, or the
577 // type-checker.
578 type ErrorKind int
579 580 const (
581 UnknownError ErrorKind = iota
582 ListError
583 ParseError
584 TypeError
585 )
586 587 func (err Error) Error() string {
588 pos := err.Pos
589 if pos == "" {
590 pos = "-" // like token.Position{}.String()
591 }
592 return pos + ": " + err.Msg
593 }
594 595 // flatPackage is the JSON form of Package
596 // It drops all the type and syntax fields, and transforms the Imports
597 //
598 // TODO(adonovan): identify this struct with Package, effectively
599 // publishing the JSON protocol.
600 type flatPackage struct {
601 ID string
602 Name string `json:",omitempty"`
603 PkgPath string `json:",omitempty"`
604 Errors []Error `json:",omitempty"`
605 GoFiles []string `json:",omitempty"`
606 CompiledGoFiles []string `json:",omitempty"`
607 OtherFiles []string `json:",omitempty"`
608 EmbedFiles []string `json:",omitempty"`
609 EmbedPatterns []string `json:",omitempty"`
610 IgnoredFiles []string `json:",omitempty"`
611 ExportFile string `json:",omitempty"`
612 Imports map[string]string `json:",omitempty"`
613 }
614 615 // MarshalJSON returns the Package in its JSON form.
616 // For the most part, the structure fields are written out unmodified, and
617 // the type and syntax fields are skipped.
618 // The imports are written out as just a map of path to package id.
619 // The errors are written using a custom type that tries to preserve the
620 // structure of error types we know about.
621 //
622 // This method exists to enable support for additional build systems. It is
623 // not intended for use by clients of the API and we may change the format.
624 func (p *Package) MarshalJSON() ([]byte, error) {
625 flat := &flatPackage{
626 ID: p.ID,
627 Name: p.Name,
628 PkgPath: p.PkgPath,
629 Errors: p.Errors,
630 GoFiles: p.GoFiles,
631 CompiledGoFiles: p.CompiledGoFiles,
632 OtherFiles: p.OtherFiles,
633 EmbedFiles: p.EmbedFiles,
634 EmbedPatterns: p.EmbedPatterns,
635 IgnoredFiles: p.IgnoredFiles,
636 ExportFile: p.ExportFile,
637 }
638 if len(p.Imports) > 0 {
639 flat.Imports = make(map[string]string, len(p.Imports))
640 for path, ipkg := range p.Imports {
641 flat.Imports[path] = ipkg.ID
642 }
643 }
644 return json.Marshal(flat)
645 }
646 647 // UnmarshalJSON reads in a Package from its JSON format.
648 // See MarshalJSON for details about the format accepted.
649 func (p *Package) UnmarshalJSON(b []byte) error {
650 flat := &flatPackage{}
651 if err := json.Unmarshal(b, &flat); err != nil {
652 return err
653 }
654 *p = Package{
655 ID: flat.ID,
656 Name: flat.Name,
657 PkgPath: flat.PkgPath,
658 Errors: flat.Errors,
659 GoFiles: flat.GoFiles,
660 CompiledGoFiles: flat.CompiledGoFiles,
661 OtherFiles: flat.OtherFiles,
662 EmbedFiles: flat.EmbedFiles,
663 EmbedPatterns: flat.EmbedPatterns,
664 IgnoredFiles: flat.IgnoredFiles,
665 ExportFile: flat.ExportFile,
666 }
667 if len(flat.Imports) > 0 {
668 p.Imports = make(map[string]*Package, len(flat.Imports))
669 for path, id := range flat.Imports {
670 p.Imports[path] = &Package{ID: id}
671 }
672 }
673 return nil
674 }
675 676 func (p *Package) String() string { return p.ID }
677 678 // loaderPackage augments Package with state used during the loading phase
679 type loaderPackage struct {
680 *Package
681 importErrors map[string]error // maps each bad import to its error
682 preds []*loaderPackage // packages that import this one
683 unfinishedSuccs atomic.Int32 // number of direct imports not yet loaded
684 color uint8 // for cycle detection
685 needsrc bool // load from source (Mode >= LoadTypes)
686 needtypes bool // type information is either requested or depended on
687 initial bool // package was matched by a pattern
688 goVersion int // minor version number of go command on PATH
689 }
690 691 // loader holds the working state of a single call to load.
692 type loader struct {
693 pkgs map[string]*loaderPackage // keyed by Package.ID
694 Config
695 sizes types.Sizes // non-nil if needed by mode
696 parseCache map[string]*parseValue
697 parseCacheMu sync.Mutex
698 exportMu sync.Mutex // enforces mutual exclusion of exportdata operations
699 700 // Config.Mode contains the implied mode (see impliedLoadMode).
701 // Implied mode contains all the fields we need the data for.
702 // In requestedMode there are the actually requested fields.
703 // We'll zero them out before returning packages to the user.
704 // This makes it easier for us to get the conditions where
705 // we need certain modes right.
706 requestedMode LoadMode
707 }
708 709 type parseValue struct {
710 f *ast.File
711 err error
712 ready chan struct{}
713 }
714 715 func newLoader(cfg *Config) *loader {
716 ld := &loader{
717 parseCache: map[string]*parseValue{},
718 }
719 if cfg != nil {
720 ld.Config = *cfg
721 // If the user has provided a logger, use it.
722 ld.Config.Logf = cfg.Logf
723 }
724 if ld.Config.Logf == nil {
725 // If the GOPACKAGESDEBUG environment variable is set to true,
726 // but the user has not provided a logger, default to log.Printf.
727 if debug {
728 ld.Config.Logf = log.Printf
729 } else {
730 ld.Config.Logf = func(format string, args ...any) {}
731 }
732 }
733 if ld.Config.Mode == 0 {
734 ld.Config.Mode = NeedName | NeedFiles | NeedCompiledGoFiles // Preserve zero behavior of Mode for backwards compatibility.
735 }
736 if ld.Config.Env == nil {
737 ld.Config.Env = os.Environ()
738 }
739 if ld.Context == nil {
740 ld.Context = context.Background()
741 }
742 if ld.Dir == "" {
743 if dir, err := os.Getwd(); err == nil {
744 ld.Dir = dir
745 }
746 }
747 748 // Save the actually requested fields. We'll zero them out before returning packages to the user.
749 ld.requestedMode = ld.Mode
750 ld.Mode = impliedLoadMode(ld.Mode)
751 752 if ld.Mode&(NeedSyntax|NeedTypes|NeedTypesInfo) != 0 {
753 if ld.Fset == nil {
754 ld.Fset = token.NewFileSet()
755 }
756 757 // ParseFile is required even in LoadTypes mode
758 // because we load source if export data is missing.
759 if ld.ParseFile == nil {
760 ld.ParseFile = func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) {
761 // We implicitly promise to keep doing ast.Object resolution. :(
762 const mode = parser.AllErrors | parser.ParseComments
763 return parser.ParseFile(fset, filename, src, mode)
764 }
765 }
766 }
767 768 return ld
769 }
770 771 // refine connects the supplied packages into a graph and then adds type
772 // and syntax information as requested by the LoadMode.
773 func (ld *loader) refine(response *DriverResponse) ([]*Package, error) {
774 roots := response.Roots
775 rootMap := make(map[string]int, len(roots))
776 for i, root := range roots {
777 rootMap[root] = i
778 }
779 ld.pkgs = make(map[string]*loaderPackage)
780 // first pass, fixup and build the map and roots
781 var initial = make([]*loaderPackage, len(roots))
782 for _, pkg := range response.Packages {
783 rootIndex := -1
784 if i, found := rootMap[pkg.ID]; found {
785 rootIndex = i
786 }
787 788 // Overlays can invalidate export data.
789 // TODO(matloob): make this check fine-grained based on dependencies on overlaid files
790 exportDataInvalid := len(ld.Overlay) > 0 || pkg.ExportFile == "" && pkg.PkgPath != "unsafe"
791 // This package needs type information if the caller requested types and the package is
792 // either a root, or it's a non-root and the user requested dependencies ...
793 needtypes := (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0))
794 // This package needs source if the call requested source (or types info, which implies source)
795 // and the package is either a root, or itas a non- root and the user requested dependencies...
796 needsrc := ((ld.Mode&(NeedSyntax|NeedTypesInfo) != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0)) ||
797 // ... or if we need types and the exportData is invalid. We fall back to (incompletely)
798 // typechecking packages from source if they fail to compile.
799 (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && exportDataInvalid)) && pkg.PkgPath != "unsafe"
800 lpkg := &loaderPackage{
801 Package: pkg,
802 needtypes: needtypes,
803 needsrc: needsrc,
804 goVersion: response.GoVersion,
805 }
806 ld.pkgs[lpkg.ID] = lpkg
807 if rootIndex >= 0 {
808 initial[rootIndex] = lpkg
809 lpkg.initial = true
810 }
811 }
812 for i, root := range roots {
813 if initial[i] == nil {
814 return nil, fmt.Errorf("root package %v is missing", root)
815 }
816 }
817 818 // Materialize the import graph if it is needed (NeedImports),
819 // or if we'll be using loadPackages (Need{Syntax|Types|TypesInfo}).
820 var leaves []*loaderPackage // packages with no unfinished successors
821 if ld.Mode&(NeedImports|NeedSyntax|NeedTypes|NeedTypesInfo) != 0 {
822 const (
823 white = 0 // new
824 grey = 1 // in progress
825 black = 2 // complete
826 )
827 828 // visit traverses the import graph, depth-first,
829 // and materializes the graph as Packages.Imports.
830 //
831 // Valid imports are saved in the Packages.Import map.
832 // Invalid imports (cycles and missing nodes) are saved in the importErrors map.
833 // Thus, even in the presence of both kinds of errors,
834 // the Import graph remains a DAG.
835 //
836 // visit returns whether the package needs src or has a transitive
837 // dependency on a package that does. These are the only packages
838 // for which we load source code.
839 var stack []*loaderPackage
840 var visit func(from, lpkg *loaderPackage) bool
841 visit = func(from, lpkg *loaderPackage) bool {
842 if lpkg.color == grey {
843 panic("internal error: grey node")
844 }
845 if lpkg.color == white {
846 lpkg.color = grey
847 stack = append(stack, lpkg) // push
848 stubs := lpkg.Imports // the structure form has only stubs with the ID in the Imports
849 lpkg.Imports = make(map[string]*Package, len(stubs))
850 for importPath, ipkg := range stubs {
851 var importErr error
852 imp := ld.pkgs[ipkg.ID]
853 if imp == nil {
854 // (includes package "C" when DisableCgo)
855 importErr = fmt.Errorf("missing package: %q", ipkg.ID)
856 } else if imp.color == grey {
857 importErr = fmt.Errorf("import cycle: %s", stack)
858 }
859 if importErr != nil {
860 if lpkg.importErrors == nil {
861 lpkg.importErrors = make(map[string]error)
862 }
863 lpkg.importErrors[importPath] = importErr
864 continue
865 }
866 867 if visit(lpkg, imp) {
868 lpkg.needsrc = true
869 }
870 lpkg.Imports[importPath] = imp.Package
871 }
872 873 // -- postorder --
874 875 // Complete type information is required for the
876 // immediate dependencies of each source package.
877 if lpkg.needsrc && ld.Mode&NeedTypes != 0 {
878 for _, ipkg := range lpkg.Imports {
879 ld.pkgs[ipkg.ID].needtypes = true
880 }
881 }
882 883 // NeedTypeSizes causes TypeSizes to be set even
884 // on packages for which types aren't needed.
885 if ld.Mode&NeedTypesSizes != 0 {
886 lpkg.TypesSizes = ld.sizes
887 }
888 889 // Add packages with no imports directly to the queue of leaves.
890 if len(lpkg.Imports) == 0 {
891 leaves = append(leaves, lpkg)
892 }
893 894 stack = stack[:len(stack)-1] // pop
895 lpkg.color = black
896 }
897 898 // Add edge from predecessor.
899 if from != nil {
900 from.unfinishedSuccs.Add(+1) // incref
901 lpkg.preds = append(lpkg.preds, from)
902 }
903 904 return lpkg.needsrc
905 }
906 907 // For each initial package, create its import DAG.
908 for _, lpkg := range initial {
909 visit(nil, lpkg)
910 }
911 912 } else {
913 // !NeedImports: drop the stub (ID-only) import packages
914 // that we are not even going to try to resolve.
915 for _, lpkg := range initial {
916 lpkg.Imports = nil
917 }
918 }
919 920 // Load type data and syntax if needed, starting at
921 // the initial packages (roots of the import DAG).
922 if ld.Mode&(NeedSyntax|NeedTypes|NeedTypesInfo) != 0 {
923 924 // We avoid using g.SetLimit to limit concurrency as
925 // it makes g.Go stop accepting work, which prevents
926 // workers from enqeuing, and thus finishing, and thus
927 // allowing the group to make progress: deadlock.
928 //
929 // Instead we use the ioLimit and cpuLimit semaphores.
930 g, _ := errgroup.WithContext(ld.Context)
931 932 // enqueues adds a package to the type-checking queue.
933 // It must have no unfinished successors.
934 var enqueue func(*loaderPackage)
935 enqueue = func(lpkg *loaderPackage) {
936 g.Go(func() error {
937 // Parse and type-check.
938 ld.loadPackage(lpkg)
939 940 // Notify each waiting predecessor,
941 // and enqueue it when it becomes a leaf.
942 for _, pred := range lpkg.preds {
943 if pred.unfinishedSuccs.Add(-1) == 0 { // decref
944 enqueue(pred)
945 }
946 }
947 948 return nil
949 })
950 }
951 952 // Load leaves first, adding new packages
953 // to the queue as they become leaves.
954 for _, leaf := range leaves {
955 enqueue(leaf)
956 }
957 958 if err := g.Wait(); err != nil {
959 return nil, err // cancelled
960 }
961 }
962 963 // If the context is done, return its error and
964 // throw out [likely] incomplete packages.
965 if err := ld.Context.Err(); err != nil {
966 return nil, err
967 }
968 969 result := make([]*Package, len(initial))
970 for i, lpkg := range initial {
971 result[i] = lpkg.Package
972 }
973 for i := range ld.pkgs {
974 // Clear all unrequested fields,
975 // to catch programs that use more than they request.
976 if ld.requestedMode&NeedName == 0 {
977 ld.pkgs[i].Name = ""
978 ld.pkgs[i].PkgPath = ""
979 }
980 if ld.requestedMode&NeedFiles == 0 {
981 ld.pkgs[i].GoFiles = nil
982 ld.pkgs[i].OtherFiles = nil
983 ld.pkgs[i].IgnoredFiles = nil
984 }
985 if ld.requestedMode&NeedEmbedFiles == 0 {
986 ld.pkgs[i].EmbedFiles = nil
987 }
988 if ld.requestedMode&NeedEmbedPatterns == 0 {
989 ld.pkgs[i].EmbedPatterns = nil
990 }
991 if ld.requestedMode&NeedCompiledGoFiles == 0 {
992 ld.pkgs[i].CompiledGoFiles = nil
993 }
994 if ld.requestedMode&NeedImports == 0 {
995 ld.pkgs[i].Imports = nil
996 }
997 if ld.requestedMode&NeedExportFile == 0 {
998 ld.pkgs[i].ExportFile = ""
999 }
1000 if ld.requestedMode&NeedTypes == 0 {
1001 ld.pkgs[i].Types = nil
1002 ld.pkgs[i].IllTyped = false
1003 }
1004 if ld.requestedMode&NeedSyntax == 0 {
1005 ld.pkgs[i].Syntax = nil
1006 }
1007 if ld.requestedMode&(NeedSyntax|NeedTypes|NeedTypesInfo) == 0 {
1008 ld.pkgs[i].Fset = nil
1009 }
1010 if ld.requestedMode&NeedTypesInfo == 0 {
1011 ld.pkgs[i].TypesInfo = nil
1012 }
1013 if ld.requestedMode&NeedTypesSizes == 0 {
1014 ld.pkgs[i].TypesSizes = nil
1015 }
1016 if ld.requestedMode&NeedModule == 0 {
1017 ld.pkgs[i].Module = nil
1018 }
1019 }
1020 1021 return result, nil
1022 }
1023 1024 // loadPackage loads/parses/typechecks the specified package.
1025 // It must be called only once per Package,
1026 // after immediate dependencies are loaded.
1027 // Precondition: ld.Mode&(NeedSyntax|NeedTypes|NeedTypesInfo) != 0.
1028 func (ld *loader) loadPackage(lpkg *loaderPackage) {
1029 if lpkg.PkgPath == "unsafe" {
1030 // To avoid surprises, fill in the blanks consistent
1031 // with other packages. (For example, some analyzers
1032 // assert that each needed types.Info map is non-nil
1033 // even when there is no syntax that would cause them
1034 // to consult the map.)
1035 lpkg.Types = types.Unsafe
1036 lpkg.Fset = ld.Fset
1037 lpkg.Syntax = []*ast.File{}
1038 lpkg.TypesInfo = ld.newTypesInfo()
1039 lpkg.TypesSizes = ld.sizes
1040 return
1041 }
1042 1043 // Call NewPackage directly with explicit name.
1044 // This avoids skew between golist and go/types when the files'
1045 // package declarations are inconsistent.
1046 lpkg.Types = types.NewPackage(lpkg.PkgPath, lpkg.Name)
1047 lpkg.Fset = ld.Fset
1048 1049 // Start shutting down if the context is done and do not load
1050 // source or export data files.
1051 // Packages that import this one will have ld.Context.Err() != nil.
1052 // ld.Context.Err() will be returned later by refine.
1053 if ld.Context.Err() != nil {
1054 return
1055 }
1056 1057 // Subtle: we populate all Types fields with an empty Package
1058 // before loading export data so that export data processing
1059 // never has to create a types.Package for an indirect dependency,
1060 // which would then require that such created packages be explicitly
1061 // inserted back into the Import graph as a final step after export data loading.
1062 // (Hence this return is after the Types assignment.)
1063 // The Diamond test exercises this case.
1064 if !lpkg.needtypes && !lpkg.needsrc {
1065 return
1066 }
1067 1068 // TODO(adonovan): this condition looks wrong:
1069 // I think it should be lpkg.needtypes && !lpg.needsrc,
1070 // so that NeedSyntax without NeedTypes can be satisfied by export data.
1071 if !lpkg.needsrc {
1072 if err := ld.loadFromExportData(lpkg); err != nil {
1073 lpkg.Errors = append(lpkg.Errors, Error{
1074 Pos: "-",
1075 Msg: err.Error(),
1076 Kind: UnknownError, // e.g. can't find/open/parse export data
1077 })
1078 }
1079 return // not a source package, don't get syntax trees
1080 }
1081 1082 appendError := func(err error) {
1083 // Convert various error types into the one true Error.
1084 var errs []Error
1085 switch err := err.(type) {
1086 case Error:
1087 // from driver
1088 errs = append(errs, err)
1089 1090 case *os.PathError:
1091 // from parser
1092 errs = append(errs, Error{
1093 Pos: err.Path + ":1",
1094 Msg: err.Err.Error(),
1095 Kind: ParseError,
1096 })
1097 1098 case scanner.ErrorList:
1099 // from parser
1100 for _, err := range err {
1101 errs = append(errs, Error{
1102 Pos: err.Pos.String(),
1103 Msg: err.Msg,
1104 Kind: ParseError,
1105 })
1106 }
1107 1108 case types.Error:
1109 // from type checker
1110 lpkg.TypeErrors = append(lpkg.TypeErrors, err)
1111 errs = append(errs, Error{
1112 Pos: err.Fset.Position(err.Pos).String(),
1113 Msg: err.Msg,
1114 Kind: TypeError,
1115 })
1116 1117 default:
1118 // unexpected impoverished error from parser?
1119 errs = append(errs, Error{
1120 Pos: "-",
1121 Msg: err.Error(),
1122 Kind: UnknownError,
1123 })
1124 1125 // If you see this error message, please file a bug.
1126 log.Printf("internal error: error %q (%T) without position", err, err)
1127 }
1128 1129 lpkg.Errors = append(lpkg.Errors, errs...)
1130 }
1131 1132 // If the go command on the PATH is newer than the runtime,
1133 // then the go/{scanner,ast,parser,types} packages from the
1134 // standard library may be unable to process the files
1135 // selected by go list.
1136 //
1137 // There is currently no way to downgrade the effective
1138 // version of the go command (see issue 52078), so we proceed
1139 // with the newer go command but, in case of parse or type
1140 // errors, we emit an additional diagnostic.
1141 //
1142 // See:
1143 // - golang.org/issue/52078 (flag to set release tags)
1144 // - golang.org/issue/50825 (gopls legacy version support)
1145 // - golang.org/issue/55883 (go/packages confusing error)
1146 //
1147 // Should we assert a hard minimum of (currently) go1.16 here?
1148 var runtimeVersion int
1149 if _, err := fmt.Sscanf(runtime.Version(), "go1.%d", &runtimeVersion); err == nil && runtimeVersion < lpkg.goVersion {
1150 defer func() {
1151 if len(lpkg.Errors) > 0 {
1152 appendError(Error{
1153 Pos: "-",
1154 Msg: fmt.Sprintf("This application uses version go1.%d of the source-processing packages but runs version go1.%d of 'go list'. It may fail to process source files that rely on newer language features. If so, rebuild the application using a newer version of Go.", runtimeVersion, lpkg.goVersion),
1155 Kind: UnknownError,
1156 })
1157 }
1158 }()
1159 }
1160 1161 if ld.Config.Mode&NeedTypes != 0 && len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" {
1162 // The config requested loading sources and types, but sources are missing.
1163 // Add an error to the package and fall back to loading from export data.
1164 appendError(Error{"-", fmt.Sprintf("sources missing for package %s", lpkg.ID), ParseError})
1165 _ = ld.loadFromExportData(lpkg) // ignore any secondary errors
1166 1167 return // can't get syntax trees for this package
1168 }
1169 1170 files, errs := ld.parseFiles(lpkg.CompiledGoFiles)
1171 for _, err := range errs {
1172 appendError(err)
1173 }
1174 1175 lpkg.Syntax = files
1176 if ld.Config.Mode&(NeedTypes|NeedTypesInfo) == 0 {
1177 return
1178 }
1179 1180 // Start shutting down if the context is done and do not type check.
1181 // Packages that import this one will have ld.Context.Err() != nil.
1182 // ld.Context.Err() will be returned later by refine.
1183 if ld.Context.Err() != nil {
1184 return
1185 }
1186 1187 lpkg.TypesInfo = ld.newTypesInfo()
1188 lpkg.TypesSizes = ld.sizes
1189 1190 importer := importerFunc(func(path string) (*types.Package, error) {
1191 if path == "unsafe" {
1192 return types.Unsafe, nil
1193 }
1194 1195 // The imports map is keyed by import path.
1196 ipkg := lpkg.Imports[path]
1197 if ipkg == nil {
1198 if err := lpkg.importErrors[path]; err != nil {
1199 return nil, err
1200 }
1201 // There was skew between the metadata and the
1202 // import declarations, likely due to an edit
1203 // race, or because the ParseFile feature was
1204 // used to supply alternative file contents.
1205 return nil, fmt.Errorf("no metadata for %s", path)
1206 }
1207 1208 if ipkg.Types != nil && ipkg.Types.Complete() {
1209 return ipkg.Types, nil
1210 }
1211 log.Fatalf("internal error: package %q without types was imported from %q", path, lpkg)
1212 panic("unreachable")
1213 })
1214 1215 // type-check
1216 tc := &types.Config{
1217 Importer: importer,
1218 1219 // Type-check bodies of functions only in initial packages.
1220 // Example: for import graph A->B->C and initial packages {A,C},
1221 // we can ignore function bodies in B.
1222 IgnoreFuncBodies: ld.Mode&NeedDeps == 0 && !lpkg.initial,
1223 1224 Error: appendError,
1225 Sizes: ld.sizes, // may be nil
1226 }
1227 if lpkg.Module != nil && lpkg.Module.GoVersion != "" {
1228 tc.GoVersion = "go" + lpkg.Module.GoVersion
1229 }
1230 if (ld.Mode & typecheckCgo) != 0 {
1231 if !typesinternal.SetUsesCgo(tc) {
1232 appendError(Error{
1233 Msg: "typecheckCgo requires Go 1.15+",
1234 Kind: ListError,
1235 })
1236 return
1237 }
1238 }
1239 1240 // Type-checking is CPU intensive.
1241 cpuLimit <- unit{} // acquire a token
1242 defer func() { <-cpuLimit }() // release a token
1243 1244 typErr := types.NewChecker(tc, ld.Fset, lpkg.Types, lpkg.TypesInfo).Files(lpkg.Syntax)
1245 lpkg.importErrors = nil // no longer needed
1246 1247 // In go/types go1.21 and go1.22, Checker.Files failed fast with a
1248 // a "too new" error, without calling tc.Error and without
1249 // proceeding to type-check the package (#66525).
1250 // We rely on the runtimeVersion error to give the suggested remedy.
1251 if typErr != nil && len(lpkg.Errors) == 0 && len(lpkg.Syntax) > 0 {
1252 if msg := typErr.Error(); strings.HasPrefix(msg, "package requires newer Go version") {
1253 appendError(types.Error{
1254 Fset: ld.Fset,
1255 Pos: lpkg.Syntax[0].Package,
1256 Msg: msg,
1257 })
1258 }
1259 }
1260 1261 // If !Cgo, the type-checker uses FakeImportC mode, so
1262 // it doesn't invoke the importer for import "C",
1263 // nor report an error for the import,
1264 // or for any undefined C.f reference.
1265 // We must detect this explicitly and correctly
1266 // mark the package as IllTyped (by reporting an error).
1267 // TODO(adonovan): if these errors are annoying,
1268 // we could just set IllTyped quietly.
1269 if tc.FakeImportC {
1270 outer:
1271 for _, f := range lpkg.Syntax {
1272 for _, imp := range f.Imports {
1273 if imp.Path.Value == `"C"` {
1274 err := types.Error{Fset: ld.Fset, Pos: imp.Pos(), Msg: `import "C" ignored`}
1275 appendError(err)
1276 break outer
1277 }
1278 }
1279 }
1280 }
1281 1282 // If types.Checker.Files had an error that was unreported,
1283 // make sure to report the unknown error so the package is illTyped.
1284 if typErr != nil && len(lpkg.Errors) == 0 {
1285 appendError(typErr)
1286 }
1287 1288 // Record accumulated errors.
1289 illTyped := len(lpkg.Errors) > 0
1290 if !illTyped {
1291 for _, imp := range lpkg.Imports {
1292 if imp.IllTyped {
1293 illTyped = true
1294 break
1295 }
1296 }
1297 }
1298 lpkg.IllTyped = illTyped
1299 }
1300 1301 func (ld *loader) newTypesInfo() *types.Info {
1302 // Populate TypesInfo only if needed, as it
1303 // causes the type checker to work much harder.
1304 if ld.Config.Mode&NeedTypesInfo == 0 {
1305 return nil
1306 }
1307 return &types.Info{
1308 Types: make(map[ast.Expr]types.TypeAndValue),
1309 Defs: make(map[*ast.Ident]types.Object),
1310 Uses: make(map[*ast.Ident]types.Object),
1311 Implicits: make(map[ast.Node]types.Object),
1312 Instances: make(map[*ast.Ident]types.Instance),
1313 Scopes: make(map[ast.Node]*types.Scope),
1314 Selections: make(map[*ast.SelectorExpr]*types.Selection),
1315 FileVersions: make(map[*ast.File]string),
1316 }
1317 }
1318 1319 // An importFunc is an implementation of the single-method
1320 // types.Importer interface based on a function value.
1321 type importerFunc func(path string) (*types.Package, error)
1322 1323 func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }
1324 1325 // We use a counting semaphore to limit
1326 // the number of parallel I/O calls or CPU threads per process.
1327 var (
1328 ioLimit = make(chan unit, 20)
1329 cpuLimit = make(chan unit, runtime.GOMAXPROCS(0))
1330 )
1331 1332 func (ld *loader) parseFile(filename string) (*ast.File, error) {
1333 ld.parseCacheMu.Lock()
1334 v, ok := ld.parseCache[filename]
1335 if ok {
1336 // cache hit
1337 ld.parseCacheMu.Unlock()
1338 <-v.ready
1339 } else {
1340 // cache miss
1341 v = &parseValue{ready: make(chan struct{})}
1342 ld.parseCache[filename] = v
1343 ld.parseCacheMu.Unlock()
1344 1345 var src []byte
1346 for f, contents := range ld.Config.Overlay {
1347 // TODO(adonovan): Inefficient for large overlays.
1348 // Do an exact name-based map lookup
1349 // (for nonexistent files) followed by a
1350 // FileID-based map lookup (for existing ones).
1351 if sameFile(f, filename) {
1352 src = contents
1353 break
1354 }
1355 }
1356 var err error
1357 if src == nil {
1358 ioLimit <- unit{} // acquire a token
1359 src, err = os.ReadFile(filename)
1360 <-ioLimit // release a token
1361 }
1362 if err != nil {
1363 v.err = err
1364 } else {
1365 // Parsing is CPU intensive.
1366 cpuLimit <- unit{} // acquire a token
1367 v.f, v.err = ld.ParseFile(ld.Fset, filename, src)
1368 <-cpuLimit // release a token
1369 }
1370 1371 close(v.ready)
1372 }
1373 return v.f, v.err
1374 }
1375 1376 // parseFiles reads and parses the Go source files and returns the ASTs
1377 // of the ones that could be at least partially parsed, along with a
1378 // list of I/O and parse errors encountered.
1379 //
1380 // Because files are scanned in parallel, the token.Pos
1381 // positions of the resulting ast.Files are not ordered.
1382 func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) {
1383 var (
1384 n = len(filenames)
1385 parsed = make([]*ast.File, n)
1386 errors = make([]error, n)
1387 )
1388 var g errgroup.Group
1389 for i, filename := range filenames {
1390 // This creates goroutines unnecessarily in the
1391 // cache-hit case, but that case is uncommon.
1392 g.Go(func() error {
1393 parsed[i], errors[i] = ld.parseFile(filename)
1394 return nil
1395 })
1396 }
1397 g.Wait()
1398 1399 // Eliminate nils, preserving order.
1400 var o int
1401 for _, f := range parsed {
1402 if f != nil {
1403 parsed[o] = f
1404 o++
1405 }
1406 }
1407 parsed = parsed[:o]
1408 1409 o = 0
1410 for _, err := range errors {
1411 if err != nil {
1412 errors[o] = err
1413 o++
1414 }
1415 }
1416 errors = errors[:o]
1417 1418 return parsed, errors
1419 }
1420 1421 // sameFile returns true if x and y have the same basename and denote
1422 // the same file.
1423 func sameFile(x, y string) bool {
1424 if x == y {
1425 // It could be the case that y doesn't exist.
1426 // For instance, it may be an overlay file that
1427 // hasn't been written to disk. To handle that case
1428 // let x == y through. (We added the exact absolute path
1429 // string to the CompiledGoFiles list, so the unwritten
1430 // overlay case implies x==y.)
1431 return true
1432 }
1433 if strings.EqualFold(filepath.Base(x), filepath.Base(y)) { // (optimisation)
1434 if xi, err := os.Stat(x); err == nil {
1435 if yi, err := os.Stat(y); err == nil {
1436 return os.SameFile(xi, yi)
1437 }
1438 }
1439 }
1440 return false
1441 }
1442 1443 // loadFromExportData ensures that type information is present for the specified
1444 // package, loading it from an export data file on the first request.
1445 // On success it sets lpkg.Types to a new Package.
1446 func (ld *loader) loadFromExportData(lpkg *loaderPackage) error {
1447 if lpkg.PkgPath == "" {
1448 log.Fatalf("internal error: Package %s has no PkgPath", lpkg)
1449 }
1450 1451 // Because gcexportdata.Read has the potential to create or
1452 // modify the types.Package for each node in the transitive
1453 // closure of dependencies of lpkg, all exportdata operations
1454 // must be sequential. (Finer-grained locking would require
1455 // changes to the gcexportdata API.)
1456 //
1457 // The exportMu lock guards the lpkg.Types field and the
1458 // types.Package it points to, for each loaderPackage in the graph.
1459 //
1460 // Not all accesses to Package.Pkg need to be protected by exportMu:
1461 // graph ordering ensures that direct dependencies of source
1462 // packages are fully loaded before the importer reads their Pkg field.
1463 ld.exportMu.Lock()
1464 defer ld.exportMu.Unlock()
1465 1466 if tpkg := lpkg.Types; tpkg != nil && tpkg.Complete() {
1467 return nil // cache hit
1468 }
1469 1470 lpkg.IllTyped = true // fail safe
1471 1472 if lpkg.ExportFile == "" {
1473 // Errors while building export data will have been printed to stderr.
1474 return fmt.Errorf("no export data file")
1475 }
1476 f, err := os.Open(lpkg.ExportFile)
1477 if err != nil {
1478 return err
1479 }
1480 defer f.Close()
1481 1482 // Read gc export data.
1483 //
1484 // We don't currently support gccgo export data because all
1485 // underlying workspaces use the gc toolchain. (Even build
1486 // systems that support gccgo don't use it for workspace
1487 // queries.)
1488 r, err := gcexportdata.NewReader(f)
1489 if err != nil {
1490 return fmt.Errorf("reading %s: %v", lpkg.ExportFile, err)
1491 }
1492 1493 // Build the view.
1494 //
1495 // The gcexportdata machinery has no concept of package ID.
1496 // It identifies packages by their PkgPath, which although not
1497 // globally unique is unique within the scope of one invocation
1498 // of the linker, type-checker, or gcexportdata.
1499 //
1500 // So, we must build a PkgPath-keyed view of the global
1501 // (conceptually ID-keyed) cache of packages and pass it to
1502 // gcexportdata. The view must contain every existing
1503 // package that might possibly be mentioned by the
1504 // current package---its transitive closure.
1505 //
1506 // In loadPackage, we unconditionally create a types.Package for
1507 // each dependency so that export data loading does not
1508 // create new ones.
1509 //
1510 // TODO(adonovan): it would be simpler and more efficient
1511 // if the export data machinery invoked a callback to
1512 // get-or-create a package instead of a map.
1513 //
1514 view := make(map[string]*types.Package) // view seen by gcexportdata
1515 seen := make(map[*loaderPackage]bool) // all visited packages
1516 var visit func(pkgs map[string]*Package)
1517 visit = func(pkgs map[string]*Package) {
1518 for _, p := range pkgs {
1519 lpkg := ld.pkgs[p.ID]
1520 if !seen[lpkg] {
1521 seen[lpkg] = true
1522 view[lpkg.PkgPath] = lpkg.Types
1523 visit(lpkg.Imports)
1524 }
1525 }
1526 }
1527 visit(lpkg.Imports)
1528 1529 viewLen := len(view) + 1 // adding the self package
1530 // Parse the export data.
1531 // (May modify incomplete packages in view but not create new ones.)
1532 tpkg, err := gcexportdata.Read(r, ld.Fset, view, lpkg.PkgPath)
1533 if err != nil {
1534 return fmt.Errorf("reading %s: %v", lpkg.ExportFile, err)
1535 }
1536 if _, ok := view["go.shape"]; ok {
1537 // Account for the pseudopackage "go.shape" that gets
1538 // created by generic code.
1539 viewLen++
1540 }
1541 if viewLen != len(view) {
1542 log.Panicf("golang.org/x/tools/go/packages: unexpected new packages during load of %s", lpkg.PkgPath)
1543 }
1544 1545 lpkg.Types = tpkg
1546 lpkg.IllTyped = false
1547 return nil
1548 }
1549 1550 // impliedLoadMode returns loadMode with its dependencies.
1551 func impliedLoadMode(loadMode LoadMode) LoadMode {
1552 if loadMode&(NeedDeps|NeedTypes|NeedTypesInfo) != 0 {
1553 // All these things require knowing the import graph.
1554 loadMode |= NeedImports
1555 }
1556 if loadMode&NeedTypes != 0 {
1557 // Types require the GoVersion from Module.
1558 loadMode |= NeedModule
1559 }
1560 1561 return loadMode
1562 }
1563 1564 func usesExportData(cfg *Config) bool {
1565 return cfg.Mode&NeedExportFile != 0 || cfg.Mode&NeedTypes != 0 && cfg.Mode&NeedDeps == 0
1566 }
1567 1568 type unit struct{}
1569