1 // Copyright 2016 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4 5 // Package gcexportdata provides functions for reading and writing
6 // export data, which is a serialized description of the API of a Go
7 // package including the names, kinds, types, and locations of all
8 // exported declarations.
9 //
10 // The standard Go compiler (cmd/compile) writes an export data file
11 // for each package it compiles, which it later reads when compiling
12 // packages that import the earlier one. The compiler must thus
13 // contain logic to both write and read export data.
14 // (See the "Export" section in the cmd/compile/README file.)
15 //
16 // The [Read] function in this package can read files produced by the
17 // compiler, producing [go/types] data structures. As a matter of
18 // policy, Read supports export data files produced by only the last
19 // two Go releases plus tip; see https://go.dev/issue/68898. The
20 // export data files produced by the compiler contain additional
21 // details related to generics, inlining, and other optimizations that
22 // cannot be decoded by the [Read] function.
23 //
24 // In files written by the compiler, the export data is not at the
25 // start of the file. Before calling Read, use [NewReader] to locate
26 // the desired portion of the file.
27 //
28 // The [Write] function in this package encodes the exported API of a
29 // Go package ([types.Package]) as a file. Such files can be later
30 // decoded by Read, but cannot be consumed by the compiler.
31 //
32 // # Future changes
33 //
34 // Although Read supports the formats written by both Write and the
35 // compiler, the two are quite different, and there is an open
36 // proposal (https://go.dev/issue/69491) to separate these APIs.
37 //
38 // Under that proposal, this package would ultimately provide only the
39 // Read operation for compiler export data, which must be defined in
40 // this module (golang.org/x/tools), not in the standard library, to
41 // avoid version skew for developer tools that need to read compiler
42 // export data both before and after a Go release, such as from Go
43 // 1.23 to Go 1.24. Because this package lives in the tools module,
44 // clients can update their version of the module some time before the
45 // Go 1.24 release and rebuild and redeploy their tools, which will
46 // then be able to consume both Go 1.23 and Go 1.24 export data files,
47 // so they will work before and after the Go update. (See discussion
48 // at https://go.dev/issue/15651.)
49 //
50 // The operations to import and export [go/types] data structures
51 // would be defined in the go/types package as Import and Export.
52 // [Write] would (eventually) delegate to Export,
53 // and [Read], when it detects a file produced by Export,
54 // would delegate to Import.
55 //
56 // # Deprecations
57 //
58 // The [NewImporter] and [Find] functions are deprecated and should
59 // not be used in new code. The [WriteBundle] and [ReadBundle]
60 // functions are experimental, and there is an open proposal to
61 // deprecate them (https://go.dev/issue/69573).
62 package gcexportdata
63 64 import (
65 "bufio"
66 "bytes"
67 "encoding/json"
68 "fmt"
69 "go/token"
70 "go/types"
71 "io"
72 "os/exec"
73 74 "golang.org/x/tools/internal/gcimporter"
75 )
76 77 // Find returns the name of an object (.o) or archive (.a) file
78 // containing type information for the specified import path,
79 // using the go command.
80 // If no file was found, an empty filename is returned.
81 //
82 // A relative srcDir is interpreted relative to the current working directory.
83 //
84 // Find also returns the package's resolved (canonical) import path,
85 // reflecting the effects of srcDir and vendoring on importPath.
86 //
87 // Deprecated: Use the higher-level API in golang.org/x/tools/go/packages,
88 // which is more efficient.
89 func Find(importPath, srcDir string) (filename, path string) {
90 cmd := exec.Command("go", "list", "-json", "-export", "--", importPath)
91 cmd.Dir = srcDir
92 out, err := cmd.Output()
93 if err != nil {
94 return "", ""
95 }
96 var data struct {
97 ImportPath string
98 Export string
99 }
100 json.Unmarshal(out, &data)
101 return data.Export, data.ImportPath
102 }
103 104 // NewReader returns a reader for the export data section of an object
105 // (.o) or archive (.a) file read from r. The new reader may provide
106 // additional trailing data beyond the end of the export data.
107 func NewReader(r io.Reader) (io.Reader, error) {
108 buf := bufio.NewReader(r)
109 size, err := gcimporter.FindExportData(buf)
110 if err != nil {
111 return nil, err
112 }
113 114 // We were given an archive and found the __.PKGDEF in it.
115 // This tells us the size of the export data, and we don't
116 // need to return the entire file.
117 return &io.LimitedReader{
118 R: buf,
119 N: size,
120 }, nil
121 }
122 123 // readAll works the same way as io.ReadAll, but avoids allocations and copies
124 // by preallocating a byte slice of the necessary size if the size is known up
125 // front. This is always possible when the input is an archive. In that case,
126 // NewReader will return the known size using an io.LimitedReader.
127 func readAll(r io.Reader) ([]byte, error) {
128 if lr, ok := r.(*io.LimitedReader); ok {
129 data := make([]byte, lr.N)
130 _, err := io.ReadFull(lr, data)
131 return data, err
132 }
133 return io.ReadAll(r)
134 }
135 136 // Read reads export data from in, decodes it, and returns type
137 // information for the package.
138 //
139 // Read is capable of reading export data produced by [Write] at the
140 // same source code version, or by the last two Go releases (plus tip)
141 // of the standard Go compiler. Reading files from older compilers may
142 // produce an error.
143 //
144 // The package path (effectively its linker symbol prefix) is
145 // specified by path, since unlike the package name, this information
146 // may not be recorded in the export data.
147 //
148 // File position information is added to fset.
149 //
150 // Read may inspect and add to the imports map to ensure that references
151 // within the export data to other packages are consistent. The caller
152 // must ensure that imports[path] does not exist, or exists but is
153 // incomplete (see types.Package.Complete), and Read inserts the
154 // resulting package into this map entry.
155 //
156 // On return, the state of the reader is undefined.
157 func Read(in io.Reader, fset *token.FileSet, imports map[string]*types.Package, path string) (*types.Package, error) {
158 data, err := readAll(in)
159 if err != nil {
160 return nil, fmt.Errorf("reading export data for %q: %v", path, err)
161 }
162 163 if bytes.HasPrefix(data, []byte("!<arch>")) {
164 return nil, fmt.Errorf("can't read export data for %q directly from an archive file (call gcexportdata.NewReader first to extract export data)", path)
165 }
166 167 // The indexed export format starts with an 'i'; the older
168 // binary export format starts with a 'c', 'd', or 'v'
169 // (from "version"). Select appropriate importer.
170 if len(data) > 0 {
171 switch data[0] {
172 case 'v', 'c', 'd':
173 // binary, produced by cmd/compile till go1.10
174 return nil, fmt.Errorf("binary (%c) import format is no longer supported", data[0])
175 176 case 'i':
177 // indexed, produced by cmd/compile till go1.19,
178 // and also by [Write].
179 //
180 // If proposal #69491 is accepted, go/types
181 // serialization will be implemented by
182 // types.Export, to which Write would eventually
183 // delegate (explicitly dropping any pretence at
184 // inter-version Write-Read compatibility).
185 // This [Read] function would delegate to types.Import
186 // when it detects that the file was produced by Export.
187 _, pkg, err := gcimporter.IImportData(fset, imports, data[1:], path)
188 return pkg, err
189 190 case 'u':
191 // unified, produced by cmd/compile since go1.20
192 _, pkg, err := gcimporter.UImportData(fset, imports, data[1:], path)
193 return pkg, err
194 195 default:
196 l := min(len(data), 10)
197 return nil, fmt.Errorf("unexpected export data with prefix %q for path %s", string(data[:l]), path)
198 }
199 }
200 return nil, fmt.Errorf("empty export data for %s", path)
201 }
202 203 // Write writes encoded type information for the specified package to out.
204 // The FileSet provides file position information for named objects.
205 func Write(out io.Writer, fset *token.FileSet, pkg *types.Package) error {
206 if _, err := io.WriteString(out, "i"); err != nil {
207 return err
208 }
209 return gcimporter.IExportData(out, fset, pkg)
210 }
211 212 // ReadBundle reads an export bundle from in, decodes it, and returns type
213 // information for the packages.
214 // File position information is added to fset.
215 //
216 // ReadBundle may inspect and add to the imports map to ensure that references
217 // within the export bundle to other packages are consistent.
218 //
219 // On return, the state of the reader is undefined.
220 //
221 // Experimental: This API is experimental and may change in the future.
222 func ReadBundle(in io.Reader, fset *token.FileSet, imports map[string]*types.Package) ([]*types.Package, error) {
223 data, err := readAll(in)
224 if err != nil {
225 return nil, fmt.Errorf("reading export bundle: %v", err)
226 }
227 return gcimporter.IImportBundle(fset, imports, data)
228 }
229 230 // WriteBundle writes encoded type information for the specified packages to out.
231 // The FileSet provides file position information for named objects.
232 //
233 // Experimental: This API is experimental and may change in the future.
234 func WriteBundle(out io.Writer, fset *token.FileSet, pkgs []*types.Package) error {
235 return gcimporter.IExportBundle(out, fset, pkgs)
236 }
237