1 // Copyright 2011 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4 5 // This file should be kept in sync with $GOROOT/src/internal/exportdata/exportdata.go.
6 // This file also additionally implements FindExportData for gcexportdata.NewReader.
7 8 package gcimporter
9 10 import (
11 "bufio"
12 "bytes"
13 "errors"
14 "fmt"
15 "go/build"
16 "io"
17 "os"
18 "os/exec"
19 "path/filepath"
20 "strings"
21 "sync"
22 )
23 24 // FindExportData positions the reader r at the beginning of the
25 // export data section of an underlying cmd/compile created archive
26 // file by reading from it. The reader must be positioned at the
27 // start of the file before calling this function.
28 // This returns the length of the export data in bytes.
29 //
30 // This function is needed by [gcexportdata.Read], which must
31 // accept inputs produced by the last two releases of cmd/compile,
32 // plus tip.
33 func FindExportData(r *bufio.Reader) (size int64, err error) {
34 arsize, err := FindPackageDefinition(r)
35 if err != nil {
36 return
37 }
38 size = int64(arsize)
39 40 objapi, headers, err := ReadObjectHeaders(r)
41 if err != nil {
42 return
43 }
44 size -= int64(len(objapi))
45 for _, h := range headers {
46 size -= int64(len(h))
47 }
48 49 // Check for the binary export data section header "$$B\n".
50 // TODO(taking): Unify with ReadExportDataHeader so that it stops at the 'u' instead of reading
51 line, err := r.ReadSlice('\n')
52 if err != nil {
53 return
54 }
55 hdr := string(line)
56 if hdr != "$$B\n" {
57 err = fmt.Errorf("unknown export data header: %q", hdr)
58 return
59 }
60 size -= int64(len(hdr))
61 62 // For files with a binary export data header "$$B\n",
63 // these are always terminated by an end-of-section marker "\n$$\n".
64 // So the last bytes must always be this constant.
65 //
66 // The end-of-section marker is not a part of the export data itself.
67 // Do not include these in size.
68 //
69 // It would be nice to have sanity check that the final bytes after
70 // the export data are indeed the end-of-section marker. The split
71 // of gcexportdata.NewReader and gcexportdata.Read make checking this
72 // ugly so gcimporter gives up enforcing this. The compiler and go/types
73 // importer do enforce this, which seems good enough.
74 const endofsection = "\n$$\n"
75 size -= int64(len(endofsection))
76 77 if size < 0 {
78 err = fmt.Errorf("invalid size (%d) in the archive file: %d bytes remain without section headers (recompile package)", arsize, size)
79 return
80 }
81 82 return
83 }
84 85 // ReadUnified reads the contents of the unified export data from a reader r
86 // that contains the contents of a GC-created archive file.
87 //
88 // On success, the reader will be positioned after the end-of-section marker "\n$$\n".
89 //
90 // Supported GC-created archive files have 4 layers of nesting:
91 // - An archive file containing a package definition file.
92 // - The package definition file contains headers followed by a data section.
93 // Headers are lines (≤ 4kb) that do not start with "$$".
94 // - The data section starts with "$$B\n" followed by export data followed
95 // by an end of section marker "\n$$\n". (The section start "$$\n" is no
96 // longer supported.)
97 // - The export data starts with a format byte ('u') followed by the <data> in
98 // the given format. (See ReadExportDataHeader for older formats.)
99 //
100 // Putting this together, the bytes in a GC-created archive files are expected
101 // to look like the following.
102 // See cmd/internal/archive for more details on ar file headers.
103 //
104 // | <!arch>\n | ar file signature
105 // | __.PKGDEF...size...\n | ar header for __.PKGDEF including size.
106 // | go object <...>\n | objabi header
107 // | <optional headers>\n | other headers such as build id
108 // | $$B\n | binary format marker
109 // | u<data>\n | unified export <data>
110 // | $$\n | end-of-section marker
111 // | [optional padding] | padding byte (0x0A) if size is odd
112 // | [ar file header] | other ar files
113 // | [ar file data] |
114 func ReadUnified(r *bufio.Reader) (data []byte, err error) {
115 // We historically guaranteed headers at the default buffer size (4096) work.
116 // This ensures we can use ReadSlice throughout.
117 const minBufferSize = 4096
118 r = bufio.NewReaderSize(r, minBufferSize)
119 120 size, err := FindPackageDefinition(r)
121 if err != nil {
122 return
123 }
124 n := size
125 126 objapi, headers, err := ReadObjectHeaders(r)
127 if err != nil {
128 return
129 }
130 n -= len(objapi)
131 for _, h := range headers {
132 n -= len(h)
133 }
134 135 hdrlen, err := ReadExportDataHeader(r)
136 if err != nil {
137 return
138 }
139 n -= hdrlen
140 141 // size also includes the end of section marker. Remove that many bytes from the end.
142 const marker = "\n$$\n"
143 n -= len(marker)
144 145 if n < 0 {
146 err = fmt.Errorf("invalid size (%d) in the archive file: %d bytes remain without section headers (recompile package)", size, n)
147 return
148 }
149 150 // Read n bytes from buf.
151 data = make([]byte, n)
152 _, err = io.ReadFull(r, data)
153 if err != nil {
154 return
155 }
156 157 // Check for marker at the end.
158 var suffix [len(marker)]byte
159 _, err = io.ReadFull(r, suffix[:])
160 if err != nil {
161 return
162 }
163 if s := string(suffix[:]); s != marker {
164 err = fmt.Errorf("read %q instead of end-of-section marker (%q)", s, marker)
165 return
166 }
167 168 return
169 }
170 171 // FindPackageDefinition positions the reader r at the beginning of a package
172 // definition file ("__.PKGDEF") within a GC-created archive by reading
173 // from it, and returns the size of the package definition file in the archive.
174 //
175 // The reader must be positioned at the start of the archive file before calling
176 // this function, and "__.PKGDEF" is assumed to be the first file in the archive.
177 //
178 // See cmd/internal/archive for details on the archive format.
179 func FindPackageDefinition(r *bufio.Reader) (size int, err error) {
180 // Uses ReadSlice to limit risk of malformed inputs.
181 182 // Read first line to make sure this is an object file.
183 line, err := r.ReadSlice('\n')
184 if err != nil {
185 err = fmt.Errorf("can't find export data (%v)", err)
186 return
187 }
188 189 // Is the first line an archive file signature?
190 if string(line) != "!<arch>\n" {
191 err = fmt.Errorf("not the start of an archive file (%q)", line)
192 return
193 }
194 195 // package export block should be first
196 size = readArchiveHeader(r, "__.PKGDEF")
197 if size <= 0 {
198 err = fmt.Errorf("not a package file")
199 return
200 }
201 202 return
203 }
204 205 // ReadObjectHeaders reads object headers from the reader. Object headers are
206 // lines that do not start with an end-of-section marker "$$". The first header
207 // is the objabi header. On success, the reader will be positioned at the beginning
208 // of the end-of-section marker.
209 //
210 // It returns an error if any header does not fit in r.Size() bytes.
211 func ReadObjectHeaders(r *bufio.Reader) (objapi string, headers []string, err error) {
212 // line is a temporary buffer for headers.
213 // Use bounded reads (ReadSlice, Peek) to limit risk of malformed inputs.
214 var line []byte
215 216 // objapi header should be the first line
217 if line, err = r.ReadSlice('\n'); err != nil {
218 err = fmt.Errorf("can't find export data (%v)", err)
219 return
220 }
221 objapi = string(line)
222 223 // objapi header begins with "go object ".
224 if !strings.HasPrefix(objapi, "go object ") {
225 err = fmt.Errorf("not a go object file: %s", objapi)
226 return
227 }
228 229 // process remaining object header lines
230 for {
231 // check for an end of section marker "$$"
232 line, err = r.Peek(2)
233 if err != nil {
234 return
235 }
236 if string(line) == "$$" {
237 return // stop
238 }
239 240 // read next header
241 line, err = r.ReadSlice('\n')
242 if err != nil {
243 return
244 }
245 headers = append(headers, string(line))
246 }
247 }
248 249 // ReadExportDataHeader reads the export data header and format from r.
250 // It returns the number of bytes read, or an error if the format is no longer
251 // supported or it failed to read.
252 //
253 // The only currently supported format is binary export data in the
254 // unified export format.
255 func ReadExportDataHeader(r *bufio.Reader) (n int, err error) {
256 // Read export data header.
257 line, err := r.ReadSlice('\n')
258 if err != nil {
259 return
260 }
261 262 hdr := string(line)
263 switch hdr {
264 case "$$\n":
265 err = fmt.Errorf("old textual export format no longer supported (recompile package)")
266 return
267 268 case "$$B\n":
269 var format byte
270 format, err = r.ReadByte()
271 if err != nil {
272 return
273 }
274 // The unified export format starts with a 'u'.
275 switch format {
276 case 'u':
277 default:
278 // Older no longer supported export formats include:
279 // indexed export format which started with an 'i'; and
280 // the older binary export format which started with a 'c',
281 // 'd', or 'v' (from "version").
282 err = fmt.Errorf("binary export format %q is no longer supported (recompile package)", format)
283 return
284 }
285 286 default:
287 err = fmt.Errorf("unknown export data header: %q", hdr)
288 return
289 }
290 291 n = len(hdr) + 1 // + 1 is for 'u'
292 return
293 }
294 295 // FindPkg returns the filename and unique package id for an import
296 // path based on package information provided by build.Import (using
297 // the build.Default build.Context). A relative srcDir is interpreted
298 // relative to the current working directory.
299 //
300 // FindPkg is only used in tests within x/tools.
301 func FindPkg(path, srcDir string) (filename, id string, err error) {
302 // TODO(taking): Move internal/exportdata.FindPkg into its own file,
303 // and then this copy into a _test package.
304 if path == "" {
305 return "", "", errors.New("path is empty")
306 }
307 308 var noext string
309 switch {
310 default:
311 // "x" -> "$GOPATH/pkg/$GOOS_$GOARCH/x.ext", "x"
312 // Don't require the source files to be present.
313 if abs, err := filepath.Abs(srcDir); err == nil { // see issue 14282
314 srcDir = abs
315 }
316 var bp *build.Package
317 bp, err = build.Import(path, srcDir, build.FindOnly|build.AllowBinary)
318 if bp.PkgObj == "" {
319 if bp.Goroot && bp.Dir != "" {
320 filename, err = lookupGorootExport(bp.Dir)
321 if err == nil {
322 _, err = os.Stat(filename)
323 }
324 if err == nil {
325 return filename, bp.ImportPath, nil
326 }
327 }
328 goto notfound
329 } else {
330 noext = strings.TrimSuffix(bp.PkgObj, ".a")
331 }
332 id = bp.ImportPath
333 334 case build.IsLocalImport(path):
335 // "./x" -> "/this/directory/x.ext", "/this/directory/x"
336 noext = filepath.Join(srcDir, path)
337 id = noext
338 339 case filepath.IsAbs(path):
340 // for completeness only - go/build.Import
341 // does not support absolute imports
342 // "/x" -> "/x.ext", "/x"
343 noext = path
344 id = path
345 }
346 347 if false { // for debugging
348 if path != id {
349 fmt.Printf("%s -> %s\n", path, id)
350 }
351 }
352 353 // try extensions
354 for _, ext := range pkgExts {
355 filename = noext + ext
356 f, statErr := os.Stat(filename)
357 if statErr == nil && !f.IsDir() {
358 return filename, id, nil
359 }
360 if err == nil {
361 err = statErr
362 }
363 }
364 365 notfound:
366 if err == nil {
367 return "", path, fmt.Errorf("can't find import: %q", path)
368 }
369 return "", path, fmt.Errorf("can't find import: %q: %w", path, err)
370 }
371 372 var pkgExts = [...]string{".a", ".o"} // a file from the build cache will have no extension
373 374 var exportMap sync.Map // package dir → func() (string, error)
375 376 // lookupGorootExport returns the location of the export data
377 // (normally found in the build cache, but located in GOROOT/pkg
378 // in prior Go releases) for the package located in pkgDir.
379 //
380 // (We use the package's directory instead of its import path
381 // mainly to simplify handling of the packages in src/vendor
382 // and cmd/vendor.)
383 //
384 // lookupGorootExport is only used in tests within x/tools.
385 func lookupGorootExport(pkgDir string) (string, error) {
386 f, ok := exportMap.Load(pkgDir)
387 if !ok {
388 var (
389 listOnce sync.Once
390 exportPath string
391 err error
392 )
393 f, _ = exportMap.LoadOrStore(pkgDir, func() (string, error) {
394 listOnce.Do(func() {
395 cmd := exec.Command(filepath.Join(build.Default.GOROOT, "bin", "go"), "list", "-export", "-f", "{{.Export}}", pkgDir)
396 cmd.Dir = build.Default.GOROOT
397 cmd.Env = append(os.Environ(), "PWD="+cmd.Dir, "GOROOT="+build.Default.GOROOT)
398 var output []byte
399 output, err = cmd.Output()
400 if err != nil {
401 if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
402 err = errors.New(string(ee.Stderr))
403 }
404 return
405 }
406 407 exports := strings.Split(string(bytes.TrimSpace(output)), "\n")
408 if len(exports) != 1 {
409 err = fmt.Errorf("go list reported %d exports; expected 1", len(exports))
410 return
411 }
412 413 exportPath = exports[0]
414 })
415 416 return exportPath, err
417 })
418 }
419 420 return f.(func() (string, error))()
421 }
422