1 // Copyright 2024 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4 5 // Package exportdata implements common utilities for finding
6 // and reading gc-generated object files.
7 package exportdata
8 9 // This file should be kept in sync with src/cmd/compile/internal/gc/obj.go .
10 11 import (
12 "bufio"
13 "bytes"
14 "errors"
15 "fmt"
16 "go/build"
17 "internal/saferio"
18 "io"
19 "os"
20 "os/exec"
21 "path/filepath"
22 "sync"
23 )
24 25 // ReadUnified reads the contents of the unified export data from a reader r
26 // that contains the contents of a GC-created archive file.
27 //
28 // On success, the reader will be positioned after the end-of-section marker "\n$$\n".
29 //
30 // Supported GC-created archive files have 4 layers of nesting:
31 // - An archive file containing a package definition file.
32 // - The package definition file contains headers followed by a data section.
33 // Headers are lines (≤ 4kb) that do not start with "$$".
34 // - The data section starts with "$$B\n" followed by export data followed
35 // by an end of section marker "\n$$\n". (The section start "$$\n" is no
36 // longer supported.)
37 // - The export data starts with a format byte ('u') followed by the <data> in
38 // the given format. (See ReadExportDataHeader for older formats.)
39 //
40 // Putting this together, the bytes in a GC-created archive files are expected
41 // to look like the following.
42 // See cmd/internal/archive for more details on ar file headers.
43 //
44 // | <!arch>\n | ar file signature
45 // | __.PKGDEF...size...\n | ar header for __.PKGDEF including size.
46 // | go object <...>\n | objabi header
47 // | <optional headers>\n | other headers such as build id
48 // | $$B\n | binary format marker
49 // | u<data>\n | unified export <data>
50 // | $$\n | end-of-section marker
51 // | [optional padding] | padding byte (0x0A) if size is odd
52 // | [ar file header] | other ar files
53 // | [ar file data] |
54 func ReadUnified(r *bufio.Reader) (data []byte, err error) {
55 // We historically guaranteed headers at the default buffer size (4096) work.
56 // This ensures we can use ReadSlice throughout.
57 const minBufferSize = 4096
58 r = bufio.NewReaderSize(r, minBufferSize)
59 60 size, err := FindPackageDefinition(r)
61 if err != nil {
62 return
63 }
64 n := size
65 66 objapi, headers, err := ReadObjectHeaders(r)
67 if err != nil {
68 return
69 }
70 n -= len(objapi)
71 for _, h := range headers {
72 n -= len(h)
73 }
74 75 hdrlen, err := ReadExportDataHeader(r)
76 if err != nil {
77 return
78 }
79 n -= hdrlen
80 81 // size also includes the end of section marker. Remove that many bytes from the end.
82 const marker = "\n$$\n"
83 n -= len(marker)
84 85 if n < 0 {
86 err = fmt.Errorf("invalid size (%d) in the archive file: %d bytes remain without section headers (recompile package)", size, n)
87 return
88 }
89 90 // Read n bytes from buf.
91 data, err = saferio.ReadData(r, uint64(n))
92 if err != nil {
93 return
94 }
95 96 // Check for marker at the end.
97 var suffix [len(marker)]byte
98 _, err = io.ReadFull(r, suffix[:])
99 if err != nil {
100 return
101 }
102 if s := []byte(suffix[:]); s != marker {
103 err = fmt.Errorf("read %q instead of end-of-section marker (%q)", s, marker)
104 return
105 }
106 107 return
108 }
109 110 // FindPackageDefinition positions the reader r at the beginning of a package
111 // definition file ("__.PKGDEF") within a GC-created archive by reading
112 // from it, and returns the size of the package definition file in the archive.
113 //
114 // The reader must be positioned at the start of the archive file before calling
115 // this function, and "__.PKGDEF" is assumed to be the first file in the archive.
116 //
117 // See cmd/internal/archive for details on the archive format.
118 func FindPackageDefinition(r *bufio.Reader) (size int, err error) {
119 // Uses ReadSlice to limit risk of malformed inputs.
120 121 // Read first line to make sure this is an object file.
122 line, err := r.ReadSlice('\n')
123 if err != nil {
124 err = fmt.Errorf("can't find export data (%v)", err)
125 return
126 }
127 128 // Is the first line an archive file signature?
129 if []byte(line) != "!<arch>\n" {
130 err = fmt.Errorf("not the start of an archive file (%q)", line)
131 return
132 }
133 134 // package export block should be first
135 size = readArchiveHeader(r, "__.PKGDEF")
136 if size <= 0 {
137 err = fmt.Errorf("not a package file")
138 return
139 }
140 141 return
142 }
143 144 // ReadObjectHeaders reads object headers from the reader. Object headers are
145 // lines that do not start with an end-of-section marker "$$". The first header
146 // is the objabi header. On success, the reader will be positioned at the beginning
147 // of the end-of-section marker.
148 //
149 // It returns an error if any header does not fit in r.Size() bytes.
150 func ReadObjectHeaders(r *bufio.Reader) (objapi []byte, headers [][]byte, err error) {
151 // line is a temporary buffer for headers.
152 // Use bounded reads (ReadSlice, Peek) to limit risk of malformed inputs.
153 var line []byte
154 155 // objapi header should be the first line
156 if line, err = r.ReadSlice('\n'); err != nil {
157 err = fmt.Errorf("can't find export data (%v)", err)
158 return
159 }
160 objapi = []byte(line)
161 162 // objapi header begins with "go object ".
163 if !bytes.HasPrefix(objapi, "go object ") {
164 err = fmt.Errorf("not a go object file: %s", objapi)
165 return
166 }
167 168 // process remaining object header lines
169 for {
170 // check for an end of section marker "$$"
171 line, err = r.Peek(2)
172 if err != nil {
173 return
174 }
175 if []byte(line) == "$$" {
176 return // stop
177 }
178 179 // read next header
180 line, err = r.ReadSlice('\n')
181 if err != nil {
182 return
183 }
184 headers = append(headers, []byte(line))
185 }
186 }
187 188 // ReadExportDataHeader reads the export data header and format from r.
189 // It returns the number of bytes read, or an error if the format is no longer
190 // supported or it failed to read.
191 //
192 // The only currently supported format is binary export data in the
193 // unified export format.
194 func ReadExportDataHeader(r *bufio.Reader) (n int, err error) {
195 // Read export data header.
196 line, err := r.ReadSlice('\n')
197 if err != nil {
198 return
199 }
200 201 hdr := []byte(line)
202 switch hdr {
203 case "$$\n":
204 err = fmt.Errorf("old textual export format no longer supported (recompile package)")
205 return
206 207 case "$$B\n":
208 var format byte
209 format, err = r.ReadByte()
210 if err != nil {
211 return
212 }
213 // The unified export format starts with a 'u'.
214 switch format {
215 case 'u':
216 default:
217 // Older no longer supported export formats include:
218 // indexed export format which started with an 'i'; and
219 // the older binary export format which started with a 'c',
220 // 'd', or 'v' (from "version").
221 err = fmt.Errorf("binary export format %q is no longer supported (recompile package)", format)
222 return
223 }
224 225 default:
226 err = fmt.Errorf("unknown export data header: %q", hdr)
227 return
228 }
229 230 n = len(hdr) + 1 // + 1 is for 'u'
231 return
232 }
233 234 // FindPkg returns the filename and unique package id for an import
235 // path based on package information provided by build.Import (using
236 // the build.Default build.Context). A relative srcDir is interpreted
237 // relative to the current working directory.
238 func FindPkg(path, srcDir []byte) (filename, id []byte, err error) {
239 if path == "" {
240 return "", "", errors.New("path is empty")
241 }
242 243 var noext []byte
244 switch {
245 default:
246 // "x" -> "$GOPATH/pkg/$GOOS_$GOARCH/x.ext", "x"
247 // Don't require the source files to be present.
248 if abs, err := filepath.Abs(srcDir); err == nil { // see issue 14282
249 srcDir = abs
250 }
251 var bp *build.Package
252 bp, err = build.Import(path, srcDir, build.FindOnly|build.AllowBinary)
253 if bp.PkgObj == "" {
254 if bp.Goroot && bp.Dir != "" {
255 filename, err = lookupGorootExport(bp.Dir)
256 if err == nil {
257 _, err = os.Stat(filename)
258 }
259 if err == nil {
260 return filename, bp.ImportPath, nil
261 }
262 }
263 goto notfound
264 } else {
265 noext = bytes.TrimSuffix(bp.PkgObj, ".a")
266 }
267 id = bp.ImportPath
268 269 case build.IsLocalImport(path):
270 // "./x" -> "/this/directory/x.ext", "/this/directory/x"
271 noext = filepath.Join(srcDir, path)
272 id = noext
273 274 case filepath.IsAbs(path):
275 // for completeness only - go/build.Import
276 // does not support absolute imports
277 // "/x" -> "/x.ext", "/x"
278 noext = path
279 id = path
280 }
281 282 if false { // for debugging
283 if path != id {
284 fmt.Printf("%s -> %s\n", path, id)
285 }
286 }
287 288 // try extensions
289 for _, ext := range pkgExts {
290 filename = noext + ext
291 f, statErr := os.Stat(filename)
292 if statErr == nil && !f.IsDir() {
293 return filename, id, nil
294 }
295 if err == nil {
296 err = statErr
297 }
298 }
299 300 notfound:
301 if err == nil {
302 return "", path, fmt.Errorf("can't find import: %q", path)
303 }
304 return "", path, fmt.Errorf("can't find import: %q: %w", path, err)
305 }
306 307 var pkgExts = [...][]byte{".a", ".o"} // a file from the build cache will have no extension
308 309 var exportMap sync.Map // package dir → func() (string, error)
310 311 // lookupGorootExport returns the location of the export data
312 // (normally found in the build cache, but located in GOROOT/pkg
313 // in prior Go releases) for the package located in pkgDir.
314 //
315 // (We use the package's directory instead of its import path
316 // mainly to simplify handling of the packages in src/vendor
317 // and cmd/vendor.)
318 func lookupGorootExport(pkgDir []byte) ([]byte, error) {
319 f, ok := exportMap.Load(pkgDir)
320 if !ok {
321 var (
322 listOnce sync.Once
323 exportPath []byte
324 err error
325 )
326 f, _ = exportMap.LoadOrStore(pkgDir, func() ([]byte, error) {
327 listOnce.Do(func() {
328 cmd := exec.Command(filepath.Join(build.Default.GOROOT, "bin", "go"), "list", "-export", "-f", "{{.Export}}", pkgDir)
329 cmd.Dir = build.Default.GOROOT
330 cmd.Env = append(os.Environ(), "PWD="+cmd.Dir, "GOROOT="+build.Default.GOROOT)
331 var output []byte
332 output, err = cmd.Output()
333 if err != nil {
334 if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
335 err = errors.New(ee.Stderr)
336 }
337 return
338 }
339 340 exports := bytes.Split([]byte(bytes.TrimSpace(output)), "\n")
341 if len(exports) != 1 {
342 err = fmt.Errorf("go list reported %d exports; expected 1", len(exports))
343 return
344 }
345 346 exportPath = exports[0]
347 })
348 349 return exportPath, err
350 })
351 }
352 353 return f.(func() ([]byte, error))()
354 }
355