funcdata_go117.go raw
1 //go:build go1.17 && !go1.18
2 // +build go1.17,!go1.18
3
4 /*
5 * Copyright 2021 ByteDance Inc.
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19
20 package loader
21
22 import (
23 `os`
24 `unsafe`
25 `sort`
26
27 `github.com/bytedance/sonic/loader/internal/rt`
28 )
29
30 const (
31 _Magic uint32 = 0xfffffffa
32 )
33
34 type pcHeader struct {
35 magic uint32 // 0xFFFFFFF0
36 pad1, pad2 uint8 // 0,0
37 minLC uint8 // min instruction size
38 ptrSize uint8 // size of a ptr in bytes
39 nfunc int // number of functions in the module
40 nfiles uint // number of entries in the file tab
41 funcnameOffset uintptr // offset to the funcnametab variable from pcHeader
42 cuOffset uintptr // offset to the cutab variable from pcHeader
43 filetabOffset uintptr // offset to the filetab variable from pcHeader
44 pctabOffset uintptr // offset to the pctab variable from pcHeader
45 pclnOffset uintptr // offset to the pclntab variable from pcHeader
46 }
47
48 type moduledata struct {
49 pcHeader *pcHeader
50 funcnametab []byte
51 cutab []uint32
52 filetab []byte
53 pctab []byte
54 pclntable []byte
55 ftab []funcTab
56 findfunctab uintptr
57 minpc, maxpc uintptr // first func address, last func address + last func size
58
59 text, etext uintptr // start/end of text, (etext-text) must be greater than MIN_FUNC
60 noptrdata, enoptrdata uintptr
61 data, edata uintptr
62 bss, ebss uintptr
63 noptrbss, enoptrbss uintptr
64 end, gcdata, gcbss uintptr
65 types, etypes uintptr
66
67 textsectmap []textSection // see runtime/symtab.go: textAddr()
68 typelinks []int32 // offsets from types
69 itablinks []*rt.GoItab
70
71 ptab []ptabEntry
72
73 pluginpath string
74 pkghashes []modulehash
75
76 modulename string
77 modulehashes []modulehash
78
79 hasmain uint8 // 1 if module contains the main function, 0 otherwise
80
81 gcdatamask, gcbssmask bitVector
82
83 typemap map[int32]*rt.GoType // offset to *_rtype in previous module
84
85 bad bool // module failed to load and should be ignored
86
87 next *moduledata
88 }
89
90 type _func struct {
91 entry uintptr // start pc, as offset from moduledata.text/pcHeader.textStart
92 nameOff int32 // function name, as index into moduledata.funcnametab.
93
94 args int32 // in/out args size
95 deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any.
96
97 pcsp uint32
98 pcfile uint32
99 pcln uint32
100 npcdata uint32
101 cuOffset uint32 // runtime.cutab offset of this function's CU
102 funcID uint8 // set for certain special runtime functions
103 _ [2]byte // pad
104 nfuncdata uint8 //
105
106 // The end of the struct is followed immediately by two variable-length
107 // arrays that reference the pcdata and funcdata locations for this
108 // function.
109
110 // pcdata contains the offset into moduledata.pctab for the start of
111 // that index's table. e.g.,
112 // &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of
113 // the unsafe point table.
114 //
115 // An offset of 0 indicates that there is no table.
116 //
117 // pcdata [npcdata]uint32
118
119 // funcdata contains the offset past moduledata.gofunc which contains a
120 // pointer to that index's funcdata. e.g.,
121 // *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is
122 // the argument pointer map.
123 //
124 // An offset of ^uint32(0) indicates that there is no entry.
125 //
126 // funcdata [nfuncdata]uint32
127 }
128
129 type funcTab struct {
130 entry uintptr
131 funcoff uintptr
132 }
133
134 type bitVector struct {
135 n int32 // # of bits
136 bytedata *uint8
137 }
138
139 type ptabEntry struct {
140 name int32
141 typ int32
142 }
143
144 type textSection struct {
145 vaddr uintptr // prelinked section vaddr
146 end uintptr // vaddr + section length
147 baseaddr uintptr // relocated section address
148 }
149
150 type modulehash struct {
151 modulename string
152 linktimehash string
153 runtimehash *string
154 }
155
156 // findfuncbucket is an array of these structures.
157 // Each bucket represents 4096 bytes of the text segment.
158 // Each subbucket represents 256 bytes of the text segment.
159 // To find a function given a pc, locate the bucket and subbucket for
160 // that pc. Add together the idx and subbucket value to obtain a
161 // function index. Then scan the functab array starting at that
162 // index to find the target function.
163 // This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead.
164 type findfuncbucket struct {
165 idx uint32
166 _SUBBUCKETS [16]byte
167 }
168
169
170 type compilationUnit struct {
171 fileNames []string
172 }
173
174 func makeFtab(funcs []_func, maxpc uintptr) (ftab []funcTab, pclntabSize int64, startLocations []uint32) {
175 // Allocate space for the pc->func table. This structure consists of a pc offset
176 // and an offset to the func structure. After that, we have a single pc
177 // value that marks the end of the last function in the binary.
178 pclntabSize = int64(len(funcs)*2*int(_PtrSize) + int(_PtrSize))
179 startLocations = make([]uint32, len(funcs))
180 for i, f := range funcs {
181 pclntabSize = rnd(pclntabSize, int64(_PtrSize))
182 //writePCToFunc
183 startLocations[i] = uint32(pclntabSize)
184 pclntabSize += int64(uint8(_FUNC_SIZE) + f.nfuncdata*_PtrSize + uint8(f.npcdata)*4)
185 }
186 ftab = make([]funcTab, 0, len(funcs)+1)
187
188 // write a map of pc->func info offsets
189 for i, f := range funcs {
190 ftab = append(ftab, funcTab{uintptr(f.entry), uintptr(startLocations[i])})
191 }
192
193 // Final entry of table is just end pc offset.
194 ftab = append(ftab, funcTab{maxpc, 0})
195
196 return
197 }
198
199 // Pcln table format: [...]funcTab + [...]_Func
200 func makePclntable(size int64, startLocations []uint32, funcs []_func, maxpc uintptr, pcdataOffs [][]uint32, funcdataAddr uintptr, funcdataOffs [][]uint32) (pclntab []byte) {
201 pclntab = make([]byte, size, size)
202
203 // write a map of pc->func info offsets
204 offs := 0
205 for i, f := range funcs {
206 byteOrder.PutUint64(pclntab[offs:offs+8], uint64(f.entry))
207 byteOrder.PutUint64(pclntab[offs+8:offs+16], uint64(startLocations[i]))
208 offs += 16
209 }
210 // Final entry of table is just end pc offset.
211 byteOrder.PutUint64(pclntab[offs:offs+8], uint64(maxpc))
212 offs += 8
213
214 // write func info table
215 for i, f := range funcs {
216 off := startLocations[i]
217
218 // write _func structure to pclntab
219 byteOrder.PutUint64(pclntab[off:off+8], uint64(f.entry))
220 off += 8
221 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.nameOff))
222 off += 4
223 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.args))
224 off += 4
225 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.deferreturn))
226 off += 4
227 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcsp))
228 off += 4
229 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcfile))
230 off += 4
231 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcln))
232 off += 4
233 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.npcdata))
234 off += 4
235 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.cuOffset))
236 off += 4
237 pclntab[off] = f.funcID
238 // NOTICE: _[2]byte alignment
239 off += 3
240 pclntab[off] = f.nfuncdata
241 off += 1
242
243 // NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3
244 for j := 3; j < len(pcdataOffs[i]); j++ {
245 byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j]))
246 off += 4
247 }
248
249 off = uint32(rnd(int64(off), int64(_PtrSize)))
250
251 // funcdata refs as offsets from gofunc
252 for _, funcdata := range funcdataOffs[i] {
253 if funcdata == _INVALID_FUNCDATA_OFFSET {
254 byteOrder.PutUint64(pclntab[off:off+8], 0)
255 } else {
256 byteOrder.PutUint64(pclntab[off:off+8], uint64(funcdataAddr)+uint64(funcdata))
257 }
258 off += 8
259 }
260 }
261
262 return
263 }
264
265 // findfunc table used to map pc to belonging func,
266 // returns the index in the func table.
267 //
268 // All text section are divided into buckets sized _BUCKETSIZE(4K):
269 // every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64),
270 // and it has a base idx to plus the offset stored in jth subbucket.
271 // see findfunc() in runtime/symtab.go
272 func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) {
273 start = len(*out)
274
275 max := ftab[len(ftab)-1].entry
276 min := ftab[0].entry
277 nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE
278 n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE
279
280 tab := make([]findfuncbucket, 0, nbuckets)
281 var s, e = 0, 0
282 for i := 0; i<int(nbuckets); i++ {
283 // store the start func of the bucket
284 var fb = findfuncbucket{idx: uint32(s)}
285
286 // find the last e-th func of the bucket
287 var pc = min + uintptr((i+1)*_BUCKETSIZE)
288 for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {}
289
290 for j := 0; j<_SUBBUCKETS && (i*_SUBBUCKETS+j)<int(n); j++ {
291 // store the start func of the subbucket
292 fb._SUBBUCKETS[j] = byte(uint32(s) - fb.idx)
293
294 // find the s-th end func of the subbucket
295 pc = min + uintptr(i*_BUCKETSIZE) + uintptr((j+1)*_SUB_BUCKETSIZE)
296 for ; s < len(ftab)-1 && ftab[s+1].entry <= pc; s++ {}
297 }
298
299 s = e
300 tab = append(tab, fb)
301 }
302
303 // write findfuncbucket
304 if len(tab) > 0 {
305 size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab)
306 *out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...)
307 }
308 return
309 }
310
311 func makeModuledata(name string, filenames []string, funcsp *[]Func, text []byte) (mod *moduledata) {
312 mod = new(moduledata)
313 mod.modulename = name
314
315 // sort funcs by entry
316 funcs := *funcsp
317 sort.Slice(funcs, func(i, j int) bool {
318 return funcs[i].EntryOff < funcs[j].EntryOff
319 })
320 *funcsp = funcs
321
322 // make filename table
323 cu := make([]string, 0, len(filenames))
324 cu = append(cu, filenames...)
325 cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}})
326 mod.cutab = cutab
327 mod.filetab = filetab
328
329 // make funcname table
330 funcnametab, nameOffs := makeFuncnameTab(funcs)
331 mod.funcnametab = funcnametab
332
333 // mmap() text and funcdata segments
334 p := os.Getpagesize()
335 size := int(rnd(int64(len(text)), int64(p)))
336 addr := mmap(size)
337 // copy the machine code
338 s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size)
339 copy(s, text)
340 // make it executable
341 mprotect(addr, size)
342
343 // assign addresses
344 mod.text = addr
345 mod.etext = addr + uintptr(size)
346 mod.minpc = addr
347 mod.maxpc = addr + uintptr(len(text))
348
349 // make pcdata table
350 // NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata
351 cuOff := cuOffs[0]
352 pctab, pcdataOffs, _funcs := makePctab(funcs, addr, cuOff, nameOffs)
353 mod.pctab = pctab
354
355 // write func data
356 // NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata
357 // TODO: estimate accurate capacity
358 cache := make([]byte, 0, len(funcs)*int(_PtrSize))
359 fstart, funcdataOffs := writeFuncdata(&cache, funcs)
360
361 // make pc->func (binary search) func table
362 ftab, pclntSize, startLocations := makeFtab(_funcs, mod.maxpc)
363 mod.ftab = ftab
364
365 // write pc->func (modmap) findfunc table
366 ffstart := writeFindfunctab(&cache, ftab)
367
368 // cache funcdata and findfuncbucket
369 moduleCache.Lock()
370 moduleCache.m[mod] = cache
371 moduleCache.Unlock()
372 mod.findfunctab = uintptr(rt.IndexByte(cache, ffstart))
373 funcdataAddr := uintptr(rt.IndexByte(cache, fstart))
374
375 // make pclnt table
376 pclntab := makePclntable(pclntSize, startLocations, _funcs, mod.maxpc, pcdataOffs, funcdataAddr, funcdataOffs)
377 mod.pclntable = pclntab
378
379 // make pc header
380 mod.pcHeader = &pcHeader {
381 magic : _Magic,
382 minLC : _MinLC,
383 ptrSize : _PtrSize,
384 nfunc : len(funcs),
385 nfiles: uint(len(cu)),
386 funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"),
387 cuOffset: getOffsetOf(moduledata{}, "cutab"),
388 filetabOffset: getOffsetOf(moduledata{}, "filetab"),
389 pctabOffset: getOffsetOf(moduledata{}, "pctab"),
390 pclnOffset: getOffsetOf(moduledata{}, "pclntable"),
391 }
392
393 // special case: gcdata and gcbss must by non-empty
394 mod.gcdata = uintptr(unsafe.Pointer(&emptyByte))
395 mod.gcbss = uintptr(unsafe.Pointer(&emptyByte))
396
397 return
398 }
399
400 // makePctab generates pcdelta->valuedelta tables for functions,
401 // and returns the table and the entry offset of every kind pcdata in the table.
402 func makePctab(funcs []Func, addr uintptr, cuOffset uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) {
403 _funcs = make([]_func, len(funcs))
404
405 // Pctab offsets of 0 are considered invalid in the runtime. We respect
406 // that by just padding a single byte at the beginning of runtime.pctab,
407 // that way no real offsets can be zero.
408 pctab = make([]byte, 1, 12*len(funcs)+1)
409 pcdataOffs = make([][]uint32, len(funcs))
410
411 for i, f := range funcs {
412 _f := &_funcs[i]
413
414 var writer = func(pc *Pcdata) {
415 var ab []byte
416 var err error
417 if pc != nil {
418 ab, err = pc.MarshalBinary()
419 if err != nil {
420 panic(err)
421 }
422 pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab)))
423 } else {
424 ab = []byte{0}
425 pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET)
426 }
427 pctab = append(pctab, ab...)
428 }
429
430 if f.Pcsp != nil {
431 _f.pcsp = uint32(len(pctab))
432 }
433 writer(f.Pcsp)
434 if f.Pcfile != nil {
435 _f.pcfile = uint32(len(pctab))
436 }
437 writer(f.Pcfile)
438 if f.Pcline != nil {
439 _f.pcln = uint32(len(pctab))
440 }
441 writer(f.Pcline)
442 writer(f.PcUnsafePoint)
443 writer(f.PcStackMapIndex)
444 writer(f.PcInlTreeIndex)
445 writer(f.PcArgLiveIndex)
446
447 _f.entry = addr + uintptr(f.EntryOff)
448 _f.nameOff = nameOffset[i]
449 _f.args = f.ArgsSize
450 _f.deferreturn = f.DeferReturn
451 // NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)]
452 _f.npcdata = uint32(_N_PCDATA)
453 _f.cuOffset = cuOffset
454 _f.funcID = f.ID
455 _f.nfuncdata = uint8(_N_FUNCDATA)
456 }
457
458 return
459 }
460
461 func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}
462