funcdata_compat.go raw
1 //go:build !go1.17 || go1.25
2 // +build !go1.17 go1.25
3
4 /*
5 * Copyright 2021 ByteDance Inc.
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19
20 package loader
21
22 import (
23 `os`
24 `unsafe`
25 `sort`
26
27 `github.com/bytedance/sonic/loader/internal/rt`
28 )
29
30 const (
31 _Magic uint32 = 0xfffffffa
32 )
33
34 type pcHeader struct {
35 magic uint32 // 0xFFFFFFF0
36 pad1, pad2 uint8 // 0,0
37 minLC uint8 // min instruction size
38 ptrSize uint8 // size of a ptr in bytes
39 nfunc int // number of functions in the module
40 nfiles uint // number of entries in the file tab
41 funcnameOffset uintptr // offset to the funcnametab variable from pcHeader
42 cuOffset uintptr // offset to the cutab variable from pcHeader
43 filetabOffset uintptr // offset to the filetab variable from pcHeader
44 pctabOffset uintptr // offset to the pctab variable from pcHeader
45 pclnOffset uintptr // offset to the pclntab variable from pcHeader
46 }
47
48 type moduledata struct {
49 pcHeader *pcHeader
50 funcnametab []byte
51 cutab []uint32
52 filetab []byte
53 pctab []byte
54 pclntable []byte
55 ftab []funcTab
56 findfunctab uintptr
57 minpc, maxpc uintptr // first func address, last func address + last func size
58
59 text, etext uintptr // start/end of text, (etext-text) must be greater than MIN_FUNC
60 noptrdata, enoptrdata uintptr
61 data, edata uintptr
62 bss, ebss uintptr
63 noptrbss, enoptrbss uintptr
64 end, gcdata, gcbss uintptr
65 types, etypes uintptr
66
67 textsectmap []textSection // see runtime/symtab.go: textAddr()
68 typelinks []int32 // offsets from types
69 itablinks []*rt.GoItab
70
71 ptab []ptabEntry
72
73 pluginpath string
74 pkghashes []modulehash
75
76 modulename string
77 modulehashes []modulehash
78
79 hasmain uint8 // 1 if module contains the main function, 0 otherwise
80
81 gcdatamask, gcbssmask bitVector
82
83 typemap map[int32]*rt.GoType // offset to *_rtype in previous module
84
85 bad bool // module failed to load and should be ignored
86
87 next *moduledata
88 }
89
90 type _func struct {
91 entry uintptr // start pc, as offset from moduledata.text/pcHeader.textStart
92 nameOff int32 // function name, as index into moduledata.funcnametab.
93
94 args int32 // in/out args size
95 deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any.
96
97 pcsp uint32
98 pcfile uint32
99 pcln uint32
100 npcdata uint32
101 cuOffset uint32 // runtime.cutab offset of this function's CU
102 funcID uint8 // set for certain special runtime functions
103 _ [2]byte // pad
104 nfuncdata uint8 //
105
106 // The end of the struct is followed immediately by two variable-length
107 // arrays that reference the pcdata and funcdata locations for this
108 // function.
109
110 // pcdata contains the offset into moduledata.pctab for the start of
111 // that index's table. e.g.,
112 // &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of
113 // the unsafe point table.
114 //
115 // An offset of 0 indicates that there is no table.
116 //
117 // pcdata [npcdata]uint32
118
119 // funcdata contains the offset past moduledata.gofunc which contains a
120 // pointer to that index's funcdata. e.g.,
121 // *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is
122 // the argument pointer map.
123 //
124 // An offset of ^uint32(0) indicates that there is no entry.
125 //
126 // funcdata [nfuncdata]uint32
127 }
128
129 type funcTab struct {
130 entry uintptr
131 funcoff uintptr
132 }
133
134 type bitVector struct {
135 n int32 // # of bits
136 bytedata *uint8
137 }
138
139 type ptabEntry struct {
140 name int32
141 typ int32
142 }
143
144 type textSection struct {
145 vaddr uintptr // prelinked section vaddr
146 end uintptr // vaddr + section length
147 baseaddr uintptr // relocated section address
148 }
149
150 type modulehash struct {
151 modulename string
152 linktimehash string
153 runtimehash *string
154 }
155
156 // findfuncbucket is an array of these structures.
157 // Each bucket represents 4096 bytes of the text segment.
158 // Each subbucket represents 256 bytes of the text segment.
159 // To find a function given a pc, locate the bucket and subbucket for
160 // that pc. Add together the idx and subbucket value to obtain a
161 // function index. Then scan the functab array starting at that
162 // index to find the target function.
163 // This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead.
164 type findfuncbucket struct {
165 idx uint32
166 _SUBBUCKETS [16]byte
167 }
168
169
170 type compilationUnit struct {
171 fileNames []string
172 }
173
174 func makeFtab(funcs []_func, maxpc uintptr) (ftab []funcTab, pclntabSize int64, startLocations []uint32) {
175 // Allocate space for the pc->func table. This structure consists of a pc offset
176 // and an offset to the func structure. After that, we have a single pc
177 // value that marks the end of the last function in the binary.
178 pclntabSize = int64(len(funcs)*2*int(_PtrSize) + int(_PtrSize))
179 startLocations = make([]uint32, len(funcs))
180 for i, f := range funcs {
181 pclntabSize = rnd(pclntabSize, int64(_PtrSize))
182 //writePCToFunc
183 startLocations[i] = uint32(pclntabSize)
184 pclntabSize += int64(uint8(_FUNC_SIZE) + f.nfuncdata*_PtrSize + uint8(f.npcdata)*4)
185 }
186 ftab = make([]funcTab, 0, len(funcs)+1)
187
188 // write a map of pc->func info offsets
189 for i, f := range funcs {
190 ftab = append(ftab, funcTab{uintptr(f.entry), uintptr(startLocations[i])})
191 }
192
193 // Final entry of table is just end pc offset.
194 ftab = append(ftab, funcTab{maxpc, 0})
195 return
196 }
197
198 // Pcln table format: [...]funcTab + [...]_Func
199 func makePclntable(size int64, startLocations []uint32, funcs []_func, maxpc uintptr, pcdataOffs [][]uint32, funcdataAddr uintptr, funcdataOffs [][]uint32) (pclntab []byte) {
200 pclntab = make([]byte, size, size)
201
202 // write a map of pc->func info offsets
203 offs := 0
204 for i, f := range funcs {
205 byteOrder.PutUint64(pclntab[offs:offs+8], uint64(f.entry))
206 byteOrder.PutUint64(pclntab[offs+8:offs+16], uint64(startLocations[i]))
207 offs += 16
208 }
209 // Final entry of table is just end pc offset.
210 byteOrder.PutUint64(pclntab[offs:offs+8], uint64(maxpc))
211 offs += 8
212
213 // write func info table
214 for i, f := range funcs {
215 off := startLocations[i]
216
217 // write _func structure to pclntab
218 byteOrder.PutUint64(pclntab[off:off+8], uint64(f.entry))
219 off += 8
220 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.nameOff))
221 off += 4
222 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.args))
223 off += 4
224 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.deferreturn))
225 off += 4
226 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcsp))
227 off += 4
228 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcfile))
229 off += 4
230 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcln))
231 off += 4
232 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.npcdata))
233 off += 4
234 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.cuOffset))
235 off += 4
236 pclntab[off] = f.funcID
237 // NOTICE: _[2]byte alignment
238 off += 3
239 pclntab[off] = f.nfuncdata
240 off += 1
241
242 // NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3
243 for j := 3; j < len(pcdataOffs[i]); j++ {
244 byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j]))
245 off += 4
246 }
247
248 off = uint32(rnd(int64(off), int64(_PtrSize)))
249
250 // funcdata refs as offsets from gofunc
251 for _, funcdata := range funcdataOffs[i] {
252 if funcdata == _INVALID_FUNCDATA_OFFSET {
253 byteOrder.PutUint64(pclntab[off:off+8], 0)
254 } else {
255 byteOrder.PutUint64(pclntab[off:off+8], uint64(funcdataAddr)+uint64(funcdata))
256 }
257 off += 8
258 }
259 }
260
261 return
262 }
263
264 // findfunc table used to map pc to belonging func,
265 // returns the index in the func table.
266 //
267 // All text section are divided into buckets sized _BUCKETSIZE(4K):
268 // every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64),
269 // and it has a base idx to plus the offset stored in jth subbucket.
270 // see findfunc() in runtime/symtab.go
271 func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) {
272 start = len(*out)
273
274 max := ftab[len(ftab)-1].entry
275 min := ftab[0].entry
276 nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE
277 n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE
278
279 tab := make([]findfuncbucket, 0, nbuckets)
280 var s, e = 0, 0
281 for i := 0; i<int(nbuckets); i++ {
282 // store the start func of the bucket
283 var fb = findfuncbucket{idx: uint32(s)}
284
285 // find the last e-th func of the bucket
286 var pc = min + uintptr((i+1)*_BUCKETSIZE)
287 for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {}
288
289 for j := 0; j<_SUBBUCKETS && (i*_SUBBUCKETS+j)<int(n); j++ {
290 // store the start func of the subbucket
291 fb._SUBBUCKETS[j] = byte(uint32(s) - fb.idx)
292
293 // find the s-th end func of the subbucket
294 pc = min + uintptr(i*_BUCKETSIZE) + uintptr((j+1)*_SUB_BUCKETSIZE)
295 for ; s < len(ftab)-1 && ftab[s+1].entry <= pc; s++ {}
296 }
297
298 s = e
299 tab = append(tab, fb)
300 }
301
302 // write findfuncbucket
303 if len(tab) > 0 {
304 size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab)
305 *out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...)
306 }
307 return
308 }
309
310 func makeModuledata(name string, filenames []string, funcsp *[]Func, text []byte) (mod *moduledata) {
311 mod = new(moduledata)
312 mod.modulename = name
313
314 // sort funcs by entry
315 funcs := *funcsp
316 sort.Slice(funcs, func(i, j int) bool {
317 return funcs[i].EntryOff < funcs[j].EntryOff
318 })
319 *funcsp = funcs
320
321 // make filename table
322 cu := make([]string, 0, len(filenames))
323 cu = append(cu, filenames...)
324 cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}})
325 mod.cutab = cutab
326 mod.filetab = filetab
327
328 // make funcname table
329 funcnametab, nameOffs := makeFuncnameTab(funcs)
330 mod.funcnametab = funcnametab
331
332 // mmap() text and funcdata segments
333 p := os.Getpagesize()
334 size := int(rnd(int64(len(text)), int64(p)))
335 addr := mmap(size)
336 // copy the machine code
337 s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size)
338 copy(s, text)
339 // make it executable
340 mprotect(addr, size)
341
342 // assign addresses
343 mod.text = addr
344 mod.etext = addr + uintptr(size)
345 mod.minpc = addr
346 mod.maxpc = addr + uintptr(len(text))
347
348 // make pcdata table
349 // NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata
350 cuOff := cuOffs[0]
351 pctab, pcdataOffs, _funcs := makePctab(funcs, addr, cuOff, nameOffs)
352 mod.pctab = pctab
353
354 // write func data
355 // NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata
356 // TODO: estimate accurate capacity
357 cache := make([]byte, 0, len(funcs)*int(_PtrSize))
358 fstart, funcdataOffs := writeFuncdata(&cache, funcs)
359
360 // make pc->func (binary search) func table
361 ftab, pclntSize, startLocations := makeFtab(_funcs, mod.maxpc)
362 mod.ftab = ftab
363
364 // write pc->func (modmap) findfunc table
365 ffstart := writeFindfunctab(&cache, ftab)
366
367 // cache funcdata and findfuncbucket
368 moduleCache.Lock()
369 moduleCache.m[mod] = cache
370 moduleCache.Unlock()
371 mod.findfunctab = uintptr(rt.IndexByte(cache, ffstart))
372 funcdataAddr := uintptr(rt.IndexByte(cache, fstart))
373
374 // make pclnt table
375 pclntab := makePclntable(pclntSize, startLocations, _funcs, mod.maxpc, pcdataOffs, funcdataAddr, funcdataOffs)
376 mod.pclntable = pclntab
377
378 // make pc header
379 mod.pcHeader = &pcHeader {
380 magic : _Magic,
381 minLC : _MinLC,
382 ptrSize : _PtrSize,
383 nfunc : len(funcs),
384 nfiles: uint(len(cu)),
385 funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"),
386 cuOffset: getOffsetOf(moduledata{}, "cutab"),
387 filetabOffset: getOffsetOf(moduledata{}, "filetab"),
388 pctabOffset: getOffsetOf(moduledata{}, "pctab"),
389 pclnOffset: getOffsetOf(moduledata{}, "pclntable"),
390 }
391
392 // special case: gcdata and gcbss must by non-empty
393 mod.gcdata = uintptr(unsafe.Pointer(&emptyByte))
394 mod.gcbss = uintptr(unsafe.Pointer(&emptyByte))
395
396 return
397 }
398
399 // makePctab generates pcdelta->valuedelta tables for functions,
400 // and returns the table and the entry offset of every kind pcdata in the table.
401 func makePctab(funcs []Func, addr uintptr, cuOffset uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) {
402 _funcs = make([]_func, len(funcs))
403
404 // Pctab offsets of 0 are considered invalid in the runtime. We respect
405 // that by just padding a single byte at the beginning of runtime.pctab,
406 // that way no real offsets can be zero.
407 pctab = make([]byte, 1, 12*len(funcs)+1)
408 pcdataOffs = make([][]uint32, len(funcs))
409
410 for i, f := range funcs {
411 _f := &_funcs[i]
412
413 var writer = func(pc *Pcdata) {
414 var ab []byte
415 var err error
416 if pc != nil {
417 ab, err = pc.MarshalBinary()
418 if err != nil {
419 panic(err)
420 }
421 pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab)))
422 } else {
423 ab = []byte{0}
424 pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET)
425 }
426 pctab = append(pctab, ab...)
427 }
428
429 if f.Pcsp != nil {
430 _f.pcsp = uint32(len(pctab))
431 }
432 writer(f.Pcsp)
433 if f.Pcfile != nil {
434 _f.pcfile = uint32(len(pctab))
435 }
436 writer(f.Pcfile)
437 if f.Pcline != nil {
438 _f.pcln = uint32(len(pctab))
439 }
440 writer(f.Pcline)
441 writer(f.PcUnsafePoint)
442 writer(f.PcStackMapIndex)
443 writer(f.PcInlTreeIndex)
444 writer(f.PcArgLiveIndex)
445
446 _f.entry = addr + uintptr(f.EntryOff)
447 _f.nameOff = nameOffset[i]
448 _f.args = f.ArgsSize
449 _f.deferreturn = f.DeferReturn
450 // NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)]
451 _f.npcdata = uint32(_N_PCDATA)
452 _f.cuOffset = cuOffset
453 _f.funcID = f.ID
454 _f.nfuncdata = uint8(_N_FUNCDATA)
455 }
456
457 return
458 }
459
460 func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}
461