codec.mx raw
1 package runtime
2
3 import "unsafe"
4
5 // Recursive value serializer with back-references for shared/cyclic pointers.
6 // Encode-side tags emitted per non-trivial value:
7 const (
8 codecTagPtr = 0x00 // non-nil pointer: inner value follows
9 codecTagStr = 0x01 // string/[]byte: uint32 len + bytes
10 codecTagSlice = 0x02 // slice: uint32 len + uint32 cap + elements
11 codecTagIface = 0x03 // interface: 8 bytes typecode + inner value
12 codecTagStruct = 0x04 // struct: uint32 nf + field encodings
13 codecTagBackref = 0x05 // back-reference: uint32 index into seen set
14 codecTagNil = 0x06 // nil pointer
15 )
16
17 type CodecBuf struct {
18 data []byte
19 pos int32
20 }
21
22 func codecBufInit(cb *CodecBuf) {
23 cb.data = []byte{:0:256}
24 cb.pos = 0
25 }
26
27 func codecWriteU32(cb *CodecBuf, v uint32) {
28 cb.data = append(cb.data, byte(v), byte(v>>8), byte(v>>16), byte(v>>24))
29 }
30
31 func codecReadU32(cb *CodecBuf) (v uint32) {
32 v = uint32(cb.data[cb.pos]) | uint32(cb.data[cb.pos+1])<<8 |
33 uint32(cb.data[cb.pos+2])<<16 | uint32(cb.data[cb.pos+3])<<24
34 cb.pos += 4; return
35 }
36
37 func codecWriteBytes(cb *CodecBuf, v []byte) {
38 codecWriteU32(cb, uint32(len(v)))
39 cb.data = append(cb.data, v...)
40 }
41
42 func codecReadBytes(cb *CodecBuf) (v []byte) {
43 n := int32(codecReadU32(cb))
44 if n == 0 { return nil }
45 if int32(len(cb.data)) < cb.pos+n { runtimePanic("codec underflow") }
46 v = make([]byte, n)
47 memcpy(unsafe.Pointer(&v[0]), unsafe.Pointer(&cb.data[cb.pos]), uintptr(n))
48 cb.pos += n; return
49 }
50
51 type codecSeenSet struct {
52 ptrs []unsafe.Pointer
53 count int32
54 }
55
56 const codecSeenCap = 65536
57
58 func codecSeenInit(ss *codecSeenSet) {
59 ss.ptrs = []unsafe.Pointer{:0:codecSeenCap}
60 ss.count = 0
61 }
62
63 func codecSeenLookup(ss *codecSeenSet, p unsafe.Pointer) int32 {
64 for i := int32(0); i < ss.count; i++ {
65 if ss.ptrs[i] == p { return i }
66 }
67 return -1
68 }
69
70 func codecSeenAdd(ss *codecSeenSet, p unsafe.Pointer) int32 {
71 if ss.count >= codecSeenCap { return -1 }
72 ss.ptrs = append(ss.ptrs, p)
73 n := ss.count
74 ss.count++
75 return n
76 }
77
78 type codecDecodeSeen struct {
79 ptrs []unsafe.Pointer
80 count int32
81 }
82
83 func codecDecodeSeenInit(ds *codecDecodeSeen) {
84 ds.ptrs = []unsafe.Pointer{:0:codecSeenCap}
85 ds.count = 0
86 }
87
88 func codecDecodeSeenAdd(ds *codecDecodeSeen, p unsafe.Pointer) int32 {
89 if ds.count >= codecSeenCap { return -1 }
90 ds.ptrs = append(ds.ptrs, p)
91 n := ds.count
92 ds.count++
93 return n
94 }
95
96 func codecDecodeSeenLookup(ds *codecDecodeSeen, idx int32) unsafe.Pointer {
97 if idx >= ds.count { return nil }
98 return ds.ptrs[idx]
99 }
100
101 func isBasicKind(k Kind) bool {
102 return k == Bool || k == Int || k == Int8 || k == Int16 || k == Int32 || k == Int64 ||
103 k == Uint || k == Uint8 || k == Uint16 || k == Uint32 || k == Uint64 || k == Uintptr ||
104 k == Float32 || k == Float64 || k == kindUnsafePointer || k == kindChan || k == kindFunc
105 }
106
107 func codecEncodeValue(cb *CodecBuf, ss *codecSeenSet, ptr unsafe.Pointer, typ *rawType) {
108 if typ == nil {
109 runtimePanic("codecEncodeValue nil typ")
110 }
111 badPtr := ptr == nil || uintptr(ptr) < 1048576
112 k := typ.kind()
113 if badPtr && k != kindStruct {
114 switch k {
115 case kindBytes:
116 cb.data = append(cb.data, byte(codecTagStr))
117 codecWriteU32(cb, 0)
118 return
119 case kindSlice:
120 cb.data = append(cb.data, byte(codecTagSlice))
121 codecWriteU32(cb, 0)
122 codecWriteU32(cb, 0)
123 return
124 case kindPointer:
125 cb.data = append(cb.data, byte(codecTagNil))
126 return
127 case kindStruct:
128 cb.data = append(cb.data, byte(codecTagStruct))
129 codecWriteU32(cb, 0)
130 return
131 case kindMap:
132 cb.data = append(cb.data, byte(codecTagNil))
133 return
134 case kindInterface:
135 cb.data = append(cb.data, byte(codecTagIface))
136 codecWriteU32(cb, 0)
137 return
138 default:
139 cb.data = append(cb.data, byte(codecTagNil))
140 return
141 }
142 }
143 if isBasicKind(k) {
144 sz := typ.size()
145 if sz > 65536 {
146 runtimePanic("codec basic bad size")
147 }
148 cb.data = append(cb.data, unsafe.Slice((*byte)(ptr), sz)...)
149 return
150 }
151 switch k {
152 case kindBytes:
153 s := *(*_string)(ptr)
154 if s.length > 0 && (s.ptr == nil || uintptr(unsafe.Pointer(s.ptr)) < 65536) {
155 cb.data = append(cb.data, byte(codecTagStr))
156 codecWriteU32(cb, 0)
157 return
158 }
159 if s.length > 2*1024*1024*1024 {
160 n := uint32(0)
161 if s.cap > 0 && s.cap <= 65536 { n = uint32(s.cap) }
162 cb.data = append(cb.data, byte(codecTagStr))
163 codecWriteU32(cb, n)
164 for i := uint32(0); i < n; i++ {
165 cb.data = append(cb.data, 0)
166 }
167 return
168 }
169 cb.data = append(cb.data, byte(codecTagStr))
170 codecWriteU32(cb, uint32(s.length))
171 if s.length > 0 {
172 cb.data = append(cb.data, unsafe.Slice((*byte)(s.ptr), s.length)...)
173 }
174 case kindSlice:
175 sl := *(*_string)(ptr)
176 cb.data = append(cb.data, byte(codecTagSlice))
177 codecWriteU32(cb, uint32(sl.length))
178 codecWriteU32(cb, uint32(sl.cap))
179 elemType := typ.elem()
180 for i := int32(0); i < int32(sl.length); i++ {
181 ep := unsafe.Add(unsafe.Pointer(sl.ptr), uintptr(i)*elemType.size())
182 codecEncodeValue(cb, ss, ep, elemType)
183 }
184 case kindPointer:
185 // Bad pointer check: if ptr looks invalid (low address, nil, etc.)
186 // emit nil instead of dereferencing.
187 if ptr == nil || uintptr(ptr) < 1048576 {
188 cb.data = append(cb.data, byte(codecTagNil))
189 return
190 }
191 p := *(*unsafe.Pointer)(ptr)
192 if p == nil {
193 cb.data = append(cb.data, byte(codecTagNil))
194 return
195 }
196 idx := codecSeenLookup(ss, p)
197 if idx >= 0 {
198 cb.data = append(cb.data, byte(codecTagBackref))
199 codecWriteU32(cb, uint32(idx))
200 return
201 }
202 codecSeenAdd(ss, p)
203 cb.data = append(cb.data, byte(codecTagPtr))
204 codecEncodeValue(cb, ss, p, typ.elem())
205 case kindInterface:
206 iface := *(*_interface)(ptr)
207 cb.data = append(cb.data, byte(codecTagIface))
208 codecWriteBytes(cb, unsafe.Slice((*byte)(unsafe.Pointer(&iface.typecode)), 8))
209 if iface.typecode != nil && uintptr(iface.typecode) > 4096 && uintptr(iface.typecode) < (uintptr(1)<<48) {
210 tc := uintptr(iface.typecode)
211 if tc > 0x1000 && tc < 0x40000000 {
212 ct := (*rawType)(iface.typecode)
213 if ct.kind() != Invalid && ct.size() > 0 && ct.size() <= 65536 {
214 if ct.size() <= unsafe.Sizeof(uintptr(0)) {
215 codecEncodeValue(cb, ss, unsafe.Pointer(&iface.value), ct)
216 } else {
217 codecEncodeValue(cb, ss, iface.value, ct)
218 }
219 }
220 }
221 }
222 case kindMap:
223 m := *(*unsafe.Pointer)(ptr)
224 if m == nil {
225 cb.data = append(cb.data, byte(codecTagNil))
226 return
227 }
228 cb.data = append(cb.data, byte(codecTagPtr))
229 hm := (*hashmap)(m)
230 fixupFlags := uint8(0)
231 kt := hashmapKeyType(typ)
232 vt := typ.elem()
233 if kt != nil && (kt.kind() == kindBytes) { fixupFlags |= 8 }
234 if vt != nil && (vt.kind() == kindBytes) { fixupFlags |= 1 }
235 if vt != nil && (vt.kind() == kindPointer) { fixupFlags |= 2 }
236 if vt != nil && (vt.kind() == kindInterface) { fixupFlags |= 4 }
237 codecEncodeMap(hm, cb, ss, fixupFlags)
238 case kindStruct:
239 if ptr == nil || uintptr(ptr) < 1048576 {
240 cb.data = append(cb.data, byte(codecTagStruct))
241 codecWriteU32(cb, 0)
242 return
243 }
244 nf := typ.numField()
245 cb.data = append(cb.data, byte(codecTagStruct))
246 codecWriteU32(cb, uint32(nf))
247 // Register all field addresses in the seen-set so that pointer
248 // fields referencing other fields within the same struct (e.g.,
249 // fmt.buf *buffer pointing to pp.buf) are handled as back-references
250 // instead of serializing duplicate copies.
251 for i := int32(0); i < nf; i++ {
252 off := typ.structFieldOffset(i)
253 fp := unsafe.Add(ptr, off)
254 codecSeenAdd(ss, fp)
255 }
256 for i := int32(0); i < nf; i++ {
257 ft := typ.structFieldType(i)
258 off := typ.structFieldOffset(i)
259 fp := unsafe.Add(ptr, off)
260 codecEncodeValue(cb, ss, fp, ft)
261 }
262 case kindArray:
263 elemType := typ.elem()
264 elemSize := elemType.size()
265 length := typ.arrayLen()
266 for i := int32(0); i < length; i++ {
267 ep := unsafe.Add(ptr, uintptr(i)*elemSize)
268 codecEncodeValue(cb, ss, ep, elemType)
269 }
270 default:
271 sz := typ.size()
272 if sz <= 0 || sz > 65536 {
273 runtimePanic("codec default bad size")
274 }
275 cb.data = append(cb.data, unsafe.Slice((*byte)(ptr), sz)...)
276 }
277 }
278
279 func codecDecodeValue(cb *CodecBuf, ds *codecDecodeSeen, typ *rawType) unsafe.Pointer {
280 k := typ.kind()
281 if isBasicKind(k) {
282 sz := typ.size()
283 buf := alloc(sz, nil)
284 memcpy(buf, unsafe.Pointer(&cb.data[cb.pos]), sz)
285 cb.pos += int32(sz)
286 return buf
287 }
288 switch k {
289 case kindBytes:
290 tag := cb.data[cb.pos]; cb.pos++
291 if tag == codecTagStr {
292 n := int32(codecReadU32(cb))
293 hdr := alloc(typ.size(), nil)
294 if n > 0 {
295 dataPtr := alloc(uintptr(n), nil)
296 memcpy(dataPtr, unsafe.Pointer(&cb.data[cb.pos]), uintptr(n))
297 cb.pos += n
298 *(*unsafe.Pointer)(hdr) = dataPtr
299 }
300 *(*uintptr)(unsafe.Add(hdr, 8)) = uintptr(n)
301 if typ.size() > 16 {
302 *(*uintptr)(unsafe.Add(hdr, 16)) = uintptr(n)
303 }
304 return hdr
305 }
306 cb.pos--
307 sz := typ.size()
308 buf := alloc(sz, nil)
309 memcpy(buf, unsafe.Pointer(&cb.data[cb.pos]), sz)
310 cb.pos += int32(sz)
311 return buf
312 case kindSlice:
313 tag := cb.data[cb.pos]; cb.pos++
314 if tag == codecTagSlice {
315 elemType := typ.elem()
316 hdr := alloc(typ.size(), nil)
317 ln := int32(codecReadU32(cb))
318 cp := int32(codecReadU32(cb))
319 if ln > 0 {
320 elemSize := elemType.size()
321 arr := alloc(uintptr(ln)*elemSize, nil)
322 *(*unsafe.Pointer)(hdr) = arr
323 for i := int32(0); i < ln; i++ {
324 ep := unsafe.Add(arr, uintptr(i)*elemSize)
325 dec := codecDecodeValue(cb, ds, elemType)
326 memcpy(ep, dec, elemSize)
327 }
328 }
329 *(*uintptr)(unsafe.Add(hdr, 8)) = uintptr(ln)
330 *(*uintptr)(unsafe.Add(hdr, 16)) = uintptr(cp)
331 return hdr
332 }
333 cb.pos--
334 sz := typ.size()
335 buf := alloc(sz, nil)
336 memcpy(buf, unsafe.Pointer(&cb.data[cb.pos]), sz)
337 cb.pos += int32(sz)
338 return buf
339 case kindPointer:
340 tag := cb.data[cb.pos]; cb.pos++
341 switch tag {
342 case codecTagNil:
343 buf := alloc(typ.size(), nil)
344 *(*unsafe.Pointer)(buf) = nil
345 return buf
346 case codecTagBackref:
347 idx := int32(codecReadU32(cb))
348 orig := codecDecodeSeenLookup(ds, idx)
349 if orig == nil {
350 runtimePanic("codec backref nil")
351 }
352 buf := alloc(typ.size(), nil)
353 *(*unsafe.Pointer)(buf) = orig
354 codecDecodeSeenAdd(ds, orig)
355 return buf
356 case codecTagPtr:
357 inner := codecDecodeValue(cb, ds, typ.elem())
358 codecDecodeSeenAdd(ds, inner)
359 buf := alloc(typ.size(), nil)
360 *(*unsafe.Pointer)(buf) = inner
361 return buf
362 }
363 runtimePanic("codec bad pointer tag")
364 return nil
365 case kindInterface:
366 tag := cb.data[cb.pos]; cb.pos++
367 if tag != codecTagIface {
368 // Buffer position check: look for codecTagIface in the
369 // surrounding bytes to detect alignment errors.
370 if cb.pos > 1 && cb.data[cb.pos-2] == codecTagIface {
371 runtimePanic("codec iface tag off by one")
372 }
373 runtimePanic("codec expected iface tag")
374 }
375 typRaw := codecReadBytes(cb)
376 buf := alloc(typ.size(), nil)
377 if len(typRaw) == 0 {
378 return buf
379 }
380 *(*unsafe.Pointer)(buf) = *(*unsafe.Pointer)(unsafe.Pointer(&typRaw[0]))
381 ct := (*rawType)(*(*unsafe.Pointer)(unsafe.Pointer(&typRaw[0])))
382 if ct == nil {
383 // nil concrete value but non-nil interface type
384 return buf
385 }
386 if ct.size() <= unsafe.Sizeof(uintptr(0)) {
387 inner := codecDecodeValue(cb, ds, ct)
388 memcpy(unsafe.Add(buf, 8), inner, ct.size())
389 } else {
390 inner := codecDecodeValue(cb, ds, ct)
391 *(*unsafe.Pointer)(unsafe.Add(buf, 8)) = inner
392 }
393 return buf
394 case kindStruct:
395 tag := cb.data[cb.pos]; cb.pos++
396 if tag != codecTagStruct {
397 printstring("codec bad struct tag: got=")
398 printuint64(uint64(tag))
399 printstring(" pos=")
400 printuint64(uint64(cb.pos))
401 printstring("\n")
402 runtimePanic("codec expected struct tag")
403 }
404 nf := int32(codecReadU32(cb))
405 buf := alloc(typ.size(), nil)
406 // Register decoded struct and all its field addresses in the
407 // decode seen-set, matching the encode side's registration order
408 // so that back-references are resolved correctly.
409 codecDecodeSeenAdd(ds, buf)
410 for i := int32(0); i < nf; i++ {
411 off := typ.structFieldOffset(i)
412 fp := unsafe.Add(buf, off)
413 codecDecodeSeenAdd(ds, fp)
414 }
415 // Decode fields in order using type descriptors
416 expectedNf := typ.numField()
417 if nf != expectedNf {
418 runtimePanic("codec struct field count mismatch")
419 }
420 for i := int32(0); i < nf; i++ {
421 ft := typ.structFieldType(i)
422 off := typ.structFieldOffset(i)
423 fp := unsafe.Add(buf, off)
424 dec := codecDecodeValue(cb, ds, ft)
425 memcpy(fp, dec, ft.size())
426 }
427 return buf
428 case kindArray:
429 elemType := typ.elem()
430 elemSize := elemType.size()
431 length := typ.arrayLen()
432 buf := alloc(typ.size(), nil)
433 for i := int32(0); i < length; i++ {
434 ep := unsafe.Add(buf, uintptr(i)*elemSize)
435 dec := codecDecodeValue(cb, ds, elemType)
436 memcpy(ep, dec, elemSize)
437 }
438 return buf
439 case kindMap:
440 buf := alloc(typ.size(), nil)
441 kt := hashmapKeyType(typ)
442 vt := typ.elem()
443 keySz := kt.size()
444 valSz := vt.size()
445 alg := uint8(0)
446 if kt.kind() == kindBytes { alg = 1 }
447 flags := uint8(0)
448 if kt.kind() == kindBytes { flags |= 8 }
449 if vt.kind() == kindBytes { flags |= 8 }
450 if vt.kind() == kindInterface { flags |= 4 }
451 decoded := codecDecodeMapValue(cb, ds, keySz, valSz, alg, flags, kt, vt)
452 *(*unsafe.Pointer)(buf) = unsafe.Pointer(decoded)
453 return buf
454 default:
455 sz := typ.size()
456 buf := alloc(sz, nil)
457 memcpy(buf, unsafe.Pointer(&cb.data[cb.pos]), sz)
458 cb.pos += int32(sz)
459 return buf
460 }
461 }
462
463 // codecDecodeMapValue decodes a map from the codec buffer, creating a new hashmap.
464 // This is a standalone function to avoid function instance specialization issues.
465 func codecDecodeMapValue(cb *CodecBuf, ds *codecDecodeSeen, keySz, valSz uintptr, alg uint8, flags uint8, kt, vt *rawType) (m *hashmap) {
466 // Read tag byte via data pointer to avoid compiler bounds check.
467 dataPtr := *(*unsafe.Pointer)(unsafe.Pointer(cb))
468 tag := *(*uint8)(unsafe.Add(dataPtr, uintptr(cb.pos)))
469 cb.pos++
470 if tag == codecTagNil { return nil }
471 if tag != codecTagPtr { runtimePanic("codec: map expected ptr tag") }
472 cnt := int32(codecReadU32(cb))
473 m = hashmapMake(keySz, valSz, uintptr(cnt), alg)
474 if cnt == 0 { return }
475 fixKey := flags&8 != 0; fixStrVal := flags&1 != 0; fixVal := flags&2 != 0; fixIface := flags&4 != 0
476 for i := int32(0); i < cnt; i++ {
477 var key, val []byte
478 if fixKey { raw := codecReadBytes(cb)
479 hdr := make([]byte, keySz)
480 *(*unsafe.Pointer)(unsafe.Pointer(&hdr[0])) = unsafe.Pointer(&raw[0])
481 *(*uintptr)(unsafe.Pointer(&hdr[8])) = uintptr(len(raw))
482 if keySz > 16 { *(*uintptr)(unsafe.Pointer(&hdr[16])) = uintptr(len(raw)) }
483 key = hdr
484 } else { key = make([]byte, keySz)
485 memcpy(unsafe.Pointer(&key[0]), unsafe.Pointer(&cb.data[cb.pos]), keySz)
486 cb.pos += int32(keySz) }
487 if fixStrVal {
488 raw := codecReadBytes(cb)
489 hdr := make([]byte, valSz)
490 *(*unsafe.Pointer)(unsafe.Pointer(&hdr[0])) = unsafe.Pointer(&raw[0])
491 *(*uintptr)(unsafe.Pointer(&hdr[8])) = uintptr(len(raw))
492 if valSz > 16 { *(*uintptr)(unsafe.Pointer(&hdr[16])) = uintptr(len(raw)) }
493 val = hdr
494 } else if fixVal {
495 tag := cb.data[cb.pos]; cb.pos++
496 if tag == codecTagNil { val = make([]byte, valSz) } else
497 if tag == codecTagPtr {
498 inner := codecDecodeValue(cb, ds, nil)
499 buf := make([]byte, valSz)
500 *(*unsafe.Pointer)(unsafe.Pointer(&buf[0])) = inner
501 val = buf
502 } else { runtimePanic("codec bad fixVal tag") }
503 } else if fixIface {
504 tag := cb.data[cb.pos]; cb.pos++
505 if tag != codecTagIface { runtimePanic("codec expected iface tag") }
506 typRaw := codecReadBytes(cb)
507 buf := make([]byte, valSz)
508 if len(typRaw) > 0 {
509 *(*unsafe.Pointer)(unsafe.Pointer(&buf[0])) = *(*unsafe.Pointer)(unsafe.Pointer(&typRaw[0]))
510 ct := (*rawType)(*(*unsafe.Pointer)(unsafe.Pointer(&typRaw[0])))
511 if ct.size() <= unsafe.Sizeof(uintptr(0)) {
512 inner := codecDecodeValue(cb, ds, ct)
513 memcpy(unsafe.Pointer(&buf[8]), inner, ct.size())
514 } else {
515 inner := codecDecodeValue(cb, ds, ct)
516 *(*unsafe.Pointer)(unsafe.Pointer(&buf[8])) = inner
517 }
518 }
519 val = buf
520 } else { val = make([]byte, valSz)
521 memcpy(unsafe.Pointer(&val[0]), unsafe.Pointer(&cb.data[cb.pos]), valSz)
522 cb.pos += int32(valSz) }
523 h := m.keyHash(unsafe.Pointer(&key[0]), keySz, m.seed)
524 hashmapSet(m, unsafe.Pointer(&key[0]), unsafe.Pointer(&val[0]), h)
525 }
526 return
527 }
528
529 func codecEncodeMap(m *hashmap, cb *CodecBuf, ss *codecSeenSet, fixupFlags uint8) {
530 // Entry count
531 cnt := int32(0)
532 if m != nil && m.bucketBits > 0 {
533 keySz := m.keySize; valSz := m.valueSize
534 bucketSz := unsafe.Sizeof(hashmapBucket{}) + keySz*8 + valSz*8
535 nb := uintptr(1) << m.bucketBits
536 for bi := uintptr(0); bi < nb; bi++ {
537 for b := (*hashmapBucket)(unsafe.Add(m.buckets, bucketSz*bi)); b != nil; b = b.next {
538 for i := uint8(0); i < 8; i++ { if b.tophash[i] != 0 { cnt++ } }
539 }
540 }
541 }
542 codecWriteU32(cb, uint32(cnt))
543 if cnt == 0 { return }
544
545 fixKey := fixupFlags&8 != 0; fixStrVal := fixupFlags&1 != 0; fixVal := fixupFlags&2 != 0; fixIface := fixupFlags&4 != 0
546 keySz := m.keySize; valSz := m.valueSize
547 bucketSz := unsafe.Sizeof(hashmapBucket{}) + keySz*8 + valSz*8
548 nb := uintptr(1) << m.bucketBits
549
550 for bi := uintptr(0); bi < nb; bi++ {
551 for b := (*hashmapBucket)(unsafe.Add(m.buckets, bucketSz*bi)); b != nil; b = b.next {
552 for i := uint8(0); i < 8; i++ {
553 if b.tophash[i] != 0 {
554 kp := hashmapSlotKey(m, b, i)
555 vp := hashmapSlotValue(m, b, i)
556 if fixKey {
557 _sh := *(*_string)(kp)
558 codecWriteBytes(cb, unsafe.Slice((*byte)(_sh.ptr), _sh.length))
559 } else {
560 cb.data = append(cb.data, unsafe.Slice((*byte)(kp), keySz)...)
561 }
562 if fixStrVal {
563 sv := *(*_string)(vp)
564 codecWriteBytes(cb, unsafe.Slice((*byte)(sv.ptr), sv.length))
565 } else if fixVal {
566 p := *(*unsafe.Pointer)(vp)
567 if p == nil {
568 cb.data = append(cb.data, byte(codecTagNil))
569 } else {
570 cb.data = append(cb.data, byte(codecTagPtr))
571 codecEncodeValue(cb, ss, p, nil)
572 }
573 } else if fixIface {
574 iface := *(*_interface)(vp)
575 cb.data = append(cb.data, byte(codecTagIface))
576 codecWriteBytes(cb, unsafe.Slice((*byte)(unsafe.Pointer(&iface.typecode)), 8))
577 if iface.typecode != nil {
578 ct := (*rawType)(iface.typecode)
579 if ct.size() <= unsafe.Sizeof(uintptr(0)) {
580 codecEncodeValue(cb, ss, unsafe.Pointer(&iface.value), ct)
581 } else {
582 codecEncodeValue(cb, ss, iface.value, ct)
583 }
584 }
585 } else {
586 cb.data = append(cb.data, unsafe.Slice((*byte)(vp), valSz)...)
587 }
588 }
589 }
590 }
591 }
592 }
593
594 // --- Compiler-generated return value encode/decode primitives ---
595 // Called from compiler-emitted code to serialize return values across
596 // arena boundaries. These replace the old 3-pass Reloc* system.
597
598 func codecWriteU8(cb *CodecBuf, v uint8) {
599 cb.data = append(cb.data, v)
600 }
601
602 func codecWriteRaw(cb *CodecBuf, ptr unsafe.Pointer, sz uintptr) {
603 if sz > 0 {
604 cb.data = append(cb.data, unsafe.Slice((*byte)(ptr), sz)...)
605 }
606 }
607
608 func codecReadU8(cb *CodecBuf) (v uint8) {
609 v = cb.data[cb.pos]; cb.pos++
610 return
611 }
612
613 func codecReadRaw(cb *CodecBuf, ptr unsafe.Pointer, sz uintptr) {
614 if sz > 0 {
615 memcpy(ptr, unsafe.Pointer(&cb.data[cb.pos]), sz)
616 cb.pos += int32(sz)
617 }
618 }
619
620 // codecEncodeStringValue encodes a string from its component fields,
621 // avoiding the alloca-pointer pattern that LLVM may optimize incorrectly.
622 func codecEncodeStringValue(cb *CodecBuf, ss *codecSeenSet, ptr unsafe.Pointer, ln uintptr) {
623 cb.data = append(cb.data, byte(codecTagStr))
624 codecWriteU32(cb, uint32(ln))
625 if ln > 0 {
626 codecWriteRaw(cb, ptr, ln)
627 }
628 }
629
630 // codecEncodeSliceValue encodes a slice from its component fields.
631 func codecEncodeSliceValue(cb *CodecBuf, ss *codecSeenSet, ptr unsafe.Pointer, ln, cp uintptr, elemType *rawType) {
632 cb.data = append(cb.data, byte(codecTagSlice))
633 codecWriteU32(cb, uint32(ln))
634 codecWriteU32(cb, uint32(cp))
635 elemSize := elemType.size()
636 for i := int32(0); i < int32(ln); i++ {
637 ep := unsafe.Add(ptr, uintptr(i)*elemSize)
638 codecEncodeValue(cb, ss, ep, elemType)
639 }
640 }
641
642 // codecEncodePtrValue encodes a pointer from its value.
643 func codecEncodePtrValue(cb *CodecBuf, ss *codecSeenSet, ptr unsafe.Pointer, elemType *rawType) {
644 if ptr == nil {
645 cb.data = append(cb.data, byte(codecTagNil))
646 return
647 }
648 idx := codecSeenLookup(ss, ptr)
649 if idx >= 0 {
650 cb.data = append(cb.data, byte(codecTagBackref))
651 codecWriteU32(cb, uint32(idx))
652 return
653 }
654 codecSeenAdd(ss, ptr)
655 cb.data = append(cb.data, byte(codecTagPtr))
656 codecEncodeValue(cb, ss, ptr, elemType)
657 }
658
659 // codecEncodeIfaceValue encodes an interface from its typecode and value fields.
660 func codecEncodeIfaceValue(cb *CodecBuf, ss *codecSeenSet, typecode, value unsafe.Pointer, typ *rawType) {
661 cb.data = append(cb.data, byte(codecTagIface))
662 codecWriteBytes(cb, unsafe.Slice((*byte)(unsafe.Pointer(&typecode)), 8))
663 if typecode != nil {
664 ct := (*rawType)(typecode)
665 if ct.size() <= unsafe.Sizeof(uintptr(0)) {
666 codecEncodeValue(cb, ss, unsafe.Pointer(&value), ct)
667 } else {
668 codecEncodeValue(cb, ss, value, ct)
669 }
670 }
671 }
672
673 // codecEncodeMapValue encodes a map from its pointer value.
674 func codecEncodeMapValue(cb *CodecBuf, ss *codecSeenSet, mptr unsafe.Pointer, typ *rawType) {
675 if mptr == nil {
676 cb.data = append(cb.data, byte(codecTagNil))
677 return
678 }
679 cb.data = append(cb.data, byte(codecTagPtr))
680 hm := (*hashmap)(mptr)
681 fixupFlags := uint8(0)
682 kt := hashmapKeyType(typ)
683 vt := typ.elem()
684 if kt != nil && (kt.kind() == kindBytes) { fixupFlags |= 8 }
685 if vt != nil && (vt.kind() == kindBytes) { fixupFlags |= 1 }
686 if vt != nil && (vt.kind() == kindPointer) { fixupFlags |= 2 }
687 if vt != nil && (vt.kind() == kindInterface) { fixupFlags |= 4 }
688 codecEncodeMap(hm, cb, ss, fixupFlags)
689 }
690
691 // codecDecodeString decodes a string from the buffer.
692 // Returns pointer to a _string header allocated in the current arena.
693 func codecDecodeString(cb *CodecBuf, ds *codecDecodeSeen) (hdr unsafe.Pointer) {
694 tag := codecReadU8(cb)
695 if tag != codecTagStr { runtimePanic("codec: expected string tag") }
696 ln := int32(codecReadU32(cb))
697 hdr = alloc(unsafe.Sizeof(_string{}), nil)
698 if ln == 0 {
699 return
700 }
701 ptr := alloc(uintptr(ln), nil)
702 codecReadRaw(cb, ptr, uintptr(ln))
703 sh := (*_string)(hdr)
704 sh.ptr = (*byte)(ptr)
705 sh.length = uintptr(ln)
706 sh.cap = uintptr(ln)
707 return
708 }
709
710 // codecDecodePtr reads a pointer from the buffer.
711 // The resulting pointer value may be nil, a backreference, or fresh allocation.
712 func codecDecodePtr(cb *CodecBuf, ds *codecDecodeSeen) (ptr unsafe.Pointer) {
713 tag := codecReadU8(cb)
714 switch tag {
715 case codecTagNil:
716 return nil
717 case codecTagBackref:
718 idx := int32(codecReadU32(cb))
719 return codecDecodeSeenLookup(ds, idx)
720 case codecTagPtr:
721 var buf [8]byte
722 codecReadRaw(cb, unsafe.Pointer(&buf[0]), unsafe.Sizeof(uintptr(0)))
723 return *(*unsafe.Pointer)(unsafe.Pointer(&buf[0]))
724 }
725 runtimePanic("codecDecodePtr: bad tag")
726 return nil
727 }
728
729 // codecDecodeSlice decodes a slice from the buffer.
730 // Returns pointer to a slice header allocated in the current arena.
731 func codecDecodeSlice(cb *CodecBuf, ds *codecDecodeSeen) (hdr unsafe.Pointer) {
732 tag := codecReadU8(cb)
733 if tag != codecTagSlice { runtimePanic("codec: expected slice tag") }
734 ln := int32(codecReadU32(cb))
735 cp := int32(codecReadU32(cb))
736 hdr = alloc(unsafe.Sizeof(_string{}), nil)
737 if ln == 0 {
738 return
739 }
740 ptr := alloc(uintptr(cp), nil)
741 codecReadRaw(cb, ptr, uintptr(ln))
742 sh := (*_string)(hdr)
743 sh.ptr = (*byte)(ptr)
744 sh.length = uintptr(ln)
745 sh.cap = uintptr(cp)
746 return
747 }
748
749 // codecDecodeInterface decodes an interface value from the buffer.
750 // Returns pointer to an _interface header in the current arena.
751 func codecDecodeInterface(cb *CodecBuf, ds *codecDecodeSeen) (ifacePtr unsafe.Pointer) {
752 tag := codecReadU8(cb)
753 if tag != codecTagIface { runtimePanic("codec: expected iface tag") }
754 ifacePtr = alloc(unsafe.Sizeof(_interface{}), nil)
755 iface := (*_interface)(ifacePtr)
756 // Read typecode (8 raw bytes)
757 var tcBuf [8]byte
758 codecReadRaw(cb, unsafe.Pointer(&tcBuf[0]), 8)
759 iface.typecode = *(*unsafe.Pointer)(unsafe.Pointer(&tcBuf[0]))
760 if iface.typecode != nil {
761 ct := (*rawType)(iface.typecode)
762 // Decode concrete value using runtime type info from typecode
763 inner := codecDecodeValue(cb, ds, ct)
764 iface.value = inner
765 }
766 return
767 }
768
769 // codecEncodeIfacePayload encodes the concrete value behind an interface.
770 // Uses the interface's own typecode (*rawType) for recursive type info.
771 func codecEncodeIfacePayload(cb *CodecBuf, ss *codecSeenSet, ifaceAddr unsafe.Pointer) {
772 iface := *(*_interface)(ifaceAddr)
773 if iface.typecode == nil {
774 return
775 }
776 t := (*rawType)(iface.typecode)
777 if t.kind() == kindPointer || t.size() > unsafe.Sizeof(uintptr(0)) {
778 // Large value: iface.value is a pointer to the data
779 codecEncodeValue(cb, ss, iface.value, t)
780 } else {
781 // Small value: iface.value IS the data
782 codecEncodeValue(cb, ss, unsafe.Pointer(&iface.value), t)
783 }
784 }
785
786 // codecReadStructTag validates a struct header (tag + field count).
787 func codecReadStructTag(cb *CodecBuf, expectedFields int32) {
788 tag := codecReadU8(cb)
789 if tag != codecTagStruct {
790 runtimePanic("codec: expected struct tag")
791 }
792 nf := int32(codecReadU32(cb))
793 if nf != expectedFields {
794 runtimePanic("codec: struct field count mismatch")
795 }
796 }
797
798 // allocRead allocates sz bytes and reads them from the codec buffer.
799 func allocRead(cb *CodecBuf, sz uintptr) (buf []byte) {
800 buf = make([]byte, sz)
801 memcpy(unsafe.Pointer(&buf[0]), unsafe.Pointer(&cb.data[cb.pos]), sz)
802 cb.pos += int32(sz)
803 return
804 }
805
806 func codecDecodeMapFlagged(cb *CodecBuf, ds *codecDecodeSeen, keySz, valSz uintptr, alg uint8, fixupFlags uint8) (m *hashmap) {
807 cnt := int32(codecReadU32(cb))
808 m = hashmapMake(keySz, valSz, uintptr(cnt), alg)
809 if cnt == 0 { return }
810 fixKey := fixupFlags&8 != 0; fixStrVal := fixupFlags&1 != 0; fixVal := fixupFlags&2 != 0; fixIface := fixupFlags&4 != 0
811 for i := int32(0); i < cnt; i++ {
812 var key, val []byte
813 if fixKey {
814 raw := codecReadBytes(cb)
815 hdr := make([]byte, keySz)
816 *(*unsafe.Pointer)(unsafe.Pointer(&hdr[0])) = unsafe.Pointer(&raw[0])
817 *(*uintptr)(unsafe.Pointer(&hdr[8])) = uintptr(len(raw))
818 if keySz > 16 {
819 *(*uintptr)(unsafe.Pointer(&hdr[16])) = uintptr(len(raw))
820 }
821 key = hdr
822 } else {
823 key = make([]byte, keySz)
824 memcpy(unsafe.Pointer(&key[0]), unsafe.Pointer(&cb.data[cb.pos]), keySz)
825 cb.pos += int32(keySz)
826 }
827 if fixStrVal {
828 raw := codecReadBytes(cb)
829 hdr := make([]byte, valSz)
830 *(*unsafe.Pointer)(unsafe.Pointer(&hdr[0])) = unsafe.Pointer(&raw[0])
831 *(*uintptr)(unsafe.Pointer(&hdr[8])) = uintptr(len(raw))
832 if valSz > 16 { *(*uintptr)(unsafe.Pointer(&hdr[16])) = uintptr(len(raw)) }
833 val = hdr
834 } else if fixVal {
835 tag := cb.data[cb.pos]; cb.pos++
836 if tag == codecTagNil {
837 val = make([]byte, valSz)
838 } else if tag == codecTagPtr {
839 // Decode pointer value; val slot gets {new_ptr}
840 inner := codecDecodeValue(cb, ds, nil)
841 buf := make([]byte, valSz)
842 *(*unsafe.Pointer)(unsafe.Pointer(&buf[0])) = inner
843 val = buf
844 } else {
845 runtimePanic("codec bad fixVal tag")
846 }
847 } else if fixIface {
848 tag := cb.data[cb.pos]; cb.pos++
849 if tag != codecTagIface {
850 runtimePanic("codec expected iface tag for fixIface")
851 }
852 typRaw := codecReadBytes(cb)
853 buf := make([]byte, valSz)
854 if len(typRaw) > 0 {
855 *(*unsafe.Pointer)(unsafe.Pointer(&buf[0])) = *(*unsafe.Pointer)(unsafe.Pointer(&typRaw[0]))
856 ct := (*rawType)(*(*unsafe.Pointer)(unsafe.Pointer(&typRaw[0])))
857 if ct.size() <= unsafe.Sizeof(uintptr(0)) {
858 inner := codecDecodeValue(cb, ds, ct)
859 memcpy(unsafe.Pointer(&buf[8]), inner, ct.size())
860 } else {
861 inner := codecDecodeValue(cb, ds, ct)
862 *(*unsafe.Pointer)(unsafe.Pointer(&buf[8])) = inner
863 }
864 }
865 val = buf
866 } else {
867 val = make([]byte, valSz)
868 memcpy(unsafe.Pointer(&val[0]), unsafe.Pointer(&cb.data[cb.pos]), valSz)
869 cb.pos += int32(valSz)
870 }
871 if uintptr(len(key)) == keySz && uintptr(len(val)) == valSz {
872 h := m.keyHash(unsafe.Pointer(&key[0]), keySz, m.seed)
873 hashmapSet(m, unsafe.Pointer(&key[0]), unsafe.Pointer(&val[0]), h)
874 }
875 }
876 return
877 }
878
879 // codecEncodeRecvWrites encodes the receiver write table into cb.
880 func codecEncodeRecvWrites(cb *CodecBuf, ss *codecSeenSet, table unsafe.Pointer, count int32) {
881 codecWriteU32(cb, uint32(count))
882 entrySize := uintptr(24)
883 for i := int32(0); i < count; i++ {
884 ep := unsafe.Add(table, uintptr(i)*entrySize)
885 valPtr := *(*unsafe.Pointer)(unsafe.Add(ep, 8))
886 typeDesc := *(*unsafe.Pointer)(unsafe.Add(ep, 16))
887 if typeDesc != nil {
888 codecEncodeValue(cb, ss, valPtr, (*rawType)(typeDesc))
889 }
890 }
891 }
892
893 // codecDecodeRecvWrites decodes receiver writes from cb and stores decoded
894 // pointers back into the receiver field addresses recorded in the table.
895 func codecDecodeRecvWrites(cb *CodecBuf, ds *codecDecodeSeen, table unsafe.Pointer) {
896 n := int32(codecReadU32(cb))
897 entrySize := uintptr(24)
898 for i := int32(0); i < n; i++ {
899 ep := unsafe.Add(table, uintptr(i)*entrySize)
900 fieldAddr := *(*unsafe.Pointer)(unsafe.Pointer(ep))
901 typeDesc := *(*unsafe.Pointer)(unsafe.Add(ep, 16))
902 if typeDesc != nil && fieldAddr != nil {
903 dec := codecDecodeValue(cb, ds, (*rawType)(typeDesc))
904 *(*unsafe.Pointer)(fieldAddr) = dec
905 }
906 }
907 }
908