ir_emit.mx raw
1 package main
2
3 import (
4 "math"
5 "os"
6 "runtime"
7 "git.smesh.lol/moxie/pkg/mxutil"
8 . "git.smesh.lol/moxie/pkg/types"
9 )
10
11 // kv is a key-value pair for accumulating cross-function map products.
12 type emitKV struct{ k, v string }
13 // emitKVI is a key-int32 pair for accumulating type ID products.
14 type emitKVI struct{ k string; v int32 }
15
16 type irEmitter struct {
17 segs []string // accumulated segments; joined in flush()
18 buf []byte // assembled output (only during flush/return)
19 outFile *os.File // streaming output; nil = accumulate in buf
20 operandBuf []SSAValue // scratch buffer for instrOperands
21 triple string
22 ptrBits int32
23 pkg *SSAPackage
24 valName map[SSAValue]string
25 nextReg int32
26 extDecls map[string]string
27 extGlobals map[string]string
28 strConst []string
29 strMap map[string]int32
30 curFunc *SSAFunction
31 typeIDs map[string]int32
32 typeIDNext int32
33 extTypeIDs map[string]bool
34 localTypeIDs map[string]bool
35 allocTypes map[SSAValue]string
36 regTypes map[string]string
37 hoisted map[SSAValue]bool
38 wasmAttrID int32
39 wasmAttrs [][3]string
40 blockExitLabel map[int32]string
41 nameUsed map[string]bool
42 missingStores map[SSAValue]SSAValue
43 globalTypes map[string]string
44 globalDeclTypes map[string]string
45 loadToGlobal map[string]*SSAGlobal
46 allocBlock map[SSAValue]int32
47 storedTo map[string]bool
48 usedAs map[string]bool
49 deferList []*SSADefer
50 deferID int32
51 blockDead bool
52 currentEmitBlock *SSABasicBlock
53 sretType string
54 pendingPhiTruncs []phiTrunc
55 // Per-function accumulators for cross-function state.
56 // Written during function emission, merged by caller after flush.
57 // All accumulator contents die with the function arena.
58 pendExtDecls []emitKV
59 pendExtGlobals []emitKV
60 pendStrConsts []string
61 pendTypeIDs []emitKVI
62 pendLocalTypeIDs []string
63 pendGlobalDeclTypes []emitKV
64 pendWasmAttrs [][3]string
65 pendDeepCopyEntries []deepCopyEntry
66 definedSyms []string // symbols that got `define` in this package
67 // Arena deep-copy state (compileArena lifetime)
68 arenaActive bool
69 deepCopyFns map[string]string
70 deepCopyQueue []deepCopyEntry
71 relocMarkVisited map[string]bool
72 relocFixupVisited map[string]bool
73 pendRelocFixupDefs []string
74 typeCache map[Type]string
75 }
76
77 type phiTrunc struct {
78 rawReg string
79 dstReg string
80 from string
81 to string
82 }
83
84 func newIREmitter(pkg *SSAPackage, triple string, srcLen int32) (p *irEmitter) {
85 ptrBits := 64
86 if len(triple) >= 4 && triple[:4] == "wasm" {
87 ptrBits = 32
88 }
89 return &irEmitter{
90 segs: []string{:0:1024},
91 triple: triple,
92 ptrBits: ptrBits,
93 pkg: pkg,
94 valName: map[SSAValue]string{},
95 extDecls: map[string]string{},
96 extGlobals: map[string]string{},
97 strMap: map[string]int32{},
98 typeIDs: map[string]int32{},
99 localTypeIDs: map[string]bool{},
100 globalDeclTypes: map[string]string{},
101 allocTypes: map[SSAValue]string{},
102 regTypes: map[string]string{},
103 blockExitLabel: map[int32]string{},
104 nameUsed: map[string]bool{},
105 wasmAttrID: 99,
106 }
107 }
108
109 func (e *irEmitter) w(s string) {
110 push(e.segs, s)
111 }
112
113 // assembleSegs joins all segments into e.buf in one allocation.
114 // After this, e.segs is reset and e.buf holds the assembled output.
115 func (e *irEmitter) assembleSegs() {
116 n := int32(0)
117 for _, s := range e.segs {
118 n += len(s)
119 }
120 e.buf = []byte{:n}
121 off := int32(0)
122 for _, s := range e.segs {
123 for j := int32(0); j < len(s); j++ {
124 e.buf[off+j] = s[j]
125 }
126 off += len(s)
127 }
128 e.segs = e.segs[:0]
129 }
130
131 // flush assembles segments, writes to output file, resets.
132 func (e *irEmitter) flush() {
133 e.assembleSegs()
134 if e.outFile != nil && len(e.buf) > 0 {
135 e.outFile.Write(e.buf)
136 }
137 e.buf = nil
138 }
139
140 // flushDedup assembles segments, deduplicates declare lines, writes to file.
141 func (e *irEmitter) flushDedup() {
142 e.assembleSegs()
143 if e.outFile == nil || len(e.buf) == 0 {
144 e.buf = nil
145 return
146 }
147 // Scan for declare lines and remove duplicates.
148 var seen []string
149 out := []byte{:0:len(e.buf)}
150 ls := int32(0)
151 n := int32(len(e.buf))
152 for ls < n {
153 le := ls
154 for le < n && e.buf[le] != '\n' {
155 le++
156 }
157 if le < n {
158 le++
159 }
160 line := string(e.buf[ls:le])
161 // Dedup declare lines.
162 isDeclare := len(line) > 8 && line[0] == 'd' && line[1] == 'e' && line[2] == 'c' && line[3] == 'l' && line[4] == 'a' && line[5] == 'r' && line[6] == 'e' && line[7] == ' '
163 // Dedup define lines (for per-type fixup functions).
164 isDefine := len(line) > 7 && line[0] == 'd' && line[1] == 'e' && line[2] == 'f' && line[3] == 'i' && line[4] == 'n' && line[5] == 'e' && line[6] == ' '
165 if isDeclare || isDefine {
166 dup := false
167 for si := int32(0); si < len(seen); si++ {
168 s := seen[si]
169 if len(s) == len(line) {
170 match := true
171 for bi := int32(0); bi < len(s); bi++ {
172 if s[bi] != line[bi] {
173 match = false
174 break
175 }
176 }
177 if match {
178 dup = true
179 break
180 }
181 }
182 }
183 if dup {
184 if isDefine {
185 // Skip entire function body until closing }.
186 for ls < n {
187 if e.buf[ls] == '}' {
188 ls++
189 if ls < n && e.buf[ls] == '\n' {
190 ls++
191 }
192 break
193 }
194 ls++
195 }
196 } else {
197 ls = le
198 }
199 continue
200 }
201 push(seen, line)
202 }
203 for j := ls; j < le; j++ {
204 push(out, e.buf[j])
205 }
206 ls = le
207 }
208 e.outFile.Write(out)
209 e.buf = nil
210 }
211
212 // hoistAllocaText assembles segments from start onward, moves alloca lines
213 // to _entry top, stores result as single segment. Segments before start
214 // are left intact (parent function already processed).
215 func (e *irEmitter) hoistAllocaText(start int32) {
216 // Assemble only segments from start to end.
217 n := int32(0)
218 for i := start; i < int32(len(e.segs)); i++ {
219 n += len(e.segs[i])
220 }
221 body := []byte{:n}
222 off := int32(0)
223 for i := start; i < int32(len(e.segs)); i++ {
224 s := e.segs[i]
225 for j := int32(0); j < len(s); j++ {
226 body[off+j] = s[j]
227 }
228 off += len(s)
229 }
230 e.segs = e.segs[:start]
231 marker := "_entry:\n"
232 mlen := int32(len(marker))
233 insert := int32(-1)
234 for i := int32(0); i+mlen <= int32(len(body)); i++ {
235 match := true
236 for k := int32(0); k < mlen; k++ {
237 if body[i+k] != marker[k] {
238 match = false
239 break
240 }
241 }
242 if match && (i == 0 || body[i-1] == '\n') {
243 insert = i + mlen
244 break
245 }
246 }
247 if insert < 0 {
248 push(e.segs, string(body))
249 return
250 }
251 // Pass 1: count alloca bytes vs rest bytes.
252 blen := int32(len(body))
253 allocaBytes := int32(0)
254 moved := false
255 ls := insert
256 for ls < blen {
257 le := ls
258 for le < blen && body[le] != '\n' {
259 le++
260 }
261 if le < blen {
262 le++
263 }
264 lineLen := le - ls
265 if lineLen > 13 && body[ls] == ' ' && body[ls+1] == ' ' && body[ls+2] == '%' {
266 for k := ls + 3; k+10 <= le; k++ {
267 if body[k] == ' ' && body[k+1] == '=' && body[k+2] == ' ' &&
268 body[k+3] == 'a' && body[k+4] == 'l' && body[k+5] == 'l' &&
269 body[k+6] == 'o' && body[k+7] == 'c' && body[k+8] == 'a' &&
270 body[k+9] == ' ' {
271 allocaBytes += lineLen
272 moved = true
273 break
274 }
275 }
276 }
277 ls = le
278 }
279 if !moved {
280 push(e.segs, string(body))
281 return
282 }
283 // Pass 2: build output - prefix, then allocas, then rest.
284 out := []byte{:int32(len(body))}
285 for i := int32(0); i < insert; i++ {
286 out[i] = body[i]
287 }
288 wp := insert // write pointer for allocas
289 wr := insert + allocaBytes // write pointer for rest
290 ls = insert
291 for ls < blen {
292 le := ls
293 for le < blen && body[le] != '\n' {
294 le++
295 }
296 if le < blen {
297 le++
298 }
299 lineLen := le - ls
300 isAlloca := false
301 if lineLen > 13 && body[ls] == ' ' && body[ls+1] == ' ' && body[ls+2] == '%' {
302 for k := ls + 3; k+10 <= le; k++ {
303 if body[k] == ' ' && body[k+1] == '=' && body[k+2] == ' ' &&
304 body[k+3] == 'a' && body[k+4] == 'l' && body[k+5] == 'l' &&
305 body[k+6] == 'o' && body[k+7] == 'c' && body[k+8] == 'a' &&
306 body[k+9] == ' ' {
307 isAlloca = true
308 break
309 }
310 }
311 }
312 if isAlloca {
313 for j := int32(0); j < lineLen; j++ {
314 out[wp+j] = body[ls+j]
315 }
316 wp += lineLen
317 } else {
318 for j := int32(0); j < lineLen; j++ {
319 out[wr+j] = body[ls+j]
320 }
321 wr += lineLen
322 }
323 ls = le
324 }
325 push(e.segs, string(out))
326 }
327
328 func (e *irEmitter) regName(v SSAValue) (s string) {
329 if v == nil {
330 e.nextReg++
331 return "%r" | irItoa(e.nextReg)
332 }
333 if nm, ok := e.valName[v]; ok {
334 return nm
335 }
336 name := safeSSAName(v)
337 if name == "" {
338 e.nextReg++
339 name = "r" | irItoa(e.nextReg)
340 }
341 n := "%" | name
342 for e.nameUsed[n] {
343 e.nextReg++
344 n = "%r" | irItoa(e.nextReg)
345 }
346 e.valName[v] = n
347 e.nameUsed[n] = true
348 return n
349 }
350
351 func (e *irEmitter) setRegType(v SSAValue, reg string, typ string) {
352 e.allocTypes[v] = typ
353 if len(reg) > 0 && reg[0] == '%' {
354 e.regTypes[reg] = typ
355 }
356 }
357
358 func (e *irEmitter) nextReg2(prefix string) (s string) {
359 e.nextReg++
360 return "%" | prefix | irItoa(e.nextReg)
361 }
362
363 func (e *irEmitter) declareRuntime(name, retType, params string) {
364 if params != "" {
365 params = params | ", ptr"
366 } else {
367 params = "ptr"
368 }
369 push(e.pendExtDecls, emitKV{name, retType | " @" | name | "(" | params | ")"})
370 }
371
372 func (e *irEmitter) declareRuntimeNoCtx(name, retType, params string) {
373 push(e.pendExtDecls, emitKV{name, retType | " @" | name | "(" | params | ")"})
374 }
375
376
377 func (e *irEmitter) declareExternalGlobal(g *SSAGlobal) {
378 if g.pkg == nil || g.pkg == e.pkg {
379 return
380 }
381 name := e.globalName(g)
382 if _, ok := e.extGlobals[name]; ok {
383 return
384 }
385 for _, p := range e.pendExtGlobals {
386 if p.k == name { return }
387 }
388 typ := e.llvmType(g.typ)
389 if p, ok := SafeUnderlying(g.typ).(*Pointer); ok {
390 if p.Base != nil {
391 typ = e.llvmType(p.Base)
392 } else {
393 typ = "ptr"
394 }
395 }
396 if typ == "void" {
397 typ = "i1"
398 }
399 push(e.pendExtGlobals, emitKV{name, typ})
400 }
401
402 func (e *irEmitter) declareExternalFunc(fn *SSAFunction) {
403 if fn == nil || len(fn.Blocks) > 0 {
404 return
405 }
406 sym := e.funcSymbol(fn)
407 if _, ok := e.extDecls[sym]; ok {
408 return
409 }
410 for _, p := range e.pendExtDecls {
411 if p.k == sym { return }
412 }
413 retType := e.funcRetType(fn)
414 useSret := needsSret(retType)
415 isIntrinsic := mxutil.HasPrefix(sym, "@llvm.") || mxutil.HasPrefix(sym, "@\"llvm.")
416 isExport := fn.externalSymbol != ""
417 skipCtx := isIntrinsic || isExport
418 params := ""
419 if useSret {
420 params = "ptr"
421 }
422 var psegs []string
423 if params != "" {
424 push(psegs, params)
425 }
426 hasRecv := fn.Signature != nil && fn.Signature.Recv != nil
427 if hasRecv {
428 push(psegs, "ptr")
429 }
430 if fn.Signature != nil && fn.Signature.Params != nil {
431 for i := 0; i < fn.Signature.Params.Len(); i++ {
432 pt := e.llvmType(fn.Signature.Params.At(i).Typ)
433 if pt == "" || pt == "void" {
434 mxutil.WriteStr(2, "BUG-DECL: empty param type in " | sym | " param " | irItoa(int32(i)) | "\n")
435 pt = "ptr"
436 }
437 push(psegs, pt)
438 }
439 }
440 if !skipCtx {
441 push(psegs, "ptr")
442 }
443 params = joinStrs(commaSep(psegs))
444 if useSret {
445 push(e.pendExtDecls, emitKV{sym, "void " | sym | "(" | params | ")"})
446 } else {
447 push(e.pendExtDecls, emitKV{sym, retType | " " | sym | "(" | params | ")"})
448 }
449 }
450
451 func (e *irEmitter) addStringConst(s string) (n int32) {
452 if existing, ok := e.strMap[s]; ok {
453 return existing
454 }
455 // Check pending list (not yet merged).
456 base := len(e.strConst)
457 for i, ps := range e.pendStrConsts {
458 if ps == s {
459 return int32(base + i)
460 }
461 }
462 idx := int32(base + len(e.pendStrConsts))
463 push(e.pendStrConsts, s)
464 return idx
465 }
466
467 func (e *irEmitter) strConstGlobal(idx int32) (s string) {
468 return "@.str." | irItoa(idx)
469 }
470
471 func (e *irEmitter) resolveAllNamedTypes() {
472 if e.pkg == nil || e.pkg.Pkg == nil || e.pkg.Pkg.Scope == nil {
473 return
474 }
475 scope := e.pkg.Pkg.Scope
476 for _, name := range scope.Names() {
477 obj := scope.Lookup(name)
478 if obj == nil {
479 continue
480 }
481 tn, ok := obj.(*TypeName)
482 if !ok || tn == nil {
483 continue
484 }
485 named, ok2 := tn.Typ.(*Named)
486 if !ok2 || named == nil {
487 continue
488 }
489 if named.Under != nil {
490 continue
491 }
492 u := SafeUnderlying(named)
493 if u != nil && u != Type(named) {
494 named.Under = u
495 }
496 }
497 }
498
499 func (e *irEmitter) emit() (s string) {
500 dl := e.dataLayout()
501 if dl != "" {
502 e.w("target datalayout = \"")
503 e.w(dl)
504 e.w("\"\n")
505 }
506 e.w("target triple = \"")
507 e.w(e.triple)
508 e.w("\"\n\n")
509
510 e.resolveAllNamedTypes()
511 e.globalTypes = map[string]string{}
512 e.globalDeclTypes = map[string]string{}
513 sortedM := e.pkgMembers()
514 for _, member := range sortedM {
515 fn, ok := member.(*SSAFunction)
516 if !ok { continue }
517 for _, b := range fn.Blocks {
518 for _, instr := range b.Instrs {
519 if st, okSt := instr.(*SSAStore); okSt && st.Addr != nil && st.Val != nil {
520 if g, ok3 := st.Addr.(*SSAGlobal); ok3 {
521 vt := e.llvmType(st.Val.SSAType())
522 if vt != "ptr" && vt != "void" && vt != "i1" && vt != "" {
523 name := e.globalName(g)
524 gt := ""
525 if p, ok4 := SafeUnderlying(g.typ).(*Pointer); ok4 {
526 gt = e.llvmType(p.Base)
527 }
528 if gt != "" && gt != "ptr" && gt != "i8" && gt[0] == '{' && vt[0] != '{' {
529 vt = gt
530 }
531 e.globalTypes[name] = vt
532 }
533 }
534 }
535 }
536 }
537 e.loadToGlobal = map[string]*SSAGlobal{}
538 for _, b := range fn.Blocks {
539 for _, instr := range b.Instrs {
540 load, okUn := instr.(*SSAUnOp)
541 if !okUn || load.Op != OpMul { continue }
542 g, ok3 := load.X.(*SSAGlobal)
543 if !ok3 { continue }
544 e.loadToGlobal[load.SSAName()] = g
545 }
546 }
547 for _, b := range fn.Blocks {
548 for _, instr := range b.Instrs {
549 if ret, okRet := instr.(*SSAReturn); okRet {
550 if fn.Signature == nil { continue }
551 rets := fn.Signature.Results
552 if rets == nil || rets.Len() == 0 { continue }
553 for i, res := range ret.Results {
554 if i >= rets.Len() { break }
555 if g, ok3 := e.loadToGlobal[res.SSAName()]; ok3 {
556 rt := rets.At(i)
557 if rt == nil { continue }
558 expectType := e.llvmType(rt.Typ)
559 if expectType != "ptr" && expectType != "void" && expectType != "i1" && expectType != "" {
560 name := e.globalName(g)
561 if _, exists := e.globalTypes[name]; !exists {
562 e.globalTypes[name] = expectType
563 }
564 }
565 }
566 }
567 }
568 call, okCall := instr.(*SSACall)
569 if !okCall { continue }
570 callee := call.Call.Value
571 if callee == nil { continue }
572 var sig *Signature
573 if cfn, ok3 := callee.(*SSAFunction); ok3 && cfn.Signature != nil {
574 sig = cfn.Signature
575 } else {
576 ct := callee.SSAType()
577 if ct != nil {
578 sig, _ = SafeUnderlying(ct).(*Signature)
579 }
580 }
581 if sig == nil { continue }
582 params := sig.Params
583 if params == nil || params.Len() == 0 { continue }
584 recvOff := 0
585 if sig.Recv != nil {
586 recvOff = 1
587 }
588 for i, arg := range call.Call.Args {
589 if arg == nil { continue }
590 sigIdx := i - recvOff
591 if sigIdx < 0 || sigIdx >= params.Len() { continue }
592 pt := params.At(sigIdx)
593 if pt == nil { continue }
594 g, found := e.loadToGlobal[arg.SSAName()]
595 if !found { continue }
596 expectType := e.llvmType(pt.Typ)
597 name := e.globalName(g)
598 if expectType != "void" && expectType != "i1" && expectType != "" {
599 if _, exists := e.globalTypes[name]; !exists {
600 e.globalTypes[name] = expectType
601 }
602 }
603 }
604 }
605 }
606 for _, b := range fn.Blocks {
607 for _, instr := range b.Instrs {
608 if rng, okRng := instr.(*SSARange); okRng && rng.X != nil {
609 if g, ok3 := e.loadToGlobal[rng.X.SSAName()]; ok3 {
610 name := e.globalName(g)
611 if _, exists := e.globalTypes[name]; !exists {
612 e.globalTypes[name] = "ptr"
613 }
614 }
615 }
616 if mu, okMu := instr.(*SSAMapUpdate); okMu && mu.Map != nil {
617 if g, ok3 := e.loadToGlobal[mu.Map.SSAName()]; ok3 {
618 name := e.globalName(g)
619 if _, exists := e.globalTypes[name]; !exists {
620 e.globalTypes[name] = "ptr"
621 }
622 }
623 }
624 if lu, okLu := instr.(*SSALookup); okLu && lu.X != nil {
625 if g, ok3 := e.loadToGlobal[lu.X.SSAName()]; ok3 {
626 name := e.globalName(g)
627 if _, exists := e.globalTypes[name]; !exists {
628 e.globalTypes[name] = "ptr"
629 }
630 }
631 }
632 bop, okBop := instr.(*SSABinOp)
633 if !okBop { continue }
634 if bop.X == nil || bop.Y == nil { continue }
635 gx, xIsGlobal := e.loadToGlobal[bop.X.SSAName()]
636 gy, yIsGlobal := e.loadToGlobal[bop.Y.SSAName()]
637 if xIsGlobal {
638 yt := e.llvmType(bop.Y.SSAType())
639 if yt != "ptr" && yt != "void" && yt != "i1" && yt != "" {
640 name := e.globalName(gx)
641 if _, exists := e.globalTypes[name]; !exists {
642 e.globalTypes[name] = yt
643 }
644 }
645 }
646 if yIsGlobal {
647 xt := e.llvmType(bop.X.SSAType())
648 if xt != "ptr" && xt != "void" && xt != "i1" && xt != "" {
649 name := e.globalName(gy)
650 if _, exists := e.globalTypes[name]; !exists {
651 e.globalTypes[name] = xt
652 }
653 }
654 }
655 }
656 }
657 }
658
659 e.flush()
660 for _, member := range e.pkgMembers() {
661 switch m := member.(type) {
662 case *SSAGlobal:
663 if m.name != "_" {
664 e.emitGlobal(m)
665 }
666 }
667 }
668 e.flush()
669 compileArena := runtime.CurrentArena()
670 // Deep copy state lives in compileArena, survives per-function resets.
671 e.deepCopyFns = map[string]string{}
672 e.deepCopyQueue = nil
673 // Pre-collect symbols that will get `define` (functions with bodies).
674 // Done in compile arena so the slice survives fn arena resets.
675 for _, member := range e.pkgMembers() {
676 if f, ok := member.(*SSAFunction); ok && f.Blocks != nil {
677 defSym := e.funcSymbol(f)
678 if len(defSym) > 0 && defSym[0] == '@' {
679 defSym = defSym[1:]
680 }
681 push(e.definedSyms, defSym)
682 }
683 }
684 fnArena := runtime.ArenaNew(64 * 1024)
685 for _, member := range e.pkgMembers() {
686 if m, ok := member.(*SSAFunction); ok {
687 checkUseAfterFree(m)
688 checkParamMutation(m)
689 runtime.SetCurrentArena(fnArena)
690 e.emitFunction(m)
691 e.emitAnonFuncs(m)
692 e.flush()
693 runtime.SetCurrentArena(compileArena)
694 e.mergeAccumulators()
695 e.segs = nil
696 e.buf = nil
697 e.operandBuf = nil
698 e.valName = nil
699 e.nameUsed = map[string]bool{}
700 e.allocTypes = nil
701 e.regTypes = nil
702 e.blockExitLabel = nil
703 e.typeCache = nil
704 e.hoisted = nil
705 e.pendingPhiTruncs = nil
706 e.deferList = nil
707 e.allocBlock = nil
708 e.storedTo = nil
709 e.usedAs = nil
710 e.missingStores = nil
711 m.Blocks = nil
712 m.Locals = nil
713 m.Params = nil
714 m.FreeVars = nil
715 m.NamedResults = nil
716 m.vars = nil
717 runtime.ArenaReset(fnArena)
718 }
719 }
720 runtime.ArenaFree(fnArena)
721 // Fresh segs for trailing declarations.
722 e.segs = []string{:0:256}
723 // Emit per-type deep copy helper functions for arena return copy.
724 e.emitDeepCopyFunctions()
725 // Merge any declares/globals generated by deep copy functions.
726 e.mergeAccumulators()
727 e.emitLibMain()
728 for i, sc := range e.strConst {
729 e.w(e.strConstGlobal(i))
730 e.w(" = private constant [")
731 e.w(irItoa(len(sc)))
732 e.w(" x i8] c\"")
733 e.w(irEscapeString(sc))
734 e.w("\"\n")
735 }
736
737
738 if len(e.extDecls) > 0 {
739 e.w("\n")
740 var edKeys []string
741 for k := range e.extDecls {
742 push(edKeys, k)
743 }
744 for i := 1; i < len(edKeys); i++ {
745 for j := i; j > 0 && edKeys[j] < edKeys[j-1]; j-- {
746 edKeys[j], edKeys[j-1] = edKeys[j-1], edKeys[j]
747 }
748 }
749 for _, k := range edKeys {
750 decl := e.extDecls[k]
751 if decl == "" {
752 continue
753 }
754 if e.isPkgFuncDefined(k) {
755 continue
756 }
757 e.w("declare ")
758 e.w(decl)
759 // Check for wasm import attributes
760 if len(cctx.wasmImportMap) > 0 {
761 sym := k
762 if len(sym) > 0 && sym[0] == '@' {
763 sym = sym[1:]
764 }
765 if len(sym) >= 2 && sym[0] == '"' && sym[len(sym)-1] == '"' {
766 sym = sym[1 : len(sym)-1]
767 }
768 if wi, ok := cctx.wasmImportMap[sym]; ok {
769 e.wasmAttrID++
770 aid := e.wasmAttrID
771 e.w(" #" | simpleItoa(aid))
772 push(e.wasmAttrs, [3]string{simpleItoa(aid), wi[0], wi[1]})
773 }
774 }
775 e.w("\n")
776 }
777 }
778
779 if len(e.extGlobals) > 0 {
780 e.w("\n")
781 var egKeys []string
782 for name := range e.extGlobals {
783 push(egKeys, name)
784 }
785 for i := 1; i < len(egKeys); i++ {
786 for j := i; j > 0 && egKeys[j] < egKeys[j-1]; j-- {
787 egKeys[j], egKeys[j-1] = egKeys[j-1], egKeys[j]
788 }
789 }
790 for _, name := range egKeys {
791 e.w(name)
792 e.w(" = ")
793 e.w(e.tlsExternalPrefix())
794 e.w(e.extGlobals[name])
795 e.w("\n")
796 }
797 }
798
799 // Emit wasm import attribute groups
800 for _, wa := range e.wasmAttrs {
801 e.w("attributes #" | wa[0] | " = { \"wasm-import-module\"=\"" | wa[1] | "\" \"wasm-import-name\"=\"" | wa[2] | "\" }\n")
802 }
803
804 if e.outFile != nil {
805 e.flushDedup()
806 return ""
807 }
808 e.assembleSegs()
809 result := string(e.buf)
810 e.buf = nil
811 return result
812 }
813
814 func (e *irEmitter) releaseAfterEmit() {}
815
816 // copyStr deep-copies a string's backing bytes into the current arena.
817 // Without this, string headers in compile-arena maps would point to
818 // fn-arena bytes that die at ArenaReset.
819 func copyStr(s string) (r string) {
820 if len(s) == 0 {
821 return ""
822 }
823 buf := []byte{:len(s)}
824 copy(buf, s)
825 return string(buf)
826 }
827
828 // mergeAccumulators deep-copies per-function products into compile-arena maps.
829 // Called on the compile arena after function emit + flush.
830 // All accumulator string bytes are in the fn arena and must be copied.
831 func (e *irEmitter) mergeAccumulators() {
832 for _, p := range e.pendExtDecls {
833 e.extDecls[copyStr(p.k)] = copyStr(p.v)
834 }
835 e.pendExtDecls = nil
836 for _, p := range e.pendExtGlobals {
837 e.extGlobals[copyStr(p.k)] = copyStr(p.v)
838 }
839 e.pendExtGlobals = nil
840 for _, s := range e.pendStrConsts {
841 cs := copyStr(s)
842 if _, ok := e.strMap[cs]; !ok {
843 e.strMap[cs] = len(e.strConst)
844 push(e.strConst, cs)
845 }
846 }
847 e.pendStrConsts = nil
848 for _, p := range e.pendTypeIDs {
849 ck := copyStr(p.k)
850 if _, ok := e.typeIDs[ck]; !ok {
851 e.typeIDs[ck] = p.v
852 }
853 }
854 e.pendTypeIDs = nil
855 for _, k := range e.pendLocalTypeIDs {
856 e.localTypeIDs[copyStr(k)] = true
857 }
858 e.pendLocalTypeIDs = nil
859 for _, p := range e.pendGlobalDeclTypes {
860 e.globalDeclTypes[copyStr(p.k)] = copyStr(p.v)
861 }
862 e.pendGlobalDeclTypes = nil
863 for _, wa := range e.pendWasmAttrs {
864 push(e.wasmAttrs, [3]string{copyStr(wa[0]), copyStr(wa[1]), copyStr(wa[2])})
865 }
866 e.pendWasmAttrs = nil
867 for _, dc := range e.pendDeepCopyEntries {
868 key := copyStr(typeKeyForDeepCopy(dc.typ))
869 sym := copyStr(dc.sym)
870 e.deepCopyFns[key] = sym
871 push(e.deepCopyQueue, deepCopyEntry{sym, dc.typ})
872 }
873 e.pendDeepCopyEntries = nil
874 for _, sym := range e.pendRelocFixupDefs {
875 push(e.definedSyms, copyStr(sym))
876 }
877 e.pendRelocFixupDefs = nil
878 }
879
880 func (e *irEmitter) pkgMembers() (ss []SSAMember) {
881 var keys []string
882 for k := range e.pkg.Members {
883 push(keys, k)
884 }
885 for i := 1; i < len(keys); i++ {
886 for j := i; j > 0 && keys[j] < keys[j-1]; j-- {
887 keys[j], keys[j-1] = keys[j-1], keys[j]
888 }
889 }
890 var members []SSAMember
891 for _, k := range keys {
892 m := e.pkg.Members[k]
893 if m != nil {
894 push(members, m)
895 }
896 }
897 return members
898 }
899
900 func (e *irEmitter) emitGlobal(g *SSAGlobal) {
901 name := e.globalName(g)
902 if ei, ok := cctx.embedVars[g.name]; ok {
903 e.emitEmbedGlobal(g, name, ei)
904 return
905 }
906 typ := e.resolveGlobalDeclType(g)
907 e.w(name)
908 e.w(" = ")
909 e.w(e.tlsPrefix())
910 e.w(typ)
911 e.w(" zeroinitializer\n")
912 }
913
914 func (e *irEmitter) emitEmbedGlobal(g *SSAGlobal, name string, ei *embedInfo) {
915 data, ok := mxutil.ReadFile(ei.path)
916 if !ok {
917 // Fail loud: a zeroinitializer embed global is a nil slice at runtime
918 // and SIGSEGVs far from the cause.
919 push(cctx.compileErrors, "embed: file not found: "|ei.path)
920 e.w(name | " = " | e.tlsPrefix() | e.sliceType() | " zeroinitializer\n")
921 return
922 }
923 elemSize := e.embedElemSize(ei.elemType)
924 if elemSize <= 0 {
925 push(cctx.compileErrors, "embed: unknown element type: "|ei.elemType|" for "|ei.path)
926 e.w(name | " = " | e.tlsPrefix() | e.sliceType() | " zeroinitializer\n")
927 return
928 }
929 count := int32(len(data)) / elemSize
930 // name may be quoted (@"pkg/path.var"); the .data suffix must land
931 // inside the quotes or the symbol is malformed IR.
932 constName := name | ".data"
933 if len(name) > 0 && name[len(name)-1] == '"' {
934 constName = name[:len(name)-1] | ".data\""
935 }
936 e.w(constName | " = private constant [" | irItoa(int32(len(data))) | " x i8] c\"")
937 hexDigit := "0123456789ABCDEF"
938 hexBuf := []byte{:int32(len(data)) * 3}
939 for i := int32(0); i < int32(len(data)); i++ {
940 b := data[i]
941 hexBuf[i*3] = '\\'
942 hexBuf[i*3+1] = hexDigit[b>>4]
943 hexBuf[i*3+2] = hexDigit[b&0x0f]
944 }
945 e.w(string(hexBuf))
946 e.w("\"\n")
947 ipt := e.intptrType()
948 e.w(name | " = " | e.tlsPrefix() | "{ptr, " | ipt | ", " | ipt | "} {ptr " | constName | ", " | ipt | " " | irItoa(count) | ", " | ipt | " " | irItoa(count) | "}\n")
949 }
950
951 func (e *irEmitter) embedElemSize(elemType string) (sz int32) {
952 if elemType == "[]Range16" {
953 return 6
954 }
955 if elemType == "[]Range32" {
956 return 12
957 }
958 if elemType == "[]CaseRange" {
959 return 20
960 }
961 if elemType == "[]foldPair" {
962 return 8
963 }
964 if elemType == "[]uint8" || elemType == "[]byte" {
965 return 1
966 }
967 if elemType == "[]uint16" {
968 return 2
969 }
970 if elemType == "[]uint32" || elemType == "[]int32" {
971 return 4
972 }
973 if elemType == "[]uint64" || elemType == "[]int64" {
974 return 8
975 }
976 return 0
977 }
978
979 func (e *irEmitter) globalName(g *SSAGlobal) (s string) {
980 pkg := e.pkg.Pkg.Path
981 if g.pkg != nil {
982 pkg = g.pkg.Pkg.Path
983 }
984 return irGlobalSymbol(pkg, g.name)
985 }
986
987 func (e *irEmitter) isPkgFuncDefined(name string) (ok bool) {
988 // Check if we emitted a define for this symbol during this package's emit.
989 for di := int32(0); di < len(e.definedSyms); di++ {
990 ds := e.definedSyms[di]
991 if len(ds) == len(name) {
992 match := true
993 for bi := int32(0); bi < len(ds); bi++ {
994 if ds[bi] != name[bi] {
995 match = false
996 break
997 }
998 }
999 if match {
1000 return true
1001 }
1002 }
1003 }
1004 return false
1005 }
1006
1007 func (e *irEmitter) isPkgFunc2(name string) (ok bool) {
1008 for _, m := range e.pkg.Members {
1009 if f, ok2 := m.(*SSAFunction); ok2 {
1010 sym := e.funcSymbol(f)
1011 if len(sym) > 2 && sym[0] == '@' && sym[1] == '"' {
1012 sym = sym[2 : len(sym)-1]
1013 } else if len(sym) > 1 && sym[0] == '@' {
1014 sym = sym[1:]
1015 }
1016 if sym == name {
1017 return true
1018 }
1019 }
1020 }
1021 return false
1022 }
1023
1024 func (e *irEmitter) funcSymbol(f *SSAFunction) (s string) {
1025 if f == nil {
1026 return "@__mxc_nil_func"
1027 }
1028 if f.externalSymbol != "" {
1029 sym := f.externalSymbol
1030 if irNeedsQuote(sym) {
1031 return "@\"" | sym | "\""
1032 }
1033 return "@" | sym
1034 }
1035 if cctx.linknameMap != nil {
1036 if target, ok := cctx.linknameMap[f.name]; ok {
1037 if irNeedsQuote(target) {
1038 return "@\"" | target | "\""
1039 }
1040 return "@" | target
1041 }
1042 }
1043 pkg := e.pkg.Pkg.Path
1044 if f.Pkg != nil {
1045 pkg = f.Pkg.Pkg.Path
1046 }
1047 return irGlobalSymbol(pkg, f.name)
1048 }
1049
1050 func (e *irEmitter) isPkgFunc(f *SSAFunction) (ok bool) {
1051 if f == nil {
1052 return false
1053 }
1054 if f.Pkg == e.pkg {
1055 return true
1056 }
1057 if f.parent != nil {
1058 return e.isPkgFunc(f.parent)
1059 }
1060 return false
1061 }
1062
1063 func (e *irEmitter) emitAnonFuncs(f *SSAFunction) {
1064 for _, af := range f.AnonFuncs {
1065 e.emitFunction(af)
1066 e.emitAnonFuncs(af)
1067 af.Blocks = nil
1068 af.Locals = nil
1069 af.Params = nil
1070 af.FreeVars = nil
1071 af.NamedResults = nil
1072 af.vars = nil
1073 }
1074 f.AnonFuncs = nil
1075 }
1076
1077 func (e *irEmitter) emitLibMain() {
1078 pkgPath := e.pkg.Pkg.Path
1079 if pkgPath == "main" || pkgPath == "command-line-arguments" {
1080 return
1081 }
1082 hasMain := false
1083 if m := e.pkg.Members["main"]; m != nil {
1084 if fn, ok := m.(*SSAFunction); ok && fn.externalSymbol == "" {
1085 hasMain = true
1086 }
1087 }
1088 if hasMain {
1089 return
1090 }
1091 sym := irGlobalSymbol(pkgPath, "main")
1092 e.w("\ndefine void ")
1093 e.w(sym)
1094 e.w("(ptr %_ctx) {\n_entry:\n ret void\n}\n")
1095 }
1096
1097 func (e *irEmitter) emitFunction(f *SSAFunction) {
1098 e.w("; [emit] " | f.name | "\n")
1099 sym := e.funcSymbol(f)
1100 if len(sym) > 5 && (mxutil.HasPrefix(sym, "@llvm.") || mxutil.HasPrefix(sym, "@\"llvm.")) {
1101 e.emitFuncDecl(f)
1102 return
1103 }
1104 if len(f.Blocks) == 0 {
1105 // If function has a linkname, the declare uses the target symbol
1106 // but consuming packages reference the original fully-qualified name.
1107 // Emit a forwarding wrapper so the linker can resolve both.
1108 if cctx.linknameMap != nil {
1109 if target, ok := cctx.linknameMap[f.name]; ok {
1110 _ = target
1111 pkg := e.pkg.Pkg.Path
1112 if f.Pkg != nil {
1113 pkg = f.Pkg.Pkg.Path
1114 }
1115 origSym := irGlobalSymbol(pkg, f.name)
1116 tgtSym := sym // already the linkname target from funcSymbol
1117 // Check if the target is defined in the same package
1118 // (has a body). If so, skip the declare to avoid
1119 // conflicting with the existing define.
1120 // If the linkname target is in the same package, the
1121 // target function is already defined - skip the declare
1122 // to avoid conflicting with the existing define.
1123 pkgDot := pkg | "."
1124 samePackage := mxutil.HasPrefix(target, pkgDot)
1125 if !samePackage {
1126 e.emitFuncDecl(f)
1127 } else if origSym == tgtSym {
1128 // Self-referential linkname: the function's linkname
1129 // target is its own fully-qualified name. Body is
1130 // provided by another package. Emit a declare.
1131 e.emitFuncDecl(f)
1132 }
1133 if origSym != tgtSym {
1134 e.emitLinknameForwarder(f, origSym, tgtSym)
1135 }
1136 return
1137 }
1138 }
1139 e.emitFuncDecl(f)
1140 return
1141 }
1142 e.curFunc = f
1143 e.arenaActive = false
1144 markReturnEscapes(f)
1145 e.nextReg = 0
1146 e.deferList = nil
1147 e.deferID = 0
1148 e.valName = map[SSAValue]string{}
1149 e.nameUsed = map[string]bool{}
1150 e.allocTypes = map[SSAValue]string{}
1151 e.regTypes = map[string]string{}
1152 e.blockExitLabel = map[int32]string{}
1153 e.typeCache = map[Type]string{}
1154 usedNames := map[string]int32{}
1155 for i, p := range f.Params {
1156 pname := p.SSAName()
1157 if pname == "" {
1158 pname = "p" | irItoa(i)
1159 }
1160 if cnt, ok := usedNames[pname]; ok {
1161 pname = pname | "." | irItoa(cnt)
1162 }
1163 usedNames[p.SSAName()]++
1164 e.valName[p] = "%" | pname
1165 e.nameUsed["%" | pname] = true
1166 }
1167
1168 e.w("\ndefine ")
1169 if f.parent != nil {
1170 e.w("hidden ")
1171 }
1172 retType := e.funcRetType(f)
1173 useSret := needsSret(retType)
1174 e.sretType = ""
1175 if useSret {
1176 e.w("void ")
1177 e.sretType = retType
1178 } else {
1179 e.w(retType)
1180 e.w(" ")
1181 }
1182 e.w(e.funcSymbol(f))
1183 e.w("(")
1184 nParam := 0
1185 if useSret {
1186 e.w("ptr sret(") ; e.w(retType) ; e.w(") %retval")
1187 nParam++
1188 }
1189 isExport := f.externalSymbol != ""
1190 for _, p := range f.Params {
1191 if nParam > 0 { e.w(", ") }
1192 pt := e.llvmType(p.SSAType())
1193 if pt == "void" {
1194 pt = "ptr"
1195 }
1196 e.w(pt)
1197 e.w(" ")
1198 e.w(e.regName(p))
1199 nParam++
1200 }
1201 if !isExport {
1202 if nParam > 0 { e.w(", ") }
1203 e.w("ptr %_ctx")
1204 }
1205 e.w(") {\n")
1206 fnBodyStart := int32(len(e.segs)) // segment index where this function's body starts
1207
1208 // Pre-scan: set allocTypes, detect cross-block alloca references
1209 e.allocBlock = map[SSAValue]int32{}
1210 for _, b := range f.Blocks {
1211 for _, instr := range b.Instrs {
1212 if n, ok := instr.(*SSANext); ok {
1213 if ri, ok2 := n.Iter.(*SSARange); ok2 {
1214 if arr, ok3 := SafeUnderlying(ri.X.SSAType()).(*Array); ok3 {
1215 elemType := e.llvmType(arr.Elem)
1216 e.allocTypes[n] = "{i1, i32, " | elemType | "}"
1217 } else if sl, ok3b := SafeUnderlying(ri.X.SSAType()).(*Slice); ok3b {
1218 elemType := e.llvmType(sl.Elem)
1219 e.allocTypes[n] = "{i1, i32, " | elemType | "}"
1220 }
1221 }
1222 }
1223 if c, ok := instr.(*SSACall); ok {
1224 if b2, ok2 := c.Call.Value.(*SSABuiltin); ok2 && b2.SSAName() == "recover" {
1225 e.allocTypes[c] = e.ifaceType()
1226 }
1227 }
1228 if a, ok := instr.(*SSAAlloc); ok {
1229 e.allocBlock[a] = b.Index
1230 }
1231 }
1232 }
1233 hoistAllocs := map[SSAValue]bool{}
1234 for _, b := range f.Blocks {
1235 for _, instr := range b.Instrs {
1236 refs := e.instrOperands(instr)
1237 for _, ref := range refs {
1238 if ab, ok := e.allocBlock[ref]; ok && ab != 0 && ab != b.Index {
1239 hoistAllocs[ref] = true
1240 }
1241 }
1242 }
1243 }
1244 e.hoisted = hoistAllocs
1245 e.missingStores = nil
1246 e.storedTo = map[string]bool{}
1247 for _, b := range f.Blocks {
1248 for _, instr := range b.Instrs {
1249 if s, ok := instr.(*SSAStore); ok && s.Addr != nil {
1250 e.storedTo[s.Addr.SSAName()] = true
1251 }
1252 }
1253 }
1254 e.usedAs = map[string]bool{}
1255 for _, b := range f.Blocks {
1256 for _, instr := range b.Instrs {
1257 refs := e.instrOperands(instr)
1258 for _, ref := range refs {
1259 if ref != nil {
1260 e.usedAs[ref.SSAName()] = true
1261 }
1262 }
1263 }
1264 }
1265 for _, b := range f.Blocks {
1266 for i := 0; i+1 < len(b.Instrs); i++ {
1267 load, isLoad := b.Instrs[i].(*SSAUnOp)
1268 if !isLoad || load.Op != OpMul {
1269 continue
1270 }
1271 alloc, isAlloc := b.Instrs[i+1].(*SSAAlloc)
1272 if !isAlloc {
1273 continue
1274 }
1275 if !e.usedAs[load.SSAName()] && !e.storedTo[alloc.SSAName()] && hoistAllocs[alloc] {
1276 srcAlloc, isSrcAlloc := load.X.(*SSAAlloc)
1277 if !isSrcAlloc {
1278 continue
1279 }
1280 srcType := e.llvmType(srcAlloc.SSAType())
1281 if p, ok := SafeUnderlying(srcAlloc.SSAType()).(*Pointer); ok && p.Base != nil {
1282 srcType = e.llvmType(p.Base)
1283 }
1284 if len(srcType) > 0 && srcType[0] == '[' {
1285 if e.missingStores == nil {
1286 e.missingStores = map[SSAValue]SSAValue{}
1287 }
1288 e.missingStores[load] = alloc
1289 e.allocTypes[alloc] = srcType
1290 }
1291 }
1292 }
1293 }
1294 var hoistList []*SSAAlloc
1295 // Collect allocas deterministically: iterate blocks/instructions in order
1296 // (both are slices). Using hoistAllocs map only as a set membership check.
1297 for _, hb := range f.Blocks {
1298 for _, hinstr := range hb.Instrs {
1299 if a, ok := hinstr.(*SSAAlloc); ok && hoistAllocs[a] {
1300 push(hoistList, a)
1301 }
1302 }
1303 }
1304 for i := 1; i < len(hoistList); i++ {
1305 for j := i; j > 0 && hoistList[j].SSAName() < hoistList[j-1].SSAName(); j-- {
1306 hoistList[j], hoistList[j-1] = hoistList[j-1], hoistList[j]
1307 }
1308 }
1309 for _, b := range f.Blocks {
1310 for _, instr := range b.Instrs {
1311 if d, ok := instr.(*SSADefer); ok {
1312 push(e.deferList, d)
1313 }
1314 }
1315 }
1316 scopeIDs := map[int32]bool{}
1317 hasHeapAllocs := false
1318 for _, b := range f.Blocks {
1319 if b.ScopeID > 0 {
1320 scopeIDs[b.ScopeID] = true
1321 }
1322 for _, instr := range b.Instrs {
1323 switch instr.(type) {
1324 case *SSAAlloc:
1325 if instr.(*SSAAlloc).Heap {
1326 hasHeapAllocs = true
1327 }
1328 case *SSAMakeSlice, *SSAMakeMap, *SSAMakeChan, *SSAMakeClosure, *SSASlice:
1329 hasHeapAllocs = true
1330 }
1331 }
1332 }
1333 for _, b := range f.Blocks {
1334 if b.Index == 0 {
1335 e.w("_entry:\n")
1336 e.currentEmitBlock = b
1337 for _, a := range hoistList {
1338 e.emitAlloc(a)
1339 }
1340 if len(e.deferList) > 0 {
1341 e.w(" %deferPtr = alloca ptr\n")
1342 e.w(" store ptr null, ptr %deferPtr\n")
1343 }
1344 if len(e.curFunc.FreeVars) > 0 {
1345 e.emitFreeVarUnpack(e.curFunc)
1346 }
1347 e.emitArenaPrologue()
1348 for _, instr := range b.Instrs {
1349 e.emitInstr(instr)
1350 }
1351 e.flushPhiTruncs()
1352 } else {
1353 e.emitBlock(b)
1354 }
1355 }
1356 e.hoisted = nil
1357 e.hoistAllocaText(fnBodyStart)
1358
1359 e.w("}\n")
1360 }
1361
1362 func (e *irEmitter) emitFuncDecl(f *SSAFunction) {
1363 sym := e.funcSymbol(f)
1364 bareKey := sym
1365 if len(bareKey) > 0 && bareKey[0] == '@' {
1366 bareKey = bareKey[1:]
1367 }
1368 if _, ok := e.extDecls[bareKey]; ok {
1369 return
1370 }
1371 for _, p := range e.pendExtDecls {
1372 if p.k == bareKey { return }
1373 }
1374 retType := e.funcRetType(f)
1375 useSret := needsSret(retType)
1376 decl := ""
1377 if useSret {
1378 decl = "void "
1379 } else {
1380 decl = retType | " "
1381 }
1382 decl = decl | sym | "("
1383 n := 0
1384 if useSret {
1385 decl = decl | "ptr"
1386 n++
1387 }
1388 isIntrinsic := mxutil.HasPrefix(sym, "@llvm.") || mxutil.HasPrefix(sym, "@\"llvm.")
1389 hasRecv := f.Signature != nil && f.Signature.Recv != nil
1390 if hasRecv {
1391 if n > 0 { decl = decl | ", " }
1392 decl = decl | "ptr"
1393 n++
1394 }
1395 if f.Signature != nil && f.Signature.Params != nil {
1396 for i := 0; i < f.Signature.Params.Len(); i++ {
1397 if n > 0 {
1398 decl = decl | ", "
1399 }
1400 pt := e.llvmType(f.Signature.Params.At(i).Typ)
1401 if pt == "" || pt == "void" {
1402 pt = "ptr"
1403 }
1404 decl = decl | pt
1405 n++
1406 }
1407 }
1408 if !isIntrinsic {
1409 if n > 0 { decl = decl | ", " }
1410 decl = decl | "ptr"
1411 }
1412 decl = decl | ")"
1413 bareKey := sym
1414 if len(bareKey) > 0 && bareKey[0] == '@' {
1415 bareKey = bareKey[1:]
1416 }
1417 push(e.pendExtDecls, emitKV{bareKey, decl})
1418 }
1419
1420 // emitLinknameForwarder emits a thin wrapper definition with origSym that
1421 // calls tgtSym. This allows consuming packages (which reference origSym)
1422 // to resolve to the linkname target (tgtSym) at link time.
1423 func (e *irEmitter) emitLinknameForwarder(f *SSAFunction, origSym, tgtSym string) {
1424 retType := e.funcRetType(f)
1425 useSret := needsSret(retType)
1426 // Build parameter list with names
1427 var paramTypes []string
1428 var paramArgs []string
1429 idx := int32(0)
1430 if useSret {
1431 pn := "%sret"
1432 push(paramTypes, "ptr " | pn)
1433 push(paramArgs, "ptr " | pn)
1434 idx++
1435 }
1436 hasRecv := f.Signature != nil && f.Signature.Recv != nil
1437 if hasRecv {
1438 pn := "%p" | irItoa(idx)
1439 push(paramTypes, "ptr " | pn)
1440 push(paramArgs, "ptr " | pn)
1441 idx++
1442 }
1443 if f.Signature != nil && f.Signature.Params != nil {
1444 for i := 0; i < f.Signature.Params.Len(); i++ {
1445 pt := e.llvmType(f.Signature.Params.At(i).Typ)
1446 if pt == "" || pt == "void" { pt = "ptr" }
1447 pn := "%p" | irItoa(idx)
1448 push(paramTypes, pt | " " | pn)
1449 push(paramArgs, pt | " " | pn)
1450 idx++
1451 }
1452 }
1453 // context pointer
1454 ctxPN := "%p" | irItoa(idx)
1455 push(paramTypes, "ptr " | ctxPN)
1456 push(paramArgs, "ptr " | ctxPN)
1457 // Emit define
1458 e.w("\ndefine ")
1459 if useSret {
1460 e.w("void ")
1461 } else {
1462 e.w(retType) ; e.w(" ")
1463 }
1464 e.w(origSym) ; e.w("(")
1465 for i, pt := range paramTypes {
1466 if i > 0 { e.w(", ") }
1467 e.w(pt)
1468 }
1469 e.w(") {\n")
1470 if useSret || retType == "void" {
1471 e.w(" call void ") ; e.w(tgtSym) ; e.w("(")
1472 for i, pa := range paramArgs {
1473 if i > 0 { e.w(", ") }
1474 e.w(pa)
1475 }
1476 e.w(")\n ret void\n")
1477 } else {
1478 e.w(" %fwd = call ") ; e.w(retType) ; e.w(" ") ; e.w(tgtSym) ; e.w("(")
1479 for i, pa := range paramArgs {
1480 if i > 0 { e.w(", ") }
1481 e.w(pa)
1482 }
1483 e.w(")\n ret ") ; e.w(retType) ; e.w(" %fwd\n")
1484 }
1485 e.w("}\n")
1486 }
1487
1488 func (e *irEmitter) emitBlock(b *SSABasicBlock) {
1489 label := "b" | irItoa(b.Index)
1490 if b.Index == 0 {
1491 label = "_entry"
1492 }
1493 e.w(label)
1494 e.w(":\n")
1495 e.blockExitLabel[b.Index] = "%" | label
1496 e.currentEmitBlock = b
1497
1498 if b.Index == 0 && len(e.curFunc.FreeVars) > 0 {
1499 e.emitFreeVarUnpack(e.curFunc)
1500 }
1501
1502 e.blockDead = false
1503 phiFlushed := false
1504 for _, instr := range b.Instrs {
1505 if e.blockDead { break }
1506 if !phiFlushed {
1507 if _, isPhi := instr.(*SSAPhi); !isPhi {
1508 phiFlushed = true
1509 e.flushPhiTruncs()
1510 }
1511 }
1512 e.emitInstr(instr)
1513 if e.missingStores != nil {
1514 if v, ok2 := instr.(SSAValue); ok2 {
1515 if dst, ok3 := e.missingStores[v]; ok3 {
1516 loadReg := e.regName(v)
1517 dstReg := e.regName(dst)
1518 arrType := e.allocTypes[dst]
1519 e.w(" store ") ; e.w(arrType) ; e.w(" ") ; e.w(loadReg) ; e.w(", ptr ") ; e.w(dstReg) ; e.w("\n")
1520 }
1521 }
1522 }
1523 }
1524
1525 e.flushPhiTruncs()
1526 hasTerminator := e.blockDead
1527 if !hasTerminator {
1528 if n := len(b.Instrs); n > 0 {
1529 switch b.Instrs[n-1].(type) {
1530 case *SSAJump, *SSAIf, *SSAReturn:
1531 hasTerminator = true
1532 }
1533 }
1534 }
1535 if !hasTerminator {
1536 e.w(" unreachable\n")
1537 }
1538 }
1539
1540 func (e *irEmitter) blockLabel(b *SSABasicBlock) (s string) {
1541 if b.Index == 0 {
1542 return "%_entry"
1543 }
1544 return "%b" | irItoa(b.Index)
1545 }
1546
1547 func (e *irEmitter) emitInstr(instr SSAInstruction) {
1548 switch i := instr.(type) {
1549 case *SSAAlloc:
1550 if e.hoisted != nil && e.hoisted[i] {
1551 break
1552 }
1553 e.emitAlloc(i)
1554 case *SSAStore:
1555 e.emitStore(i)
1556 case *SSABinOp:
1557 e.emitBinOp(i)
1558 case *SSAUnOp:
1559 e.emitUnOp(i)
1560 case *SSACall:
1561 e.emitCall(i)
1562 case *SSAPhi:
1563 e.emitPhi(i)
1564 case *SSAReturn:
1565 e.emitReturn(i)
1566 case *SSAJump:
1567 e.emitJump(i)
1568 case *SSAIf:
1569 e.emitIf(i)
1570 case *SSAConvert:
1571 e.emitConvert(i)
1572 case *SSAChangeType:
1573 e.emitChangeType(i)
1574 case *SSAFieldAddr:
1575 e.emitFieldAddr(i)
1576 case *SSAIndexAddr:
1577 e.emitIndexAddr(i)
1578 case *SSAExtract:
1579 e.emitExtract(i)
1580 case *SSAMakeSlice:
1581 e.emitMakeSlice(i)
1582 case *SSASlice:
1583 e.emitSliceOp(i)
1584 case *SSAMakeInterface:
1585 e.emitMakeInterface(i)
1586 case *SSAInvoke:
1587 e.emitInvoke(i)
1588 case *SSATypeAssert:
1589 e.emitTypeAssert(i)
1590 case *SSAMakeMap:
1591 e.emitMakeMap(i)
1592 case *SSAMapUpdate:
1593 e.emitMapUpdate(i)
1594 case *SSALookup:
1595 e.emitLookup(i)
1596 case *SSAMakeClosure:
1597 e.emitMakeClosure(i)
1598 case *SSAPanic:
1599 e.emitPanic(i)
1600 case *SSARunDefers:
1601 e.emitRunDefers()
1602 case *SSADefer:
1603 e.emitDefer(i)
1604 case *SSASend:
1605 e.emitChanSend(i)
1606 case *SSAGo:
1607 e.w(" ; go\n")
1608 case *SSASelect:
1609 e.emitSelect(i)
1610 case *SSARange:
1611 e.emitRange(i)
1612 case *SSANext:
1613 e.emitNext(i)
1614 case *SSAMakeChan:
1615 e.emitMakeChan(i)
1616 }
1617 }
1618
1619 func (e *irEmitter) emitStore(s *SSAStore) {
1620 if s.Val == nil || s.Addr == nil {
1621 e.w(" ; store with nil val/addr\n")
1622 return
1623 }
1624 valType := e.llvmType(s.Val.SSAType())
1625 val := e.operand(s.Val)
1626 if load, ok := s.Val.(*SSAUnOp); ok && load.Op == OpMul {
1627 if g, ok2 := load.X.(*SSAGlobal); ok2 {
1628 valType = e.resolveGlobalDeclType(g)
1629 }
1630 }
1631 _, isStoreAlloc := s.Val.(*SSAAlloc)
1632 _, isStoreIA := s.Val.(*SSAIndexAddr)
1633 if !isStoreAlloc && !isStoreIA {
1634 if at, ok := e.allocTypes[s.Val]; ok && at != valType {
1635 bothScalar := len(valType) > 0 && valType[0] == 'i' && len(at) > 0 && at[0] == 'i'
1636 if !bothScalar {
1637 valType = at
1638 if val == "null" && valType != "ptr" {
1639 val = "zeroinitializer"
1640 }
1641 } else if irParseIntWidth(at) > irParseIntWidth(valType) {
1642 valType = at
1643 }
1644 }
1645 }
1646 if len(valType) > 0 && (valType[0] == '[' || valType[0] == '{') {
1647 if addrAt, ok := e.allocTypes[s.Addr]; ok && addrAt != valType {
1648 if len(valType) >= len(addrAt) || (valType[0] == '[' && addrAt[0] == '{') {
1649 e.allocTypes[s.Addr] = valType
1650 }
1651 }
1652 }
1653 if valType == "void" {
1654 if at, ok := e.allocTypes[s.Addr]; ok && at != "ptr" && at != "void" {
1655 valType = at
1656 if val == "null" && valType != "ptr" {
1657 val = "zeroinitializer"
1658 }
1659 }
1660 } else if valType == "ptr" {
1661 if uop, ok := s.Val.(*SSAUnOp); ok && uop.Op == OpMul {
1662 if at, ok2 := e.allocTypes[s.Addr]; ok2 && at != "ptr" && at != "void" {
1663 valType = at
1664 if val == "null" && valType != "ptr" {
1665 val = "zeroinitializer"
1666 }
1667 }
1668 }
1669 }
1670 if valType == "void" {
1671 if _, isFV := s.Addr.(*SSAFreeVar); isFV {
1672 valType = e.llvmType(s.Addr.SSAType())
1673 } else if p, ok := SafeUnderlying(s.Addr.SSAType()).(*Pointer); ok {
1674 valType = e.llvmType(p.Base)
1675 }
1676 if valType == "void" {
1677 valType = "ptr"
1678 }
1679 if val == "null" && valType != "ptr" {
1680 val = "zeroinitializer"
1681 }
1682 }
1683 addr := e.operand(s.Addr)
1684 if at, ok := e.allocTypes[s.Addr]; ok && (at == "double" || at == "float") && len(valType) > 0 && valType[0] == 'i' {
1685 if isConstOperand(val) {
1686 val = ensureFloatLit(val)
1687 } else {
1688 e.nextReg++
1689 conv := "%si2f" | irItoa(e.nextReg)
1690 e.w(" ") ; e.w(conv) ; e.w(" = sitofp ") ; e.w(valType) ; e.w(" ") ; e.w(val) ; e.w(" to ") ; e.w(at) ; e.w("\n")
1691 val = conv
1692 }
1693 valType = at
1694 }
1695 if at, ok2 := e.allocTypes[s.Addr]; ok2 && len(at) > 0 && at[0] == '{' && len(valType) > 0 && valType[0] == 'i' {
1696 if val == "0" || val == "zeroinitializer" {
1697 val = "zeroinitializer"
1698 valType = at
1699 }
1700 }
1701 if at, ok2 := e.allocTypes[s.Addr]; ok2 && len(at) > 0 && at[0] == 'i' && len(valType) > 0 && valType[0] == '{' {
1702 valType = at
1703 val = "zeroinitializer"
1704 }
1705 if p, ok := SafeUnderlying(s.Addr.SSAType()).(*Pointer); ok {
1706 elemT := e.llvmType(p.Base)
1707 if len(elemT) > 1 && elemT[0] == 'i' && len(valType) > 1 && valType[0] == 'i' && elemT != valType {
1708 vw := irParseIntWidth(valType)
1709 ew := irParseIntWidth(elemT)
1710 if ew > 0 && vw > ew {
1711 e.nextReg++
1712 trunc := "%tr" | irItoa(e.nextReg)
1713 e.w(" ")
1714 e.w(trunc)
1715 e.w(" = trunc ")
1716 e.w(valType)
1717 e.w(" ")
1718 e.w(val)
1719 e.w(" to ")
1720 e.w(elemT)
1721 e.w("\n")
1722 val = trunc
1723 valType = elemT
1724 } else if ew > 0 && vw > 0 && vw < ew {
1725 if c, ok2 := s.Val.(*SSAConst); ok2 {
1726 if ci, ok3 := c.val.(*ConstInt); ok3 {
1727 val = irItoa64(ci.V)
1728 valType = elemT
1729 } else {
1730 valType = elemT
1731 }
1732 } else {
1733 e.nextReg++
1734 ext := "%se" | irItoa(e.nextReg)
1735 // Source signedness decides the extension (Go semantics).
1736 extOp := "sext"
1737 if b, ok4 := SafeUnderlying(s.Val.SSAType()).(*Basic); ok4 && b.Info&IsUnsigned != 0 {
1738 extOp = "zext"
1739 }
1740 e.w(" ")
1741 e.w(ext)
1742 e.w(" = ") ; e.w(extOp) ; e.w(" ")
1743 e.w(valType)
1744 e.w(" ")
1745 e.w(val)
1746 e.w(" to ")
1747 e.w(elemT)
1748 e.w("\n")
1749 val = ext
1750 valType = elemT
1751 }
1752 }
1753 }
1754 }
1755 e.emitScopeRelocateOnStore(s, val, valType)
1756 e.w(" store ")
1757 e.w(valType)
1758 e.w(" ")
1759 e.w(val)
1760 e.w(", ptr ")
1761 e.w(addr)
1762 e.w("\n")
1763 }
1764
1765 func (e *irEmitter) emitPhi(p *SSAPhi) {
1766 reg := e.regName(p)
1767 typ := e.llvmType(p.SSAType())
1768 blk := p.InstrBlock()
1769 if blk == nil {
1770 return
1771 }
1772 // Collect operands and check for i1/i32 mismatch.
1773 ops := []string{:len(p.Edges):len(p.Edges)}
1774 widen := false
1775 if typ == "i1" {
1776 for i, edge := range p.Edges {
1777 ops[i] = e.operand(edge)
1778 if !widen && len(ops[i]) > 0 && ops[i][0] == '%' {
1779 if rt, ok := e.regTypes[ops[i]]; ok && rt == "i32" {
1780 widen = true
1781 }
1782 }
1783 }
1784 }
1785 if !widen {
1786 // Normal path: emit phi as-is.
1787 for i, edge := range p.Edges {
1788 if ops[i] == "" {
1789 ops[i] = e.operand(edge)
1790 }
1791 }
1792 e.w(" ") ; e.w(reg) ; e.w(" = phi ") ; e.w(typ) ; e.w(" ")
1793 for i, op := range ops {
1794 if i > 0 { e.w(", ") }
1795 e.w("[") ; e.w(op) ; e.w(", ")
1796 e.emitPhiPred(blk, i)
1797 e.w("]")
1798 }
1799 e.w("\n")
1800 return
1801 }
1802 // Widen i1 phi to i32 to match edge types, trunc after all phis.
1803 rawReg := e.nextReg2("phw")
1804 e.w(" ") ; e.w(rawReg) ; e.w(" = phi i32 ")
1805 for i, op := range ops {
1806 if i > 0 { e.w(", ") }
1807 e.w("[")
1808 if op == "false" { op = "0" }
1809 if op == "true" { op = "1" }
1810 e.w(op) ; e.w(", ")
1811 e.emitPhiPred(blk, i)
1812 e.w("]")
1813 }
1814 e.w("\n")
1815 push(e.pendingPhiTruncs, phiTrunc{rawReg, reg, "i32", "i1"})
1816 }
1817
1818 func (e *irEmitter) emitPhiPred(blk *SSABasicBlock, i int32) {
1819 if i < len(blk.Preds) {
1820 pred := blk.Preds[i]
1821 if pred != nil {
1822 if exitLbl, ok := e.blockExitLabel[pred.Index]; ok {
1823 e.w(exitLbl)
1824 } else {
1825 e.w(e.blockLabel(pred))
1826 }
1827 return
1828 }
1829 }
1830 e.w("%unknown")
1831 }
1832
1833 func (e *irEmitter) flushPhiTruncs() {
1834 for _, pt := range e.pendingPhiTruncs {
1835 e.w(" ") ; e.w(pt.dstReg) ; e.w(" = trunc ") ; e.w(pt.from) ; e.w(" ") ; e.w(pt.rawReg) ; e.w(" to ") ; e.w(pt.to) ; e.w("\n")
1836 }
1837 e.pendingPhiTruncs = e.pendingPhiTruncs[:0]
1838 }
1839
1840 func (e *irEmitter) emitReturn(r *SSAReturn) {
1841 if len(e.deferList) > 0 {
1842 e.emitRunDefers()
1843 }
1844 e.scopeBeforeReturn(r)
1845 frt := e.funcRetType(e.curFunc)
1846 if len(r.Results) == 0 {
1847 rt := e.funcRetType(e.curFunc)
1848 if rt == "void" {
1849 e.w(" ret void\n")
1850 } else if len(e.curFunc.NamedResults) > 0 {
1851 if len(e.curFunc.NamedResults) == 1 {
1852 nr := e.curFunc.NamedResults[0]
1853 nrt := e.namedResultType(nr)
1854 e.nextReg++
1855 tmp := "%nr" | irItoa(e.nextReg)
1856 e.w(" ") ; e.w(tmp) ; e.w(" = load ") ; e.w(nrt) ; e.w(", ptr ") ; e.w(e.regName(nr)) ; e.w("\n")
1857 if e.sretType != "" {
1858 e.w(" store ") ; e.w(nrt) ; e.w(" ") ; e.w(tmp) ; e.w(", ptr %retval\n")
1859 e.w(" ret void\n")
1860 } else {
1861 e.emitRetWithArenaCopy(nrt, tmp)
1862 }
1863 } else {
1864 aggRT := rt
1865 e.nextReg++
1866 agg := "%nr" | irItoa(e.nextReg)
1867 e.w(" ") ; e.w(agg) ; e.w(" = alloca ") ; e.w(aggRT) ; e.w("\n")
1868 e.w(" store ") ; e.w(aggRT) ; e.w(" zeroinitializer, ptr ") ; e.w(agg) ; e.w("\n")
1869 for i, nr := range e.curFunc.NamedResults {
1870 nrt := e.namedResultType(nr)
1871 e.nextReg++
1872 tmp := "%nr" | irItoa(e.nextReg)
1873 e.w(" ") ; e.w(tmp) ; e.w(" = load ") ; e.w(nrt) ; e.w(", ptr ") ; e.w(e.regName(nr)) ; e.w("\n")
1874 e.nextReg++
1875 gep := "%nr" | irItoa(e.nextReg)
1876 e.w(" ") ; e.w(gep) ; e.w(" = getelementptr ") ; e.w(aggRT) ; e.w(", ptr ") ; e.w(agg) ; e.w(", i32 0, i32 ") ; e.w(irItoa(i)) ; e.w("\n")
1877 e.w(" store ") ; e.w(nrt) ; e.w(" ") ; e.w(tmp) ; e.w(", ptr ") ; e.w(gep) ; e.w("\n")
1878 }
1879 e.nextReg++
1880 rv := "%nr" | irItoa(e.nextReg)
1881 e.w(" ") ; e.w(rv) ; e.w(" = load ") ; e.w(aggRT) ; e.w(", ptr ") ; e.w(agg) ; e.w("\n")
1882 if e.sretType != "" {
1883 e.w(" store ") ; e.w(aggRT) ; e.w(" ") ; e.w(rv) ; e.w(", ptr %retval\n")
1884 e.w(" ret void\n")
1885 } else {
1886 e.emitRetWithArenaCopy(aggRT, rv)
1887 }
1888 }
1889 } else {
1890 if e.sretType != "" {
1891 e.w(" store ") ; e.w(rt) ; e.w(" zeroinitializer, ptr %retval\n")
1892 e.w(" ret void\n")
1893 } else {
1894 e.w(" ret ") ; e.w(rt) ; e.w(" zeroinitializer\n")
1895 }
1896 }
1897 return
1898 }
1899 sig := e.curFunc.Signature
1900 if len(r.Results) == 1 {
1901 typ := e.llvmType(r.Results[0].SSAType())
1902 val := e.operand(r.Results[0])
1903 expectType := typ
1904 if sig != nil && sig.Results != nil && sig.Results.Len() == 1 {
1905 expectType = e.llvmType(sig.Results.At(0).Typ)
1906 }
1907 if typ == "void" { typ = frt }
1908 if expectType == "void" { expectType = frt }
1909 if val == "null" && expectType != "ptr" {
1910 val = "zeroinitializer"
1911 } else {
1912 val = e.coerceInt(val, typ, expectType)
1913 }
1914 if typ != expectType && val != "zeroinitializer" {
1915 if expectType == "ptr" && e.intBits(typ) > 0 {
1916 e.nextReg++
1917 rc := "%rc" | irItoa(e.nextReg)
1918 e.w(" ") ; e.w(rc) ; e.w(" = inttoptr ") ; e.w(typ) ; e.w(" ") ; e.w(val) ; e.w(" to ptr\n")
1919 val = rc
1920 typ = "ptr"
1921 } else if typ == "ptr" && e.intBits(expectType) > 0 {
1922 e.nextReg++
1923 rc := "%rc" | irItoa(e.nextReg)
1924 e.w(" ") ; e.w(rc) ; e.w(" = ptrtoint ptr ") ; e.w(val) ; e.w(" to ") ; e.w(expectType) ; e.w("\n")
1925 val = rc
1926 typ = expectType
1927 }
1928 if typ != expectType {
1929 val = "zeroinitializer"
1930 }
1931 }
1932 if e.sretType != "" {
1933 e.w(" store ") ; e.w(expectType) ; e.w(" ") ; e.w(val) ; e.w(", ptr %retval\n")
1934 e.w(" ret void\n")
1935 } else {
1936 e.emitRetWithArenaCopy(expectType, val)
1937 }
1938 return
1939 }
1940 var expectTypes []string
1941 if sig != nil && sig.Results != nil {
1942 for i := 0; i < sig.Results.Len(); i++ {
1943 push(expectTypes, e.resolveResultType(sig.Results.At(i).Typ))
1944 }
1945 }
1946 rtSegs := []string{"{"}
1947 for i := 0; i < len(r.Results); i++ {
1948 res := r.Results[i]
1949 if i > 0 {
1950 push(rtSegs, ", ")
1951 }
1952 if i < len(expectTypes) {
1953 push(rtSegs, expectTypes[i])
1954 } else {
1955 push(rtSegs, e.llvmType(res.SSAType()))
1956 }
1957 }
1958 push(rtSegs, "}")
1959 retType := joinStrs(rtSegs)
1960 prev := "undef"
1961 for i := 0; i < len(r.Results); i++ {
1962 res := r.Results[i]
1963 valType := e.llvmType(res.SSAType())
1964 valOp := e.operand(res)
1965 elemType := valType
1966 if i < len(expectTypes) {
1967 elemType = expectTypes[i]
1968 if valType != elemType && len(valType) > 0 && valType[0] == '{' {
1969 e.nextReg++
1970 ex := "%rv" | irItoa(e.nextReg)
1971 e.w(" ") ; e.w(ex) ; e.w(" = extractvalue ") ; e.w(valType) ; e.w(" ") ; e.w(valOp) ; e.w(", 0\n")
1972 valOp = ex
1973 valType = elemType
1974 }
1975 if valType == "ptr" && len(elemType) > 0 && elemType[0] == '{' {
1976 e.nextReg++
1977 ld := "%rv" | irItoa(e.nextReg)
1978 e.w(" ") ; e.w(ld) ; e.w(" = load ") ; e.w(elemType) ; e.w(", ptr ") ; e.w(valOp) ; e.w("\n")
1979 valOp = ld
1980 valType = elemType
1981 }
1982 if valOp == "null" && elemType != "ptr" {
1983 valOp = "zeroinitializer"
1984 } else if (elemType == "double" || elemType == "float") && isConstOperand(valOp) {
1985 valOp = ensureFloatLit(valOp)
1986 } else if (elemType == "double" || elemType == "float") && e.intBits(valType) > 0 {
1987 valOp = e.intToFloat(valOp, valType, elemType)
1988 } else {
1989 valOp = e.coerceInt(valOp, valType, elemType)
1990 }
1991 if valType == "double" && elemType == "float" {
1992 e.nextReg++
1993 rc := "%rc" | irItoa(e.nextReg)
1994 e.w(" ") ; e.w(rc) ; e.w(" = fptrunc double ") ; e.w(valOp) ; e.w(" to float\n")
1995 valOp = rc
1996 } else if valType == "float" && elemType == "double" {
1997 e.nextReg++
1998 rc := "%rc" | irItoa(e.nextReg)
1999 e.w(" ") ; e.w(rc) ; e.w(" = fpext float ") ; e.w(valOp) ; e.w(" to double\n")
2000 valOp = rc
2001 }
2002 }
2003 e.nextReg++
2004 cur := "%rv" | irItoa(e.nextReg)
2005 e.w(" ")
2006 e.w(cur)
2007 e.w(" = insertvalue ")
2008 e.w(retType)
2009 e.w(" ")
2010 e.w(prev)
2011 e.w(", ")
2012 e.w(elemType)
2013 e.w(" ")
2014 e.w(valOp)
2015 e.w(", ")
2016 e.w(irItoa(i))
2017 e.w("\n")
2018 prev = cur
2019 }
2020 if e.sretType != "" {
2021 e.w(" store ") ; e.w(retType) ; e.w(" ") ; e.w(prev) ; e.w(", ptr %retval\n")
2022 e.w(" ret void\n")
2023 } else {
2024 e.emitRetWithArenaCopy(retType, prev)
2025 }
2026 }
2027
2028 func (e *irEmitter) emitJump(j *SSAJump) {
2029 blk := j.InstrBlock()
2030 if blk == nil {
2031 return
2032 }
2033 if len(blk.Succs) > 0 {
2034 target := blk.Succs[0]
2035 if blk.ScopeID > 0 && target.ScopeID != blk.ScopeID && !blockHasReturn(target) {
2036 e.emitScopeFreesAt(blk.ScopeID)
2037 }
2038 e.w(" br label ")
2039 e.w(e.blockLabel(target))
2040 e.w("\n")
2041 }
2042 }
2043
2044 func (e *irEmitter) emitIf(i *SSAIf) {
2045 blk := i.InstrBlock()
2046 if i.Cond == nil {
2047 if len(blk.Succs) >= 2 {
2048 e.w(" br label ")
2049 e.w(e.blockLabel(blk.Succs[1]))
2050 e.w("\n")
2051 } else {
2052 e.w(" unreachable\n")
2053 }
2054 return
2055 }
2056 cond := e.operand(i.Cond)
2057 condType := e.llvmType(i.Cond.SSAType())
2058 if at, ok := e.allocTypes[i.Cond]; ok {
2059 condType = at
2060 }
2061 if bop, ok := i.Cond.(*SSABinOp); ok && isComparisonOp(bop.Op) {
2062 condType = "i1"
2063 }
2064 if condType != "i1" && condType != "" && condType != "void" {
2065 e.nextReg++
2066 truncReg := "%ift" | irItoa(e.nextReg)
2067 if condType == "ptr" {
2068 e.w(" ") ; e.w(truncReg) ; e.w(" = icmp ne ptr ") ; e.w(cond) ; e.w(", null\n")
2069 } else if len(condType) > 0 && condType[0] == '{' {
2070 curType := condType
2071 curVal := cond
2072 for len(curType) > 0 && curType[0] == '{' {
2073 e.nextReg++
2074 extReg := "%ife" | irItoa(e.nextReg)
2075 e.w(" ") ; e.w(extReg) ; e.w(" = extractvalue ") ; e.w(curType) ; e.w(" ") ; e.w(curVal) ; e.w(", 0\n")
2076 curType = e.structField0Type(curType)
2077 curVal = extReg
2078 }
2079 e.w(" ") ; e.w(truncReg) ; e.w(" = icmp ne ptr ") ; e.w(curVal) ; e.w(", null\n")
2080 } else {
2081 e.w(" ") ; e.w(truncReg) ; e.w(" = trunc ") ; e.w(condType) ; e.w(" ") ; e.w(cond) ; e.w(" to i1\n")
2082 }
2083 cond = truncReg
2084 }
2085 if len(blk.Succs) >= 2 {
2086 e.w(" br i1 ")
2087 e.w(cond)
2088 e.w(", label ")
2089 e.w(e.blockLabel(blk.Succs[0]))
2090 e.w(", label ")
2091 e.w(e.blockLabel(blk.Succs[1]))
2092 e.w("\n")
2093 }
2094 }
2095
2096 func (e *irEmitter) emitConvert(c *SSAConvert) {
2097 reg := e.regName(c)
2098 srcType := e.llvmType(c.X.SSAType())
2099 dstType := e.llvmType(c.SSAType())
2100 val := e.operand(c.X)
2101
2102 if srcType != "ptr" {
2103 resolved := e.resolvedType(c.X, srcType)
2104 if resolved != srcType {
2105 srcType = resolved
2106 }
2107 }
2108
2109 if srcType == "void" || c.X.SSAType() == nil {
2110 if dstType == "ptr" {
2111 e.valName[c] = "null"
2112 } else {
2113 e.valName[c] = "zeroinitializer"
2114 }
2115 return
2116 }
2117
2118 if srcType == dstType {
2119 e.valName[c] = val
2120 e.allocTypes[c] = srcType
2121 return
2122 }
2123
2124 // Constant int->int conversions fold at emission. Materializing an
2125 // untyped constant at its default width and extending loses high bits
2126 // (e.g. uint64(0xFFFFFFFFFFFFFFFF) emitted as zext i32 -1 -> 0xFFFFFFFF).
2127 if k, ok := c.X.(*SSAConst); ok {
2128 if ci, ok2 := k.val.(*ConstInt); ok2 {
2129 sb := e.intBits(srcType)
2130 db := e.intBits(dstType)
2131 if sb > 0 && db > 0 {
2132 v := ci.V
2133 if db < 64 {
2134 v = v & ((int64(1) << uint32(db)) - 1)
2135 }
2136 e.valName[c] = irItoa64(v)
2137 return
2138 }
2139 }
2140 }
2141
2142 if dstType == e.ifaceType() {
2143 _, dstIsIface := SafeUnderlying(c.SSAType()).(*TCInterface)
2144 if dstIsIface {
2145 var valPtr string
2146 if srcType == "ptr" {
2147 valPtr = val
2148 } else if e.isScalarType(srcType) {
2149 ipt := e.intptrType()
2150 srcBits := e.intBits(srcType)
2151 dstBits := e.intBits(ipt)
2152 if srcBits > dstBits {
2153 // Value doesn't fit in intptr - heap allocate
2154 sz := e.nextReg2("cv")
2155 e.w(" ") ; e.w(sz) ; e.w(" = ptrtoint ptr getelementptr (") ; e.w(srcType) ; e.w(", ptr null, i32 1) to ") ; e.w(ipt) ; e.w("\n")
2156 valPtr = e.nextReg2("cv")
2157 e.w(" ") ; e.w(valPtr) ; e.w(" = call ptr @runtime.alloc(") ; e.w(ipt) ; e.w(" ") ; e.w(sz) ; e.w(", ptr null, ptr null)\n")
2158 e.w(" store ") ; e.w(srcType) ; e.w(" ") ; e.w(val) ; e.w(", ptr ") ; e.w(valPtr) ; e.w("\n")
2159 e.declareRuntime("runtime.alloc", "ptr", ipt | ", ptr")
2160 } else {
2161 ext := e.nextReg2("cv")
2162 if srcBits == dstBits {
2163 ext = val
2164 } else {
2165 e.w(" ") ; e.w(ext) ; e.w(" = zext ") ; e.w(srcType) ; e.w(" ") ; e.w(val) ; e.w(" to ") ; e.w(ipt) ; e.w("\n")
2166 }
2167 valPtr = e.nextReg2("cv")
2168 e.w(" ") ; e.w(valPtr) ; e.w(" = inttoptr ") ; e.w(ipt) ; e.w(" ") ; e.w(ext) ; e.w(" to ptr\n")
2169 }
2170 } else {
2171 ipt := e.intptrType()
2172 sz := e.nextReg2("cv")
2173 e.w(" ") ; e.w(sz) ; e.w(" = ptrtoint ptr getelementptr (") ; e.w(srcType) ; e.w(", ptr null, i32 1) to ") ; e.w(ipt) ; e.w("\n")
2174 valPtr = e.nextReg2("cv")
2175 e.w(" ") ; e.w(valPtr) ; e.w(" = call ptr @runtime.alloc(") ; e.w(ipt) ; e.w(" ") ; e.w(sz) ; e.w(", ptr null, ptr null)\n")
2176 e.w(" store ") ; e.w(srcType) ; e.w(" ") ; e.w(val) ; e.w(", ptr ") ; e.w(valPtr) ; e.w("\n")
2177 e.declareRuntime("runtime.alloc", "ptr", ipt | ", ptr")
2178 }
2179 typeid := e.typeIDHash(c.X.SSAType())
2180 t1 := e.nextReg2("cv")
2181 e.w(" ") ; e.w(t1) ; e.w(" = insertvalue " | e.ifaceType() | " undef, " | e.intptrType() | " ") ; e.w(typeid) ; e.w(", 0\n")
2182 e.w(" ") ; e.w(reg) ; e.w(" = insertvalue " | e.ifaceType() | " ") ; e.w(t1) ; e.w(", ptr ") ; e.w(valPtr) ; e.w(", 1\n")
2183 return
2184 }
2185 }
2186
2187 srcIsInt := false
2188 if b, ok := SafeUnderlying(c.X.SSAType()).(*Basic); ok {
2189 srcIsInt = b.Info&IsInteger != 0
2190 }
2191 if !srcIsInt && len(srcType) > 0 && srcType[0] == 'i' {
2192 srcIsInt = true
2193 }
2194 if e.isStringLike(c.SSAType()) && srcIsInt {
2195 if k, ok := c.X.(*SSAConst); ok {
2196 rv := int64(0)
2197 if ci, ok2 := k.val.(*ConstInt); ok2 {
2198 rv = ci.V
2199 }
2200 s := runeToUTF8(rune(rv))
2201 idx := e.addStringConst(s)
2202 ipt := e.intptrType()
2203 slen := irItoa64(int64(len(s)))
2204 e.valName[c] = "{ ptr " | e.strConstGlobal(idx) | ", " | ipt | " " | slen | ", " | ipt | " " | slen | " }"
2205 return
2206 }
2207 e.declareRuntime("runtime.stringFromUnicode", e.sliceType(), "i32")
2208 srcVal := val
2209 if srcType != "i32" {
2210 e.nextReg++
2211 srcVal = "%cv" | irItoa(e.nextReg)
2212 if e.typeBits(c.X.SSAType()) < 32 {
2213 e.w(" ") ; e.w(srcVal) ; e.w(" = sext ") ; e.w(srcType) ; e.w(" ") ; e.w(val) ; e.w(" to i32\n")
2214 } else if e.typeBits(c.X.SSAType()) > 32 {
2215 e.w(" ") ; e.w(srcVal) ; e.w(" = trunc ") ; e.w(srcType) ; e.w(" ") ; e.w(val) ; e.w(" to i32\n")
2216 }
2217 }
2218 e.w(" ") ; e.w(reg) ; e.w(" = call ") ; e.w(e.sliceType()) ; e.w(" @runtime.stringFromUnicode(i32 ") ; e.w(srcVal) ; e.w(", ptr null)\n")
2219 return
2220 }
2221
2222 srcIsSlice := false
2223 if _, slOK := SafeUnderlying(c.X.SSAType()).(*Slice); slOK {
2224 srcIsSlice = true
2225 } else if e.isStringLike(c.X.SSAType()) {
2226 srcIsSlice = true
2227 } else if srcType == e.sliceType() {
2228 srcIsSlice = true
2229 }
2230 if srcIsSlice {
2231 if _, arOK := SafeUnderlying(c.SSAType()).(*Array); arOK {
2232 dp := e.nextReg2("cv")
2233 e.w(" ") ; e.w(dp) ; e.w(" = extractvalue ") ; e.w(srcType) ; e.w(" ") ; e.w(val) ; e.w(", 0\n")
2234 e.w(" ") ; e.w(reg) ; e.w(" = load ") ; e.w(dstType) ; e.w(", ptr ") ; e.w(dp) ; e.w("\n")
2235 return
2236 }
2237 }
2238
2239 op := e.conversionOp(c.X.SSAType(), c.SSAType())
2240 srcBitsLLVM := e.intBits(srcType)
2241 dstBitsLLVM := e.intBits(dstType)
2242 if (op == "sext" || op == "zext") && srcBitsLLVM > 0 && dstBitsLLVM > 0 && srcBitsLLVM > dstBitsLLVM {
2243 op = "trunc"
2244 } else if op == "trunc" && srcBitsLLVM > 0 && dstBitsLLVM > 0 && srcBitsLLVM < dstBitsLLVM {
2245 op = "sext"
2246 }
2247 srcIsFloat := srcType == "double" || srcType == "float"
2248 dstIsFloat := dstType == "double" || dstType == "float"
2249 if op == "trunc" && srcIsFloat && !dstIsFloat {
2250 op = "fptosi"
2251 } else if op == "trunc" && !srcIsFloat && dstIsFloat {
2252 op = "sitofp"
2253 } else if (op == "sext" || op == "zext") && !srcIsFloat && dstIsFloat {
2254 op = "sitofp"
2255 } else if (op == "sext" || op == "zext") && srcIsFloat && !dstIsFloat {
2256 op = "fptosi"
2257 } else if op == "bitcast" && srcIsFloat != dstIsFloat {
2258 if srcIsFloat {
2259 op = "fptosi"
2260 } else {
2261 op = "sitofp"
2262 }
2263 } else if (op == "sext" || op == "zext" || op == "trunc") && srcIsFloat && dstIsFloat {
2264 if e.intBits(srcType) < e.intBits(dstType) {
2265 op = "fpext"
2266 } else {
2267 op = "fptrunc"
2268 }
2269 }
2270 if op == "ptrtoint" && e.intBits(dstType) == 0 {
2271 if dstType == e.ifaceType() {
2272 typeid := e.typeIDHash(c.X.SSAType())
2273 t1 := e.nextReg2("cv")
2274 e.w(" ") ; e.w(t1) ; e.w(" = insertvalue " | e.ifaceType() | " undef, " | e.intptrType() | " ") ; e.w(typeid) ; e.w(", 0\n")
2275 e.w(" ") ; e.w(reg) ; e.w(" = insertvalue " | e.ifaceType() | " ") ; e.w(t1) ; e.w(", ptr ") ; e.w(val) ; e.w(", 1\n")
2276 } else {
2277 e.valName[c] = "zeroinitializer"
2278 }
2279 return
2280 }
2281 if op == "inttoptr" && e.intBits(srcType) == 0 {
2282 if srcType == e.ifaceType() {
2283 e.nextReg++
2284 r := "%cv" | irItoa(e.nextReg)
2285 e.w(" ") ; e.w(r) ; e.w(" = extractvalue ") ; e.w(srcType) ; e.w(" ") ; e.w(val) ; e.w(", 1\n")
2286 e.valName[c] = r
2287 } else {
2288 e.valName[c] = "null"
2289 }
2290 return
2291 }
2292 e.w(" ")
2293 e.w(reg)
2294 e.w(" = ")
2295 e.w(op)
2296 e.w(" ")
2297 e.w(srcType)
2298 e.w(" ")
2299 e.w(val)
2300 e.w(" to ")
2301 e.w(dstType)
2302 e.w("\n")
2303 if e.intBits(dstType) > 0 || dstType == "ptr" {
2304 e.setRegType(c, reg, dstType)
2305 }
2306 }
2307
2308 func (e *irEmitter) emitChangeType(c *SSAChangeType) {
2309 srcType := e.llvmType(c.X.SSAType())
2310 dstType := e.llvmType(c.SSAType())
2311 if at, ok := e.allocTypes[c.X]; ok && at != "ptr" && at != "void" {
2312 srcType = at
2313 }
2314 if srcType == dstType || (srcType == "ptr" && dstType == "ptr") {
2315 e.valName[c] = e.operand(c.X)
2316 return
2317 }
2318 reg := e.regName(c)
2319 val := e.operand(c.X)
2320 e.nextReg++
2321 tmp := "%ct" | irItoa(e.nextReg)
2322 e.w(" ") ; e.w(tmp) ; e.w(" = alloca ") ; e.w(dstType) ; e.w("\n")
2323 e.w(" store ") ; e.w(srcType) ; e.w(" ") ; e.w(val) ; e.w(", ptr ") ; e.w(tmp) ; e.w("\n")
2324 e.w(" ") ; e.w(reg) ; e.w(" = load ") ; e.w(dstType) ; e.w(", ptr ") ; e.w(tmp) ; e.w("\n")
2325 }
2326
2327 func (e *irEmitter) resolveFieldAddrBase(f *SSAFieldAddr) (s string) {
2328 baseType := e.llvmType(f.X.SSAType())
2329 if p, ok := SafeUnderlying(f.X.SSAType()).(*Pointer); ok && p.Base != nil {
2330 elem := p.Base
2331 if p2, ok2 := SafeUnderlying(elem).(*Pointer); ok2 && p2.Base != nil {
2332 baseType = e.llvmType(p2.Base)
2333 } else {
2334 baseType = e.llvmType(elem)
2335 }
2336 }
2337 if at, ok := e.allocTypes[f.X]; ok && at != "ptr" && at != "void" {
2338 baseType = at
2339 }
2340 return baseType
2341 }
2342
2343 func (e *irEmitter) emitFieldAddr(f *SSAFieldAddr) {
2344 reg := e.regName(f)
2345 baseType := e.resolveFieldAddrBase(f)
2346 base := e.operand(f.X)
2347 if uop, ok := f.X.(*SSAUnOp); ok {
2348 _, isFreeVar := uop.X.(*SSAFreeVar)
2349 addrType := e.llvmType(uop.X.SSAType())
2350 useSource := false
2351 if p, ok2 := SafeUnderlying(uop.X.SSAType()).(*Pointer); ok2 && p.Base != nil {
2352 elem := p.Base
2353 if _, ok3 := SafeUnderlying(elem).(*Pointer); ok3 {
2354 // double-pointer: alloca holds **T, keep the loaded *T as base
2355 } else {
2356 baseType = e.llvmType(elem)
2357 useSource = true
2358 }
2359 }
2360 if useSource && !isFreeVar && addrType == "ptr" && baseType != "ptr" && baseType != "void" {
2361 base = e.operand(uop.X)
2362 }
2363 }
2364 if baseType == "ptr" || baseType == "void" {
2365 e.w(" ") ; e.w(reg) ; e.w(" = getelementptr inbounds i8, ptr ") ; e.w(base)
2366 e.w(", i32 0\n")
2367 return
2368 }
2369 e.w(" ")
2370 e.w(reg)
2371 e.w(" = getelementptr inbounds ")
2372 e.w(baseType)
2373 e.w(", ptr ")
2374 e.w(base)
2375 e.w(", i32 0, i32 ")
2376 e.w(irItoa(f.Field))
2377 e.w("\n")
2378 }
2379
2380 func (e *irEmitter) emitIndexAddr(idx *SSAIndexAddr) {
2381 reg := e.regName(idx)
2382 elemType := e.llvmType(idx.SSAType())
2383 if p, ok := SafeUnderlying(idx.SSAType()).(*Pointer); ok {
2384 elemType = e.llvmType(p.Base)
2385 }
2386 base := e.operand(idx.X)
2387 index := e.operand(idx.Index)
2388 baseType := e.llvmType(idx.X.SSAType())
2389 resolvedBase := e.resolvedType(idx.X, baseType)
2390 _, isSlice := SafeUnderlying(idx.X.SSAType()).(*Slice)
2391 if !isSlice {
2392 if b, ok := SafeUnderlying(idx.X.SSAType()).(*Basic); ok && b.Info&IsString != 0 {
2393 isSlice = true
2394 }
2395 }
2396 if at, ok4 := e.allocTypes[idx.X]; ok4 && len(at) > 0 && at[0] == '[' {
2397 isSlice = false
2398 } else if !isSlice && (baseType == e.sliceType() || resolvedBase == e.sliceType()) {
2399 isSlice = true
2400 }
2401 if isSlice && (elemType == "void" || elemType == "i8" || elemType == "ptr") {
2402 baseU := SafeUnderlying(idx.X.SSAType())
2403 if sl, ok2 := baseU.(*Slice); ok2 && sl.Elem != nil {
2404 elemType = e.llvmType(sl.Elem)
2405 } else if b, ok3 := baseU.(*Basic); ok3 && b.Info&IsString != 0 {
2406 elemType = "i8"
2407 }
2408 if elemType == "void" {
2409 elemType = "i8"
2410 }
2411 }
2412 idxType := e.resolvedType(idx.Index, e.llvmType(idx.Index.SSAType()))
2413 // GEP indices are interpreted as signed: a raw i8/i16 index from a
2414 // uint8/uint16 value >= half-range goes negative and reads garbage.
2415 // Normalize narrow indices to i32 with the extension matching the
2416 // source signedness.
2417 if idxType == "i8" || idxType == "i16" {
2418 ext := "sext"
2419 if b, okx := SafeUnderlying(idx.Index.SSAType()).(*Basic); okx && b.Info&IsUnsigned != 0 {
2420 ext = "zext"
2421 }
2422 e.nextReg++
2423 widened := "%ixw" | irItoa(e.nextReg)
2424 e.w(" ") ; e.w(widened) ; e.w(" = ") ; e.w(ext) ; e.w(" ") ; e.w(idxType) ; e.w(" ") ; e.w(index) ; e.w(" to i32\n")
2425 index = widened
2426 idxType = "i32"
2427 }
2428 if isSlice {
2429 e.nextReg++
2430 dataPtr := "%sp" | irItoa(e.nextReg)
2431 e.w(" ")
2432 e.w(dataPtr)
2433 e.w(" = extractvalue ")
2434 e.w(e.sliceType())
2435 e.w(" ")
2436 e.w(base)
2437 e.w(", 0\n")
2438 e.w(" ")
2439 e.w(reg)
2440 e.w(" = getelementptr inbounds ")
2441 e.w(elemType)
2442 e.w(", ptr ")
2443 e.w(dataPtr)
2444 e.w(", ")
2445 e.w(idxType)
2446 e.w(" ")
2447 e.w(index)
2448 e.w("\n")
2449 if elemType != "i8" {
2450 e.allocTypes[idx] = elemType
2451 }
2452 return
2453 }
2454 arr, isArray := SafeUnderlying(idx.X.SSAType()).(*Array)
2455 if !isArray {
2456 if at, ok4 := e.allocTypes[idx.X]; ok4 && len(at) > 0 && at[0] == '[' {
2457 isArray = true
2458 }
2459 }
2460 if !isArray {
2461 if alloc, ok4 := idx.X.(*SSAAlloc); ok4 {
2462 if p, ok5 := SafeUnderlying(alloc.SSAType()).(*Pointer); ok5 && p.Base != nil {
2463 if ar, ok6 := SafeUnderlying(p.Base).(*Array); ok6 && ar.Len > 0 {
2464 isArray = true
2465 arrT := e.llvmType(p.Base)
2466 if len(arrT) > 0 && arrT[0] == '[' {
2467 e.allocTypes[alloc] = arrT
2468 }
2469 }
2470 }
2471 }
2472 }
2473 if !isArray {
2474 if load, ok4 := idx.X.(*SSAUnOp); ok4 && load.Op == OpMul {
2475 if at, ok5 := e.allocTypes[load.X]; ok5 && len(at) > 0 && at[0] == '[' {
2476 isArray = true
2477 e.allocTypes[idx.X] = at
2478 allocBase := e.operand(load.X)
2479 e.w(" ") ; e.w(reg) ; e.w(" = getelementptr inbounds ")
2480 e.w(at) ; e.w(", ptr ") ; e.w(allocBase) ; e.w(", i32 0, ")
2481 e.w(e.llvmType(idx.Index.SSAType())) ; e.w(" ") ; e.w(index) ; e.w("\n")
2482 aet := e.arrayElemType(at)
2483 if aet != "" { e.setRegType(idx, reg, aet) }
2484 return
2485 }
2486 }
2487 }
2488 if isArray {
2489 arrType := e.llvmType(idx.X.SSAType())
2490 if at, ok4 := e.allocTypes[idx.X]; ok4 && len(at) > 0 && at[0] == '[' {
2491 arrType = at
2492 }
2493 if arrType == "ptr" || arrType == "void" {
2494 if p, ok4 := SafeUnderlying(idx.X.SSAType()).(*Pointer); ok4 && p.Base != nil {
2495 arrType = e.llvmType(p.Base)
2496 }
2497 }
2498 _, isGlobal := idx.X.(*SSAGlobal)
2499 _, isAlloc := idx.X.(*SSAAlloc)
2500 _, isFieldAddr := idx.X.(*SSAFieldAddr)
2501 if isGlobal || isAlloc || isFieldAddr {
2502 _ = arr
2503 e.w(" ") ; e.w(reg) ; e.w(" = getelementptr inbounds ")
2504 e.w(arrType) ; e.w(", ptr ") ; e.w(base) ; e.w(", i32 0, ")
2505 e.w(idxType) ; e.w(" ") ; e.w(index) ; e.w("\n")
2506 return
2507 }
2508 if load, ok4 := idx.X.(*SSAUnOp); ok4 && load.Op == OpMul {
2509 _, srcAlloc := load.X.(*SSAAlloc)
2510 _, srcGlobal := load.X.(*SSAGlobal)
2511 _, srcField := load.X.(*SSAFieldAddr)
2512 // A load of an index/field-addr result is a load from an
2513 // addressable chain (e.g. global[i][j]): forwarding to the
2514 // source GEP keeps the element addressable so mutations
2515 // through method receivers hit the original, not a spill copy.
2516 _, srcIndex := load.X.(*SSAIndexAddr)
2517 if srcAlloc || srcGlobal || srcField || srcIndex {
2518 // Index into a load-of-array: GEP the source address
2519 // instead of spilling a copy. The spill alloca lands in
2520 // the current block - inside a loop that grows the frame
2521 // per iteration until the stack is gone.
2522 allocBase := e.operand(load.X)
2523 e.w(" ") ; e.w(reg) ; e.w(" = getelementptr inbounds ")
2524 e.w(arrType) ; e.w(", ptr ") ; e.w(allocBase) ; e.w(", i32 0, ")
2525 e.w(idxType) ; e.w(" ") ; e.w(index) ; e.w("\n")
2526 aet0 := e.arrayElemType(arrType)
2527 if aet0 != "" {
2528 e.setRegType(idx, reg, aet0)
2529 }
2530 return
2531 }
2532 }
2533 e.nextReg++
2534 arrPtr := "%ai" | irItoa(e.nextReg)
2535 e.w(" ") ; e.w(arrPtr) ; e.w(" = alloca ") ; e.w(arrType) ; e.w("\n")
2536 e.w(" store ") ; e.w(arrType) ; e.w(" ") ; e.w(base) ; e.w(", ptr ") ; e.w(arrPtr) ; e.w("\n")
2537 e.w(" ") ; e.w(reg) ; e.w(" = getelementptr inbounds ")
2538 e.w(arrType) ; e.w(", ptr ") ; e.w(arrPtr) ; e.w(", i32 0, ")
2539 e.w(idxType) ; e.w(" ") ; e.w(index) ; e.w("\n")
2540 aet := e.arrayElemType(arrType)
2541 if aet != "" {
2542 e.setRegType(idx, reg, aet)
2543 }
2544 return
2545 }
2546 if len(elemType) > 0 && elemType[0] == '[' {
2547 aet := e.arrayElemType(elemType)
2548 e.w(" ") ; e.w(reg) ; e.w(" = getelementptr inbounds ")
2549 e.w(elemType) ; e.w(", ptr ") ; e.w(base) ; e.w(", i32 0, ")
2550 e.w(idxType) ; e.w(" ") ; e.w(index) ; e.w("\n")
2551 e.setRegType(idx, reg, aet)
2552 return
2553 }
2554 e.w(" ")
2555 e.w(reg)
2556 e.w(" = getelementptr inbounds ")
2557 e.w(elemType)
2558 e.w(", ptr ")
2559 e.w(base)
2560 e.w(", ")
2561 e.w(idxType)
2562 e.w(" ")
2563 e.w(index)
2564 e.w("\n")
2565 }
2566
2567 func (e *irEmitter) emitExtract(ex *SSAExtract) {
2568 reg := e.regName(ex)
2569 tupType := e.llvmType(ex.Tuple.SSAType())
2570 if at, ok := e.allocTypes[ex.Tuple]; ok {
2571 tupType = at
2572 }
2573 if n, ok := ex.Tuple.(*SSANext); ok {
2574 rangeInstr := n.Iter.(*SSARange)
2575 if mt, ok2 := SafeUnderlying(rangeInstr.X.SSAType()).(*TCMap); ok2 {
2576 tupType = "{i1, " | e.llvmType(mt.Key) | ", " | e.llvmType(mt.Elem) | "}"
2577 } else if arr, ok3 := SafeUnderlying(rangeInstr.X.SSAType()).(*Array); ok3 {
2578 tupType = "{i1, i32, " | e.llvmType(arr.Elem) | "}"
2579 } else if p, ok4 := SafeUnderlying(rangeInstr.X.SSAType()).(*Pointer); ok4 {
2580 if ar, ok5 := SafeUnderlying(p.Base).(*Array); ok5 {
2581 tupType = "{i1, i32, " | e.llvmType(ar.Elem) | "}"
2582 }
2583 } else {
2584 et := "i32"
2585 if sl, ok6 := SafeUnderlying(rangeInstr.X.SSAType()).(*Slice); ok6 {
2586 et = e.llvmType(sl.Elem)
2587 }
2588 tupType = "{i1, i32, " | et | "}"
2589 }
2590 }
2591 val := e.operand(ex.Tuple)
2592 // Track extracted element type for downstream alloc/store consistency
2593 extractedType := extractTupleField(tupType, ex.Index)
2594 if extractedType != "" {
2595 ssaType := e.llvmType(ex.SSAType())
2596 if extractedType != ssaType {
2597 e.allocTypes[ex] = extractedType
2598 }
2599 }
2600 if tupType == "ptr" || tupType == "void" {
2601 elemType := e.llvmType(ex.SSAType())
2602 if elemType == "void" { elemType = "ptr" }
2603 e.nextReg++
2604 castReg := "%ev" | irItoa(e.nextReg)
2605 e.w(" ") ; e.w(castReg) ; e.w(" = getelementptr inbounds i8, ptr ") ; e.w(val) ; e.w(", i32 0\n")
2606 e.w(" ") ; e.w(reg) ; e.w(" = load ") ; e.w(elemType) ; e.w(", ptr ") ; e.w(castReg) ; e.w("\n")
2607 e.allocTypes[ex] = elemType
2608 return
2609 }
2610 e.w(" ")
2611 e.w(reg)
2612 e.w(" = extractvalue ")
2613 e.w(tupType)
2614 e.w(" ")
2615 e.w(val)
2616 e.w(", ")
2617 e.w(irItoa(ex.Index))
2618 e.w("\n")
2619 }
2620
2621 func (e *irEmitter) sextToIpt(val SSAValue, op string) (s string) {
2622 ipt := e.intptrType()
2623 if val == nil {
2624 return op
2625 }
2626 valType := e.llvmType(val.SSAType())
2627 if valType == ipt {
2628 return op
2629 }
2630 e.nextReg++
2631 ext := "%sx" | irItoa(e.nextReg)
2632 srcBits := e.intBits(valType)
2633 dstBits := e.intBits(ipt)
2634 if srcBits > dstBits {
2635 e.w(" ") ; e.w(ext) ; e.w(" = trunc ") ; e.w(valType) ; e.w(" ") ; e.w(op) ; e.w(" to ") ; e.w(ipt) ; e.w("\n")
2636 return ext
2637 }
2638 extOp := "sext"
2639 if b, ok := SafeUnderlying(val.SSAType()).(*Basic); ok && b.Info&IsUnsigned != 0 {
2640 extOp = "zext"
2641 }
2642 e.w(" ") ; e.w(ext) ; e.w(" = ") ; e.w(extOp) ; e.w(" ") ; e.w(valType) ; e.w(" ") ; e.w(op) ; e.w(" to ") ; e.w(ipt) ; e.w("\n")
2643 return ext
2644 }
2645
2646 func (e *irEmitter) emitMakeSlice(m *SSAMakeSlice) {
2647 reg := e.regName(m)
2648 ipt := e.intptrType()
2649 sty := e.sliceType()
2650 lenOp := e.sextToIpt(m.Len, e.operand(m.Len))
2651 capOp := lenOp
2652 if m.Cap != nil {
2653 capOp = e.sextToIpt(m.Cap, e.operand(m.Cap))
2654 }
2655 var dataPtr string
2656 if m.Data != nil {
2657 dataPtr = e.operand(m.Data)
2658 } else {
2659 elemType := "i8"
2660 if sl, ok := SafeUnderlying(m.SSAType()).(*Slice); ok {
2661 elemType = e.llvmType(sl.Elem)
2662 } else if b, ok2 := SafeUnderlying(m.SSAType()).(*Basic); ok2 && b.Info&IsString != 0 {
2663 elemType = "i8"
2664 }
2665 e.nextReg++
2666 elemSz := "%ms" | irItoa(e.nextReg)
2667 e.w(" ")
2668 e.w(elemSz)
2669 e.w(" = ptrtoint ptr getelementptr (")
2670 e.w(elemType)
2671 e.w(", ptr null, i32 1) to ")
2672 e.w(ipt)
2673 e.w("\n")
2674 e.nextReg++
2675 allocSz := "%ms" | irItoa(e.nextReg)
2676 e.w(" ")
2677 e.w(allocSz)
2678 e.w(" = mul ")
2679 e.w(ipt)
2680 e.w(" ")
2681 e.w(elemSz)
2682 e.w(", ")
2683 e.w(capOp)
2684 e.w("\n")
2685 e.nextReg++
2686 dataPtr = "%ms" | irItoa(e.nextReg)
2687 e.w(" ")
2688 e.w(dataPtr)
2689 e.w(" = call ptr @runtime.alloc(")
2690 e.w(ipt)
2691 e.w(" ")
2692 e.w(allocSz)
2693 e.w(", ptr null, ptr null)\n")
2694 e.declareRuntime("runtime.alloc", "ptr", ipt | ", ptr")
2695 e.scopeTrackAlloc(dataPtr)
2696 }
2697 e.nextReg++
2698 s1 := "%ms" | irItoa(e.nextReg)
2699 e.w(" ")
2700 e.w(s1)
2701 e.w(" = insertvalue ")
2702 e.w(sty)
2703 e.w(" undef, ptr ")
2704 e.w(dataPtr)
2705 e.w(", 0\n")
2706 e.nextReg++
2707 s2 := "%ms" | irItoa(e.nextReg)
2708 e.w(" ")
2709 e.w(s2)
2710 e.w(" = insertvalue ")
2711 e.w(sty)
2712 e.w(" ")
2713 e.w(s1)
2714 e.w(", ")
2715 e.w(ipt)
2716 e.w(" ")
2717 e.w(lenOp)
2718 e.w(", 1\n")
2719 e.w(" ")
2720 e.w(reg)
2721 e.w(" = insertvalue ")
2722 e.w(sty)
2723 e.w(" ")
2724 e.w(s2)
2725 e.w(", ")
2726 e.w(ipt)
2727 e.w(" ")
2728 e.w(capOp)
2729 e.w(", 2\n")
2730 }
2731
2732 func (e *irEmitter) emitSliceOp(s *SSASlice) {
2733 reg := e.regName(s)
2734 ipt := e.intptrType()
2735 sty := e.sliceType()
2736 src := e.operand(s.X)
2737 var oldPtr, oldLen, oldCap string
2738 srcType := SafeUnderlying(s.X.SSAType())
2739 ptrToArr := false
2740 if p, ok := srcType.(*Pointer); ok && p.Base != nil {
2741 if arr, ok2 := SafeUnderlying(p.Base).(*Array); ok2 {
2742 oldPtr = src
2743 oldLen = irItoa64(arr.Len)
2744 oldCap = oldLen
2745 ptrToArr = true
2746 }
2747 }
2748 if !ptrToArr {
2749 if arr, ok := srcType.(*Array); ok {
2750 // Heap-allocate the array copy: the resulting slice can escape
2751 // (e.g. returned), so a stack alloca would dangle.
2752 arrType := e.llvmType(s.X.SSAType())
2753 e.nextReg++
2754 sz := "%sl" | irItoa(e.nextReg)
2755 e.w(" ") ; e.w(sz) ; e.w(" = ptrtoint ptr getelementptr (") ; e.w(arrType) ; e.w(", ptr null, i32 1) to ") ; e.w(ipt) ; e.w("\n")
2756 e.nextReg++
2757 tmp := "%sl" | irItoa(e.nextReg)
2758 e.w(" ") ; e.w(tmp) ; e.w(" = call ptr @runtime.alloc(") ; e.w(ipt) ; e.w(" ") ; e.w(sz) ; e.w(", ptr null, ptr null)\n")
2759 e.declareRuntime("runtime.alloc", "ptr", ipt | ", ptr")
2760 e.w(" store ") ; e.w(arrType) ; e.w(" ") ; e.w(src) ; e.w(", ptr ") ; e.w(tmp) ; e.w("\n")
2761 oldPtr = tmp
2762 oldLen = irItoa64(arr.Len)
2763 oldCap = oldLen
2764 } else {
2765 e.nextReg++
2766 oldPtr = "%sl" | irItoa(e.nextReg)
2767 e.w(" ")
2768 e.w(oldPtr)
2769 e.w(" = extractvalue ")
2770 e.w(sty)
2771 e.w(" ")
2772 e.w(src)
2773 e.w(", 0\n")
2774 e.nextReg++
2775 oldLen = "%sl" | irItoa(e.nextReg)
2776 e.w(" ")
2777 e.w(oldLen)
2778 e.w(" = extractvalue ")
2779 e.w(sty)
2780 e.w(" ")
2781 e.w(src)
2782 e.w(", 1\n")
2783 e.nextReg++
2784 oldCap = "%sl" | irItoa(e.nextReg)
2785 e.w(" ")
2786 e.w(oldCap)
2787 e.w(" = extractvalue ")
2788 e.w(sty)
2789 e.w(" ")
2790 e.w(src)
2791 e.w(", 2\n")
2792 }
2793 }
2794 low := "0"
2795 if s.Low != nil {
2796 low = e.sliceIdxToIpt(s.Low, ipt)
2797 }
2798 high := oldLen
2799 if s.High != nil {
2800 high = e.sliceIdxToIpt(s.High, ipt)
2801 }
2802 _ = oldCap
2803 if s.Max != nil {
2804 _ = e.sliceIdxToIpt(s.Max, ipt)
2805 }
2806 elemType := "i8"
2807 if sl, ok := SafeUnderlying(s.X.SSAType()).(*Slice); ok {
2808 elemType = e.llvmType(sl.Elem)
2809 } else if ar, ok2 := SafeUnderlying(s.X.SSAType()).(*Array); ok2 {
2810 elemType = e.llvmType(ar.Elem)
2811 } else if p, ok3 := SafeUnderlying(s.X.SSAType()).(*Pointer); ok3 && p.Base != nil {
2812 if ar2, ok4 := SafeUnderlying(p.Base).(*Array); ok4 {
2813 elemType = e.llvmType(ar2.Elem)
2814 }
2815 }
2816 newPtr := e.nextReg2("sl")
2817 e.w(" ") ; e.w(newPtr) ; e.w(" = getelementptr inbounds ") ; e.w(elemType)
2818 e.w(", ptr ") ; e.w(oldPtr) ; e.w(", ") ; e.w(ipt) ; e.w(" ") ; e.w(low) ; e.w("\n")
2819 newLen := e.nextReg2("sl")
2820 e.w(" ") ; e.w(newLen) ; e.w(" = sub ") ; e.w(ipt) ; e.w(" ") ; e.w(high) ; e.w(", ") ; e.w(low) ; e.w("\n")
2821 newCap := e.nextReg2("sl")
2822 e.w(" ") ; e.w(newCap) ; e.w(" = sub ") ; e.w(ipt) ; e.w(" ") ; e.w(oldCap) ; e.w(", ") ; e.w(low) ; e.w("\n")
2823 s1 := e.nextReg2("sl")
2824 e.w(" ") ; e.w(s1) ; e.w(" = insertvalue ") ; e.w(sty) ; e.w(" undef, ptr ") ; e.w(newPtr) ; e.w(", 0\n")
2825 s2 := e.nextReg2("sl")
2826 e.w(" ") ; e.w(s2) ; e.w(" = insertvalue ") ; e.w(sty) ; e.w(" ") ; e.w(s1) ; e.w(", ") ; e.w(ipt) ; e.w(" ") ; e.w(newLen) ; e.w(", 1\n")
2827 e.w(" ") ; e.w(reg) ; e.w(" = insertvalue ") ; e.w(sty) ; e.w(" ") ; e.w(s2) ; e.w(", ") ; e.w(ipt) ; e.w(" ") ; e.w(newCap) ; e.w(", 2\n")
2828 }
2829
2830 func (e *irEmitter) sliceIdxToIpt(val SSAValue, ipt string) (s string) {
2831 operandStr := e.operand(val)
2832 valType := e.llvmType(val.SSAType())
2833 if valType == ipt {
2834 return operandStr
2835 }
2836 e.nextReg++
2837 ext := "%sl" | irItoa(e.nextReg)
2838 srcBits := e.intBits(valType)
2839 dstBits := e.intBits(ipt)
2840 if srcBits > dstBits {
2841 e.w(" ") ; e.w(ext) ; e.w(" = trunc ") ; e.w(valType) ; e.w(" ") ; e.w(operandStr) ; e.w(" to ") ; e.w(ipt) ; e.w("\n")
2842 return ext
2843 }
2844 op := "sext"
2845 if b, ok2 := SafeUnderlying(val.SSAType()).(*Basic); ok2 && b.Info&IsUnsigned != 0 {
2846 op = "zext"
2847 }
2848 e.w(" ") ; e.w(ext) ; e.w(" = ") ; e.w(op) ; e.w(" ") ; e.w(valType) ; e.w(" ") ; e.w(operandStr) ; e.w(" to ") ; e.w(ipt) ; e.w("\n")
2849 return ext
2850 }
2851
2852 func (e *irEmitter) emitPanic(p *SSAPanic) {
2853 if c, ok := p.X.(*SSAConst); ok && e.isStringLike(c.SSAType()) {
2854 arg := e.operand(c)
2855 sty := e.sliceType()
2856 e.w(" call void @runtime._panicstr(") ; e.w(sty) ; e.w(" ") ; e.w(arg) ; e.w(", ptr null)\n")
2857 e.declareRuntime("runtime._panicstr", "void", sty)
2858 e.w(" unreachable\n")
2859 return
2860 }
2861 ity := e.ifaceType()
2862 e.w(" call void @runtime._panic(" | ity | " zeroinitializer, ptr null)\n")
2863 e.declareRuntime("runtime._panic", "void", ity)
2864 e.w(" unreachable\n")
2865 }
2866
2867 func (e *irEmitter) operandNoSideEffect(v SSAValue) (s string) {
2868 if v == nil {
2869 return "zeroinitializer"
2870 }
2871 if c, ok := v.(*SSAConst); ok {
2872 return e.constOperand(c)
2873 }
2874 if n, ok := e.valName[v]; ok {
2875 return n
2876 }
2877 return ""
2878 }
2879
2880 func (e *irEmitter) operand(v SSAValue) (s string) {
2881 if v == nil {
2882 return "zeroinitializer"
2883 }
2884 if c, ok := v.(*SSAConst); ok {
2885 return e.constOperand(c)
2886 }
2887 if b, ok := v.(*SSABuiltin); ok {
2888 return "@runtime." | b.SSAName()
2889 }
2890 if f, ok := v.(*SSAFunction); ok {
2891 if !e.isPkgFunc(f) {
2892 // Func used as a value: ensure the symbol is declared, the
2893 // call-emission path is never reached for it.
2894 e.declareExternalFunc(f)
2895 }
2896 return "{ ptr null, ptr " | e.funcSymbol(f) | " }"
2897 }
2898 if g, ok := v.(*SSAGlobal); ok {
2899 e.declareExternalGlobal(g)
2900 return e.globalName(g)
2901 }
2902 return e.regName(v)
2903 }
2904
2905 func (e *irEmitter) constOperand(c *SSAConst) (s string) {
2906 if c.val == nil {
2907 if c.typ == nil {
2908 return "null"
2909 }
2910 typ := e.llvmType(c.typ)
2911 if typ == "ptr" {
2912 return "null"
2913 }
2914 if typ == "i1" {
2915 return "false"
2916 }
2917 return "zeroinitializer"
2918 }
2919 b := underlyingBasic(c.typ)
2920 if b != nil {
2921 switch b.Kind {
2922 case Bool, UntypedBool:
2923 if cb, ok := c.val.(*ConstBool); ok {
2924 if cb.V {
2925 return "true"
2926 }
2927 return "false"
2928 }
2929 sv := c.val.String()
2930 if sv == "true" {
2931 return "true"
2932 }
2933 return "false"
2934 case Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32, Uint64,
2935 UntypedInt, UntypedRune:
2936 if ci, ok := c.val.(*ConstInt); ok {
2937 v := ci.V
2938 switch b.Kind {
2939 case Int8:
2940 v = int64(int8(v))
2941 case Uint8:
2942 v = int64(uint8(v))
2943 case Int16:
2944 v = int64(int16(v))
2945 case Uint16:
2946 v = int64(uint16(v))
2947 case Int32:
2948 v = int64(int32(v))
2949 case Uint32:
2950 v = int64(uint32(v))
2951 case UntypedInt, UntypedRune:
2952 if v < -2147483648 || v > 4294967295 {
2953 e.allocTypes[c] = "i64"
2954 }
2955 }
2956 return irItoa64(v)
2957 }
2958 if cf, ok := c.val.(*ConstFloat); ok {
2959 return irItoa64(int64(cf.V))
2960 }
2961 if cs, ok := c.val.(*ConstStr); ok {
2962 if len(cs.S) == 0 {
2963 return "zeroinitializer"
2964 }
2965 idx := e.addStringConst(cs.S)
2966 ipt := e.intptrType()
2967 slen := irItoa64(int64(len(cs.S)))
2968 e.allocTypes[c] = e.sliceType()
2969 return "{ ptr " | e.strConstGlobal(idx) | ", " | ipt | " " | slen | ", " | ipt | " " | slen | " }"
2970 }
2971 return c.val.String()
2972 case Float32:
2973 if cf, ok := c.val.(*ConstFloat); ok {
2974 if cf.Lit != "" {
2975 return NormalizeLLVMFloat(cf.Lit)
2976 }
2977 bits := math.Float64bits(float64(float32(cf.V)))
2978 return "0x" | irHex64(bits)
2979 }
2980 return "0.0"
2981 case Float64, UntypedFloat:
2982 if cf, ok := c.val.(*ConstFloat); ok {
2983 if cf.Lit != "" {
2984 return NormalizeLLVMFloat(cf.Lit)
2985 }
2986 bits := math.Float64bits(cf.V)
2987 return "0x" | irHex64(bits)
2988 }
2989 return "0.0"
2990 case TCString, UntypedString:
2991 if cs, ok := c.val.(*ConstStr); ok {
2992 if len(cs.S) == 0 {
2993 return "zeroinitializer"
2994 }
2995 idx := e.addStringConst(cs.S)
2996 ipt := e.intptrType()
2997 slen := irItoa64(int64(len(cs.S)))
2998 return "{ ptr " | e.strConstGlobal(idx) | ", " | ipt | " " | slen | ", " | ipt | " " | slen | " }"
2999 }
3000 return "zeroinitializer"
3001 }
3002 }
3003 if c.typ == nil {
3004 return c.val.String()
3005 }
3006 return "zeroinitializer"
3007 }
3008
3009 func (e *irEmitter) instrOperands(instr SSAInstruction) (ss []SSAValue) {
3010 // Reuse scratch slice to avoid per-instruction allocation.
3011 e.operandBuf = e.operandBuf[:0]
3012 switch i := instr.(type) {
3013 case *SSAStore:
3014 push(e.operandBuf, i.Addr, i.Val)
3015 case *SSAUnOp:
3016 push(e.operandBuf, i.X)
3017 case *SSABinOp:
3018 push(e.operandBuf, i.X, i.Y)
3019 case *SSACall:
3020 push(e.operandBuf, i.Call.Value)
3021 for _, a := range i.Call.Args {
3022 push(e.operandBuf, a)
3023 }
3024 case *SSAFieldAddr:
3025 push(e.operandBuf, i.X)
3026 case *SSAIndexAddr:
3027 push(e.operandBuf, i.X, i.Index)
3028 case *SSAExtract:
3029 push(e.operandBuf, i.Tuple)
3030 case *SSAPhi:
3031 return i.Edges
3032 case *SSAReturn:
3033 for _, r := range i.Results {
3034 push(e.operandBuf, r)
3035 }
3036 case *SSAIf:
3037 push(e.operandBuf, i.Cond)
3038 case *SSAConvert:
3039 push(e.operandBuf, i.X)
3040 case *SSAChangeType:
3041 push(e.operandBuf, i.X)
3042 case *SSAMakeInterface:
3043 push(e.operandBuf, i.X)
3044 case *SSATypeAssert:
3045 push(e.operandBuf, i.X)
3046 case *SSASlice:
3047 push(e.operandBuf, i.X)
3048 if i.Low != nil { push(e.operandBuf, i.Low) }
3049 if i.High != nil { push(e.operandBuf, i.High) }
3050 if i.Max != nil { push(e.operandBuf, i.Max) }
3051 case *SSAMapUpdate:
3052 push(e.operandBuf, i.Map, i.Key, i.Value)
3053 case *SSALookup:
3054 push(e.operandBuf, i.X, i.Index)
3055 case *SSARange:
3056 push(e.operandBuf, i.X)
3057 case *SSANext:
3058 push(e.operandBuf, i.Iter)
3059 case *SSASend:
3060 push(e.operandBuf, i.Chan, i.X)
3061 case *SSAMakeSlice:
3062 push(e.operandBuf, i.Len)
3063 if i.Cap != nil { push(e.operandBuf, i.Cap) }
3064 if i.Data != nil { push(e.operandBuf, i.Data) }
3065 }
3066 return e.operandBuf
3067 return nil
3068 }
3069
3070