assembler_regabi_amd64.go raw
1 //go:build go1.17 && !go1.25
2 // +build go1.17,!go1.25
3
4 /*
5 * Copyright 2021 ByteDance Inc.
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 */
19
20 package x86
21
22 import (
23 "fmt"
24 "reflect"
25 "strconv"
26 "unsafe"
27
28 "github.com/bytedance/sonic/internal/cpu"
29 "github.com/bytedance/sonic/internal/encoder/alg"
30 "github.com/bytedance/sonic/internal/encoder/ir"
31 "github.com/bytedance/sonic/internal/encoder/vars"
32 "github.com/bytedance/sonic/internal/jit"
33 "github.com/bytedance/sonic/internal/native/types"
34 "github.com/twitchyliquid64/golang-asm/obj"
35 "github.com/twitchyliquid64/golang-asm/obj/x86"
36
37 "github.com/bytedance/sonic/internal/native"
38 "github.com/bytedance/sonic/internal/rt"
39 )
40
41 /** Register Allocations
42 *
43 * State Registers:
44 *
45 * %rbx : stack base
46 * %rdi : result pointer
47 * %rsi : result length
48 * %rdx : result capacity
49 * %r12 : sp->p
50 * %r13 : sp->q
51 * %r14 : sp->x
52 * %r15 : sp->f
53 *
54 * Error Registers:
55 *
56 * %r10 : error type register
57 * %r11 : error pointer register
58 */
59
60 /** Function Prototype & Stack Map
61 *
62 * func (buf *[]byte, p unsafe.Pointer, sb *_Stack, fv uint64) (err error)
63 *
64 * buf : (FP)
65 * p : 8(FP)
66 * sb : 16(FP)
67 * fv : 24(FP)
68 * err.vt : 32(FP)
69 * err.vp : 40(FP)
70 */
71
72 const (
73 _S_cond = iota
74 _S_init
75 )
76
77 const (
78 _FP_args = 32 // 32 bytes for spill registers of arguments
79 _FP_fargs = 40 // 40 bytes for passing arguments to other Go functions
80 _FP_saves = 64 // 64 bytes for saving the registers before CALL instructions
81 _FP_locals = 24 // 24 bytes for local variables
82 )
83
84 const (
85 _FP_loffs = _FP_fargs + _FP_saves
86 FP_offs = _FP_loffs + _FP_locals
87 // _FP_offs = _FP_loffs + _FP_locals + _FP_debug
88 _FP_size = FP_offs + 8 // 8 bytes for the parent frame pointer
89 _FP_base = _FP_size + 8 // 8 bytes for the return address
90 )
91
92 const (
93 _FM_exp32 = 0x7f800000
94 _FM_exp64 = 0x7ff0000000000000
95 )
96
97 const (
98 _IM_null = 0x6c6c756e // 'null'
99 _IM_true = 0x65757274 // 'true'
100 _IM_fals = 0x736c6166 // 'fals' ('false' without the 'e')
101 _IM_open = 0x00225c22 // '"\"∅'
102 _IM_array = 0x5d5b // '[]'
103 _IM_object = 0x7d7b // '{}'
104 _IM_mulv = -0x5555555555555555
105 )
106
107 const (
108 _LB_more_space = "_more_space"
109 _LB_more_space_return = "_more_space_return_"
110 )
111
112 const (
113 _LB_error = "_error"
114 _LB_error_too_deep = "_error_too_deep"
115 _LB_error_invalid_number = "_error_invalid_number"
116 _LB_error_nan_or_infinite = "_error_nan_or_infinite"
117 _LB_panic = "_panic"
118 )
119
120 var (
121 _AX = jit.Reg("AX")
122 _BX = jit.Reg("BX")
123 _CX = jit.Reg("CX")
124 _DX = jit.Reg("DX")
125 _DI = jit.Reg("DI")
126 _SI = jit.Reg("SI")
127 _BP = jit.Reg("BP")
128 _SP = jit.Reg("SP")
129 _R8 = jit.Reg("R8")
130 _R9 = jit.Reg("R9")
131 )
132
133 var (
134 _X0 = jit.Reg("X0")
135 _Y0 = jit.Reg("Y0")
136 )
137
138 var (
139 _ST = jit.Reg("R15") // can't use R14 since it's always scratched by Go...
140 _RP = jit.Reg("DI")
141 _RL = jit.Reg("SI")
142 _RC = jit.Reg("DX")
143 )
144
145 var (
146 _LR = jit.Reg("R9")
147 _ET = jit.Reg("AX")
148 _EP = jit.Reg("BX")
149 )
150
151 var (
152 _SP_p = jit.Reg("R10") // saved on BX when call_c
153 _SP_q = jit.Reg("R11") // saved on BP when call_c
154 _SP_x = jit.Reg("R12")
155 _SP_f = jit.Reg("R13")
156 )
157
158 var (
159 _ARG_rb = jit.Ptr(_SP, _FP_base)
160 _ARG_vp = jit.Ptr(_SP, _FP_base+8)
161 _ARG_sb = jit.Ptr(_SP, _FP_base+16)
162 _ARG_fv = jit.Ptr(_SP, _FP_base+24)
163 )
164
165 var (
166 _RET_et = _ET
167 _RET_ep = _EP
168 )
169
170 var (
171 _VAR_sp = jit.Ptr(_SP, _FP_fargs+_FP_saves)
172 _VAR_dn = jit.Ptr(_SP, _FP_fargs+_FP_saves+8)
173 _VAR_vp = jit.Ptr(_SP, _FP_fargs+_FP_saves+16)
174 )
175
176 var (
177 _REG_ffi = []obj.Addr{_RP, _RL, _RC, _SP_q}
178 _REG_b64 = []obj.Addr{_SP_p, _SP_q}
179
180 _REG_all = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q, _RP, _RL, _RC}
181 _REG_ms = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q, _LR}
182 _REG_enc = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q, _RL}
183 )
184
185 type Assembler struct {
186 Name string
187 jit.BaseAssembler
188 p ir.Program
189 x int
190 }
191
192 func NewAssembler(p ir.Program) *Assembler {
193 return new(Assembler).Init(p)
194 }
195
196 /** Assembler Interface **/
197
198 func (self *Assembler) Load() vars.Encoder {
199 return ptoenc(self.BaseAssembler.Load("encode_"+self.Name, _FP_size, _FP_args, vars.ArgPtrs, vars.LocalPtrs))
200 }
201
202 func (self *Assembler) Init(p ir.Program) *Assembler {
203 self.p = p
204 self.BaseAssembler.Init(self.compile)
205 return self
206 }
207
208 func (self *Assembler) compile() {
209 self.prologue()
210 self.instrs()
211 self.epilogue()
212 self.builtins()
213 }
214
215 /** Assembler Stages **/
216
217 var _OpFuncTab = [256]func(*Assembler, *ir.Instr){
218 ir.OP_null: (*Assembler)._asm_OP_null,
219 ir.OP_empty_arr: (*Assembler)._asm_OP_empty_arr,
220 ir.OP_empty_obj: (*Assembler)._asm_OP_empty_obj,
221 ir.OP_bool: (*Assembler)._asm_OP_bool,
222 ir.OP_i8: (*Assembler)._asm_OP_i8,
223 ir.OP_i16: (*Assembler)._asm_OP_i16,
224 ir.OP_i32: (*Assembler)._asm_OP_i32,
225 ir.OP_i64: (*Assembler)._asm_OP_i64,
226 ir.OP_u8: (*Assembler)._asm_OP_u8,
227 ir.OP_u16: (*Assembler)._asm_OP_u16,
228 ir.OP_u32: (*Assembler)._asm_OP_u32,
229 ir.OP_u64: (*Assembler)._asm_OP_u64,
230 ir.OP_f32: (*Assembler)._asm_OP_f32,
231 ir.OP_f64: (*Assembler)._asm_OP_f64,
232 ir.OP_str: (*Assembler)._asm_OP_str,
233 ir.OP_bin: (*Assembler)._asm_OP_bin,
234 ir.OP_quote: (*Assembler)._asm_OP_quote,
235 ir.OP_number: (*Assembler)._asm_OP_number,
236 ir.OP_eface: (*Assembler)._asm_OP_eface,
237 ir.OP_iface: (*Assembler)._asm_OP_iface,
238 ir.OP_byte: (*Assembler)._asm_OP_byte,
239 ir.OP_text: (*Assembler)._asm_OP_text,
240 ir.OP_deref: (*Assembler)._asm_OP_deref,
241 ir.OP_index: (*Assembler)._asm_OP_index,
242 ir.OP_load: (*Assembler)._asm_OP_load,
243 ir.OP_save: (*Assembler)._asm_OP_save,
244 ir.OP_drop: (*Assembler)._asm_OP_drop,
245 ir.OP_drop_2: (*Assembler)._asm_OP_drop_2,
246 ir.OP_recurse: (*Assembler)._asm_OP_recurse,
247 ir.OP_is_nil: (*Assembler)._asm_OP_is_nil,
248 ir.OP_is_nil_p1: (*Assembler)._asm_OP_is_nil_p1,
249 ir.OP_is_zero_1: (*Assembler)._asm_OP_is_zero_1,
250 ir.OP_is_zero_2: (*Assembler)._asm_OP_is_zero_2,
251 ir.OP_is_zero_4: (*Assembler)._asm_OP_is_zero_4,
252 ir.OP_is_zero_8: (*Assembler)._asm_OP_is_zero_8,
253 ir.OP_is_zero_map: (*Assembler)._asm_OP_is_zero_map,
254 ir.OP_goto: (*Assembler)._asm_OP_goto,
255 ir.OP_map_iter: (*Assembler)._asm_OP_map_iter,
256 ir.OP_map_stop: (*Assembler)._asm_OP_map_stop,
257 ir.OP_map_check_key: (*Assembler)._asm_OP_map_check_key,
258 ir.OP_map_write_key: (*Assembler)._asm_OP_map_write_key,
259 ir.OP_map_value_next: (*Assembler)._asm_OP_map_value_next,
260 ir.OP_slice_len: (*Assembler)._asm_OP_slice_len,
261 ir.OP_slice_next: (*Assembler)._asm_OP_slice_next,
262 ir.OP_marshal: (*Assembler)._asm_OP_marshal,
263 ir.OP_marshal_p: (*Assembler)._asm_OP_marshal_p,
264 ir.OP_marshal_text: (*Assembler)._asm_OP_marshal_text,
265 ir.OP_marshal_text_p: (*Assembler)._asm_OP_marshal_text_p,
266 ir.OP_cond_set: (*Assembler)._asm_OP_cond_set,
267 ir.OP_cond_testc: (*Assembler)._asm_OP_cond_testc,
268 ir.OP_unsupported: (*Assembler)._asm_OP_unsupported,
269 ir.OP_is_zero: (*Assembler)._asm_OP_is_zero,
270 }
271
272 func (self *Assembler) instr(v *ir.Instr) {
273 if fn := _OpFuncTab[v.Op()]; fn != nil {
274 fn(self, v)
275 } else {
276 panic(fmt.Sprintf("invalid opcode: %d", v.Op()))
277 }
278 }
279
280 func (self *Assembler) instrs() {
281 for i, v := range self.p {
282 self.Mark(i)
283 self.instr(&v)
284 self.debug_instr(i, &v)
285 }
286 }
287
288 func (self *Assembler) builtins() {
289 self.more_space()
290 self.error_too_deep()
291 self.error_invalid_number()
292 self.error_nan_or_infinite()
293 self.go_panic()
294 }
295
296 func (self *Assembler) epilogue() {
297 self.Mark(len(self.p))
298 self.Emit("XORL", _ET, _ET)
299 self.Emit("XORL", _EP, _EP)
300 self.Link(_LB_error)
301 self.Emit("MOVQ", _ARG_rb, _CX) // MOVQ rb<>+0(FP), CX
302 self.Emit("MOVQ", _RL, jit.Ptr(_CX, 8)) // MOVQ RL, 8(CX)
303 self.Emit("MOVQ", jit.Imm(0), _ARG_rb) // MOVQ AX, rb<>+0(FP)
304 self.Emit("MOVQ", jit.Imm(0), _ARG_vp) // MOVQ BX, vp<>+8(FP)
305 self.Emit("MOVQ", jit.Imm(0), _ARG_sb) // MOVQ CX, sb<>+16(FP)
306 self.Emit("MOVQ", jit.Ptr(_SP, FP_offs), _BP) // MOVQ _FP_offs(SP), BP
307 self.Emit("ADDQ", jit.Imm(_FP_size), _SP) // ADDQ $_FP_size, SP
308 self.Emit("RET") // RET
309 }
310
311 func (self *Assembler) prologue() {
312 self.Emit("SUBQ", jit.Imm(_FP_size), _SP) // SUBQ $_FP_size, SP
313 self.Emit("MOVQ", _BP, jit.Ptr(_SP, FP_offs)) // MOVQ BP, _FP_offs(SP)
314 self.Emit("LEAQ", jit.Ptr(_SP, FP_offs), _BP) // LEAQ _FP_offs(SP), BP
315 self.Emit("MOVQ", _AX, _ARG_rb) // MOVQ AX, rb<>+0(FP)
316 self.Emit("MOVQ", _BX, _ARG_vp) // MOVQ BX, vp<>+8(FP)
317 self.Emit("MOVQ", _CX, _ARG_sb) // MOVQ CX, sb<>+16(FP)
318 self.Emit("MOVQ", _DI, _ARG_fv) // MOVQ DI, rb<>+24(FP)
319 self.Emit("MOVQ", jit.Ptr(_AX, 0), _RP) // MOVQ (AX) , DI
320 self.Emit("MOVQ", jit.Ptr(_AX, 8), _RL) // MOVQ 8(AX) , SI
321 self.Emit("MOVQ", jit.Ptr(_AX, 16), _RC) // MOVQ 16(AX), DX
322 self.Emit("MOVQ", _BX, _SP_p) // MOVQ BX, R10
323 self.Emit("MOVQ", _CX, _ST) // MOVQ CX, R8
324 self.Emit("XORL", _SP_x, _SP_x) // XORL R10, R12
325 self.Emit("XORL", _SP_f, _SP_f) // XORL R11, R13
326 self.Emit("XORL", _SP_q, _SP_q) // XORL R13, R11
327 }
328
329 /** Assembler Inline Functions **/
330
331 func (self *Assembler) xsave(reg ...obj.Addr) {
332 for i, v := range reg {
333 if i > _FP_saves/8-1 {
334 panic("too many registers to save")
335 } else {
336 self.Emit("MOVQ", v, jit.Ptr(_SP, _FP_fargs+int64(i)*8))
337 }
338 }
339 }
340
341 func (self *Assembler) xload(reg ...obj.Addr) {
342 for i, v := range reg {
343 if i > _FP_saves/8-1 {
344 panic("too many registers to load")
345 } else {
346 self.Emit("MOVQ", jit.Ptr(_SP, _FP_fargs+int64(i)*8), v)
347 }
348 }
349 }
350
351 func (self *Assembler) rbuf_di() {
352 if _RP.Reg != x86.REG_DI {
353 panic("register allocation messed up: RP != DI")
354 } else {
355 self.Emit("ADDQ", _RL, _RP)
356 }
357 }
358
359 func (self *Assembler) store_int(nd int, fn obj.Addr, ins string) {
360 self.check_size(nd)
361 self.save_c() // SAVE $C_regs
362 self.rbuf_di() // MOVQ RP, DI
363 self.Emit(ins, jit.Ptr(_SP_p, 0), _SI) // $ins (SP.p), SI
364 self.call_c(fn) // CALL_C $fn
365 self.Emit("ADDQ", _AX, _RL) // ADDQ AX, RL
366 }
367
368 func (self *Assembler) store_str(s string) {
369 i := 0
370 m := rt.Str2Mem(s)
371
372 /* 8-byte stores */
373 for i <= len(m)-8 {
374 self.Emit("MOVQ", jit.Imm(rt.Get64(m[i:])), _AX) // MOVQ $s[i:], AX
375 self.Emit("MOVQ", _AX, jit.Sib(_RP, _RL, 1, int64(i))) // MOVQ AX, i(RP)(RL)
376 i += 8
377 }
378
379 /* 4-byte stores */
380 if i <= len(m)-4 {
381 self.Emit("MOVL", jit.Imm(int64(rt.Get32(m[i:]))), jit.Sib(_RP, _RL, 1, int64(i))) // MOVL $s[i:], i(RP)(RL)
382 i += 4
383 }
384
385 /* 2-byte stores */
386 if i <= len(m)-2 {
387 self.Emit("MOVW", jit.Imm(int64(rt.Get16(m[i:]))), jit.Sib(_RP, _RL, 1, int64(i))) // MOVW $s[i:], i(RP)(RL)
388 i += 2
389 }
390
391 /* last byte */
392 if i < len(m) {
393 self.Emit("MOVB", jit.Imm(int64(m[i])), jit.Sib(_RP, _RL, 1, int64(i))) // MOVB $s[i:], i(RP)(RL)
394 }
395 }
396
397 func (self *Assembler) check_size(n int) {
398 self.check_size_rl(jit.Ptr(_RL, int64(n)))
399 }
400
401 func (self *Assembler) check_size_r(r obj.Addr, d int) {
402 self.check_size_rl(jit.Sib(_RL, r, 1, int64(d)))
403 }
404
405 func (self *Assembler) check_size_rl(v obj.Addr) {
406 idx := self.x
407 key := _LB_more_space_return + strconv.Itoa(idx)
408
409 /* the following code relies on LR == R9 to work */
410 if _LR.Reg != x86.REG_R9 {
411 panic("register allocation messed up: LR != R9")
412 }
413
414 /* check for buffer capacity */
415 self.x++
416 self.Emit("LEAQ", v, _AX) // LEAQ $v, AX
417 self.Emit("CMPQ", _AX, _RC) // CMPQ AX, RC
418 self.Sjmp("JBE", key) // JBE _more_space_return_{n}
419 self.slice_grow_ax(key) // GROW $key
420 self.Link(key) // _more_space_return_{n}:
421 }
422
423 func (self *Assembler) slice_grow_ax(ret string) {
424 self.Byte(0x4c, 0x8d, 0x0d) // LEAQ ?(PC), R9
425 self.Sref(ret, 4) // .... &ret
426 self.Sjmp("JMP", _LB_more_space) // JMP _more_space
427 }
428
429 /** State Stack Helpers **/
430
431
432
433 func (self *Assembler) save_state() {
434 self.Emit("MOVQ", jit.Ptr(_ST, 0), _CX) // MOVQ (ST), CX
435 self.Emit("LEAQ", jit.Ptr(_CX, vars.StateSize), _R9) // LEAQ vars.StateSize(CX), R9
436 self.Emit("CMPQ", _R9, jit.Imm(vars.StackLimit)) // CMPQ R9, $vars.StackLimit
437 self.Sjmp("JAE", _LB_error_too_deep) // JA _error_too_deep
438 self.Emit("MOVQ", _SP_x, jit.Sib(_ST, _CX, 1, 8)) // MOVQ SP.x, 8(ST)(CX)
439 self.Emit("MOVQ", _SP_f, jit.Sib(_ST, _CX, 1, 16)) // MOVQ SP.f, 16(ST)(CX)
440 self.WritePtr(0, _SP_p, jit.Sib(_ST, _CX, 1, 24)) // MOVQ SP.p, 24(ST)(CX)
441 self.WritePtr(1, _SP_q, jit.Sib(_ST, _CX, 1, 32)) // MOVQ SP.q, 32(ST)(CX)
442 self.Emit("MOVQ", _R9, jit.Ptr(_ST, 0)) // MOVQ R9, (ST)
443 }
444
445 func (self *Assembler) drop_state(decr int64) {
446 self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX
447 self.Emit("SUBQ", jit.Imm(decr), _AX) // SUBQ $decr, AX
448 self.Emit("MOVQ", _AX, jit.Ptr(_ST, 0)) // MOVQ AX, (ST)
449 self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 8), _SP_x) // MOVQ 8(ST)(AX), SP.x
450 self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 16), _SP_f) // MOVQ 16(ST)(AX), SP.f
451 self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 24), _SP_p) // MOVQ 24(ST)(AX), SP.p
452 self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 32), _SP_q) // MOVQ 32(ST)(AX), SP.q
453 self.Emit("PXOR", _X0, _X0) // PXOR X0, X0
454 self.Emit("MOVOU", _X0, jit.Sib(_ST, _AX, 1, 8)) // MOVOU X0, 8(ST)(AX)
455 self.Emit("MOVOU", _X0, jit.Sib(_ST, _AX, 1, 24)) // MOVOU X0, 24(ST)(AX)
456 }
457
458 /** Buffer Helpers **/
459
460 func (self *Assembler) add_char(ch byte) {
461 self.Emit("MOVB", jit.Imm(int64(ch)), jit.Sib(_RP, _RL, 1, 0)) // MOVB $ch, (RP)(RL)
462 self.Emit("ADDQ", jit.Imm(1), _RL) // ADDQ $1, RL
463 }
464
465 func (self *Assembler) add_long(ch uint32, n int64) {
466 self.Emit("MOVL", jit.Imm(int64(ch)), jit.Sib(_RP, _RL, 1, 0)) // MOVL $ch, (RP)(RL)
467 self.Emit("ADDQ", jit.Imm(n), _RL) // ADDQ $n, RL
468 }
469
470 func (self *Assembler) add_text(ss string) {
471 self.store_str(ss) // TEXT $ss
472 self.Emit("ADDQ", jit.Imm(int64(len(ss))), _RL) // ADDQ ${len(ss)}, RL
473 }
474
475 // get *buf at AX
476 func (self *Assembler) prep_buffer_AX() {
477 self.Emit("MOVQ", _ARG_rb, _AX) // MOVQ rb<>+0(FP), AX
478 self.Emit("MOVQ", _RL, jit.Ptr(_AX, 8)) // MOVQ RL, 8(AX)
479 }
480
481 func (self *Assembler) save_buffer() {
482 self.Emit("MOVQ", _ARG_rb, _CX) // MOVQ rb<>+0(FP), CX
483 self.Emit("MOVQ", _RP, jit.Ptr(_CX, 0)) // MOVQ RP, (CX)
484 self.Emit("MOVQ", _RL, jit.Ptr(_CX, 8)) // MOVQ RL, 8(CX)
485 self.Emit("MOVQ", _RC, jit.Ptr(_CX, 16)) // MOVQ RC, 16(CX)
486 }
487
488 // get *buf at AX
489 func (self *Assembler) load_buffer_AX() {
490 self.Emit("MOVQ", _ARG_rb, _AX) // MOVQ rb<>+0(FP), AX
491 self.Emit("MOVQ", jit.Ptr(_AX, 0), _RP) // MOVQ (AX), RP
492 self.Emit("MOVQ", jit.Ptr(_AX, 8), _RL) // MOVQ 8(AX), RL
493 self.Emit("MOVQ", jit.Ptr(_AX, 16), _RC) // MOVQ 16(AX), RC
494 }
495
496 /** Function Interface Helpers **/
497
498 func (self *Assembler) call(pc obj.Addr) {
499 self.Emit("MOVQ", pc, _LR) // MOVQ $pc, AX
500 self.Rjmp("CALL", _LR) // CALL AX
501 }
502
503 func (self *Assembler) save_c() {
504 self.xsave(_REG_ffi...) // SAVE $REG_ffi
505 }
506
507 func (self *Assembler) call_b64(pc obj.Addr) {
508 self.xsave(_REG_b64...) // SAVE $REG_all
509 self.call(pc) // CALL $pc
510 self.xload(_REG_b64...) // LOAD $REG_ffi
511 }
512
513 func (self *Assembler) call_c(pc obj.Addr) {
514 self.Emit("XCHGQ", _SP_p, _BX)
515 self.call(pc) // CALL $pc
516 self.xload(_REG_ffi...) // LOAD $REG_ffi
517 self.Emit("XCHGQ", _SP_p, _BX)
518 }
519
520 func (self *Assembler) call_go(pc obj.Addr) {
521 self.xsave(_REG_all...) // SAVE $REG_all
522 self.call(pc) // CALL $pc
523 self.xload(_REG_all...) // LOAD $REG_all
524 }
525
526 func (self *Assembler) call_more_space(pc obj.Addr) {
527 self.xsave(_REG_ms...) // SAVE $REG_all
528 self.call(pc) // CALL $pc
529 self.xload(_REG_ms...) // LOAD $REG_all
530 }
531
532 func (self *Assembler) call_encoder(pc obj.Addr) {
533 self.xsave(_REG_enc...) // SAVE $REG_all
534 self.call(pc) // CALL $pc
535 self.xload(_REG_enc...) // LOAD $REG_all
536 }
537
538 func (self *Assembler) call_marshaler(fn obj.Addr, it *rt.GoType, vt reflect.Type) {
539 switch vt.Kind() {
540 case reflect.Interface:
541 self.call_marshaler_i(fn, it)
542 case reflect.Ptr, reflect.Map:
543 self.call_marshaler_v(fn, it, vt, true)
544 // struct/array of 1 direct iface type can be direct
545 default:
546 self.call_marshaler_v(fn, it, vt, !rt.UnpackType(vt).Indirect())
547 }
548 }
549
550 var (
551 _F_assertI2I = jit.Func(rt.AssertI2I)
552 )
553
554 func (self *Assembler) call_marshaler_i(fn obj.Addr, it *rt.GoType) {
555 self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _AX) // MOVQ (SP.p), AX
556 self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
557 self.Sjmp("JZ", "_null_{n}") // JZ _null_{n}
558 self.Emit("MOVQ", _AX, _BX) // MOVQ AX, BX
559 self.Emit("MOVQ", jit.Ptr(_SP_p, 8), _CX) // MOVQ 8(SP.p), CX
560 self.Emit("MOVQ", jit.Gtype(it), _AX) // MOVQ $it, AX
561 self.call_go(_F_assertI2I) // CALL_GO assertI2I
562 self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
563 self.Sjmp("JZ", "_null_{n}") // JZ _null_{n}
564 self.Emit("MOVQ", _BX, _CX) // MOVQ BX, CX
565 self.Emit("MOVQ", _AX, _BX) // MOVQ AX, BX
566 self.prep_buffer_AX()
567 self.Emit("MOVQ", _ARG_fv, _DI) // MOVQ ARG.fv, DI
568 self.call_go(fn) // CALL $fn
569 self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET
570 self.Sjmp("JNZ", _LB_error) // JNZ _error
571 self.load_buffer_AX()
572 self.Sjmp("JMP", "_done_{n}") // JMP _done_{n}
573 self.Link("_null_{n}") // _null_{n}:
574 self.check_size(4) // SIZE $4
575 self.Emit("MOVL", jit.Imm(_IM_null), jit.Sib(_RP, _RL, 1, 0)) // MOVL $'null', (RP)(RL*1)
576 self.Emit("ADDQ", jit.Imm(4), _RL) // ADDQ $4, RL
577 self.Link("_done_{n}") // _done_{n}:
578 }
579
580 func (self *Assembler) call_marshaler_v(fn obj.Addr, it *rt.GoType, vt reflect.Type, deref bool) {
581 self.prep_buffer_AX() // MOVE {buf}, (SP)
582 self.Emit("MOVQ", jit.Itab(it, vt), _BX) // MOVQ $(itab(it, vt)), BX
583
584 /* dereference the pointer if needed */
585 if !deref {
586 self.Emit("MOVQ", _SP_p, _CX) // MOVQ SP.p, CX
587 } else {
588 self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _CX) // MOVQ 0(SP.p), CX
589 }
590
591 /* call the encoder, and perform error checks */
592 self.Emit("MOVQ", _ARG_fv, _DI) // MOVQ ARG.fv, DI
593 self.call_go(fn) // CALL $fn
594 self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET
595 self.Sjmp("JNZ", _LB_error) // JNZ _error
596 self.load_buffer_AX()
597 }
598
599 /** Builtin: _more_space **/
600
601 var (
602 _T_byte = jit.Type(vars.ByteType)
603 _F_growslice = jit.Func(rt.GrowSlice)
604
605 _T_json_Marshaler = rt.UnpackType(vars.JsonMarshalerType)
606 _T_encoding_TextMarshaler = rt.UnpackType(vars.EncodingTextMarshalerType)
607 )
608
609 // AX must saving n
610 func (self *Assembler) more_space() {
611 self.Link(_LB_more_space)
612 self.Emit("MOVQ", _RP, _BX) // MOVQ DI, BX
613 self.Emit("MOVQ", _RL, _CX) // MOVQ SI, CX
614 self.Emit("MOVQ", _RC, _DI) // MOVQ DX, DI
615 self.Emit("MOVQ", _AX, _SI) // MOVQ AX, SI
616 self.Emit("MOVQ", _T_byte, _AX) // MOVQ $_T_byte, AX
617 self.call_more_space(_F_growslice) // CALL $pc
618 self.Emit("MOVQ", _AX, _RP) // MOVQ AX, DI
619 self.Emit("MOVQ", _BX, _RL) // MOVQ BX, SI
620 self.Emit("MOVQ", _CX, _RC) // MOVQ CX, DX
621 self.save_buffer() // SAVE {buf}
622 self.Rjmp("JMP", _LR) // JMP LR
623 }
624
625 /** Builtin Errors **/
626
627 var (
628 _V_ERR_too_deep = jit.Imm(int64(uintptr(unsafe.Pointer(vars.ERR_too_deep))))
629 _V_ERR_nan_or_infinite = jit.Imm(int64(uintptr(unsafe.Pointer(vars.ERR_nan_or_infinite))))
630 _I_json_UnsupportedValueError = jit.Itab(rt.UnpackType(vars.ErrorType), vars.JsonUnsupportedValueType)
631 )
632
633 func (self *Assembler) error_too_deep() {
634 self.Link(_LB_error_too_deep)
635 self.Emit("MOVQ", _V_ERR_too_deep, _EP) // MOVQ $_V_ERR_too_deep, EP
636 self.Emit("MOVQ", _I_json_UnsupportedValueError, _ET) // MOVQ $_I_json_UnsupportedValuError, ET
637 self.Sjmp("JMP", _LB_error) // JMP _error
638 }
639
640 func (self *Assembler) error_invalid_number() {
641 self.Link(_LB_error_invalid_number)
642 self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _AX) // MOVQ 0(SP), AX
643 self.Emit("MOVQ", jit.Ptr(_SP_p, 8), _BX) // MOVQ 8(SP), BX
644 self.call_go(_F_error_number) // CALL_GO error_number
645 self.Sjmp("JMP", _LB_error) // JMP _error
646 }
647
648 func (self *Assembler) error_nan_or_infinite() {
649 self.Link(_LB_error_nan_or_infinite)
650 self.Emit("MOVQ", _V_ERR_nan_or_infinite, _EP) // MOVQ $_V_ERR_nan_or_infinite, EP
651 self.Emit("MOVQ", _I_json_UnsupportedValueError, _ET) // MOVQ $_I_json_UnsupportedValuError, ET
652 self.Sjmp("JMP", _LB_error) // JMP _error
653 }
654
655 /** String Encoding Routine **/
656
657 var (
658 _F_quote = jit.Imm(int64(native.S_quote))
659 _F_panic = jit.Func(vars.GoPanic)
660 )
661
662 func (self *Assembler) go_panic() {
663 self.Link(_LB_panic)
664 self.Emit("MOVQ", _SP_p, _BX)
665 self.call_go(_F_panic)
666 }
667
668 func (self *Assembler) encode_string(doubleQuote bool) {
669 self.Emit("MOVQ", jit.Ptr(_SP_p, 8), _AX) // MOVQ 8(SP.p), AX
670 self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
671 self.Sjmp("JZ", "_str_empty_{n}") // JZ _str_empty_{n}
672 self.Emit("CMPQ", jit.Ptr(_SP_p, 0), jit.Imm(0))
673 self.Sjmp("JNE", "_str_next_{n}")
674 self.Emit("MOVQ", jit.Imm(int64(vars.PanicNilPointerOfNonEmptyString)), _AX)
675 self.Sjmp("JMP", _LB_panic)
676 self.Link("_str_next_{n}")
677
678 /* opening quote, check for double quote */
679 if !doubleQuote {
680 self.check_size_r(_AX, 2) // SIZE $2
681 self.add_char('"') // CHAR $'"'
682 } else {
683 self.check_size_r(_AX, 6) // SIZE $6
684 self.add_long(_IM_open, 3) // TEXT $`"\"`
685 }
686
687 /* quoting loop */
688 self.Emit("XORL", _AX, _AX) // XORL AX, AX
689 self.Emit("MOVQ", _AX, _VAR_sp) // MOVQ AX, sp
690 self.Link("_str_loop_{n}") // _str_loop_{n}:
691 self.save_c() // SAVE $REG_ffi
692
693 /* load the output buffer first, and then input buffer,
694 * because the parameter registers collide with RP / RL / RC */
695 self.Emit("MOVQ", _RC, _CX) // MOVQ RC, CX
696 self.Emit("SUBQ", _RL, _CX) // SUBQ RL, CX
697 self.Emit("MOVQ", _CX, _VAR_dn) // MOVQ CX, dn
698 self.Emit("LEAQ", jit.Sib(_RP, _RL, 1, 0), _DX) // LEAQ (RP)(RL), DX
699 self.Emit("LEAQ", _VAR_dn, _CX) // LEAQ dn, CX
700 self.Emit("MOVQ", _VAR_sp, _AX) // MOVQ sp, AX
701 self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _DI) // MOVQ (SP.p), DI
702 self.Emit("MOVQ", jit.Ptr(_SP_p, 8), _SI) // MOVQ 8(SP.p), SI
703 self.Emit("ADDQ", _AX, _DI) // ADDQ AX, DI
704 self.Emit("SUBQ", _AX, _SI) // SUBQ AX, SI
705
706 /* set the flags based on `doubleQuote` */
707 if !doubleQuote {
708 self.Emit("XORL", _R8, _R8) // XORL R8, R8
709 } else {
710 self.Emit("MOVL", jit.Imm(types.F_DOUBLE_UNQUOTE), _R8) // MOVL ${types.F_DOUBLE_UNQUOTE}, R8
711 }
712
713 /* call the native quoter */
714 self.call_c(_F_quote) // CALL quote
715 self.Emit("ADDQ", _VAR_dn, _RL) // ADDQ dn, RL
716
717 self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
718 self.Sjmp("JS", "_str_space_{n}") // JS _str_space_{n}
719
720 /* close the string, check for double quote */
721 if !doubleQuote {
722 self.check_size(1) // SIZE $1
723 self.add_char('"') // CHAR $'"'
724 self.Sjmp("JMP", "_str_end_{n}") // JMP _str_end_{n}
725 } else {
726 self.check_size(3) // SIZE $3
727 self.add_text("\\\"\"") // TEXT $'\""'
728 self.Sjmp("JMP", "_str_end_{n}") // JMP _str_end_{n}
729 }
730
731 /* not enough space to contain the quoted string */
732 self.Link("_str_space_{n}") // _str_space_{n}:
733 self.Emit("NOTQ", _AX) // NOTQ AX
734 self.Emit("ADDQ", _AX, _VAR_sp) // ADDQ AX, sp
735 self.Emit("LEAQ", jit.Sib(_RC, _RC, 1, 0), _AX) // LEAQ (RC)(RC), AX
736 self.slice_grow_ax("_str_loop_{n}") // GROW _str_loop_{n}
737
738 /* empty string, check for double quote */
739 if !doubleQuote {
740 self.Link("_str_empty_{n}") // _str_empty_{n}:
741 self.check_size(2) // SIZE $2
742 self.add_text("\"\"") // TEXT $'""'
743 self.Link("_str_end_{n}") // _str_end_{n}:
744 } else {
745 self.Link("_str_empty_{n}") // _str_empty_{n}:
746 self.check_size(6) // SIZE $6
747 self.add_text("\"\\\"\\\"\"") // TEXT $'"\"\""'
748 self.Link("_str_end_{n}") // _str_end_{n}:
749 }
750 }
751
752 /** OpCode Assembler Functions **/
753
754
755
756 var (
757 _F_f64toa = jit.Imm(int64(native.S_f64toa))
758 _F_f32toa = jit.Imm(int64(native.S_f32toa))
759 _F_i64toa = jit.Imm(int64(native.S_i64toa))
760 _F_u64toa = jit.Imm(int64(native.S_u64toa))
761 _F_b64encode = jit.Imm(int64(rt.SubrB64Encode))
762 )
763
764 var (
765 _F_memmove = jit.Func(rt.Memmove)
766 _F_error_number = jit.Func(vars.Error_number)
767 _F_isValidNumber = jit.Func(rt.IsValidNumber)
768 )
769
770 var (
771 _F_iteratorStop = jit.Func(alg.IteratorStop)
772 _F_iteratorNext = jit.Func(alg.IteratorNext)
773 _F_iteratorStart = jit.Func(alg.IteratorStart)
774 )
775
776 var (
777 _F_encodeTypedPointer obj.Addr
778 _F_encodeJsonMarshaler obj.Addr
779 _F_encodeTextMarshaler obj.Addr
780 )
781
782 const (
783 _MODE_AVX2 = 1 << 2
784 )
785
786 func init() {
787 _F_encodeJsonMarshaler = jit.Func(alg.EncodeJsonMarshaler)
788 _F_encodeTextMarshaler = jit.Func(alg.EncodeTextMarshaler)
789 _F_encodeTypedPointer = jit.Func(EncodeTypedPointer)
790 }
791
792 func (self *Assembler) _asm_OP_null(_ *ir.Instr) {
793 self.check_size(4)
794 self.Emit("MOVL", jit.Imm(_IM_null), jit.Sib(_RP, _RL, 1, 0)) // MOVL $'null', (RP)(RL*1)
795 self.Emit("ADDQ", jit.Imm(4), _RL) // ADDQ $4, RL
796 }
797
798 func (self *Assembler) _asm_OP_empty_arr(_ *ir.Instr) {
799 self.Emit("BTQ", jit.Imm(int64(alg.BitNoNullSliceOrMap)), _ARG_fv)
800 self.Sjmp("JC", "_empty_arr_{n}")
801 self._asm_OP_null(nil)
802 self.Sjmp("JMP", "_empty_arr_end_{n}")
803 self.Link("_empty_arr_{n}")
804 self.check_size(2)
805 self.Emit("MOVW", jit.Imm(_IM_array), jit.Sib(_RP, _RL, 1, 0))
806 self.Emit("ADDQ", jit.Imm(2), _RL)
807 self.Link("_empty_arr_end_{n}")
808 }
809
810 func (self *Assembler) _asm_OP_empty_obj(_ *ir.Instr) {
811 self.Emit("BTQ", jit.Imm(int64(alg.BitNoNullSliceOrMap)), _ARG_fv)
812 self.Sjmp("JC", "_empty_obj_{n}")
813 self._asm_OP_null(nil)
814 self.Sjmp("JMP", "_empty_obj_end_{n}")
815 self.Link("_empty_obj_{n}")
816 self.check_size(2)
817 self.Emit("MOVW", jit.Imm(_IM_object), jit.Sib(_RP, _RL, 1, 0))
818 self.Emit("ADDQ", jit.Imm(2), _RL)
819 self.Link("_empty_obj_end_{n}")
820 }
821
822 func (self *Assembler) _asm_OP_bool(_ *ir.Instr) {
823 self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPB (SP.p), $0
824 self.Sjmp("JE", "_false_{n}") // JE _false_{n}
825 self.check_size(4) // SIZE $4
826 self.Emit("MOVL", jit.Imm(_IM_true), jit.Sib(_RP, _RL, 1, 0)) // MOVL $'true', (RP)(RL*1)
827 self.Emit("ADDQ", jit.Imm(4), _RL) // ADDQ $4, RL
828 self.Sjmp("JMP", "_end_{n}") // JMP _end_{n}
829 self.Link("_false_{n}") // _false_{n}:
830 self.check_size(5) // SIZE $5
831 self.Emit("MOVL", jit.Imm(_IM_fals), jit.Sib(_RP, _RL, 1, 0)) // MOVL $'fals', (RP)(RL*1)
832 self.Emit("MOVB", jit.Imm('e'), jit.Sib(_RP, _RL, 1, 4)) // MOVB $'e', 4(RP)(RL*1)
833 self.Emit("ADDQ", jit.Imm(5), _RL) // ADDQ $5, RL
834 self.Link("_end_{n}") // _end_{n}:
835 }
836
837 func (self *Assembler) _asm_OP_i8(_ *ir.Instr) {
838 self.store_int(4, _F_i64toa, "MOVBQSX")
839 }
840
841 func (self *Assembler) _asm_OP_i16(_ *ir.Instr) {
842 self.store_int(6, _F_i64toa, "MOVWQSX")
843 }
844
845 func (self *Assembler) _asm_OP_i32(_ *ir.Instr) {
846 self.store_int(17, _F_i64toa, "MOVLQSX")
847 }
848
849 func (self *Assembler) _asm_OP_i64(_ *ir.Instr) {
850 self.store_int(21, _F_i64toa, "MOVQ")
851 }
852
853 func (self *Assembler) _asm_OP_u8(_ *ir.Instr) {
854 self.store_int(3, _F_u64toa, "MOVBQZX")
855 }
856
857 func (self *Assembler) _asm_OP_u16(_ *ir.Instr) {
858 self.store_int(5, _F_u64toa, "MOVWQZX")
859 }
860
861 func (self *Assembler) _asm_OP_u32(_ *ir.Instr) {
862 self.store_int(16, _F_u64toa, "MOVLQZX")
863 }
864
865 func (self *Assembler) _asm_OP_u64(_ *ir.Instr) {
866 self.store_int(20, _F_u64toa, "MOVQ")
867 }
868
869 func (self *Assembler) _asm_OP_f32(_ *ir.Instr) {
870 self.check_size(32)
871 self.Emit("MOVL", jit.Ptr(_SP_p, 0), _AX) // MOVL (SP.p), AX
872 self.Emit("ANDL", jit.Imm(_FM_exp32), _AX) // ANDL $_FM_exp32, AX
873 self.Emit("XORL", jit.Imm(_FM_exp32), _AX) // XORL $_FM_exp32, AX
874 self.Sjmp("JNZ", "_encode_normal_f32_{n}")// JNZ _encode_normal_f32_{n}
875 self.Emit("BTQ", jit.Imm(alg.BitEncodeNullForInfOrNan), _ARG_fv) // BTQ ${BitEncodeNullForInfOrNan}, fv
876 self.Sjmp("JNC", _LB_error_nan_or_infinite) // JNC _error_nan_or_infinite
877 self._asm_OP_null(nil)
878 self.Sjmp("JMP", "_encode_f32_end_{n}") // JMP _encode_f32_end_{n}
879 self.Link("_encode_normal_f32_{n}")
880 self.save_c() // SAVE $C_regs
881 self.rbuf_di() // MOVQ RP, DI
882 self.Emit("MOVSS", jit.Ptr(_SP_p, 0), _X0) // MOVSS (SP.p), X0
883 self.call_c(_F_f32toa) // CALL_C f32toa
884 self.Emit("ADDQ", _AX, _RL) // ADDQ AX, RL
885 self.Link("_encode_f32_end_{n}")
886 }
887
888 func (self *Assembler) _asm_OP_f64(_ *ir.Instr) {
889 self.check_size(32)
890 self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _AX) // MOVQ (SP.p), AX
891 self.Emit("MOVQ", jit.Imm(_FM_exp64), _CX) // MOVQ $_FM_exp64, CX
892 self.Emit("ANDQ", _CX, _AX) // ANDQ CX, AX
893 self.Emit("XORQ", _CX, _AX) // XORQ CX, AX
894 self.Sjmp("JNZ", "_encode_normal_f64_{n}")// JNZ _encode_normal_f64_{n}
895 self.Emit("BTQ", jit.Imm(alg.BitEncodeNullForInfOrNan), _ARG_fv) // BTQ ${BitEncodeNullForInfOrNan}, fv
896 self.Sjmp("JNC", _LB_error_nan_or_infinite)// JNC _error_nan_or_infinite
897 self._asm_OP_null(nil)
898 self.Sjmp("JMP", "_encode_f64_end_{n}") // JMP _encode_f64_end_{n}
899 self.Link("_encode_normal_f64_{n}")
900 self.save_c() // SAVE $C_regs
901 self.rbuf_di() // MOVQ RP, DI
902 self.Emit("MOVSD", jit.Ptr(_SP_p, 0), _X0) // MOVSD (SP.p), X0
903 self.call_c(_F_f64toa) // CALL_C f64toa
904 self.Emit("ADDQ", _AX, _RL) // ADDQ AX, RL
905 self.Link("_encode_f64_end_{n}")
906 }
907
908 func (self *Assembler) _asm_OP_str(_ *ir.Instr) {
909 self.encode_string(false)
910 }
911
912 func (self *Assembler) _asm_OP_bin(_ *ir.Instr) {
913 self.Emit("MOVQ", jit.Ptr(_SP_p, 8), _AX) // MOVQ 8(SP.p), AX
914 self.Emit("ADDQ", jit.Imm(2), _AX) // ADDQ $2, AX
915 self.Emit("MOVQ", jit.Imm(_IM_mulv), _CX) // MOVQ $_MF_mulv, CX
916 self.Emit("MOVQ", _DX, _BX) // MOVQ DX, BX
917 self.From("MULQ", _CX) // MULQ CX
918 self.Emit("LEAQ", jit.Sib(_DX, _DX, 1, 1), _AX) // LEAQ 1(DX)(DX), AX
919 self.Emit("ORQ", jit.Imm(2), _AX) // ORQ $2, AX
920 self.Emit("MOVQ", _BX, _DX) // MOVQ BX, DX
921 self.check_size_r(_AX, 0) // SIZE AX
922 self.add_char('"') // CHAR $'"'
923 self.Emit("MOVQ", _ARG_rb, _DI) // MOVQ rb<>+0(FP), DI
924 self.Emit("MOVQ", _RL, jit.Ptr(_DI, 8)) // MOVQ SI, 8(DI)
925 self.Emit("MOVQ", _SP_p, _SI) // MOVQ SP.p, SI
926
927 /* check for AVX2 support */
928 if !cpu.HasAVX2 {
929 self.Emit("XORL", _DX, _DX) // XORL DX, DX
930 } else {
931 self.Emit("MOVL", jit.Imm(_MODE_AVX2), _DX) // MOVL $_MODE_AVX2, DX
932 }
933
934 /* call the encoder */
935 self.call_b64(_F_b64encode) // CALL b64encode
936 self.load_buffer_AX() // LOAD {buf}
937 self.add_char('"') // CHAR $'"'
938 }
939
940 func (self *Assembler) _asm_OP_quote(_ *ir.Instr) {
941 self.encode_string(true)
942 }
943
944 func (self *Assembler) _asm_OP_number(_ *ir.Instr) {
945 self.Emit("MOVQ", jit.Ptr(_SP_p, 8), _BX) // MOVQ (SP.p), BX
946 self.Emit("TESTQ", _BX, _BX) // TESTQ BX, BX
947 self.Sjmp("JZ", "_empty_{n}")
948 self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _AX) // MOVQ (SP.p), AX
949 self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
950 self.Sjmp("JNZ", "_number_next_{n}")
951 self.Emit("MOVQ", jit.Imm(int64(vars.PanicNilPointerOfNonEmptyString)), _AX)
952 self.Sjmp("JMP", _LB_panic)
953 self.Link("_number_next_{n}")
954 self.call_go(_F_isValidNumber) // CALL_GO isValidNumber
955 self.Emit("CMPB", _AX, jit.Imm(0)) // CMPB AX, $0
956 self.Sjmp("JE", _LB_error_invalid_number) // JE _error_invalid_number
957 self.Emit("MOVQ", jit.Ptr(_SP_p, 8), _BX) // MOVQ (SP.p), BX
958 self.check_size_r(_BX, 0) // SIZE BX
959 self.Emit("LEAQ", jit.Sib(_RP, _RL, 1, 0), _AX) // LEAQ (RP)(RL), AX
960 self.Emit("ADDQ", jit.Ptr(_SP_p, 8), _RL) // ADDQ 8(SP.p), RL
961 self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _BX) // MOVOU (SP.p), BX
962 self.Emit("MOVQ", jit.Ptr(_SP_p, 8), _CX) // MOVOU X0, 8(SP)
963 self.call_go(_F_memmove) // CALL_GO memmove
964 self.Emit("MOVQ", _ARG_rb, _AX) // MOVQ rb<>+0(FP), AX
965 self.Emit("MOVQ", _RL, jit.Ptr(_AX, 8)) // MOVQ RL, 8(AX)
966 self.Sjmp("JMP", "_done_{n}") // JMP _done_{n}
967 self.Link("_empty_{n}") // _empty_{n}
968 self.check_size(1) // SIZE $1
969 self.add_char('0') // CHAR $'0'
970 self.Link("_done_{n}") // _done_{n}:
971 }
972
973 func (self *Assembler) _asm_OP_eface(_ *ir.Instr) {
974 self.prep_buffer_AX() // MOVE {buf}, AX
975 self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _BX) // MOVQ (SP.p), BX
976 self.Emit("LEAQ", jit.Ptr(_SP_p, 8), _CX) // LEAQ 8(SP.p), CX
977 self.Emit("MOVQ", _ST, _DI) // MOVQ ST, DI
978 self.Emit("MOVQ", _ARG_fv, _SI) // MOVQ fv, AX
979 self.call_encoder(_F_encodeTypedPointer) // CALL encodeTypedPointer
980 self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET
981 self.Sjmp("JNZ", _LB_error) // JNZ _error
982 self.load_buffer_AX()
983 }
984
985 func (self *Assembler) _asm_OP_iface(_ *ir.Instr) {
986 self.prep_buffer_AX() // MOVE {buf}, AX
987 self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _CX) // MOVQ (SP.p), CX
988 self.Emit("MOVQ", jit.Ptr(_CX, 8), _BX) // MOVQ 8(CX), BX
989 self.Emit("LEAQ", jit.Ptr(_SP_p, 8), _CX) // LEAQ 8(SP.p), CX
990 self.Emit("MOVQ", _ST, _DI) // MOVQ ST, DI
991 self.Emit("MOVQ", _ARG_fv, _SI) // MOVQ fv, AX
992 self.call_encoder(_F_encodeTypedPointer) // CALL encodeTypedPointer
993 self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET
994 self.Sjmp("JNZ", _LB_error) // JNZ _error
995 self.load_buffer_AX()
996 }
997
998 func (self *Assembler) _asm_OP_byte(p *ir.Instr) {
999 self.check_size(1)
1000 self.Emit("MOVB", jit.Imm(p.I64()), jit.Sib(_RP, _RL, 1, 0)) // MOVL p.Vi(), (RP)(RL*1)
1001 self.Emit("ADDQ", jit.Imm(1), _RL) // ADDQ $1, RL
1002 }
1003
1004 func (self *Assembler) _asm_OP_text(p *ir.Instr) {
1005 self.check_size(len(p.Vs())) // SIZE ${len(p.Vs())}
1006 self.add_text(p.Vs()) // TEXT ${p.Vs()}
1007 }
1008
1009 func (self *Assembler) _asm_OP_deref(_ *ir.Instr) {
1010 self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _SP_p) // MOVQ (SP.p), SP.p
1011 }
1012
1013 func (self *Assembler) _asm_OP_index(p *ir.Instr) {
1014 self.Emit("MOVQ", jit.Imm(p.I64()), _AX) // MOVQ $p.Vi(), AX
1015 self.Emit("ADDQ", _AX, _SP_p) // ADDQ AX, SP.p
1016 }
1017
1018 func (self *Assembler) _asm_OP_load(_ *ir.Instr) {
1019 self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX
1020 self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, -24), _SP_x) // MOVQ -24(ST)(AX), SP.x
1021 self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, -8), _SP_p) // MOVQ -8(ST)(AX), SP.p
1022 self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 0), _SP_q) // MOVQ (ST)(AX), SP.q
1023 }
1024
1025 func (self *Assembler) _asm_OP_save(_ *ir.Instr) {
1026 self.save_state()
1027 }
1028
1029 func (self *Assembler) _asm_OP_drop(_ *ir.Instr) {
1030 self.drop_state(vars.StateSize)
1031 }
1032
1033 func (self *Assembler) _asm_OP_drop_2(_ *ir.Instr) {
1034 self.drop_state(vars.StateSize * 2) // DROP $(vars.StateSize * 2)
1035 self.Emit("MOVOU", _X0, jit.Sib(_ST, _AX, 1, 56)) // MOVOU X0, 56(ST)(AX)
1036 }
1037
1038 func (self *Assembler) _asm_OP_recurse(p *ir.Instr) {
1039 self.prep_buffer_AX() // MOVE {buf}, (SP)
1040 vt, pv := p.Vp()
1041 self.Emit("MOVQ", jit.Type(vt), _BX) // MOVQ $(type(p.Vt())), BX
1042
1043 /* check for indirection */
1044 if !rt.UnpackType(vt).Indirect() {
1045 self.Emit("MOVQ", _SP_p, _CX) // MOVQ SP.p, CX
1046 } else {
1047 self.Emit("MOVQ", _SP_p, _VAR_vp) // MOVQ SP.p, VAR.vp
1048 self.Emit("LEAQ", _VAR_vp, _CX) // LEAQ VAR.vp, CX
1049 }
1050
1051 /* call the encoder */
1052 self.Emit("MOVQ", _ST, _DI) // MOVQ ST, DI
1053 self.Emit("MOVQ", _ARG_fv, _SI) // MOVQ $fv, SI
1054 if pv {
1055 self.Emit("BTSQ", jit.Imm(alg.BitPointerValue), _SI) // BTSQ $1, SI
1056 }
1057
1058 self.call_encoder(_F_encodeTypedPointer) // CALL encodeTypedPointer
1059 self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET
1060 self.Sjmp("JNZ", _LB_error) // JNZ _error
1061 self.load_buffer_AX()
1062 }
1063
1064 func (self *Assembler) _asm_OP_is_nil(p *ir.Instr) {
1065 self.Emit("CMPQ", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPQ (SP.p), $0
1066 self.Xjmp("JE", p.Vi()) // JE p.Vi()
1067 }
1068
1069 func (self *Assembler) _asm_OP_is_nil_p1(p *ir.Instr) {
1070 self.Emit("CMPQ", jit.Ptr(_SP_p, 8), jit.Imm(0)) // CMPQ 8(SP.p), $0
1071 self.Xjmp("JE", p.Vi()) // JE p.Vi()
1072 }
1073
1074 func (self *Assembler) _asm_OP_is_zero_1(p *ir.Instr) {
1075 self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPB (SP.p), $0
1076 self.Xjmp("JE", p.Vi()) // JE p.Vi()
1077 }
1078
1079 func (self *Assembler) _asm_OP_is_zero_2(p *ir.Instr) {
1080 self.Emit("CMPW", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPW (SP.p), $0
1081 self.Xjmp("JE", p.Vi()) // JE p.Vi()
1082 }
1083
1084 func (self *Assembler) _asm_OP_is_zero_4(p *ir.Instr) {
1085 self.Emit("CMPL", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPL (SP.p), $0
1086 self.Xjmp("JE", p.Vi()) // JE p.Vi()
1087 }
1088
1089 func (self *Assembler) _asm_OP_is_zero_8(p *ir.Instr) {
1090 self.Emit("CMPQ", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPQ (SP.p), $0
1091 self.Xjmp("JE", p.Vi()) // JE p.Vi()
1092 }
1093
1094 func (self *Assembler) _asm_OP_is_zero_map(p *ir.Instr) {
1095 self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _AX) // MOVQ (SP.p), AX
1096 self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX
1097 self.Xjmp("JZ", p.Vi()) // JZ p.Vi()
1098 self.Emit("CMPQ", jit.Ptr(_AX, 0), jit.Imm(0)) // CMPQ (AX), $0
1099 self.Xjmp("JE", p.Vi()) // JE p.Vi()
1100 }
1101
1102 var (
1103 _F_is_zero = jit.Func(alg.IsZero)
1104 _T_reflect_Type = rt.UnpackIface(reflect.Type(nil))
1105 )
1106
1107 func (self *Assembler) _asm_OP_is_zero(p *ir.Instr) {
1108 fv := p.VField()
1109 self.Emit("MOVQ", _SP_p, _AX) // ptr
1110 self.Emit("MOVQ", jit.ImmPtr(unsafe.Pointer(fv)), _BX) // fv
1111 self.call_go(_F_is_zero) // CALL $fn
1112 self.Emit("CMPB", _AX, jit.Imm(0)) // CMPB (SP.p), $0
1113 self.Xjmp("JNE", p.Vi()) // JE p.Vi()
1114 }
1115
1116 func (self *Assembler) _asm_OP_goto(p *ir.Instr) {
1117 self.Xjmp("JMP", p.Vi())
1118 }
1119
1120 func (self *Assembler) _asm_OP_map_iter(p *ir.Instr) {
1121 self.Emit("MOVQ", jit.Type(p.Vt()), _AX) // MOVQ $p.Vt(), AX
1122 self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _BX) // MOVQ (SP.p), BX
1123 self.Emit("MOVQ", _ARG_fv, _CX) // MOVQ fv, CX
1124 self.call_go(_F_iteratorStart) // CALL_GO iteratorStart
1125 self.Emit("MOVQ", _AX, _SP_q) // MOVQ AX, SP.q
1126 self.Emit("MOVQ", _BX, _ET) // MOVQ 32(SP), ET
1127 self.Emit("MOVQ", _CX, _EP) // MOVQ 40(SP), EP
1128 self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET
1129 self.Sjmp("JNZ", _LB_error) // JNZ _error
1130 }
1131
1132 func (self *Assembler) _asm_OP_map_stop(_ *ir.Instr) {
1133 self.Emit("MOVQ", _SP_q, _AX) // MOVQ SP.q, AX
1134 self.call_go(_F_iteratorStop) // CALL_GO iteratorStop
1135 self.Emit("XORL", _SP_q, _SP_q) // XORL SP.q, SP.q
1136 }
1137
1138 func (self *Assembler) _asm_OP_map_check_key(p *ir.Instr) {
1139 self.Emit("MOVQ", jit.Ptr(_SP_q, 0), _SP_p) // MOVQ (SP.q), SP.p
1140 self.Emit("TESTQ", _SP_p, _SP_p) // TESTQ SP.p, SP.p
1141 self.Xjmp("JZ", p.Vi()) // JNZ p.Vi()
1142 }
1143
1144 func (self *Assembler) _asm_OP_map_write_key(p *ir.Instr) {
1145 self.Emit("BTQ", jit.Imm(alg.BitSortMapKeys), _ARG_fv) // BTQ ${SortMapKeys}, fv
1146 self.Sjmp("JNC", "_unordered_key_{n}") // JNC _unordered_key_{n}
1147 self.encode_string(false) // STR $false
1148 self.Xjmp("JMP", p.Vi()) // JMP ${p.Vi()}
1149 self.Link("_unordered_key_{n}") // _unordered_key_{n}:
1150 }
1151
1152 func (self *Assembler) _asm_OP_map_value_next(_ *ir.Instr) {
1153 self.Emit("MOVQ", jit.Ptr(_SP_q, 8), _SP_p) // MOVQ 8(SP.q), SP.p
1154 self.Emit("MOVQ", _SP_q, _AX) // MOVQ SP.q, AX
1155 self.call_go(_F_iteratorNext) // CALL_GO iteratorNext
1156 }
1157
1158 func (self *Assembler) _asm_OP_slice_len(_ *ir.Instr) {
1159 self.Emit("MOVQ", jit.Ptr(_SP_p, 8), _SP_x) // MOVQ 8(SP.p), SP.x
1160 self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _SP_p) // MOVQ (SP.p), SP.p
1161 self.Emit("ORQ", jit.Imm(1<<_S_init), _SP_f) // ORQ $(1<<_S_init), SP.f
1162 }
1163
1164 func (self *Assembler) _asm_OP_slice_next(p *ir.Instr) {
1165 self.Emit("TESTQ", _SP_x, _SP_x) // TESTQ SP.x, SP.x
1166 self.Xjmp("JZ", p.Vi()) // JZ p.Vi()
1167 self.Emit("SUBQ", jit.Imm(1), _SP_x) // SUBQ $1, SP.x
1168 self.Emit("BTRQ", jit.Imm(_S_init), _SP_f) // BTRQ $_S_init, SP.f
1169 self.Emit("LEAQ", jit.Ptr(_SP_p, int64(p.Vlen())), _AX) // LEAQ $(p.vlen())(SP.p), AX
1170 self.Emit("CMOVQCC", _AX, _SP_p) // CMOVQNC AX, SP.p
1171 }
1172
1173 func (self *Assembler) _asm_OP_marshal(p *ir.Instr) {
1174 self.call_marshaler(_F_encodeJsonMarshaler, _T_json_Marshaler, p.Vt())
1175 }
1176
1177 func (self *Assembler) _asm_OP_marshal_p(p *ir.Instr) {
1178 if p.Vk() != reflect.Ptr {
1179 panic("marshal_p: invalid type")
1180 } else {
1181 self.call_marshaler_v(_F_encodeJsonMarshaler, _T_json_Marshaler, p.Vt(), false)
1182 }
1183 }
1184
1185 func (self *Assembler) _asm_OP_marshal_text(p *ir.Instr) {
1186 self.call_marshaler(_F_encodeTextMarshaler, _T_encoding_TextMarshaler, p.Vt())
1187 }
1188
1189 func (self *Assembler) _asm_OP_marshal_text_p(p *ir.Instr) {
1190 if p.Vk() != reflect.Ptr {
1191 panic("marshal_text_p: invalid type")
1192 } else {
1193 self.call_marshaler_v(_F_encodeTextMarshaler, _T_encoding_TextMarshaler, p.Vt(), false)
1194 }
1195 }
1196
1197 func (self *Assembler) _asm_OP_cond_set(_ *ir.Instr) {
1198 self.Emit("ORQ", jit.Imm(1<<_S_cond), _SP_f) // ORQ $(1<<_S_cond), SP.f
1199 }
1200
1201 func (self *Assembler) _asm_OP_cond_testc(p *ir.Instr) {
1202 self.Emit("BTRQ", jit.Imm(_S_cond), _SP_f) // BTRQ $_S_cond, SP.f
1203 self.Xjmp("JC", p.Vi())
1204 }
1205
1206 var _F_error_unsupported = jit.Func(vars.Error_unsuppoted)
1207
1208 func (self *Assembler) _asm_OP_unsupported(i *ir.Instr) {
1209 typ := int64(uintptr(unsafe.Pointer(i.GoType())))
1210 self.Emit("MOVQ", jit.Imm(typ), _AX)
1211 self.call_go(_F_error_unsupported)
1212 self.Sjmp("JMP", _LB_error)
1213 }
1214
1215 func (self *Assembler) print_gc(i int, p1 *ir.Instr, p2 *ir.Instr) {
1216 self.Emit("MOVQ", jit.Imm(int64(p2.Op())), _CX) // MOVQ $(p2.Op()), AX
1217 self.Emit("MOVQ", jit.Imm(int64(p1.Op())), _BX) // MOVQ $(p1.Op()), BX
1218 self.Emit("MOVQ", jit.Imm(int64(i)), _AX) // MOVQ $(i), CX
1219 self.call_go(_F_println)
1220 }
1221