atomic_loong64.s raw
1 // Copyright 2022 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "textflag.h"
7
8 // func Cas(ptr *int32, old, new int32) bool
9 // Atomically:
10 // if *ptr == old {
11 // *ptr = new
12 // return true
13 // } else {
14 // return false
15 // }
16 TEXT ·Cas(SB), NOSPLIT, $0-17
17 MOVV ptr+0(FP), R4
18 MOVW old+8(FP), R5
19 MOVW new+12(FP), R6
20
21 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8
22 BEQ R8, cas_again
23 MOVV R5, R7 // backup old value
24 AMCASDBW R6, (R4), R5
25 BNE R7, R5, cas_fail0
26 MOVV $1, R4
27 MOVB R4, ret+16(FP)
28 RET
29 cas_fail0:
30 MOVB R0, ret+16(FP)
31 RET
32
33 // Implemented using the ll-sc instruction pair
34 DBAR $0x14 // LoadAcquire barrier
35 cas_again:
36 MOVV R6, R7
37 LL (R4), R8
38 BNE R5, R8, cas_fail1
39 SC R7, (R4)
40 BEQ R7, cas_again
41 MOVV $1, R4
42 MOVB R4, ret+16(FP)
43 DBAR $0x12 // StoreRelease barrier
44 RET
45 cas_fail1:
46 MOVV $0, R4
47 JMP -4(PC)
48
49 // func Cas64(ptr *uint64, old, new uint64) bool
50 // Atomically:
51 // if *ptr == old {
52 // *ptr = new
53 // return true
54 // } else {
55 // return false
56 // }
57 TEXT ·Cas64(SB), NOSPLIT, $0-25
58 MOVV ptr+0(FP), R4
59 MOVV old+8(FP), R5
60 MOVV new+16(FP), R6
61
62 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8
63 BEQ R8, cas64_again
64 MOVV R5, R7 // backup old value
65 AMCASDBV R6, (R4), R5
66 BNE R7, R5, cas64_fail0
67 MOVV $1, R4
68 MOVB R4, ret+24(FP)
69 RET
70 cas64_fail0:
71 MOVB R0, ret+24(FP)
72 RET
73
74 // Implemented using the ll-sc instruction pair
75 DBAR $0x14
76 cas64_again:
77 MOVV R6, R7
78 LLV (R4), R8
79 BNE R5, R8, cas64_fail1
80 SCV R7, (R4)
81 BEQ R7, cas64_again
82 MOVV $1, R4
83 MOVB R4, ret+24(FP)
84 DBAR $0x12
85 RET
86 cas64_fail1:
87 MOVV $0, R4
88 JMP -4(PC)
89
90 TEXT ·Casint32(SB),NOSPLIT,$0-17
91 JMP ·Cas(SB)
92
93 TEXT ·Casint64(SB),NOSPLIT,$0-25
94 JMP ·Cas64(SB)
95
96 TEXT ·Casuintptr(SB), NOSPLIT, $0-25
97 JMP ·Cas64(SB)
98
99 TEXT ·CasRel(SB), NOSPLIT, $0-17
100 JMP ·Cas(SB)
101
102 TEXT ·Loaduintptr(SB), NOSPLIT|NOFRAME, $0-16
103 JMP ·Load64(SB)
104
105 TEXT ·Loaduint(SB), NOSPLIT|NOFRAME, $0-16
106 JMP ·Load64(SB)
107
108 TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
109 JMP ·Store64(SB)
110
111 TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
112 JMP ·Xadd64(SB)
113
114 TEXT ·Loadint64(SB), NOSPLIT, $0-16
115 JMP ·Load64(SB)
116
117 TEXT ·Xaddint32(SB),NOSPLIT,$0-20
118 JMP ·Xadd(SB)
119
120 TEXT ·Xaddint64(SB), NOSPLIT, $0-24
121 JMP ·Xadd64(SB)
122
123 // func Casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool
124 // Atomically:
125 // if *ptr == old {
126 // *ptr = new
127 // return true
128 // } else {
129 // return false
130 // }
131 TEXT ·Casp1(SB), NOSPLIT, $0-25
132 JMP ·Cas64(SB)
133
134 // uint32 Xadd(uint32 volatile *ptr, int32 delta)
135 // Atomically:
136 // *val += delta;
137 // return *val;
138 TEXT ·Xadd(SB), NOSPLIT, $0-20
139 MOVV ptr+0(FP), R4
140 MOVW delta+8(FP), R5
141 AMADDDBW R5, (R4), R6
142 ADDV R6, R5, R4
143 MOVW R4, ret+16(FP)
144 RET
145
146 // func Xadd64(ptr *uint64, delta int64) uint64
147 TEXT ·Xadd64(SB), NOSPLIT, $0-24
148 MOVV ptr+0(FP), R4
149 MOVV delta+8(FP), R5
150 AMADDDBV R5, (R4), R6
151 ADDV R6, R5, R4
152 MOVV R4, ret+16(FP)
153 RET
154
155 // uint8 Xchg8(ptr *uint8, new uint8)
156 // Atomically:
157 // old := *ptr;
158 // *ptr = new;
159 // return old;
160 TEXT ·Xchg8(SB), NOSPLIT, $0-17
161 MOVV ptr+0(FP), R4
162 MOVBU new+8(FP), R5
163
164 // R6 = ((ptr & 3) * 8)
165 AND $3, R4, R6
166 SLLV $3, R6
167
168 // R7 = ((0xFF) << R6) ^ (-1)
169 MOVV $0xFF, R8
170 SLLV R6, R8, R7
171 XOR $-1, R7
172
173 // R4 = ptr & (~3)
174 MOVV $~3, R8
175 AND R8, R4
176
177 // R5 = ((val) << R6)
178 SLLV R6, R5
179
180 DBAR $0x14 // LoadAcquire barrier
181 _xchg8_again:
182 LL (R4), R8
183 MOVV R8, R9 // backup old val
184 AND R7, R8
185 OR R5, R8
186 SC R8, (R4)
187 BEQ R8, _xchg8_again
188 DBAR $0x12 // StoreRelease barrier
189 SRLV R6, R9, R9
190 MOVBU R9, ret+16(FP)
191 RET
192
193 // func Xchg(ptr *uint32, new uint32) uint32
194 TEXT ·Xchg(SB), NOSPLIT, $0-20
195 MOVV ptr+0(FP), R4
196 MOVW new+8(FP), R5
197 AMSWAPDBW R5, (R4), R6
198 MOVW R6, ret+16(FP)
199 RET
200
201 // func Xchg64(ptr *uint64, new uint64) uint64
202 TEXT ·Xchg64(SB), NOSPLIT, $0-24
203 MOVV ptr+0(FP), R4
204 MOVV new+8(FP), R5
205 AMSWAPDBV R5, (R4), R6
206 MOVV R6, ret+16(FP)
207 RET
208
209 TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
210 JMP ·Xchg64(SB)
211
212 // func Xchgint32(ptr *int32, new int32) int32
213 TEXT ·Xchgint32(SB), NOSPLIT, $0-20
214 JMP ·Xchg(SB)
215
216 // func Xchgint64(ptr *int64, new int64) int64
217 TEXT ·Xchgint64(SB), NOSPLIT, $0-24
218 JMP ·Xchg64(SB)
219
220 TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
221 JMP ·Store64(SB)
222
223 TEXT ·StoreRel(SB), NOSPLIT, $0-12
224 JMP ·Store(SB)
225
226 TEXT ·StoreRel64(SB), NOSPLIT, $0-16
227 JMP ·Store64(SB)
228
229 TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
230 JMP ·Store64(SB)
231
232 TEXT ·Store(SB), NOSPLIT, $0-12
233 MOVV ptr+0(FP), R4
234 MOVW val+8(FP), R5
235 AMSWAPDBW R5, (R4), R0
236 RET
237
238 TEXT ·Store8(SB), NOSPLIT, $0-9
239 MOVV ptr+0(FP), R4
240 MOVB val+8(FP), R5
241 MOVBU internal∕cpu·Loong64+const_offsetLoong64HasLAM_BH(SB), R6
242 BEQ R6, _legacy_store8_
243 AMSWAPDBB R5, (R4), R0
244 RET
245 _legacy_store8_:
246 // StoreRelease barrier
247 DBAR $0x12
248 MOVB R5, 0(R4)
249 DBAR $0x18
250 RET
251
252 TEXT ·Store64(SB), NOSPLIT, $0-16
253 MOVV ptr+0(FP), R4
254 MOVV val+8(FP), R5
255 AMSWAPDBV R5, (R4), R0
256 RET
257
258 // void Or8(byte volatile*, byte);
259 TEXT ·Or8(SB), NOSPLIT, $0-9
260 MOVV ptr+0(FP), R4
261 MOVBU val+8(FP), R5
262 // R6 = ptr & (~3)
263 MOVV $~3, R6
264 AND R4, R6
265 // R7 = ((ptr & 3) * 8)
266 AND $3, R4, R7
267 SLLV $3, R7
268 // R5 = val << R7
269 SLLV R7, R5
270 AMORDBW R5, (R6), R0
271 RET
272
273 // void And8(byte volatile*, byte);
274 TEXT ·And8(SB), NOSPLIT, $0-9
275 MOVV ptr+0(FP), R4
276 MOVBU val+8(FP), R5
277 // R6 = ptr & (~3)
278 MOVV $~3, R6
279 AND R4, R6
280 // R7 = ((ptr & 3) * 8)
281 AND $3, R4, R7
282 SLLV $3, R7
283 // R5 = ((val ^ 0xFF) << R7) ^ (-1)
284 XOR $255, R5
285 SLLV R7, R5
286 XOR $-1, R5
287 AMANDDBW R5, (R6), R0
288 RET
289
290 // func Or(addr *uint32, v uint32)
291 TEXT ·Or(SB), NOSPLIT, $0-12
292 MOVV ptr+0(FP), R4
293 MOVW val+8(FP), R5
294 AMORDBW R5, (R4), R0
295 RET
296
297 // func And(addr *uint32, v uint32)
298 TEXT ·And(SB), NOSPLIT, $0-12
299 MOVV ptr+0(FP), R4
300 MOVW val+8(FP), R5
301 AMANDDBW R5, (R4), R0
302 RET
303
304 // func Or32(addr *uint32, v uint32) old uint32
305 TEXT ·Or32(SB), NOSPLIT, $0-20
306 MOVV ptr+0(FP), R4
307 MOVW val+8(FP), R5
308 AMORDBW R5, (R4), R6
309 MOVW R6, ret+16(FP)
310 RET
311
312 // func And32(addr *uint32, v uint32) old uint32
313 TEXT ·And32(SB), NOSPLIT, $0-20
314 MOVV ptr+0(FP), R4
315 MOVW val+8(FP), R5
316 AMANDDBW R5, (R4), R6
317 MOVW R6, ret+16(FP)
318 RET
319
320 // func Or64(addr *uint64, v uint64) old uint64
321 TEXT ·Or64(SB), NOSPLIT, $0-24
322 MOVV ptr+0(FP), R4
323 MOVV val+8(FP), R5
324 AMORDBV R5, (R4), R6
325 MOVV R6, ret+16(FP)
326 RET
327
328 // func And64(addr *uint64, v uint64) old uint64
329 TEXT ·And64(SB), NOSPLIT, $0-24
330 MOVV ptr+0(FP), R4
331 MOVV val+8(FP), R5
332 AMANDDBV R5, (R4), R6
333 MOVV R6, ret+16(FP)
334 RET
335
336 // func Anduintptr(addr *uintptr, v uintptr) old uintptr
337 TEXT ·Anduintptr(SB), NOSPLIT, $0-24
338 JMP ·And64(SB)
339
340 // func Oruintptr(addr *uintptr, v uintptr) old uintptr
341 TEXT ·Oruintptr(SB), NOSPLIT, $0-24
342 JMP ·Or64(SB)
343
344 // uint32 internal∕runtime∕atomic·Load(uint32 volatile* ptr)
345 TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12
346 MOVV ptr+0(FP), R19
347 MOVWU 0(R19), R19
348 DBAR $0x14 // LoadAcquire barrier
349 MOVW R19, ret+8(FP)
350 RET
351
352 // uint8 internal∕runtime∕atomic·Load8(uint8 volatile* ptr)
353 TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9
354 MOVV ptr+0(FP), R19
355 MOVBU 0(R19), R19
356 DBAR $0x14
357 MOVB R19, ret+8(FP)
358 RET
359
360 // uint64 internal∕runtime∕atomic·Load64(uint64 volatile* ptr)
361 TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16
362 MOVV ptr+0(FP), R19
363 MOVV 0(R19), R19
364 DBAR $0x14
365 MOVV R19, ret+8(FP)
366 RET
367
368 // void *internal∕runtime∕atomic·Loadp(void *volatile *ptr)
369 TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-16
370 JMP ·Load64(SB)
371
372 // uint32 internal∕runtime∕atomic·LoadAcq(uint32 volatile* ptr)
373 TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12
374 JMP ·Load(SB)
375
376 // uint64 ·LoadAcq64(uint64 volatile* ptr)
377 TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16
378 JMP ·Load64(SB)
379
380 // uintptr ·LoadAcquintptr(uintptr volatile* ptr)
381 TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16
382 JMP ·Load64(SB)
383
384