atomic_arm64.s raw
1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "textflag.h"
7
8 TEXT ·Casint32(SB), NOSPLIT, $0-17
9 B ·Cas(SB)
10
11 TEXT ·Casint64(SB), NOSPLIT, $0-25
12 B ·Cas64(SB)
13
14 TEXT ·Casuintptr(SB), NOSPLIT, $0-25
15 B ·Cas64(SB)
16
17 TEXT ·CasRel(SB), NOSPLIT, $0-17
18 B ·Cas(SB)
19
20 TEXT ·Loadint32(SB), NOSPLIT, $0-12
21 B ·Load(SB)
22
23 TEXT ·Loadint64(SB), NOSPLIT, $0-16
24 B ·Load64(SB)
25
26 TEXT ·Loaduintptr(SB), NOSPLIT, $0-16
27 B ·Load64(SB)
28
29 TEXT ·Loaduint(SB), NOSPLIT, $0-16
30 B ·Load64(SB)
31
32 TEXT ·Storeint32(SB), NOSPLIT, $0-12
33 B ·Store(SB)
34
35 TEXT ·Storeint64(SB), NOSPLIT, $0-16
36 B ·Store64(SB)
37
38 TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
39 B ·Store64(SB)
40
41 TEXT ·Xaddint32(SB), NOSPLIT, $0-20
42 B ·Xadd(SB)
43
44 TEXT ·Xaddint64(SB), NOSPLIT, $0-24
45 B ·Xadd64(SB)
46
47 TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
48 B ·Xadd64(SB)
49
50 TEXT ·Casp1(SB), NOSPLIT, $0-25
51 B ·Cas64(SB)
52
53 // uint32 ·Load(uint32 volatile* addr)
54 TEXT ·Load(SB),NOSPLIT,$0-12
55 MOVD ptr+0(FP), R0
56 LDARW (R0), R0
57 MOVW R0, ret+8(FP)
58 RET
59
60 // uint8 ·Load8(uint8 volatile* addr)
61 TEXT ·Load8(SB),NOSPLIT,$0-9
62 MOVD ptr+0(FP), R0
63 LDARB (R0), R0
64 MOVB R0, ret+8(FP)
65 RET
66
67 // uint64 ·Load64(uint64 volatile* addr)
68 TEXT ·Load64(SB),NOSPLIT,$0-16
69 MOVD ptr+0(FP), R0
70 LDAR (R0), R0
71 MOVD R0, ret+8(FP)
72 RET
73
74 // void *·Loadp(void *volatile *addr)
75 TEXT ·Loadp(SB),NOSPLIT,$0-16
76 MOVD ptr+0(FP), R0
77 LDAR (R0), R0
78 MOVD R0, ret+8(FP)
79 RET
80
81 // uint32 ·LoadAcq(uint32 volatile* addr)
82 TEXT ·LoadAcq(SB),NOSPLIT,$0-12
83 B ·Load(SB)
84
85 // uint64 ·LoadAcquintptr(uint64 volatile* addr)
86 TEXT ·LoadAcq64(SB),NOSPLIT,$0-16
87 B ·Load64(SB)
88
89 // uintptr ·LoadAcq64(uintptr volatile* addr)
90 TEXT ·LoadAcquintptr(SB),NOSPLIT,$0-16
91 B ·Load64(SB)
92
93 TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
94 B ·Store64(SB)
95
96 TEXT ·StoreRel(SB), NOSPLIT, $0-12
97 B ·Store(SB)
98
99 TEXT ·StoreRel64(SB), NOSPLIT, $0-16
100 B ·Store64(SB)
101
102 TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
103 B ·Store64(SB)
104
105 TEXT ·Store(SB), NOSPLIT, $0-12
106 MOVD ptr+0(FP), R0
107 MOVW val+8(FP), R1
108 STLRW R1, (R0)
109 RET
110
111 TEXT ·Store8(SB), NOSPLIT, $0-9
112 MOVD ptr+0(FP), R0
113 MOVB val+8(FP), R1
114 STLRB R1, (R0)
115 RET
116
117 TEXT ·Store64(SB), NOSPLIT, $0-16
118 MOVD ptr+0(FP), R0
119 MOVD val+8(FP), R1
120 STLR R1, (R0)
121 RET
122
123 // uint8 Xchg(ptr *uint8, new uint8)
124 // Atomically:
125 // old := *ptr;
126 // *ptr = new;
127 // return old;
128 TEXT ·Xchg8(SB), NOSPLIT, $0-17
129 MOVD ptr+0(FP), R0
130 MOVB new+8(FP), R1
131 #ifndef GOARM64_LSE
132 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
133 CBZ R4, load_store_loop
134 #endif
135 SWPALB R1, (R0), R2
136 MOVB R2, ret+16(FP)
137 RET
138 #ifndef GOARM64_LSE
139 load_store_loop:
140 LDAXRB (R0), R2
141 STLXRB R1, (R0), R3
142 CBNZ R3, load_store_loop
143 MOVB R2, ret+16(FP)
144 RET
145 #endif
146
147 // uint32 Xchg(ptr *uint32, new uint32)
148 // Atomically:
149 // old := *ptr;
150 // *ptr = new;
151 // return old;
152 TEXT ·Xchg(SB), NOSPLIT, $0-20
153 MOVD ptr+0(FP), R0
154 MOVW new+8(FP), R1
155 #ifndef GOARM64_LSE
156 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
157 CBZ R4, load_store_loop
158 #endif
159 SWPALW R1, (R0), R2
160 MOVW R2, ret+16(FP)
161 RET
162 #ifndef GOARM64_LSE
163 load_store_loop:
164 LDAXRW (R0), R2
165 STLXRW R1, (R0), R3
166 CBNZ R3, load_store_loop
167 MOVW R2, ret+16(FP)
168 RET
169 #endif
170
171 // uint64 Xchg64(ptr *uint64, new uint64)
172 // Atomically:
173 // old := *ptr;
174 // *ptr = new;
175 // return old;
176 TEXT ·Xchg64(SB), NOSPLIT, $0-24
177 MOVD ptr+0(FP), R0
178 MOVD new+8(FP), R1
179 #ifndef GOARM64_LSE
180 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
181 CBZ R4, load_store_loop
182 #endif
183 SWPALD R1, (R0), R2
184 MOVD R2, ret+16(FP)
185 RET
186 #ifndef GOARM64_LSE
187 load_store_loop:
188 LDAXR (R0), R2
189 STLXR R1, (R0), R3
190 CBNZ R3, load_store_loop
191 MOVD R2, ret+16(FP)
192 RET
193 #endif
194
195 // func Cas(ptr *uint32, old, new uint32) bool
196 // Atomically:
197 // if *ptr == old {
198 // *ptr = new
199 // return true
200 // } else {
201 // return false
202 // }
203 TEXT ·Cas(SB), NOSPLIT, $0-17
204 MOVD ptr+0(FP), R0
205 MOVW old+8(FP), R1
206 MOVW new+12(FP), R2
207 #ifndef GOARM64_LSE
208 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
209 CBZ R4, load_store_loop
210 #endif
211 MOVD R1, R3
212 CASALW R3, (R0), R2
213 CMP R1, R3
214 CSET EQ, R0
215 MOVB R0, ret+16(FP)
216 RET
217 #ifndef GOARM64_LSE
218 load_store_loop:
219 LDAXRW (R0), R3
220 CMPW R1, R3
221 BNE ok
222 STLXRW R2, (R0), R3
223 CBNZ R3, load_store_loop
224 ok:
225 CSET EQ, R0
226 MOVB R0, ret+16(FP)
227 RET
228 #endif
229
230 // func Cas64(ptr *uint64, old, new uint64) bool
231 // Atomically:
232 // if *ptr == old {
233 // *ptr = new
234 // return true
235 // } else {
236 // return false
237 // }
238 TEXT ·Cas64(SB), NOSPLIT, $0-25
239 MOVD ptr+0(FP), R0
240 MOVD old+8(FP), R1
241 MOVD new+16(FP), R2
242 #ifndef GOARM64_LSE
243 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
244 CBZ R4, load_store_loop
245 #endif
246 MOVD R1, R3
247 CASALD R3, (R0), R2
248 CMP R1, R3
249 CSET EQ, R0
250 MOVB R0, ret+24(FP)
251 RET
252 #ifndef GOARM64_LSE
253 load_store_loop:
254 LDAXR (R0), R3
255 CMP R1, R3
256 BNE ok
257 STLXR R2, (R0), R3
258 CBNZ R3, load_store_loop
259 ok:
260 CSET EQ, R0
261 MOVB R0, ret+24(FP)
262 RET
263 #endif
264
265 // uint32 xadd(uint32 volatile *ptr, int32 delta)
266 // Atomically:
267 // *val += delta;
268 // return *val;
269 TEXT ·Xadd(SB), NOSPLIT, $0-20
270 MOVD ptr+0(FP), R0
271 MOVW delta+8(FP), R1
272 #ifndef GOARM64_LSE
273 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
274 CBZ R4, load_store_loop
275 #endif
276 LDADDALW R1, (R0), R2
277 ADD R1, R2
278 MOVW R2, ret+16(FP)
279 RET
280 #ifndef GOARM64_LSE
281 load_store_loop:
282 LDAXRW (R0), R2
283 ADDW R2, R1, R2
284 STLXRW R2, (R0), R3
285 CBNZ R3, load_store_loop
286 MOVW R2, ret+16(FP)
287 RET
288 #endif
289
290 // uint64 Xadd64(uint64 volatile *ptr, int64 delta)
291 // Atomically:
292 // *val += delta;
293 // return *val;
294 TEXT ·Xadd64(SB), NOSPLIT, $0-24
295 MOVD ptr+0(FP), R0
296 MOVD delta+8(FP), R1
297 #ifndef GOARM64_LSE
298 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
299 CBZ R4, load_store_loop
300 #endif
301 LDADDALD R1, (R0), R2
302 ADD R1, R2
303 MOVD R2, ret+16(FP)
304 RET
305 #ifndef GOARM64_LSE
306 load_store_loop:
307 LDAXR (R0), R2
308 ADD R2, R1, R2
309 STLXR R2, (R0), R3
310 CBNZ R3, load_store_loop
311 MOVD R2, ret+16(FP)
312 RET
313 #endif
314
315 TEXT ·Xchgint32(SB), NOSPLIT, $0-20
316 B ·Xchg(SB)
317
318 TEXT ·Xchgint64(SB), NOSPLIT, $0-24
319 B ·Xchg64(SB)
320
321 TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
322 B ·Xchg64(SB)
323
324 TEXT ·And8(SB), NOSPLIT, $0-9
325 MOVD ptr+0(FP), R0
326 MOVB val+8(FP), R1
327 #ifndef GOARM64_LSE
328 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
329 CBZ R4, load_store_loop
330 #endif
331 MVN R1, R2
332 LDCLRALB R2, (R0), R3
333 RET
334 #ifndef GOARM64_LSE
335 load_store_loop:
336 LDAXRB (R0), R2
337 AND R1, R2
338 STLXRB R2, (R0), R3
339 CBNZ R3, load_store_loop
340 RET
341 #endif
342
343 TEXT ·Or8(SB), NOSPLIT, $0-9
344 MOVD ptr+0(FP), R0
345 MOVB val+8(FP), R1
346 #ifndef GOARM64_LSE
347 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
348 CBZ R4, load_store_loop
349 #endif
350 LDORALB R1, (R0), R2
351 RET
352 #ifndef GOARM64_LSE
353 load_store_loop:
354 LDAXRB (R0), R2
355 ORR R1, R2
356 STLXRB R2, (R0), R3
357 CBNZ R3, load_store_loop
358 RET
359 #endif
360
361 // func And(addr *uint32, v uint32)
362 TEXT ·And(SB), NOSPLIT, $0-12
363 MOVD ptr+0(FP), R0
364 MOVW val+8(FP), R1
365 #ifndef GOARM64_LSE
366 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
367 CBZ R4, load_store_loop
368 #endif
369 MVN R1, R2
370 LDCLRALW R2, (R0), R3
371 RET
372 #ifndef GOARM64_LSE
373 load_store_loop:
374 LDAXRW (R0), R2
375 AND R1, R2
376 STLXRW R2, (R0), R3
377 CBNZ R3, load_store_loop
378 RET
379 #endif
380
381 // func Or(addr *uint32, v uint32)
382 TEXT ·Or(SB), NOSPLIT, $0-12
383 MOVD ptr+0(FP), R0
384 MOVW val+8(FP), R1
385 #ifndef GOARM64_LSE
386 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
387 CBZ R4, load_store_loop
388 #endif
389 LDORALW R1, (R0), R2
390 RET
391 #ifndef GOARM64_LSE
392 load_store_loop:
393 LDAXRW (R0), R2
394 ORR R1, R2
395 STLXRW R2, (R0), R3
396 CBNZ R3, load_store_loop
397 RET
398 #endif
399
400 // func Or32(addr *uint32, v uint32) old uint32
401 TEXT ·Or32(SB), NOSPLIT, $0-20
402 MOVD ptr+0(FP), R0
403 MOVW val+8(FP), R1
404 #ifndef GOARM64_LSE
405 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
406 CBZ R4, load_store_loop
407 #endif
408 LDORALW R1, (R0), R2
409 MOVD R2, ret+16(FP)
410 RET
411 #ifndef GOARM64_LSE
412 load_store_loop:
413 LDAXRW (R0), R2
414 ORR R1, R2, R3
415 STLXRW R3, (R0), R4
416 CBNZ R4, load_store_loop
417 MOVD R2, ret+16(FP)
418 RET
419 #endif
420
421 // func And32(addr *uint32, v uint32) old uint32
422 TEXT ·And32(SB), NOSPLIT, $0-20
423 MOVD ptr+0(FP), R0
424 MOVW val+8(FP), R1
425 #ifndef GOARM64_LSE
426 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
427 CBZ R4, load_store_loop
428 #endif
429 MVN R1, R2
430 LDCLRALW R2, (R0), R3
431 MOVD R3, ret+16(FP)
432 RET
433 #ifndef GOARM64_LSE
434 load_store_loop:
435 LDAXRW (R0), R2
436 AND R1, R2, R3
437 STLXRW R3, (R0), R4
438 CBNZ R4, load_store_loop
439 MOVD R2, ret+16(FP)
440 RET
441 #endif
442
443 // func Or64(addr *uint64, v uint64) old uint64
444 TEXT ·Or64(SB), NOSPLIT, $0-24
445 MOVD ptr+0(FP), R0
446 MOVD val+8(FP), R1
447 #ifndef GOARM64_LSE
448 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
449 CBZ R4, load_store_loop
450 #endif
451 LDORALD R1, (R0), R2
452 MOVD R2, ret+16(FP)
453 RET
454 #ifndef GOARM64_LSE
455 load_store_loop:
456 LDAXR (R0), R2
457 ORR R1, R2, R3
458 STLXR R3, (R0), R4
459 CBNZ R4, load_store_loop
460 MOVD R2, ret+16(FP)
461 RET
462 #endif
463
464 // func And64(addr *uint64, v uint64) old uint64
465 TEXT ·And64(SB), NOSPLIT, $0-24
466 MOVD ptr+0(FP), R0
467 MOVD val+8(FP), R1
468 #ifndef GOARM64_LSE
469 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
470 CBZ R4, load_store_loop
471 #endif
472 MVN R1, R2
473 LDCLRALD R2, (R0), R3
474 MOVD R3, ret+16(FP)
475 RET
476 #ifndef GOARM64_LSE
477 load_store_loop:
478 LDAXR (R0), R2
479 AND R1, R2, R3
480 STLXR R3, (R0), R4
481 CBNZ R4, load_store_loop
482 MOVD R2, ret+16(FP)
483 RET
484 #endif
485
486 // func Anduintptr(addr *uintptr, v uintptr) old uintptr
487 TEXT ·Anduintptr(SB), NOSPLIT, $0-24
488 B ·And64(SB)
489
490 // func Oruintptr(addr *uintptr, v uintptr) old uintptr
491 TEXT ·Oruintptr(SB), NOSPLIT, $0-24
492 B ·Or64(SB)
493