atomic_arm.s raw
1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "textflag.h"
7 #include "funcdata.h"
8
9 // func armcas(ptr *int32, old, new int32) bool
10 // Atomically:
11 // if *ptr == old {
12 // *ptr = new
13 // return true
14 // } else {
15 // return false
16 // }
17 //
18 // To implement ·cas in sys_$GOOS_arm.s
19 // using the native instructions, use:
20 //
21 // TEXT ·cas(SB),NOSPLIT,$0
22 // B ·armcas(SB)
23 //
24 TEXT ·armcas(SB),NOSPLIT,$0-13
25 MOVW ptr+0(FP), R1
26 MOVW old+4(FP), R2
27 MOVW new+8(FP), R3
28 casl:
29 LDREX (R1), R0
30 CMP R0, R2
31 BNE casfail
32
33 #ifndef GOARM_7
34 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
35 CMP $0, R11
36 BEQ 2(PC)
37 #endif
38 DMB MB_ISHST
39
40 STREX R3, (R1), R0
41 CMP $0, R0
42 BNE casl
43 MOVW $1, R0
44
45 #ifndef GOARM_7
46 CMP $0, R11
47 BEQ 2(PC)
48 #endif
49 DMB MB_ISH
50
51 MOVB R0, ret+12(FP)
52 RET
53 casfail:
54 MOVW $0, R0
55 MOVB R0, ret+12(FP)
56 RET
57
58 // stubs
59
60 TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-8
61 B ·Load(SB)
62
63 TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-8
64 B ·Load(SB)
65
66 TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-8
67 B ·Load(SB)
68
69 TEXT ·Casint32(SB),NOSPLIT,$0-13
70 B ·Cas(SB)
71
72 TEXT ·Casint64(SB),NOSPLIT,$-4-21
73 B ·Cas64(SB)
74
75 TEXT ·Casuintptr(SB),NOSPLIT,$0-13
76 B ·Cas(SB)
77
78 TEXT ·Casp1(SB),NOSPLIT,$0-13
79 B ·Cas(SB)
80
81 TEXT ·CasRel(SB),NOSPLIT,$0-13
82 B ·Cas(SB)
83
84 TEXT ·Loadint32(SB),NOSPLIT,$0-8
85 B ·Load(SB)
86
87 TEXT ·Loadint64(SB),NOSPLIT,$-4-12
88 B ·Load64(SB)
89
90 TEXT ·Loaduintptr(SB),NOSPLIT,$0-8
91 B ·Load(SB)
92
93 TEXT ·Loaduint(SB),NOSPLIT,$0-8
94 B ·Load(SB)
95
96 TEXT ·Storeint32(SB),NOSPLIT,$0-8
97 B ·Store(SB)
98
99 TEXT ·Storeint64(SB),NOSPLIT,$0-12
100 B ·Store64(SB)
101
102 TEXT ·Storeuintptr(SB),NOSPLIT,$0-8
103 B ·Store(SB)
104
105 TEXT ·StorepNoWB(SB),NOSPLIT,$0-8
106 B ·Store(SB)
107
108 TEXT ·StoreRel(SB),NOSPLIT,$0-8
109 B ·Store(SB)
110
111 TEXT ·StoreReluintptr(SB),NOSPLIT,$0-8
112 B ·Store(SB)
113
114 TEXT ·Xaddint32(SB),NOSPLIT,$0-12
115 B ·Xadd(SB)
116
117 TEXT ·Xaddint64(SB),NOSPLIT,$-4-20
118 B ·Xadd64(SB)
119
120 TEXT ·Xadduintptr(SB),NOSPLIT,$0-12
121 B ·Xadd(SB)
122
123 TEXT ·Xchgint32(SB),NOSPLIT,$0-12
124 B ·Xchg(SB)
125
126 TEXT ·Xchgint64(SB),NOSPLIT,$-4-20
127 B ·Xchg64(SB)
128
129 // 64-bit atomics
130 // The native ARM implementations use LDREXD/STREXD, which are
131 // available on ARMv6k or later. We use them only on ARMv7.
132 // On older ARM, we use Go implementations which simulate 64-bit
133 // atomics with locks.
134 TEXT armCas64<>(SB),NOSPLIT,$0-21
135 // addr is already in R1
136 MOVW old_lo+4(FP), R2
137 MOVW old_hi+8(FP), R3
138 MOVW new_lo+12(FP), R4
139 MOVW new_hi+16(FP), R5
140 cas64loop:
141 LDREXD (R1), R6 // loads R6 and R7
142 CMP R2, R6
143 BNE cas64fail
144 CMP R3, R7
145 BNE cas64fail
146
147 DMB MB_ISHST
148
149 STREXD R4, (R1), R0 // stores R4 and R5
150 CMP $0, R0
151 BNE cas64loop
152 MOVW $1, R0
153
154 DMB MB_ISH
155
156 MOVBU R0, swapped+20(FP)
157 RET
158 cas64fail:
159 MOVW $0, R0
160 MOVBU R0, swapped+20(FP)
161 RET
162
163 TEXT armXadd64<>(SB),NOSPLIT,$0-20
164 // addr is already in R1
165 MOVW delta_lo+4(FP), R2
166 MOVW delta_hi+8(FP), R3
167
168 add64loop:
169 LDREXD (R1), R4 // loads R4 and R5
170 ADD.S R2, R4
171 ADC R3, R5
172
173 DMB MB_ISHST
174
175 STREXD R4, (R1), R0 // stores R4 and R5
176 CMP $0, R0
177 BNE add64loop
178
179 DMB MB_ISH
180
181 MOVW R4, new_lo+12(FP)
182 MOVW R5, new_hi+16(FP)
183 RET
184
185 TEXT armXchg64<>(SB),NOSPLIT,$0-20
186 // addr is already in R1
187 MOVW new_lo+4(FP), R2
188 MOVW new_hi+8(FP), R3
189
190 swap64loop:
191 LDREXD (R1), R4 // loads R4 and R5
192
193 DMB MB_ISHST
194
195 STREXD R2, (R1), R0 // stores R2 and R3
196 CMP $0, R0
197 BNE swap64loop
198
199 DMB MB_ISH
200
201 MOVW R4, old_lo+12(FP)
202 MOVW R5, old_hi+16(FP)
203 RET
204
205 TEXT armLoad64<>(SB),NOSPLIT,$0-12
206 // addr is already in R1
207
208 LDREXD (R1), R2 // loads R2 and R3
209 DMB MB_ISH
210
211 MOVW R2, val_lo+4(FP)
212 MOVW R3, val_hi+8(FP)
213 RET
214
215 TEXT armStore64<>(SB),NOSPLIT,$0-12
216 // addr is already in R1
217 MOVW val_lo+4(FP), R2
218 MOVW val_hi+8(FP), R3
219
220 store64loop:
221 LDREXD (R1), R4 // loads R4 and R5
222
223 DMB MB_ISHST
224
225 STREXD R2, (R1), R0 // stores R2 and R3
226 CMP $0, R0
227 BNE store64loop
228
229 DMB MB_ISH
230 RET
231
232 TEXT armAnd8<>(SB),NOSPLIT,$0-5
233 // addr is already in R1
234 MOVB v+4(FP), R2
235
236 and8loop:
237 LDREXB (R1), R6
238
239 DMB MB_ISHST
240
241 AND R2, R6
242 STREXB R6, (R1), R0
243 CMP $0, R0
244 BNE and8loop
245
246 DMB MB_ISH
247
248 RET
249
250 TEXT armOr8<>(SB),NOSPLIT,$0-5
251 // addr is already in R1
252 MOVB v+4(FP), R2
253
254 or8loop:
255 LDREXB (R1), R6
256
257 DMB MB_ISHST
258
259 ORR R2, R6
260 STREXB R6, (R1), R0
261 CMP $0, R0
262 BNE or8loop
263
264 DMB MB_ISH
265
266 RET
267
268 TEXT armXchg8<>(SB),NOSPLIT,$0-9
269 // addr is already in R1
270 MOVB v+4(FP), R2
271 xchg8loop:
272 LDREXB (R1), R6
273
274 DMB MB_ISHST
275
276 STREXB R2, (R1), R0
277 CMP $0, R0
278 BNE xchg8loop
279
280 DMB MB_ISH
281
282 MOVB R6, ret+8(FP)
283 RET
284
285 // The following functions all panic if their address argument isn't
286 // 8-byte aligned. Since we're calling back into Go code to do this,
287 // we have to cooperate with stack unwinding. In the normal case, the
288 // functions tail-call into the appropriate implementation, which
289 // means they must not open a frame. Hence, when they go down the
290 // panic path, at that point they push the LR to create a real frame
291 // (they don't need to pop it because panic won't return; however, we
292 // do need to set the SP delta back).
293
294 // Check if R1 is 8-byte aligned, panic if not.
295 // Clobbers R2.
296 #define CHECK_ALIGN \
297 AND.S $7, R1, R2 \
298 BEQ 4(PC) \
299 MOVW.W R14, -4(R13) /* prepare a real frame */ \
300 BL ·panicUnaligned(SB) \
301 ADD $4, R13 /* compensate SP delta */
302
303 TEXT ·Cas64(SB),NOSPLIT,$-4-21
304 NO_LOCAL_POINTERS
305 MOVW addr+0(FP), R1
306 CHECK_ALIGN
307
308 #ifndef GOARM_7
309 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
310 CMP $1, R11
311 BEQ 2(PC)
312 JMP ·goCas64(SB)
313 #endif
314 JMP armCas64<>(SB)
315
316 TEXT ·Xadd64(SB),NOSPLIT,$-4-20
317 NO_LOCAL_POINTERS
318 MOVW addr+0(FP), R1
319 CHECK_ALIGN
320
321 #ifndef GOARM_7
322 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
323 CMP $1, R11
324 BEQ 2(PC)
325 JMP ·goXadd64(SB)
326 #endif
327 JMP armXadd64<>(SB)
328
329 TEXT ·Xchg64(SB),NOSPLIT,$-4-20
330 NO_LOCAL_POINTERS
331 MOVW addr+0(FP), R1
332 CHECK_ALIGN
333
334 #ifndef GOARM_7
335 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
336 CMP $1, R11
337 BEQ 2(PC)
338 JMP ·goXchg64(SB)
339 #endif
340 JMP armXchg64<>(SB)
341
342 TEXT ·Load64(SB),NOSPLIT,$-4-12
343 NO_LOCAL_POINTERS
344 MOVW addr+0(FP), R1
345 CHECK_ALIGN
346
347 #ifndef GOARM_7
348 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
349 CMP $1, R11
350 BEQ 2(PC)
351 JMP ·goLoad64(SB)
352 #endif
353 JMP armLoad64<>(SB)
354
355 TEXT ·Store64(SB),NOSPLIT,$-4-12
356 NO_LOCAL_POINTERS
357 MOVW addr+0(FP), R1
358 CHECK_ALIGN
359
360 #ifndef GOARM_7
361 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
362 CMP $1, R11
363 BEQ 2(PC)
364 JMP ·goStore64(SB)
365 #endif
366 JMP armStore64<>(SB)
367
368 TEXT ·And8(SB),NOSPLIT,$-4-5
369 NO_LOCAL_POINTERS
370 MOVW addr+0(FP), R1
371
372 // Uses STREXB/LDREXB that is armv6k or later.
373 // For simplicity we only enable this on armv7.
374 #ifndef GOARM_7
375 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
376 CMP $1, R11
377 BEQ 2(PC)
378 JMP ·goAnd8(SB)
379 #endif
380 JMP armAnd8<>(SB)
381
382 TEXT ·Or8(SB),NOSPLIT,$-4-5
383 NO_LOCAL_POINTERS
384 MOVW addr+0(FP), R1
385
386 // Uses STREXB/LDREXB that is armv6k or later.
387 // For simplicity we only enable this on armv7.
388 #ifndef GOARM_7
389 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
390 CMP $1, R11
391 BEQ 2(PC)
392 JMP ·goOr8(SB)
393 #endif
394 JMP armOr8<>(SB)
395
396 TEXT ·Xchg8(SB),NOSPLIT,$-4-9
397 NO_LOCAL_POINTERS
398 MOVW addr+0(FP), R1
399
400 // Uses STREXB/LDREXB that is armv6k or later.
401 // For simplicity we only enable this on armv7.
402 #ifndef GOARM_7
403 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
404 CMP $1, R11
405 BEQ 2(PC)
406 JMP ·goXchg8(SB)
407 #endif
408 JMP armXchg8<>(SB)
409