atomic_riscv64.s raw
1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 // RISC-V's atomic operations have two bits, aq ("acquire") and rl ("release"),
6 // which may be toggled on and off. Their precise semantics are defined in
7 // section 6.3 of the specification, but the basic idea is as follows:
8 //
9 // - If neither aq nor rl is set, the CPU may reorder the atomic arbitrarily.
10 // It guarantees only that it will execute atomically.
11 //
12 // - If aq is set, the CPU may move the instruction backward, but not forward.
13 //
14 // - If rl is set, the CPU may move the instruction forward, but not backward.
15 //
16 // - If both are set, the CPU may not reorder the instruction at all.
17 //
18 // These four modes correspond to other well-known memory models on other CPUs.
19 // On ARM, aq corresponds to a dmb ishst, aq+rl corresponds to a dmb ish. On
20 // Intel, aq corresponds to an lfence, rl to an sfence, and aq+rl to an mfence
21 // (or a lock prefix).
22 //
23 // Go's memory model requires that
24 // - if a read happens after a write, the read must observe the write, and
25 // that
26 // - if a read happens concurrently with a write, the read may observe the
27 // write.
28 // aq is sufficient to guarantee this, so that's what we use here. (This jibes
29 // with ARM, which uses dmb ishst.)
30
31 #include "textflag.h"
32
33 // func Cas(ptr *uint64, old, new uint64) bool
34 // Atomically:
35 // if *ptr == old {
36 // *ptr = new
37 // return true
38 // } else {
39 // return false
40 // }
41 TEXT ·Cas(SB), NOSPLIT, $0-17
42 MOV ptr+0(FP), A0
43 MOVW old+8(FP), A1
44 MOVW new+12(FP), A2
45 cas_again:
46 LRW (A0), A3
47 BNE A3, A1, cas_fail
48 SCW A2, (A0), A4
49 BNE A4, ZERO, cas_again
50 MOV $1, A0
51 MOVB A0, ret+16(FP)
52 RET
53 cas_fail:
54 MOV $0, A0
55 MOV A0, ret+16(FP)
56 RET
57
58 // func Cas64(ptr *uint64, old, new uint64) bool
59 TEXT ·Cas64(SB), NOSPLIT, $0-25
60 MOV ptr+0(FP), A0
61 MOV old+8(FP), A1
62 MOV new+16(FP), A2
63 cas_again:
64 LRD (A0), A3
65 BNE A3, A1, cas_fail
66 SCD A2, (A0), A4
67 BNE A4, ZERO, cas_again
68 MOV $1, A0
69 MOVB A0, ret+24(FP)
70 RET
71 cas_fail:
72 MOVB ZERO, ret+24(FP)
73 RET
74
75 // func Load(ptr *uint32) uint32
76 TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12
77 MOV ptr+0(FP), A0
78 LRW (A0), A0
79 MOVW A0, ret+8(FP)
80 RET
81
82 // func Load8(ptr *uint8) uint8
83 TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9
84 MOV ptr+0(FP), A0
85 FENCE
86 MOVBU (A0), A1
87 FENCE
88 MOVB A1, ret+8(FP)
89 RET
90
91 // func Load64(ptr *uint64) uint64
92 TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16
93 MOV ptr+0(FP), A0
94 LRD (A0), A0
95 MOV A0, ret+8(FP)
96 RET
97
98 // func Store(ptr *uint32, val uint32)
99 TEXT ·Store(SB), NOSPLIT, $0-12
100 MOV ptr+0(FP), A0
101 MOVW val+8(FP), A1
102 AMOSWAPW A1, (A0), ZERO
103 RET
104
105 // func Store8(ptr *uint8, val uint8)
106 TEXT ·Store8(SB), NOSPLIT, $0-9
107 MOV ptr+0(FP), A0
108 MOVBU val+8(FP), A1
109 FENCE
110 MOVB A1, (A0)
111 FENCE
112 RET
113
114 // func Store64(ptr *uint64, val uint64)
115 TEXT ·Store64(SB), NOSPLIT, $0-16
116 MOV ptr+0(FP), A0
117 MOV val+8(FP), A1
118 AMOSWAPD A1, (A0), ZERO
119 RET
120
121 TEXT ·Casp1(SB), NOSPLIT, $0-25
122 JMP ·Cas64(SB)
123
124 TEXT ·Casint32(SB),NOSPLIT,$0-17
125 JMP ·Cas(SB)
126
127 TEXT ·Casint64(SB),NOSPLIT,$0-25
128 JMP ·Cas64(SB)
129
130 TEXT ·Casuintptr(SB),NOSPLIT,$0-25
131 JMP ·Cas64(SB)
132
133 TEXT ·CasRel(SB), NOSPLIT, $0-17
134 JMP ·Cas(SB)
135
136 TEXT ·Loaduintptr(SB),NOSPLIT,$0-16
137 JMP ·Load64(SB)
138
139 TEXT ·Storeint32(SB),NOSPLIT,$0-12
140 JMP ·Store(SB)
141
142 TEXT ·Storeint64(SB),NOSPLIT,$0-16
143 JMP ·Store64(SB)
144
145 TEXT ·Storeuintptr(SB),NOSPLIT,$0-16
146 JMP ·Store64(SB)
147
148 TEXT ·Loaduint(SB),NOSPLIT,$0-16
149 JMP ·Loaduintptr(SB)
150
151 TEXT ·Loadint32(SB),NOSPLIT,$0-12
152 JMP ·Load(SB)
153
154 TEXT ·Loadint64(SB),NOSPLIT,$0-16
155 JMP ·Load64(SB)
156
157 TEXT ·Xaddint32(SB),NOSPLIT,$0-20
158 JMP ·Xadd(SB)
159
160 TEXT ·Xaddint64(SB),NOSPLIT,$0-24
161 MOV ptr+0(FP), A0
162 MOV delta+8(FP), A1
163 AMOADDD A1, (A0), A0
164 ADD A0, A1, A0
165 MOVW A0, ret+16(FP)
166 RET
167
168 TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12
169 JMP ·Load(SB)
170
171 TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16
172 JMP ·Load64(SB)
173
174 TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16
175 JMP ·Load64(SB)
176
177 // func Loadp(ptr unsafe.Pointer) unsafe.Pointer
178 TEXT ·Loadp(SB),NOSPLIT,$0-16
179 JMP ·Load64(SB)
180
181 // func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer)
182 TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
183 JMP ·Store64(SB)
184
185 TEXT ·StoreRel(SB), NOSPLIT, $0-12
186 JMP ·Store(SB)
187
188 TEXT ·StoreRel64(SB), NOSPLIT, $0-16
189 JMP ·Store64(SB)
190
191 TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
192 JMP ·Store64(SB)
193
194 // func Xchg(ptr *uint32, new uint32) uint32
195 TEXT ·Xchg(SB), NOSPLIT, $0-20
196 MOV ptr+0(FP), A0
197 MOVW new+8(FP), A1
198 AMOSWAPW A1, (A0), A1
199 MOVW A1, ret+16(FP)
200 RET
201
202 // func Xchg8(ptr *uint8, new uint8) uint8
203 TEXT ·Xchg8(SB), NOSPLIT, $0-17
204 MOV ptr+0(FP), A0
205 MOVBU new+8(FP), A1
206 AND $3, A0, A2
207 SLL $3, A2
208 MOV $255, A4
209 SLL A2, A4
210 NOT A4
211 AND $~3, A0
212 SLL A2, A1
213 xchg8_again:
214 LRW (A0), A5
215 AND A4, A5, A3
216 OR A1, A3
217 SCW A3, (A0), A6
218 BNEZ A6, xchg8_again
219 SRL A2, A5
220 MOVB A5, ret+16(FP)
221 RET
222
223 // func Xchg64(ptr *uint64, new uint64) uint64
224 TEXT ·Xchg64(SB), NOSPLIT, $0-24
225 MOV ptr+0(FP), A0
226 MOV new+8(FP), A1
227 AMOSWAPD A1, (A0), A1
228 MOV A1, ret+16(FP)
229 RET
230
231 // Atomically:
232 // *val += delta;
233 // return *val;
234
235 // func Xadd(ptr *uint32, delta int32) uint32
236 TEXT ·Xadd(SB), NOSPLIT, $0-20
237 MOV ptr+0(FP), A0
238 MOVW delta+8(FP), A1
239 AMOADDW A1, (A0), A2
240 ADD A2,A1,A0
241 MOVW A0, ret+16(FP)
242 RET
243
244 // func Xadd64(ptr *uint64, delta int64) uint64
245 TEXT ·Xadd64(SB), NOSPLIT, $0-24
246 MOV ptr+0(FP), A0
247 MOV delta+8(FP), A1
248 AMOADDD A1, (A0), A2
249 ADD A2, A1, A0
250 MOV A0, ret+16(FP)
251 RET
252
253 // func Xadduintptr(ptr *uintptr, delta uintptr) uintptr
254 TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
255 JMP ·Xadd64(SB)
256
257 // func Xchgint32(ptr *int32, new int32) int32
258 TEXT ·Xchgint32(SB), NOSPLIT, $0-20
259 JMP ·Xchg(SB)
260
261 // func Xchgint64(ptr *int64, new int64) int64
262 TEXT ·Xchgint64(SB), NOSPLIT, $0-24
263 JMP ·Xchg64(SB)
264
265 // func Xchguintptr(ptr *uintptr, new uintptr) uintptr
266 TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
267 JMP ·Xchg64(SB)
268
269 // func And8(ptr *uint8, val uint8)
270 TEXT ·And8(SB), NOSPLIT, $0-9
271 MOV ptr+0(FP), A0
272 MOVBU val+8(FP), A1
273 AND $3, A0, A2
274 AND $-4, A0
275 SLL $3, A2
276 XOR $255, A1
277 SLL A2, A1
278 XOR $-1, A1
279 AMOANDW A1, (A0), ZERO
280 RET
281
282 // func Or8(ptr *uint8, val uint8)
283 TEXT ·Or8(SB), NOSPLIT, $0-9
284 MOV ptr+0(FP), A0
285 MOVBU val+8(FP), A1
286 AND $3, A0, A2
287 AND $-4, A0
288 SLL $3, A2
289 SLL A2, A1
290 AMOORW A1, (A0), ZERO
291 RET
292
293 // func And(ptr *uint32, val uint32)
294 TEXT ·And(SB), NOSPLIT, $0-12
295 MOV ptr+0(FP), A0
296 MOVW val+8(FP), A1
297 AMOANDW A1, (A0), ZERO
298 RET
299
300 // func Or(ptr *uint32, val uint32)
301 TEXT ·Or(SB), NOSPLIT, $0-12
302 MOV ptr+0(FP), A0
303 MOVW val+8(FP), A1
304 AMOORW A1, (A0), ZERO
305 RET
306
307 // func Or32(ptr *uint32, val uint32) uint32
308 TEXT ·Or32(SB), NOSPLIT, $0-20
309 MOV ptr+0(FP), A0
310 MOVW val+8(FP), A1
311 AMOORW A1, (A0), A2
312 MOVW A2, ret+16(FP)
313 RET
314
315 // func And32(ptr *uint32, val uint32) uint32
316 TEXT ·And32(SB), NOSPLIT, $0-20
317 MOV ptr+0(FP), A0
318 MOVW val+8(FP), A1
319 AMOANDW A1, (A0), A2
320 MOVW A2, ret+16(FP)
321 RET
322
323 // func Or64(ptr *uint64, val uint64) uint64
324 TEXT ·Or64(SB), NOSPLIT, $0-24
325 MOV ptr+0(FP), A0
326 MOV val+8(FP), A1
327 AMOORD A1, (A0), A2
328 MOV A2, ret+16(FP)
329 RET
330
331 // func And64(ptr *uint64, val uint64) uint64
332 TEXT ·And64(SB), NOSPLIT, $0-24
333 MOV ptr+0(FP), A0
334 MOV val+8(FP), A1
335 AMOANDD A1, (A0), A2
336 MOV A2, ret+16(FP)
337 RET
338
339 // func Anduintptr(ptr *uintptr, val uintptr) uintptr
340 TEXT ·Anduintptr(SB), NOSPLIT, $0-24
341 JMP ·And64(SB)
342
343 // func Oruintptr(ptr *uintptr, val uintptr) uintptr
344 TEXT ·Oruintptr(SB), NOSPLIT, $0-24
345 JMP ·Or64(SB)
346