gc_atomic_ops.h raw
1 /*
2 * Copyright (c) 2017 Ivan Maidanski
3 *
4 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
5 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
6 *
7 * Permission is hereby granted to use or copy this program
8 * for any purpose, provided the above notices are retained on all copies.
9 * Permission to modify the code and to distribute modified code is granted,
10 * provided the above notices are retained, and a notice that the code was
11 * modified is included with the above copyright notice.
12 */
13
14 /*
15 * This is a private collector header which provides an implementation of
16 * `libatomic_ops` subset primitives sufficient for the collector assuming
17 * that gcc atomic intrinsics are available (and have the correct
18 * implementation). This is enabled by defining `GC_BUILTIN_ATOMIC` macro.
19 * Otherwise, `libatomic_ops` library is used to define the primitives.
20 */
21
22 #ifndef GC_ATOMIC_OPS_H
23 #define GC_ATOMIC_OPS_H
24
25 #ifdef GC_BUILTIN_ATOMIC
26
27 # include "gc/gc.h" /*< for `size_t` */
28
29 # ifdef __cplusplus
30 extern "C" {
31 # endif
32
33 typedef size_t AO_t;
34
35 # ifdef GC_PRIVATE_H /*< i.e. have `GC_INLINE` */
36 # define AO_INLINE GC_INLINE
37 # else
38 # define AO_INLINE static __inline
39 # endif
40
41 # if !defined(THREAD_SANITIZER) && !defined(GC_PRIVATE_H)
42 /* Similar to that in `gcconfig.h` file. */
43 # if defined(__has_feature)
44 # if __has_feature(thread_sanitizer)
45 # define THREAD_SANITIZER
46 # endif
47 # elif defined(__SANITIZE_THREAD__)
48 # define THREAD_SANITIZER
49 # endif
50 # endif /* !THREAD_SANITIZER && !GC_PRIVATE_H */
51
52 typedef unsigned char AO_TS_t;
53 # define AO_TS_CLEAR 0
54 # define AO_TS_INITIALIZER ((AO_TS_t)AO_TS_CLEAR)
55 # if defined(__GCC_ATOMIC_TEST_AND_SET_TRUEVAL) && !defined(CPPCHECK)
56 # define AO_TS_SET __GCC_ATOMIC_TEST_AND_SET_TRUEVAL
57 # else
58 # define AO_TS_SET (AO_TS_t)1 /*< true */
59 # endif
60 # define AO_CLEAR(p) __atomic_clear(p, __ATOMIC_RELEASE)
61 # define AO_test_and_set_acquire(p) \
62 (__atomic_test_and_set(p, __ATOMIC_ACQUIRE) ? AO_TS_SET : AO_TS_CLEAR)
63 # define AO_HAVE_test_and_set_acquire
64
65 # define AO_compiler_barrier() __atomic_signal_fence(__ATOMIC_SEQ_CST)
66
67 # if defined(THREAD_SANITIZER) && !defined(AO_USE_ATOMIC_THREAD_FENCE)
68 /*
69 * Workaround a compiler warning (reported by gcc-11, at least) that
70 * `__atomic_thread_fence` is unsupported with thread sanitizer.
71 */
72 AO_INLINE void
73 AO_nop_full(void)
74 {
75 volatile AO_TS_t dummy = AO_TS_INITIALIZER;
76 (void)__atomic_test_and_set(&dummy, __ATOMIC_SEQ_CST);
77 }
78 # else
79 # define AO_nop_full() __atomic_thread_fence(__ATOMIC_SEQ_CST)
80 # endif
81 # define AO_HAVE_nop_full
82
83 # define AO_fetch_and_add(p, v) __atomic_fetch_add(p, v, __ATOMIC_RELAXED)
84 # define AO_HAVE_fetch_and_add
85 # define AO_fetch_and_add1(p) AO_fetch_and_add(p, 1)
86 # define AO_HAVE_fetch_and_add1
87 # define AO_fetch_and_sub1(p) AO_fetch_and_add(p, ~(AO_t)0 /* -1 */)
88 # define AO_HAVE_fetch_and_sub1
89
90 # define AO_or(p, v) (void)__atomic_or_fetch(p, v, __ATOMIC_RELAXED)
91 # define AO_HAVE_or
92
93 # define AO_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
94 # define AO_HAVE_load
95 # define AO_load_acquire(p) __atomic_load_n(p, __ATOMIC_ACQUIRE)
96 # define AO_HAVE_load_acquire
97 /*
98 * `AO_load_acquire_read(p)` is not defined as it is unused, but we need
99 * its `AO_HAVE_` macro defined.
100 */
101 # define AO_HAVE_load_acquire_read
102
103 # define AO_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
104 # define AO_HAVE_store
105 # define AO_store_release(p, v) __atomic_store_n(p, v, __ATOMIC_RELEASE)
106 # define AO_HAVE_store_release
107 # define AO_store_release_write(p, v) AO_store_release(p, v)
108 # define AO_HAVE_store_release_write
109
110 # define AO_char_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
111 # define AO_HAVE_char_load
112 # define AO_char_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
113 # define AO_HAVE_char_store
114 # define AO_char_fetch_and_add1(p) __atomic_fetch_add(p, 1, __ATOMIC_RELAXED)
115 # define AO_HAVE_char_fetch_and_add1
116
117 # ifdef AO_REQUIRE_CAS
118 AO_INLINE int
119 AO_compare_and_swap_release(volatile AO_t *p, AO_t ov, AO_t nv)
120 {
121 return (int)__atomic_compare_exchange_n(p, &ov, nv, 0, __ATOMIC_RELEASE,
122 __ATOMIC_RELAXED /* on fail */);
123 }
124 # define AO_HAVE_compare_and_swap_release
125 # endif
126
127 # ifdef __cplusplus
128 } /* extern "C" */
129 # endif
130
131 # ifndef NO_LOCKFREE_AO_OR
132 /* `__atomic_or_fetch()` is assumed to be lock-free. */
133 # define HAVE_LOCKFREE_AO_OR 1
134 # endif
135
136 #else
137 /* Fall back to `libatomic_ops`. */
138 # include "atomic_ops.h"
139
140 /*
141 * `AO_compiler_barrier`, `AO_load` and `AO_store` should be defined
142 * for all targets; the rest of the primitives are guaranteed to exist
143 * only if `AO_REQUIRE_CAS` is defined (or if the corresponding
144 * `AO_HAVE_` macro is defined). i686 and x86_64 targets have
145 * `AO_nop_full`, `AO_load_acquire`, `AO_store_release`, at least.
146 */
147 # if (!defined(AO_HAVE_load) || !defined(AO_HAVE_store)) && !defined(CPPCHECK)
148 # error AO_load or AO_store is missing; probably old version of atomic_ops
149 # endif
150
151 #endif /* !GC_BUILTIN_ATOMIC */
152
153 #if defined(GC_BUILTIN_ATOMIC) || defined(__CHERI_PURE_CAPABILITY__)
154 /*
155 * Assume that gcc atomic intrinsics are available (and have correct
156 * implementation). `p` should be of a pointer to `ptr_t` (`char *`) value.
157 */
158 # define GC_cptr_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
159 # define GC_cptr_load_acquire(p) __atomic_load_n(p, __ATOMIC_ACQUIRE)
160 # define GC_cptr_load_acquire_read(p) GC_cptr_load_acquire(p)
161 # define GC_cptr_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
162 # define GC_cptr_store_release(p, v) __atomic_store_n(p, v, __ATOMIC_RELEASE)
163 # define GC_cptr_store_release_write(p, v) GC_cptr_store_release(p, v)
164 # ifdef AO_REQUIRE_CAS
165 AO_INLINE int
166 GC_cptr_compare_and_swap(char *volatile *p, char *ov, char *nv)
167 {
168 return (int)__atomic_compare_exchange_n(p, &ov, nv, 0, __ATOMIC_RELAXED,
169 __ATOMIC_RELAXED);
170 }
171 # endif
172 #else
173 /*
174 * Redirect to the `AO_` primitives. Assume the size of `AO_t` matches
175 * that of a pointer.
176 */
177 # define GC_cptr_load(p) (char *)AO_load((volatile AO_t *)(p))
178 # define GC_cptr_load_acquire(p) (char *)AO_load_acquire((volatile AO_t *)(p))
179 # define GC_cptr_load_acquire_read(p) \
180 (char *)AO_load_acquire_read((volatile AO_t *)(p))
181 # define GC_cptr_store(p, v) AO_store((volatile AO_t *)(p), (AO_t)(v))
182 # define GC_cptr_store_release(p, v) \
183 AO_store_release((volatile AO_t *)(p), (AO_t)(v))
184 # define GC_cptr_store_release_write(p, v) \
185 AO_store_release_write((volatile AO_t *)(p), (AO_t)(v))
186 # ifdef AO_REQUIRE_CAS
187 # define GC_cptr_compare_and_swap(p, ov, nv) \
188 AO_compare_and_swap((volatile AO_t *)(p), (AO_t)(ov), (AO_t)(nv))
189 # endif
190 #endif /* !GC_BUILTIN_ATOMIC */
191
192 #endif /* GC_ATOMIC_OPS_H */
193