pthread_stop_world.c raw
1 /*
2 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
3 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
4 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
5 * Copyright (c) 2000-2009 by Hewlett-Packard Development Company.
6 * All rights reserved.
7 * Copyright (c) 2008-2022 Ivan Maidanski
8 *
9 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
10 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
11 *
12 * Permission is hereby granted to use or copy this program
13 * for any purpose, provided the above notices are retained on all copies.
14 * Permission to modify the code and to distribute modified code is granted,
15 * provided the above notices are retained, and a notice that the code was
16 * modified is included with the above copyright notice.
17 */
18
19 #include "private/pthread_support.h"
20
21 #ifdef PTHREAD_STOP_WORLD_IMPL
22
23 # ifdef NACL
24 # include <sys/time.h>
25 # else
26 # include <errno.h>
27 # include <semaphore.h>
28 # include <signal.h>
29 # include <time.h>
30 # endif /* !NACL */
31
32 # ifdef E2K
33 # include <alloca.h>
34 # endif
35
36 GC_INLINE void
37 GC_usleep(unsigned us)
38 {
39 # if defined(LINT2) || defined(THREAD_SANITIZER)
40 /*
41 * Workaround "waiting while holding a lock" static analyzer warning.
42 * Workaround a rare hang in `usleep()` trying to acquire TSan Lock.
43 */
44 while (us-- > 0) {
45 /* Sleep for a moment, pretending it takes 1us. */
46 sched_yield();
47 }
48 # elif defined(CPPCHECK) /* `|| _POSIX_C_SOURCE >= 199309L` */
49 struct timespec ts;
50
51 ts.tv_sec = 0;
52 ts.tv_nsec = (unsigned32)us * 1000;
53 /* This requires `_POSIX_TIMERS` feature. */
54 (void)nanosleep(&ts, NULL);
55 # else
56 usleep(us);
57 # endif
58 }
59
60 # ifdef NACL
61
62 STATIC int GC_nacl_num_gc_threads = 0;
63 STATIC volatile int GC_nacl_park_threads_now = 0;
64 STATIC volatile pthread_t GC_nacl_thread_parker = -1;
65
66 STATIC __thread int GC_nacl_thread_idx = -1;
67
68 /* TODO: Use `GC_get_tlfs()` instead. */
69 STATIC __thread GC_thread GC_nacl_gc_thread_self = NULL;
70
71 volatile int GC_nacl_thread_parked[MAX_NACL_GC_THREADS];
72 int GC_nacl_thread_used[MAX_NACL_GC_THREADS];
73
74 # else /* !NACL */
75
76 # if (!defined(AO_HAVE_load_acquire) || !defined(AO_HAVE_store_release)) \
77 && !defined(CPPCHECK)
78 # error AO_load_acquire and/or AO_store_release are missing;
79 # error please define AO_REQUIRE_CAS manually
80 # endif
81
82 # ifdef DEBUG_THREADS
83 /* It is safe to call original `pthread_sigmask()` here. */
84 # undef pthread_sigmask
85
86 # ifndef NSIG
87 # ifdef CPPCHECK
88 # define NSIG 32
89 # elif defined(MAXSIG)
90 # define NSIG (MAXSIG + 1)
91 # elif defined(_NSIG)
92 # define NSIG _NSIG
93 # elif defined(__SIGRTMAX)
94 # define NSIG (__SIGRTMAX + 1)
95 # else
96 # error define NSIG
97 # endif
98 # endif
99
100 void
101 GC_print_sig_mask(void)
102 {
103 sigset_t blocked;
104 int i;
105
106 if (pthread_sigmask(SIG_BLOCK, NULL, &blocked) != 0)
107 ABORT("pthread_sigmask failed");
108 for (i = 1; i < NSIG; i++) {
109 if (sigismember(&blocked, i))
110 GC_printf("Signal blocked: %d\n", i);
111 }
112 }
113 # endif /* DEBUG_THREADS */
114
115 /*
116 * Remove the signals that we want to allow in thread stopping handler
117 * from a set.
118 */
119 STATIC void
120 GC_remove_allowed_signals(sigset_t *set)
121 {
122 if (sigdelset(set, SIGINT) != 0 || sigdelset(set, SIGQUIT) != 0
123 || sigdelset(set, SIGABRT) != 0 || sigdelset(set, SIGTERM) != 0) {
124 ABORT("sigdelset failed");
125 }
126
127 # ifdef MPROTECT_VDB
128 /*
129 * The handlers write to the thread structure, which is in the heap,
130 * and hence can trigger a protection fault.
131 */
132 if (sigdelset(set, SIGSEGV) != 0
133 # ifdef HAVE_SIGBUS
134 || sigdelset(set, SIGBUS) != 0
135 # endif
136 ) {
137 ABORT("sigdelset failed");
138 }
139 # endif
140 }
141
142 static sigset_t suspend_handler_mask;
143
144 # define THREAD_RESTARTED 0x1
145
146 /*
147 * Incremented (to the nearest even value) at the beginning of
148 * `GC_stop_world()` (or when a thread is requested to be suspended by
149 * `GC_suspend_thread`) and once more (to an odd value) at the
150 * beginning of `GC_start_world`. The lowest bit is `THREAD_RESTARTED`
151 * one which, if set, means it is safe for threads to restart, i.e.
152 * they will see another suspend signal before they are expected to
153 * stop (unless they have stopped voluntarily).
154 */
155 STATIC volatile AO_t GC_stop_count;
156
157 STATIC GC_bool GC_retry_signals = FALSE;
158
159 /*
160 * We use signals to stop threads during GC.
161 *
162 * Suspended threads wait in signal handler for `SIG_THR_RESTART`.
163 * That is more portable than semaphores or condition variables.
164 * (We do use `sem_post()` from a signal handler, but that should be
165 * portable.)
166 *
167 * The thread suspension signal `SIG_SUSPEND` is now defined in `gc_priv.h`
168 * file. Note that we cannot just stop a thread; we need it to save its
169 * stack pointers and acknowledge.
170 */
171 # ifndef SIG_THR_RESTART
172 # if defined(SUSPEND_HANDLER_NO_CONTEXT) || defined(GC_REUSE_SIG_SUSPEND)
173 /* Reuse the suspend signal. */
174 # define SIG_THR_RESTART SIG_SUSPEND
175 # elif defined(HPUX) || defined(NETBSD) || defined(OSF1) \
176 || defined(GC_USESIGRT_SIGNALS)
177 # if defined(_SIGRTMIN) && !defined(CPPCHECK)
178 # define SIG_THR_RESTART (_SIGRTMIN + 5)
179 # else
180 # define SIG_THR_RESTART (SIGRTMIN + 5)
181 # endif
182 # elif defined(FREEBSD) && defined(__GLIBC__)
183 # define SIG_THR_RESTART (32 + 5)
184 # elif defined(FREEBSD) || defined(HURD) || defined(RTEMS)
185 # define SIG_THR_RESTART SIGUSR2
186 # else
187 # define SIG_THR_RESTART SIGXCPU
188 # endif
189 # endif /* !SIG_THR_RESTART */
190
191 # define SIGNAL_UNSET (-1)
192
193 /*
194 * Since `SIG_SUSPEND` and/or `SIG_THR_RESTART` could represent
195 * a non-constant expression (e.g., in case of `SIGRTMIN`), actual
196 * signal numbers are determined by `GC_stop_init()` unless manually
197 * set (before the collector initialization). Might be set to the
198 * same signal number.
199 */
200 STATIC int GC_sig_suspend = SIGNAL_UNSET;
201 STATIC int GC_sig_thr_restart = SIGNAL_UNSET;
202
203 GC_API void GC_CALL
204 GC_set_suspend_signal(int sig)
205 {
206 if (GC_is_initialized)
207 return;
208
209 GC_sig_suspend = sig;
210 }
211
212 GC_API void GC_CALL
213 GC_set_thr_restart_signal(int sig)
214 {
215 if (GC_is_initialized)
216 return;
217
218 GC_sig_thr_restart = sig;
219 }
220
221 GC_API int GC_CALL
222 GC_get_suspend_signal(void)
223 {
224 return GC_sig_suspend != SIGNAL_UNSET ? GC_sig_suspend : SIG_SUSPEND;
225 }
226
227 GC_API int GC_CALL
228 GC_get_thr_restart_signal(void)
229 {
230 return GC_sig_thr_restart != SIGNAL_UNSET ? GC_sig_thr_restart
231 : SIG_THR_RESTART;
232 }
233
234 # ifdef BASE_ATOMIC_OPS_EMULATED
235 /*
236 * The AO primitives emulated with locks cannot be used inside signal
237 * handlers as this could cause a deadlock or a double lock.
238 * The following "async" macro definitions are correct only for
239 * an uniprocessor case and are provided for a test purpose.
240 */
241 # define ao_load_acquire_async(p) (*(p))
242 # define ao_load_async(p) ao_load_acquire_async(p)
243 # define ao_store_release_async(p, v) (void)(*(p) = (v))
244 # define ao_cptr_store_async(p, v) (void)(*(p) = (v))
245 # else
246 # define ao_load_acquire_async(p) AO_load_acquire(p)
247 # define ao_load_async(p) AO_load(p)
248 # define ao_store_release_async(p, v) AO_store_release(p, v)
249 # define ao_cptr_store_async(p, v) GC_cptr_store(p, v)
250 # endif /* !BASE_ATOMIC_OPS_EMULATED */
251
252 /* Note: this is also used to acknowledge restart. */
253 STATIC sem_t GC_suspend_ack_sem;
254
255 STATIC void GC_suspend_handler_inner(ptr_t dummy, void *context);
256
257 # ifdef SUSPEND_HANDLER_NO_CONTEXT
258 STATIC void
259 GC_suspend_handler(int sig)
260 # else
261 STATIC void
262 GC_suspend_sigaction(int sig, siginfo_t *info, void *context)
263 # endif
264 {
265 int old_errno = errno;
266
267 if (sig != GC_sig_suspend) {
268 # ifdef FREEBSD
269 /* Workaround "deferred signal handling" bug in FreeBSD 9.2. */
270 if (0 == sig)
271 return;
272 # endif
273 ABORT("Bad signal in suspend_handler");
274 }
275
276 # ifdef SUSPEND_HANDLER_NO_CONTEXT
277 /* A quick check if the signal is called to restart the world. */
278 if ((ao_load_async(&GC_stop_count) & THREAD_RESTARTED) != 0)
279 return;
280 GC_with_callee_saves_pushed(GC_suspend_handler_inner, NULL);
281 # else
282 UNUSED_ARG(info);
283 /*
284 * We believe that in this case the full context is already in the
285 * signal handler frame.
286 */
287 GC_suspend_handler_inner(NULL, context);
288 # endif
289 errno = old_errno;
290 }
291
292 /*
293 * The lookup here is safe, since this is done on behalf of a thread which
294 * holds the allocator lock in order to stop the world. Thus concurrent
295 * modification of the data structure is impossible. Unfortunately, we
296 * have to instruct TSan that the lookup is safe.
297 */
298 # ifdef THREAD_SANITIZER
299 /*
300 * Almost same as `GC_self_thread_inner()` except for the `no_sanitize`
301 * attribute added and the result is never `NULL`.
302 */
303 GC_ATTR_NO_SANITIZE_THREAD
304 static GC_thread
305 GC_lookup_self_thread_async(void)
306 {
307 thread_id_t self_id = thread_id_self();
308 GC_thread p = GC_threads[THREAD_TABLE_INDEX(self_id)];
309
310 for (;; p = p->tm.next) {
311 if (THREAD_EQUAL(p->id, self_id))
312 break;
313 }
314 return p;
315 }
316 # else
317 # define GC_lookup_self_thread_async() GC_self_thread_inner()
318 # endif
319
320 GC_INLINE void
321 GC_store_stack_ptr(GC_stack_context_t crtn)
322 {
323 /*
324 * There is no data race between the suspend handler (storing
325 * `stack_ptr`) and `GC_push_all_stacks` (fetching `stack_ptr`)
326 * because `GC_push_all_stacks` is executed after `GC_stop_world`
327 * exits and the latter runs `sem_wait()` repeatedly waiting for all
328 * the suspended threads to call `sem_post()`. Nonetheless,
329 * `stack_ptr` is stored (here) and fetched (by `GC_push_all_stacks`)
330 * using the atomic primitives to avoid the related TSan warning.
331 */
332 # ifdef SPARC
333 ao_cptr_store_async(&crtn->stack_ptr, GC_save_regs_in_stack());
334 /* TODO: Registers saving already done by `GC_with_callee_saves_pushed`. */
335 # else
336 # ifdef IA64
337 crtn->backing_store_ptr = GC_save_regs_in_stack();
338 # endif
339 ao_cptr_store_async(&crtn->stack_ptr, GC_approx_sp());
340 # endif
341 }
342
343 STATIC void
344 GC_suspend_handler_inner(ptr_t dummy, void *context)
345 {
346 GC_thread me;
347 GC_stack_context_t crtn;
348 # ifdef E2K
349 ptr_t bs_lo;
350 size_t stack_size;
351 # endif
352 IF_CANCEL(int cancel_state;)
353 # ifdef GC_ENABLE_SUSPEND_THREAD
354 AO_t suspend_cnt;
355 # endif
356 AO_t my_stop_count = ao_load_acquire_async(&GC_stop_count);
357 /*
358 * After the barrier, this thread should see the actual content of
359 * `GC_threads`.
360 */
361
362 UNUSED_ARG(dummy);
363 UNUSED_ARG(context);
364 if ((my_stop_count & THREAD_RESTARTED) != 0) {
365 /* Restarting the world. */
366 return;
367 }
368
369 /*
370 * `pthread_setcancelstate()` is not defined to be async-signal-safe.
371 * But the `glibc` version appears to be defined, in the absence of
372 * asynchronous cancellation. And since this signal handler to block
373 * on `sigsuspend`, which is both async-signal-safe and
374 * a cancellation point, there seems to be no obvious way out of it.
375 * In fact, it looks like an async-signal-safe cancellation point
376 * is inherently a problem, unless there is some way to disable
377 * cancellation in the handler.
378 */
379 DISABLE_CANCEL(cancel_state);
380
381 # ifdef DEBUG_THREADS
382 GC_log_printf("Suspending %p\n", PTHREAD_TO_VPTR(pthread_self()));
383 # endif
384 me = GC_lookup_self_thread_async();
385 if ((me->last_stop_count & ~(word)THREAD_RESTARTED) == my_stop_count) {
386 /* Duplicate signal. OK if we are retrying. */
387 if (!GC_retry_signals) {
388 WARN("Duplicate suspend signal in thread %p\n", pthread_self());
389 }
390 RESTORE_CANCEL(cancel_state);
391 return;
392 }
393 crtn = me->crtn;
394 GC_store_stack_ptr(crtn);
395 # ifdef E2K
396 GC_ASSERT(NULL == crtn->backing_store_end);
397 GET_PROCEDURE_STACK_LOCAL(crtn->ps_ofs, &bs_lo, &stack_size);
398 crtn->backing_store_end = bs_lo;
399 crtn->backing_store_ptr = bs_lo + stack_size;
400 # endif
401 # ifdef GC_ENABLE_SUSPEND_THREAD
402 suspend_cnt = ao_load_async(&me->ext_suspend_cnt);
403 # endif
404
405 /*
406 * Tell the thread that wants to stop the world that this thread has
407 * been stopped. Note that `sem_post()` is the only async-signal-safe
408 * primitive in LinuxThreads.
409 */
410 sem_post(&GC_suspend_ack_sem);
411 ao_store_release_async(&me->last_stop_count, my_stop_count);
412
413 /*
414 * Wait until that thread tells us to restart by sending this thread
415 * a `GC_sig_thr_restart` signal (should be masked at this point thus
416 * there is no race). We do not continue until we receive that signal,
417 * but we do not take that as authoritative. (We may be accidentally
418 * restarted by one of the user signals we do not block.)
419 * After we receive the signal, we use a primitive and an expensive
420 * mechanism to wait until it is really safe to proceed.
421 */
422 do {
423 sigsuspend(&suspend_handler_mask);
424 /* Iterate while not restarting the world or thread is suspended. */
425 } while (ao_load_acquire_async(&GC_stop_count) == my_stop_count
426 # ifdef GC_ENABLE_SUSPEND_THREAD
427 || ((suspend_cnt & 1) != 0
428 && ao_load_async(&me->ext_suspend_cnt) == suspend_cnt)
429 # endif
430 );
431
432 # ifdef DEBUG_THREADS
433 GC_log_printf("Resuming %p\n", PTHREAD_TO_VPTR(pthread_self()));
434 # endif
435 # ifdef E2K
436 GC_ASSERT(crtn->backing_store_end == bs_lo);
437 crtn->backing_store_ptr = NULL;
438 crtn->backing_store_end = NULL;
439 # endif
440
441 # ifndef GC_NETBSD_THREADS_WORKAROUND
442 if (GC_retry_signals || GC_sig_suspend == GC_sig_thr_restart)
443 # endif
444 {
445 /*
446 * If the `SIG_THR_RESTART` signal loss is possible (though it should
447 * be less likely than losing the `SIG_SUSPEND` signal as we do not
448 * do much between the first `sem_post` and `sigsuspend` calls), more
449 * handshaking is provided to work around it.
450 */
451 sem_post(&GC_suspend_ack_sem);
452 /* Set the flag that the thread has been restarted. */
453 if (GC_retry_signals)
454 ao_store_release_async(&me->last_stop_count,
455 my_stop_count | THREAD_RESTARTED);
456 }
457 RESTORE_CANCEL(cancel_state);
458 }
459
460 static void
461 suspend_restart_barrier(int n_live_threads)
462 {
463 int i;
464
465 for (i = 0; i < n_live_threads; i++) {
466 while (sem_wait(&GC_suspend_ack_sem) == -1) {
467 /*
468 * On Linux, `sem_wait()` is documented to always return zero.
469 * But the documentation appears to be incorrect.
470 * `EINTR` seems to happen with some versions of gdb.
471 */
472 if (errno != EINTR)
473 ABORT("sem_wait failed");
474 }
475 }
476 # ifdef GC_ASSERTIONS
477 sem_getvalue(&GC_suspend_ack_sem, &i);
478 GC_ASSERT(0 == i);
479 # endif
480 }
481
482 # define WAIT_UNIT 3000 /* us */
483
484 static int
485 resend_lost_signals(int n_live_threads, int (*suspend_restart_all)(void))
486 {
487 # define RESEND_SIGNALS_LIMIT 150
488 # define RETRY_INTERVAL 100000 /* us */
489
490 if (n_live_threads > 0) {
491 unsigned long wait_usecs = 0; /*< total wait since retry */
492 int retry = 0;
493 int prev_sent = 0;
494
495 for (;;) {
496 int ack_count;
497
498 sem_getvalue(&GC_suspend_ack_sem, &ack_count);
499 if (ack_count == n_live_threads)
500 break;
501 if (wait_usecs > RETRY_INTERVAL) {
502 int newly_sent = suspend_restart_all();
503
504 if (newly_sent != prev_sent) {
505 /* Restart the counter. */
506 retry = 0;
507 } else if (++retry >= RESEND_SIGNALS_LIMIT) {
508 /* No progress. */
509 ABORT_ARG1("Signals delivery fails constantly", " at GC #%lu",
510 (unsigned long)GC_gc_no);
511 }
512
513 GC_COND_LOG_PRINTF("Resent %d signals after timeout, retry: %d\n",
514 newly_sent, retry);
515 sem_getvalue(&GC_suspend_ack_sem, &ack_count);
516 if (newly_sent < n_live_threads - ack_count) {
517 WARN("Lost some threads while stopping or starting world?!\n", 0);
518 n_live_threads = ack_count + newly_sent;
519 }
520 prev_sent = newly_sent;
521 wait_usecs = 0;
522 }
523 GC_usleep(WAIT_UNIT);
524 wait_usecs += WAIT_UNIT;
525 }
526 }
527 return n_live_threads;
528 }
529
530 # ifdef HAVE_CLOCK_GETTIME
531 # define TS_NSEC_ADD(ts, ns) \
532 (ts.tv_nsec += (ns), \
533 (void)(ts.tv_nsec >= 1000000L * 1000 \
534 ? (ts.tv_nsec -= 1000000L * 1000, ts.tv_sec++, 0) \
535 : 0))
536 # endif
537
538 static void
539 resend_lost_signals_retry(int n_live_threads, int (*suspend_restart_all)(void))
540 {
541 # if defined(HAVE_CLOCK_GETTIME) && !defined(DONT_TIMEDWAIT_ACK_SEM)
542 # define TIMEOUT_BEFORE_RESEND 10000 /* us */
543 struct timespec ts;
544
545 if (n_live_threads > 0 && clock_gettime(CLOCK_REALTIME, &ts) == 0) {
546 int i;
547
548 TS_NSEC_ADD(ts, TIMEOUT_BEFORE_RESEND * (unsigned32)1000);
549 /*
550 * First, try to wait for the semaphore with some timeout.
551 * On failure, fall back to `WAIT_UNIT` pause and resend of the signal.
552 */
553 for (i = 0; i < n_live_threads; i++) {
554 if (sem_timedwait(&GC_suspend_ack_sem, &ts) == -1)
555 break; /*< wait timed out or any other error */
556 }
557 /* Update the count of threads to wait the acknowledge from. */
558 n_live_threads -= i;
559 }
560 # endif
561 n_live_threads = resend_lost_signals(n_live_threads, suspend_restart_all);
562 suspend_restart_barrier(n_live_threads);
563 }
564
565 STATIC void
566 GC_restart_handler(int sig)
567 {
568 # ifdef DEBUG_THREADS
569 /* Preserve `errno` value. */
570 int old_errno = errno;
571 # endif
572
573 if (sig != GC_sig_thr_restart) {
574 ABORT("Bad signal in restart handler");
575 }
576
577 /*
578 * Note: even if we do not do anything useful here, it would still
579 * be necessary to have a signal handler, rather than ignoring the
580 * signals, otherwise the signals will not be delivered at all, and
581 * will thus not interrupt the `sigsuspend()` above.
582 */
583 # ifdef DEBUG_THREADS
584 GC_log_printf("In GC_restart_handler for %p\n",
585 PTHREAD_TO_VPTR(pthread_self()));
586 errno = old_errno;
587 # endif
588 }
589
590 # ifdef USE_TKILL_ON_ANDROID
591 EXTERN_C_BEGIN
592 extern int tkill(pid_t tid, int sig); /*< from platform `sys/linux-unistd.h` */
593 EXTERN_C_END
594 # define THREAD_SYSTEM_ID(t) (t)->kernel_id
595 # else
596 # define THREAD_SYSTEM_ID(t) (t)->id
597 # endif
598
599 # ifndef RETRY_TKILL_EAGAIN_LIMIT
600 # define RETRY_TKILL_EAGAIN_LIMIT 16
601 # endif
602
603 static int
604 raise_signal(GC_thread p, int sig)
605 {
606 int res;
607 # ifdef RETRY_TKILL_ON_EAGAIN
608 int retry;
609 # endif
610 # if defined(SIMULATE_LOST_SIGNALS) && !defined(GC_ENABLE_SUSPEND_THREAD)
611 # ifndef LOST_SIGNALS_RATIO
612 # define LOST_SIGNALS_RATIO 25
613 # endif
614 /* Note: race is OK, it is for test purpose only. */
615 static int signal_cnt;
616
617 if (GC_retry_signals && (++signal_cnt) % LOST_SIGNALS_RATIO == 0) {
618 /* Simulate the signal is sent but lost. */
619 return 0;
620 }
621 # endif
622 # ifdef RETRY_TKILL_ON_EAGAIN
623 for (retry = 0;; retry++)
624 # endif
625 {
626 # ifdef USE_TKILL_ON_ANDROID
627 int old_errno = errno;
628
629 res = tkill(THREAD_SYSTEM_ID(p), sig);
630 if (res < 0) {
631 res = errno;
632 errno = old_errno;
633 }
634 # else
635 res = pthread_kill(THREAD_SYSTEM_ID(p), sig);
636 # endif
637 # ifdef RETRY_TKILL_ON_EAGAIN
638 if (res != EAGAIN || retry >= RETRY_TKILL_EAGAIN_LIMIT)
639 break;
640 /* A temporal overflow of the real-time signal queue. */
641 GC_usleep(WAIT_UNIT);
642 # endif
643 }
644 return res;
645 }
646
647 # ifdef GC_ENABLE_SUSPEND_THREAD
648 # include <sys/time.h>
649
650 /* This is to get the prototypes as `extern "C"`. */
651 # include "gc/javaxfc.h"
652
653 STATIC void
654 GC_brief_async_signal_safe_sleep(void)
655 {
656 struct timeval tv;
657 tv.tv_sec = 0;
658 # if defined(GC_TIME_LIMIT) && !defined(CPPCHECK)
659 tv.tv_usec = 1000 * GC_TIME_LIMIT / 2;
660 # else
661 tv.tv_usec = 1000 * 15 / 2;
662 # endif
663 (void)select(0, 0, 0, 0, &tv);
664 }
665
666 GC_INNER void
667 GC_suspend_self_inner(GC_thread me, size_t suspend_cnt)
668 {
669 IF_CANCEL(int cancel_state;)
670
671 GC_ASSERT((suspend_cnt & 1) != 0);
672 DISABLE_CANCEL(cancel_state);
673 # ifdef DEBUG_THREADS
674 GC_log_printf("Suspend self: %p\n", THREAD_ID_TO_VPTR(me->id));
675 # endif
676 while (ao_load_acquire_async(&me->ext_suspend_cnt) == suspend_cnt) {
677 /* TODO: Use `sigsuspend()` even for self-suspended threads. */
678 GC_brief_async_signal_safe_sleep();
679 }
680 # ifdef DEBUG_THREADS
681 GC_log_printf("Resume self: %p\n", THREAD_ID_TO_VPTR(me->id));
682 # endif
683 RESTORE_CANCEL(cancel_state);
684 }
685
686 GC_API void GC_CALL
687 GC_suspend_thread(GC_SUSPEND_THREAD_ID thread)
688 {
689 GC_thread t;
690 AO_t next_stop_count;
691 AO_t suspend_cnt;
692 IF_CANCEL(int cancel_state;)
693
694 LOCK();
695 t = GC_lookup_by_pthread((pthread_t)thread);
696 if (NULL == t) {
697 UNLOCK();
698 return;
699 }
700 suspend_cnt = t->ext_suspend_cnt;
701 if ((suspend_cnt & 1) != 0) /*< already suspended? */ {
702 GC_ASSERT(!THREAD_EQUAL((pthread_t)thread, pthread_self()));
703 UNLOCK();
704 return;
705 }
706 if ((t->flags & (FINISHED | DO_BLOCKING)) != 0) {
707 t->ext_suspend_cnt = suspend_cnt | 1; /*< suspend */
708 /* Terminated but not joined yet, or in do-blocking state. */
709 UNLOCK();
710 return;
711 }
712
713 if (THREAD_EQUAL((pthread_t)thread, pthread_self())) {
714 t->ext_suspend_cnt = suspend_cnt | 1;
715 GC_with_callee_saves_pushed(GC_suspend_self_blocked, (ptr_t)t);
716 UNLOCK();
717 return;
718 }
719
720 /* `GC_suspend_thread()` is not a cancellation point. */
721 DISABLE_CANCEL(cancel_state);
722 # ifdef PARALLEL_MARK
723 /*
724 * Ensure we do not suspend a thread while it is rebuilding a free list,
725 * otherwise such a deadlock is possible:
726 * - thread 1 is blocked in `GC_wait_for_reclaim` holding the allocator
727 * lock;
728 * - thread 2 is suspended in `GC_reclaim_generic` invoked from
729 * `GC_generic_malloc_many` (with `GC_fl_builder_count` greater than
730 * zero);
731 * - thread 3 is blocked acquiring the allocator lock in
732 * `GC_resume_thread`.
733 */
734 if (GC_parallel)
735 GC_wait_for_reclaim();
736 # endif
737
738 if (GC_manual_vdb) {
739 /* See the relevant comment in `GC_stop_world()`. */
740 GC_acquire_dirty_lock();
741 }
742 /*
743 * Else do not acquire the dirty lock as the write fault handler
744 * might be trying to acquire it too, and the suspend handler
745 * execution is deferred until the write fault handler completes.
746 */
747
748 next_stop_count = GC_stop_count + THREAD_RESTARTED;
749 GC_ASSERT((next_stop_count & THREAD_RESTARTED) == 0);
750 AO_store(&GC_stop_count, next_stop_count);
751
752 /* Set the flag making the change visible to the signal handler. */
753 AO_store_release(&t->ext_suspend_cnt, suspend_cnt | 1);
754
755 /* TODO: Support `GC_retry_signals` (not needed for TSan). */
756 switch (raise_signal(t, GC_sig_suspend)) {
757 /* `ESRCH` cannot happen as terminated threads are handled above. */
758 case 0:
759 break;
760 default:
761 ABORT("pthread_kill failed");
762 }
763
764 /*
765 * Wait for the thread to complete threads table lookup and
766 * `stack_ptr` assignment.
767 */
768 GC_ASSERT(GC_thr_initialized);
769 suspend_restart_barrier(1);
770 if (GC_manual_vdb)
771 GC_release_dirty_lock();
772 AO_store(&GC_stop_count, next_stop_count | THREAD_RESTARTED);
773
774 RESTORE_CANCEL(cancel_state);
775 UNLOCK();
776 }
777
778 GC_API void GC_CALL
779 GC_resume_thread(GC_SUSPEND_THREAD_ID thread)
780 {
781 GC_thread t;
782
783 LOCK();
784 t = GC_lookup_by_pthread((pthread_t)thread);
785 if (t != NULL) {
786 AO_t suspend_cnt = t->ext_suspend_cnt;
787
788 if ((suspend_cnt & 1) != 0) { /*< is suspended? */
789 GC_ASSERT((GC_stop_count & THREAD_RESTARTED) != 0);
790 /* Mark the thread as not suspended - it will be resumed shortly. */
791 AO_store(&t->ext_suspend_cnt, suspend_cnt + 1);
792
793 if ((t->flags & (FINISHED | DO_BLOCKING)) == 0) {
794 int result = raise_signal(t, GC_sig_thr_restart);
795
796 /* TODO: Support signal resending on `GC_retry_signals`. */
797 if (result != 0)
798 ABORT_ARG1("pthread_kill failed in GC_resume_thread",
799 ": errcode= %d", result);
800 # ifndef GC_NETBSD_THREADS_WORKAROUND
801 if (GC_retry_signals || GC_sig_suspend == GC_sig_thr_restart)
802 # endif
803 {
804 IF_CANCEL(int cancel_state;)
805
806 DISABLE_CANCEL(cancel_state);
807 suspend_restart_barrier(1);
808 RESTORE_CANCEL(cancel_state);
809 }
810 }
811 }
812 }
813 UNLOCK();
814 }
815
816 GC_API int GC_CALL
817 GC_is_thread_suspended(GC_SUSPEND_THREAD_ID thread)
818 {
819 GC_thread t;
820 int is_suspended = 0;
821
822 READER_LOCK();
823 t = GC_lookup_by_pthread((pthread_t)thread);
824 if (t != NULL && (t->ext_suspend_cnt & 1) != 0)
825 is_suspended = (int)TRUE;
826 READER_UNLOCK();
827 return is_suspended;
828 }
829 # endif /* GC_ENABLE_SUSPEND_THREAD */
830
831 # undef ao_cptr_store_async
832 # undef ao_load_acquire_async
833 # undef ao_load_async
834 # undef ao_store_release_async
835 # endif /* !NACL */
836
837 GC_INNER void
838 GC_push_all_stacks(void)
839 {
840 GC_bool found_me = FALSE;
841 size_t nthreads = 0;
842 int i;
843 GC_thread p;
844 ptr_t lo; /*< stack top (`sp`) */
845 ptr_t hi; /*< bottom */
846 # if defined(E2K) || defined(IA64)
847 /* We also need to scan the register backing store. */
848 ptr_t bs_lo, bs_hi;
849 # endif
850 struct GC_traced_stack_sect_s *traced_stack_sect;
851 pthread_t self = pthread_self();
852 word total_size = 0;
853
854 GC_ASSERT(I_HOLD_LOCK());
855 GC_ASSERT(GC_thr_initialized);
856 # ifdef DEBUG_THREADS
857 GC_log_printf("Pushing stacks from thread %p\n", PTHREAD_TO_VPTR(self));
858 # endif
859 for (i = 0; i < THREAD_TABLE_SZ; i++) {
860 for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
861 # if defined(E2K) || defined(IA64)
862 GC_bool is_self = FALSE;
863 # endif
864 GC_stack_context_t crtn = p->crtn;
865
866 GC_ASSERT(THREAD_TABLE_INDEX(p->id) == i);
867 if (KNOWN_FINISHED(p))
868 continue;
869 ++nthreads;
870 traced_stack_sect = crtn->traced_stack_sect;
871 if (THREAD_EQUAL(p->id, self)) {
872 GC_ASSERT((p->flags & DO_BLOCKING) == 0);
873 # ifdef SPARC
874 lo = GC_save_regs_in_stack();
875 # else
876 lo = GC_approx_sp();
877 # ifdef IA64
878 bs_hi = GC_save_regs_in_stack();
879 # elif defined(E2K)
880 {
881 size_t stack_size;
882
883 GC_ASSERT(NULL == crtn->backing_store_end);
884 GET_PROCEDURE_STACK_LOCAL(crtn->ps_ofs, &bs_lo, &stack_size);
885 bs_hi = bs_lo + stack_size;
886 }
887 # endif
888 # endif
889 found_me = TRUE;
890 # if defined(E2K) || defined(IA64)
891 is_self = TRUE;
892 # endif
893 } else {
894 lo = GC_cptr_load(&crtn->stack_ptr);
895 # ifdef IA64
896 bs_hi = crtn->backing_store_ptr;
897 # elif defined(E2K)
898 bs_lo = crtn->backing_store_end;
899 bs_hi = crtn->backing_store_ptr;
900 # endif
901 if (traced_stack_sect != NULL
902 && traced_stack_sect->saved_stack_ptr == lo) {
903 /*
904 * If the thread has never been stopped since the recent
905 * `GC_call_with_gc_active` invocation, then skip the top
906 * "stack section" as `stack_ptr` already points to.
907 */
908 traced_stack_sect = traced_stack_sect->prev;
909 }
910 }
911 hi = crtn->stack_end;
912 # ifdef IA64
913 bs_lo = crtn->backing_store_end;
914 # endif
915 # ifdef DEBUG_THREADS
916 # ifdef STACK_GROWS_UP
917 GC_log_printf("Stack for thread %p is (%p,%p]\n",
918 THREAD_ID_TO_VPTR(p->id), (void *)hi, (void *)lo);
919 # else
920 GC_log_printf("Stack for thread %p is [%p,%p)\n",
921 THREAD_ID_TO_VPTR(p->id), (void *)lo, (void *)hi);
922 # endif
923 # endif
924 if (NULL == lo)
925 ABORT("GC_push_all_stacks: sp not set!");
926 if (crtn->altstack != NULL && ADDR_GE(lo, crtn->altstack)
927 && ADDR_GE(crtn->altstack + crtn->altstack_size, lo)) {
928 # ifdef STACK_GROWS_UP
929 hi = crtn->altstack;
930 # else
931 hi = crtn->altstack + crtn->altstack_size;
932 # endif
933 /* FIXME: Need to scan the normal stack too, but how? */
934 }
935 # ifdef STACKPTR_CORRECTOR_AVAILABLE
936 if (GC_sp_corrector != 0)
937 GC_sp_corrector((void **)&lo, THREAD_ID_TO_VPTR(p->id));
938 # endif
939 GC_push_all_stack_sections(lo, hi, traced_stack_sect);
940 # ifdef STACK_GROWS_UP
941 total_size += lo - hi;
942 # else
943 total_size += hi - lo; /*< `lo` is not greater than `hi` */
944 # endif
945 # ifdef NACL
946 /* Push `reg_storage` as roots, this will cover the reg context. */
947 GC_push_all_stack((ptr_t)p->reg_storage,
948 (ptr_t)(p->reg_storage + NACL_GC_REG_STORAGE_SIZE));
949 total_size += NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t);
950 # endif
951 # ifdef E2K
952 if ((GC_stop_count & THREAD_RESTARTED) != 0
953 # ifdef GC_ENABLE_SUSPEND_THREAD
954 && (p->ext_suspend_cnt & 1) == 0
955 # endif
956 && !is_self && (p->flags & DO_BLOCKING) == 0) {
957 /* Procedure stack buffer has already been freed. */
958 continue;
959 }
960 # endif
961 # if defined(E2K) || defined(IA64)
962 # ifdef DEBUG_THREADS
963 GC_log_printf("Reg stack for thread %p is [%p,%p)\n",
964 THREAD_ID_TO_VPTR(p->id), (void *)bs_lo, (void *)bs_hi);
965 # endif
966 GC_ASSERT(bs_lo != NULL && bs_hi != NULL);
967 /*
968 * FIXME: This (if `is_self`) may add an unbounded number of entries,
969 * and hence overflow the mark stack, which is bad.
970 */
971 # ifdef IA64
972 GC_push_all_register_sections(bs_lo, bs_hi, is_self, traced_stack_sect);
973 # else
974 if (is_self) {
975 GC_push_all_eager(bs_lo, bs_hi);
976 } else {
977 GC_push_all_stack(bs_lo, bs_hi);
978 }
979 # endif
980 total_size += bs_hi - bs_lo; /*< `bs_lo` is not greater than `bs_hi` */
981 # endif
982 }
983 }
984 GC_VERBOSE_LOG_PRINTF("Pushed %d thread stacks\n", (int)nthreads);
985 if (!found_me && !GC_in_thread_creation)
986 ABORT("Collecting from unknown thread");
987 GC_total_stacksize = total_size;
988 }
989
990 # ifdef DEBUG_THREADS
991 /*
992 * There seems to be a very rare thread stopping problem.
993 * To help us debug that, we save the ids of the stopping thread.
994 */
995 pthread_t GC_stopping_thread;
996 int GC_stopping_pid = 0;
997 # endif
998
999 /*
1000 * Suspend all threads that might still be running. Return the number
1001 * of suspend signals that were sent.
1002 */
1003 STATIC int
1004 GC_suspend_all(void)
1005 {
1006 int n_live_threads = 0;
1007 int i;
1008 # ifndef NACL
1009 GC_thread p;
1010 pthread_t self = pthread_self();
1011 int result;
1012
1013 GC_ASSERT((GC_stop_count & THREAD_RESTARTED) == 0);
1014 GC_ASSERT(I_HOLD_LOCK());
1015 for (i = 0; i < THREAD_TABLE_SZ; i++) {
1016 for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
1017 if (!THREAD_EQUAL(p->id, self)) {
1018 if ((p->flags & (FINISHED | DO_BLOCKING)) != 0)
1019 continue;
1020 # ifdef GC_ENABLE_SUSPEND_THREAD
1021 if ((p->ext_suspend_cnt & 1) != 0)
1022 continue;
1023 # endif
1024 if (AO_load(&p->last_stop_count) == GC_stop_count) {
1025 /* Matters only if `GC_retry_signals`. */
1026 continue;
1027 }
1028 n_live_threads++;
1029 # ifdef DEBUG_THREADS
1030 GC_log_printf("Sending suspend signal to %p\n",
1031 THREAD_ID_TO_VPTR(p->id));
1032 # endif
1033
1034 /*
1035 * The synchronization between `GC_dirty` (based on test-and-set)
1036 * and the signal-based thread suspension is performed in
1037 * `GC_stop_world()` because `GC_release_dirty_lock()` cannot be
1038 * called before acknowledging that the thread is really suspended.
1039 */
1040 result = raise_signal(p, GC_sig_suspend);
1041 switch (result) {
1042 case ESRCH:
1043 /* Not really there anymore. Possible? */
1044 n_live_threads--;
1045 break;
1046 case 0:
1047 if (GC_on_thread_event) {
1048 /* Note: thread id might be truncated. */
1049 GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED,
1050 THREAD_ID_TO_VPTR(THREAD_SYSTEM_ID(p)));
1051 }
1052 break;
1053 default:
1054 ABORT_ARG1("pthread_kill failed at suspend", ": errcode= %d",
1055 result);
1056 }
1057 }
1058 }
1059 }
1060
1061 # else /* NACL */
1062 # ifndef NACL_PARK_WAIT_USEC
1063 # define NACL_PARK_WAIT_USEC 100 /* us */
1064 # endif
1065 unsigned long num_sleeps = 0;
1066
1067 GC_ASSERT(I_HOLD_LOCK());
1068 # ifdef DEBUG_THREADS
1069 GC_log_printf("pthread_stop_world: number of threads: %d\n",
1070 GC_nacl_num_gc_threads - 1);
1071 # endif
1072 GC_nacl_thread_parker = pthread_self();
1073 GC_nacl_park_threads_now = 1;
1074
1075 if (GC_manual_vdb)
1076 GC_acquire_dirty_lock();
1077 for (;;) {
1078 int num_threads_parked = 0;
1079 int num_used = 0;
1080
1081 /* Check the "parked" flag for each thread the collector knows about. */
1082 for (i = 0; i < MAX_NACL_GC_THREADS && num_used < GC_nacl_num_gc_threads;
1083 i++) {
1084 if (GC_nacl_thread_used[i] == 1) {
1085 num_used++;
1086 if (GC_nacl_thread_parked[i] == 1) {
1087 num_threads_parked++;
1088 if (GC_on_thread_event)
1089 GC_on_thread_event(GC_EVENT_THREAD_SUSPENDED, NUMERIC_TO_VPTR(i));
1090 }
1091 }
1092 }
1093 /* Note: -1 for the current thread. */
1094 if (num_threads_parked >= GC_nacl_num_gc_threads - 1)
1095 break;
1096 # ifdef DEBUG_THREADS
1097 GC_log_printf("Sleep waiting for %d threads to park...\n",
1098 GC_nacl_num_gc_threads - num_threads_parked - 1);
1099 # endif
1100 GC_usleep(NACL_PARK_WAIT_USEC);
1101 if (++num_sleeps > (1000 * 1000) / NACL_PARK_WAIT_USEC) {
1102 WARN("GC appears stalled waiting for %" WARN_PRIdPTR
1103 " threads to park...\n",
1104 GC_nacl_num_gc_threads - num_threads_parked - 1);
1105 num_sleeps = 0;
1106 }
1107 }
1108 if (GC_manual_vdb)
1109 GC_release_dirty_lock();
1110 # endif /* NACL */
1111 return n_live_threads;
1112 }
1113
1114 GC_INNER void
1115 GC_stop_world(void)
1116 {
1117 # ifndef NACL
1118 int n_live_threads;
1119 # endif
1120 GC_ASSERT(I_HOLD_LOCK());
1121 /*
1122 * Make sure all free list construction has stopped before we start.
1123 * No new construction can start, since it is required to acquire and
1124 * release the allocator lock before start.
1125 */
1126
1127 GC_ASSERT(GC_thr_initialized);
1128 # ifdef DEBUG_THREADS
1129 GC_stopping_thread = pthread_self();
1130 GC_stopping_pid = getpid();
1131 GC_log_printf("Stopping the world from %p\n",
1132 PTHREAD_TO_VPTR(GC_stopping_thread));
1133 # endif
1134 # ifdef PARALLEL_MARK
1135 if (GC_parallel) {
1136 GC_acquire_mark_lock();
1137 GC_ASSERT(GC_fl_builder_count == 0);
1138 /* We should have previously waited for it to become zero. */
1139 }
1140 # endif /* PARALLEL_MARK */
1141
1142 # ifdef NACL
1143 (void)GC_suspend_all();
1144 # else
1145 /* Note: only concurrent reads are possible. */
1146 AO_store(&GC_stop_count, GC_stop_count + THREAD_RESTARTED);
1147 if (GC_manual_vdb) {
1148 GC_acquire_dirty_lock();
1149 /*
1150 * The write fault handler cannot be called if `GC_manual_vdb` (thus
1151 * double-locking should not occur in `async_set_pht_entry_from_index`
1152 * based on test-and-set).
1153 */
1154 }
1155 n_live_threads = GC_suspend_all();
1156 if (GC_retry_signals) {
1157 resend_lost_signals_retry(n_live_threads, GC_suspend_all);
1158 } else {
1159 suspend_restart_barrier(n_live_threads);
1160 }
1161 if (GC_manual_vdb) {
1162 /* Note: cannot be done in `GC_suspend_all`. */
1163 GC_release_dirty_lock();
1164 }
1165 # endif
1166
1167 # ifdef PARALLEL_MARK
1168 if (GC_parallel)
1169 GC_release_mark_lock();
1170 # endif
1171 # ifdef DEBUG_THREADS
1172 GC_log_printf("World stopped from %p\n", PTHREAD_TO_VPTR(pthread_self()));
1173 GC_stopping_thread = 0;
1174 # endif
1175 }
1176
1177 # ifdef NACL
1178 # if defined(__x86_64__)
1179 # define NACL_STORE_REGS() \
1180 do { \
1181 __asm__ __volatile__("push %rbx"); \
1182 __asm__ __volatile__("push %rbp"); \
1183 __asm__ __volatile__("push %r12"); \
1184 __asm__ __volatile__("push %r13"); \
1185 __asm__ __volatile__("push %r14"); \
1186 __asm__ __volatile__("push %r15"); \
1187 __asm__ __volatile__( \
1188 "mov %%esp, %0" \
1189 : "=m"(GC_nacl_gc_thread_self->crtn->stack_ptr)); \
1190 BCOPY(GC_nacl_gc_thread_self->crtn->stack_ptr, \
1191 GC_nacl_gc_thread_self->reg_storage, \
1192 NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t)); \
1193 __asm__ __volatile__("naclasp $48, %r15"); \
1194 } while (0)
1195 # elif defined(__i386__)
1196 # define NACL_STORE_REGS() \
1197 do { \
1198 __asm__ __volatile__("push %ebx"); \
1199 __asm__ __volatile__("push %ebp"); \
1200 __asm__ __volatile__("push %esi"); \
1201 __asm__ __volatile__("push %edi"); \
1202 __asm__ __volatile__( \
1203 "mov %%esp, %0" \
1204 : "=m"(GC_nacl_gc_thread_self->crtn->stack_ptr)); \
1205 BCOPY(GC_nacl_gc_thread_self->crtn->stack_ptr, \
1206 GC_nacl_gc_thread_self->reg_storage, \
1207 NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t)); \
1208 __asm__ __volatile__("add $16, %esp"); \
1209 } while (0)
1210 # elif defined(__arm__)
1211 # define NACL_STORE_REGS() \
1212 do { \
1213 __asm__ __volatile__("push {r4-r8,r10-r12,lr}"); \
1214 __asm__ __volatile__( \
1215 "mov r0, %0" \
1216 : \
1217 : "r"(&GC_nacl_gc_thread_self->crtn->stack_ptr)); \
1218 __asm__ __volatile__("bic r0, r0, #0xc0000000"); \
1219 __asm__ __volatile__("str sp, [r0]"); \
1220 BCOPY(GC_nacl_gc_thread_self->crtn->stack_ptr, \
1221 GC_nacl_gc_thread_self->reg_storage, \
1222 NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t)); \
1223 __asm__ __volatile__("add sp, sp, #40"); \
1224 __asm__ __volatile__("bic sp, sp, #0xc0000000"); \
1225 } while (0)
1226 # else
1227 # error TODO Please port NACL_STORE_REGS
1228 # endif
1229
1230 GC_API_OSCALL void
1231 nacl_pre_syscall_hook(void)
1232 {
1233 if (GC_nacl_thread_idx != -1) {
1234 NACL_STORE_REGS();
1235 GC_nacl_gc_thread_self->crtn->stack_ptr = GC_approx_sp();
1236 GC_nacl_thread_parked[GC_nacl_thread_idx] = 1;
1237 }
1238 }
1239
1240 GC_API_OSCALL void
1241 __nacl_suspend_thread_if_needed(void)
1242 {
1243 if (!GC_nacl_park_threads_now)
1244 return;
1245
1246 /* Do not try to park the thread parker. */
1247 if (GC_nacl_thread_parker == pthread_self())
1248 return;
1249
1250 /*
1251 * This can happen when a thread is created outside of the collector
1252 * system (`wthread` mostly).
1253 */
1254 if (GC_nacl_thread_idx < 0)
1255 return;
1256
1257 /*
1258 * If it was already "parked", we are returning from a `syscall`,
1259 * so do not bother storing registers again, the collector has a set of.
1260 */
1261 if (!GC_nacl_thread_parked[GC_nacl_thread_idx]) {
1262 NACL_STORE_REGS();
1263 GC_nacl_gc_thread_self->crtn->stack_ptr = GC_approx_sp();
1264 }
1265 GC_nacl_thread_parked[GC_nacl_thread_idx] = 1;
1266 while (GC_nacl_park_threads_now) {
1267 /* Just spin. */
1268 }
1269 GC_nacl_thread_parked[GC_nacl_thread_idx] = 0;
1270
1271 /* Clear out the `reg_storage` for the next suspend. */
1272 BZERO(GC_nacl_gc_thread_self->reg_storage,
1273 NACL_GC_REG_STORAGE_SIZE * sizeof(ptr_t));
1274 }
1275
1276 GC_API_OSCALL void
1277 nacl_post_syscall_hook(void)
1278 {
1279 /*
1280 * Calling `__nacl_suspend_thread_if_needed` right away should guarantee
1281 * we do not mutate the collector set.
1282 */
1283 __nacl_suspend_thread_if_needed();
1284 if (GC_nacl_thread_idx != -1) {
1285 GC_nacl_thread_parked[GC_nacl_thread_idx] = 0;
1286 }
1287 }
1288
1289 STATIC GC_bool GC_nacl_thread_parking_inited = FALSE;
1290 STATIC pthread_mutex_t GC_nacl_thread_alloc_lock = PTHREAD_MUTEX_INITIALIZER;
1291
1292 struct nacl_irt_blockhook {
1293 int (*register_block_hooks)(void (*pre)(void), void (*post)(void));
1294 };
1295
1296 EXTERN_C_BEGIN
1297 extern size_t nacl_interface_query(const char *interface_ident, void *table,
1298 size_t tablesize);
1299 EXTERN_C_END
1300
1301 GC_INNER void
1302 GC_nacl_initialize_gc_thread(GC_thread me)
1303 {
1304 int i;
1305 static struct nacl_irt_blockhook gc_hook;
1306
1307 GC_ASSERT(NULL == GC_nacl_gc_thread_self);
1308 GC_nacl_gc_thread_self = me;
1309 pthread_mutex_lock(&GC_nacl_thread_alloc_lock);
1310 if (UNLIKELY(!GC_nacl_thread_parking_inited)) {
1311 BZERO(GC_nacl_thread_parked, sizeof(GC_nacl_thread_parked));
1312 BZERO(GC_nacl_thread_used, sizeof(GC_nacl_thread_used));
1313 /*
1314 * TODO: Replace with the public "register hook" function when
1315 * available from `glibc`.
1316 */
1317 nacl_interface_query("nacl-irt-blockhook-0.1", &gc_hook, sizeof(gc_hook));
1318 gc_hook.register_block_hooks(nacl_pre_syscall_hook,
1319 nacl_post_syscall_hook);
1320 GC_nacl_thread_parking_inited = TRUE;
1321 }
1322 GC_ASSERT(GC_nacl_num_gc_threads <= MAX_NACL_GC_THREADS);
1323 for (i = 0; i < MAX_NACL_GC_THREADS; i++) {
1324 if (GC_nacl_thread_used[i] == 0) {
1325 GC_nacl_thread_used[i] = 1;
1326 GC_nacl_thread_idx = i;
1327 GC_nacl_num_gc_threads++;
1328 break;
1329 }
1330 }
1331 pthread_mutex_unlock(&GC_nacl_thread_alloc_lock);
1332 }
1333
1334 GC_INNER void
1335 GC_nacl_shutdown_gc_thread(void)
1336 {
1337 GC_ASSERT(GC_nacl_gc_thread_self != NULL);
1338 pthread_mutex_lock(&GC_nacl_thread_alloc_lock);
1339 GC_ASSERT(GC_nacl_thread_idx >= 0);
1340 GC_ASSERT(GC_nacl_thread_idx < MAX_NACL_GC_THREADS);
1341 GC_ASSERT(GC_nacl_thread_used[GC_nacl_thread_idx] != 0);
1342 GC_nacl_thread_used[GC_nacl_thread_idx] = 0;
1343 GC_nacl_thread_idx = -1;
1344 GC_nacl_num_gc_threads--;
1345 pthread_mutex_unlock(&GC_nacl_thread_alloc_lock);
1346 GC_nacl_gc_thread_self = NULL;
1347 }
1348
1349 # else /* !NACL */
1350
1351 static GC_bool in_resend_restart_signals;
1352
1353 /*
1354 * Restart all threads that were suspended by the collector.
1355 * Return the number of restart signals that were sent.
1356 */
1357 STATIC int
1358 GC_restart_all(void)
1359 {
1360 int n_live_threads = 0;
1361 int i;
1362 pthread_t self = pthread_self();
1363 GC_thread p;
1364 int result;
1365
1366 GC_ASSERT(I_HOLD_LOCK());
1367 GC_ASSERT((GC_stop_count & THREAD_RESTARTED) != 0);
1368 for (i = 0; i < THREAD_TABLE_SZ; i++) {
1369 for (p = GC_threads[i]; p != NULL; p = p->tm.next) {
1370 if (!THREAD_EQUAL(p->id, self)) {
1371 if ((p->flags & (FINISHED | DO_BLOCKING)) != 0)
1372 continue;
1373 # ifdef GC_ENABLE_SUSPEND_THREAD
1374 if ((p->ext_suspend_cnt & 1) != 0)
1375 continue;
1376 # endif
1377 if (GC_retry_signals
1378 && AO_load(&p->last_stop_count) == GC_stop_count) {
1379 /* The thread has been restarted. */
1380 if (!in_resend_restart_signals) {
1381 /*
1382 * Some user signal (which we do not block, e.g. `SIGQUIT`)
1383 * has already restarted the thread, but nonetheless we need to
1384 * count the latter in `n_live_threads`, so that to decrement
1385 * the semaphore's value proper amount of times. (We are also
1386 * sending the restart signal to the thread, it is not needed
1387 * actually but does not hurt.)
1388 */
1389 } else {
1390 continue;
1391 /*
1392 * FIXME: Still, an extremely low chance exists that the user
1393 * signal restarts the thread after the restart signal has been
1394 * lost (causing `sem_timedwait()` to fail) while retrying,
1395 * causing finally a mismatch between `GC_suspend_ack_sem` and
1396 * `n_live_threads`.
1397 */
1398 }
1399 }
1400 n_live_threads++;
1401 # ifdef DEBUG_THREADS
1402 GC_log_printf("Sending restart signal to %p\n",
1403 THREAD_ID_TO_VPTR(p->id));
1404 # endif
1405 result = raise_signal(p, GC_sig_thr_restart);
1406 switch (result) {
1407 case ESRCH:
1408 /* Not really there anymore. Possible? */
1409 n_live_threads--;
1410 break;
1411 case 0:
1412 if (GC_on_thread_event)
1413 GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED,
1414 THREAD_ID_TO_VPTR(THREAD_SYSTEM_ID(p)));
1415 break;
1416 default:
1417 ABORT_ARG1("pthread_kill failed at resume", ": errcode= %d", result);
1418 }
1419 }
1420 }
1421 }
1422 return n_live_threads;
1423 }
1424 # endif /* !NACL */
1425
1426 GC_INNER void
1427 GC_start_world(void)
1428 {
1429 # ifndef NACL
1430 int n_live_threads;
1431
1432 /* The allocator lock is held continuously since the world stopped. */
1433 GC_ASSERT(I_HOLD_LOCK());
1434
1435 # ifdef DEBUG_THREADS
1436 GC_log_printf("World starting\n");
1437 # endif
1438 /*
1439 * Note: the updated value should now be visible to the signal handler
1440 * (note that `pthread_kill` is not on the list of functions that
1441 * synchronize memory).
1442 */
1443 AO_store_release(&GC_stop_count, GC_stop_count + THREAD_RESTARTED);
1444
1445 GC_ASSERT(!in_resend_restart_signals);
1446 n_live_threads = GC_restart_all();
1447 if (GC_retry_signals) {
1448 in_resend_restart_signals = TRUE;
1449 resend_lost_signals_retry(n_live_threads, GC_restart_all);
1450 in_resend_restart_signals = FALSE;
1451 } else {
1452 # ifndef GC_NETBSD_THREADS_WORKAROUND
1453 if (GC_sig_suspend == GC_sig_thr_restart)
1454 # endif
1455 {
1456 suspend_restart_barrier(n_live_threads);
1457 }
1458 }
1459 # ifdef DEBUG_THREADS
1460 GC_log_printf("World started\n");
1461 # endif
1462 # else /* NACL */
1463 # ifdef DEBUG_THREADS
1464 GC_log_printf("World starting...\n");
1465 # endif
1466 GC_nacl_park_threads_now = 0;
1467 if (GC_on_thread_event) {
1468 GC_on_thread_event(GC_EVENT_THREAD_UNSUSPENDED, NULL);
1469 /* TODO: Send event for every unsuspended thread. */
1470 }
1471 # endif
1472 }
1473
1474 GC_INNER void
1475 GC_stop_init(void)
1476 {
1477 # ifndef NACL
1478 struct sigaction act;
1479 char *str;
1480
1481 if (SIGNAL_UNSET == GC_sig_suspend)
1482 GC_sig_suspend = SIG_SUSPEND;
1483 if (SIGNAL_UNSET == GC_sig_thr_restart)
1484 GC_sig_thr_restart = SIG_THR_RESTART;
1485
1486 if (sem_init(&GC_suspend_ack_sem, GC_SEM_INIT_PSHARED, 0) == -1)
1487 ABORT("sem_init failed");
1488 GC_stop_count = THREAD_RESTARTED; /*< i.e. the world is not stopped */
1489
1490 if (sigfillset(&act.sa_mask) != 0) {
1491 ABORT("sigfillset failed");
1492 }
1493 # ifdef RTEMS
1494 if (sigprocmask(SIG_UNBLOCK, &act.sa_mask, NULL) != 0) {
1495 ABORT("sigprocmask failed");
1496 }
1497 # endif
1498 GC_remove_allowed_signals(&act.sa_mask);
1499 /*
1500 * `GC_sig_thr_restart` is set in the resulting mask.
1501 * It is unmasked by the handler when necessary.
1502 */
1503
1504 # ifdef SA_RESTART
1505 act.sa_flags = SA_RESTART;
1506 # else
1507 act.sa_flags = 0;
1508 # endif
1509 # ifdef SUSPEND_HANDLER_NO_CONTEXT
1510 act.sa_handler = GC_suspend_handler;
1511 # else
1512 act.sa_flags |= SA_SIGINFO;
1513 act.sa_sigaction = GC_suspend_sigaction;
1514 # endif
1515 /* `act.sa_restorer` is deprecated and should not be initialized. */
1516 if (sigaction(GC_sig_suspend, &act, NULL) != 0) {
1517 ABORT("Cannot set SIG_SUSPEND handler");
1518 }
1519
1520 if (GC_sig_suspend != GC_sig_thr_restart) {
1521 # ifndef SUSPEND_HANDLER_NO_CONTEXT
1522 act.sa_flags &= ~SA_SIGINFO;
1523 # endif
1524 act.sa_handler = GC_restart_handler;
1525 if (sigaction(GC_sig_thr_restart, &act, NULL) != 0)
1526 ABORT("Cannot set SIG_THR_RESTART handler");
1527 } else {
1528 GC_COND_LOG_PRINTF("Using same signal for suspend and restart\n");
1529 }
1530
1531 /* Initialize `suspend_handler_mask` (excluding `GC_sig_thr_restart`). */
1532 if (sigfillset(&suspend_handler_mask) != 0)
1533 ABORT("sigfillset failed");
1534 GC_remove_allowed_signals(&suspend_handler_mask);
1535 if (sigdelset(&suspend_handler_mask, GC_sig_thr_restart) != 0)
1536 ABORT("sigdelset failed");
1537
1538 # ifndef NO_RETRY_SIGNALS
1539 /*
1540 * Any platform could lose signals, so let's be conservative and
1541 * always enable signals retry logic.
1542 */
1543 GC_retry_signals = TRUE;
1544 # endif
1545 /* Override the default value of `GC_retry_signals`. */
1546 str = GETENV("GC_RETRY_SIGNALS");
1547 if (str != NULL) {
1548 /* Do not retry if the environment variable is set to "0". */
1549 GC_retry_signals = *str != '0' || *(str + 1) != '\0';
1550 }
1551 if (GC_retry_signals) {
1552 GC_COND_LOG_PRINTF(
1553 "Will retry suspend and restart signals if necessary\n");
1554 }
1555 # ifndef NO_SIGNALS_UNBLOCK_IN_MAIN
1556 /* Explicitly unblock the signals once before new threads creation. */
1557 GC_unblock_gc_signals();
1558 # endif
1559 # endif /* !NACL */
1560 }
1561
1562 #endif /* PTHREAD_STOP_WORLD_IMPL */
1563