os_dep.c raw
1 /*
2 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
3 * Copyright (c) 1991-1995 by Xerox Corporation. All rights reserved.
4 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
5 * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved.
6 * Copyright (c) 2008-2022 Ivan Maidanski
7 *
8 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
9 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
10 *
11 * Permission is hereby granted to use or copy this program
12 * for any purpose, provided the above notices are retained on all copies.
13 * Permission to modify the code and to distribute modified code is granted,
14 * provided the above notices are retained, and a notice that the code was
15 * modified is included with the above copyright notice.
16 */
17
18 #include "private/gc_priv.h"
19
20 #if (defined(MPROTECT_VDB) && !defined(MSWIN32) && !defined(MSWINCE)) \
21 || (defined(SOLARIS) && defined(THREADS)) || defined(OPENBSD)
22 # include <signal.h>
23 #endif
24
25 #if defined(UNIX_LIKE) || defined(CYGWIN32) || defined(NACL) \
26 || defined(SYMBIAN)
27 # include <fcntl.h>
28 #endif
29
30 #ifdef LINUX
31 # include <ctype.h>
32 #endif
33
34 /*
35 * Blatantly OS-dependent routines, except for those that are related
36 * to dynamic loading.
37 */
38
39 #ifdef IRIX5
40 # include <malloc.h> /*< for locking */
41 # include <sys/uio.h>
42 #endif
43
44 #if defined(MMAP_SUPPORTED) || defined(ADD_HEAP_GUARD_PAGES)
45 # if defined(USE_MUNMAP) && !defined(USE_MMAP) && !defined(CPPCHECK)
46 # error Invalid config: USE_MUNMAP requires USE_MMAP
47 # endif
48 # include <sys/mman.h>
49 # include <sys/stat.h>
50 #endif
51
52 #if defined(LINUX) && defined(SPECIFIC_MAIN_STACKBOTTOM) \
53 || defined(ADD_HEAP_GUARD_PAGES) || defined(MMAP_SUPPORTED) \
54 || defined(NEED_PROC_MAPS)
55 # include <errno.h>
56 #endif
57
58 #if defined(DARWIN) && !defined(DYNAMIC_LOADING) \
59 && !defined(GC_DONT_REGISTER_MAIN_STATIC_DATA)
60 # include <mach-o/getsect.h> /*< for `get_etext` and friends */
61 #endif
62
63 #ifdef DJGPP
64 /*
65 * Apparently necessary for djgpp 2.01. May cause problems with
66 * other versions.
67 */
68 typedef long unsigned int caddr_t;
69 #endif
70
71 #if !defined(NO_EXECUTE_PERMISSION)
72 STATIC GC_bool GC_pages_executable = TRUE;
73 #else
74 STATIC GC_bool GC_pages_executable = FALSE;
75 #endif
76
77 /* Note: it is undefined later on `GC_pages_executable` real use. */
78 #define IGNORE_PAGES_EXECUTABLE 1
79
80 #if ((defined(LINUX) && defined(SPECIFIC_MAIN_STACKBOTTOM) \
81 || defined(NEED_PROC_MAPS) || defined(PROC_VDB) || defined(SOFT_VDB)) \
82 && !defined(PROC_READ)) \
83 || defined(CPPCHECK)
84 /* Note: should probably call the real `read()`, if later is wrapped. */
85 # define PROC_READ read
86 #endif
87
88 #if defined(LINUX) && defined(SPECIFIC_MAIN_STACKBOTTOM) \
89 || defined(NEED_PROC_MAPS)
90 /*
91 * Repeatedly perform a `read()` call until the buffer is filled up,
92 * or we encounter EOF (end of file) or an error.
93 */
94 STATIC ssize_t
95 GC_repeat_read(int f, char *buf, size_t count)
96 {
97 size_t num_read = 0;
98
99 ASSERT_CANCEL_DISABLED();
100 while (num_read < count) {
101 ssize_t result = PROC_READ(f, buf + num_read, count - num_read);
102
103 if (result < 0)
104 return result;
105 if (0 == result)
106 break;
107 # ifdef LINT2
108 if ((size_t)result > count - num_read)
109 ABORT("read() result cannot be bigger than requested length");
110 # endif
111 num_read += (size_t)result;
112 }
113 return num_read;
114 }
115 #endif /* LINUX && SPECIFIC_MAIN_STACKBOTTOM || NEED_PROC_MAPS */
116
117 #ifdef NEED_PROC_MAPS
118 /*
119 * We need to parse `/proc/self/maps` pseudo-file, either to find
120 * dynamic libraries, and/or to find the register backing store
121 * base (the IA-64 case). Do it once here.
122 */
123
124 # ifndef SINGLE_THREADED_PROCESS
125 /*
126 * Determine the length of a file by incrementally reading it into a buffer.
127 * This would be silly to use it on a file supporting `lseek`, but Linux
128 * `/proc` files usually do not. As of Linux 4.15.0, `lseek(SEEK_END)` fails
129 * for `/proc/self/maps` file.
130 */
131 STATIC size_t
132 GC_get_file_len(int f)
133 {
134 size_t total = 0;
135 # define GET_FILE_LEN_BUF_SZ 500
136 char buf[GET_FILE_LEN_BUF_SZ];
137
138 ASSERT_CANCEL_DISABLED();
139 for (;;) {
140 ssize_t result = PROC_READ(f, buf, sizeof(buf));
141
142 if (result < 0) {
143 /* An error has occurred. */
144 return 0;
145 }
146 if (0 == result)
147 break;
148 # ifdef LINT2
149 if ((size_t)result >= GC_SIZE_MAX - total)
150 ABORT("Too big file is passed to GC_get_file_len");
151 # endif
152 total += (size_t)result;
153 }
154 return total;
155 }
156
157 STATIC size_t
158 GC_get_maps_len(void)
159 {
160 int f = open("/proc/self/maps", O_RDONLY);
161 size_t result;
162
163 if (f < 0) {
164 /* Treat missing file as empty. */
165 return 0;
166 }
167 result = GC_get_file_len(f);
168 close(f);
169 return result;
170 }
171 # endif /* !SINGLE_THREADED_PROCESS */
172
173 GC_INNER const char *
174 GC_get_maps(void)
175 {
176 ssize_t result;
177 static char *maps_buf = NULL;
178 static size_t maps_buf_sz = 1;
179 size_t maps_size;
180 # ifndef SINGLE_THREADED_PROCESS
181 size_t old_maps_size = 0;
182 # endif
183
184 /* The buffer is essentially `static`, so there must be a single client. */
185 GC_ASSERT(I_HOLD_LOCK());
186
187 /*
188 * Note that in the presence of threads in the process (even if the
189 * collector itself is built single-threaded), the `maps` file can
190 * essentially shrink asynchronously and unexpectedly as threads
191 * that we already think of as dead release their stacks.
192 * And there is no easy way to read the entire file atomically.
193 * This is arguably a misfeature of the `/proc/self/maps` interface.
194 * Since we expect the file can grow asynchronously in rare cases,
195 * it should suffice to first determine the size (using `read()`),
196 * and then to reread the file. If the size is inconsistent, then
197 * we have to retry. This only matters with threads enabled, and
198 * if we use this to locate the data roots (not the default).
199 */
200
201 # ifndef SINGLE_THREADED_PROCESS
202 /* Determine the initial size of `/proc/self/maps` file. */
203 maps_size = GC_get_maps_len();
204 if (0 == maps_size)
205 ABORT("Cannot determine length of /proc/self/maps");
206 # else
207 maps_size = 4000; /*< guess */
208 # endif
209
210 /*
211 * Read `/proc/self/maps` file, growing `maps_buf` as necessary.
212 * Note that we may not allocate conventionally, and thus cannot
213 * use `stdio` functionality.
214 */
215 do {
216 int f;
217
218 while (maps_size >= maps_buf_sz) {
219 # ifdef LINT2
220 /* Workaround passing tainted `maps_buf` to a tainted sink. */
221 GC_noop1_ptr(maps_buf);
222 # else
223 GC_scratch_recycle_no_gww(maps_buf, maps_buf_sz);
224 # endif
225 /* Grow only by powers of 2, since we leak "too small" buffers. */
226 while (maps_size >= maps_buf_sz)
227 maps_buf_sz *= 2;
228 maps_buf = GC_scratch_alloc(maps_buf_sz);
229 if (NULL == maps_buf)
230 ABORT_ARG1("Insufficient space for /proc/self/maps buffer",
231 ", %lu bytes requested", (unsigned long)maps_buf_sz);
232 # ifndef SINGLE_THREADED_PROCESS
233 /*
234 * Recompute initial length, since we allocated.
235 * This can only happen a few times per program execution.
236 */
237 maps_size = GC_get_maps_len();
238 if (0 == maps_size)
239 ABORT("Cannot determine length of /proc/self/maps");
240 # endif
241 }
242 GC_ASSERT(maps_buf_sz >= maps_size + 1);
243 f = open("/proc/self/maps", O_RDONLY);
244 if (-1 == f)
245 ABORT_ARG1("Cannot open /proc/self/maps", ": errno= %d", errno);
246 # ifndef SINGLE_THREADED_PROCESS
247 old_maps_size = maps_size;
248 # endif
249 maps_size = 0;
250 do {
251 result = GC_repeat_read(f, maps_buf, maps_buf_sz - 1);
252 if (result < 0) {
253 ABORT_ARG1("Failed to read /proc/self/maps", ": errno= %d", errno);
254 }
255 maps_size += (size_t)result;
256 } while ((size_t)result == maps_buf_sz - 1);
257 close(f);
258 if (0 == maps_size)
259 ABORT("Empty /proc/self/maps");
260 # ifndef SINGLE_THREADED_PROCESS
261 if (maps_size > old_maps_size) {
262 /* This might be caused by e.g. thread creation. */
263 WARN("Unexpected asynchronous /proc/self/maps growth"
264 " (to %" WARN_PRIuPTR " bytes)\n",
265 maps_size);
266 }
267 # endif
268 } while (maps_size >= maps_buf_sz
269 # ifndef SINGLE_THREADED_PROCESS
270 || maps_size < old_maps_size
271 # endif
272 );
273 maps_buf[maps_size] = '\0';
274 return maps_buf;
275 }
276
277 /*
278 * `GC_parse_map_entry` parses an entry from `/proc/self/maps` file so we
279 * can locate all writable data segments that belong to shared libraries.
280 * The format of one of these entries and the fields we care about
281 * is as follows:
282 * ```
283 * XXXXXXXX-XXXXXXXX r-xp 00000000 30:05 260537 name-of-mapping...\n
284 * ^^^^^^^^ ^^^^^^^^ ^^^^ ^^
285 * *p_start *p_end *p_prot *p_maj_dev
286 * ```
287 *
288 * Note that since about August 2003 kernels, the columns no longer have
289 * fixed offsets on 64-bit kernels. Hence we no longer rely on fixed
290 * offsets anywhere, which is safer anyway.
291 */
292
293 # if defined(DYNAMIC_LOADING) && defined(USE_PROC_FOR_LIBRARIES) \
294 || defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) \
295 || (defined(CHECK_SOFT_VDB) && defined(MPROTECT_VDB)) \
296 || defined(REDIR_MALLOC_AND_LINUXTHREADS)
297 GC_INNER const char *
298 GC_parse_map_entry(const char *maps_ptr, ptr_t *p_start, ptr_t *p_end,
299 const char **p_prot, unsigned *p_maj_dev,
300 const char **p_mapping_name)
301 {
302 const unsigned char *start_start, *end_start, *maj_dev_start;
303 const unsigned char *p; /*< unsigned for `isspace`, `isxdigit` */
304
305 if (maps_ptr == NULL || *maps_ptr == '\0') {
306 return NULL;
307 }
308
309 p = (const unsigned char *)maps_ptr;
310 while (isspace(*p))
311 ++p;
312 start_start = p;
313 GC_ASSERT(isxdigit(*start_start));
314 *p_start = (ptr_t)strtoul((const char *)start_start, (char **)&p, 16);
315 GC_ASSERT(*p == '-');
316
317 ++p;
318 end_start = p;
319 GC_ASSERT(isxdigit(*end_start));
320 *p_end = (ptr_t)strtoul((const char *)end_start, (char **)&p, 16);
321 GC_ASSERT(isspace(*p));
322
323 while (isspace(*p))
324 ++p;
325 GC_ASSERT(*p == 'r' || *p == '-');
326 *p_prot = (const char *)p;
327 /* Skip past protection field to offset field. */
328 while (!isspace(*p))
329 ++p;
330 while (isspace(*p))
331 p++;
332 GC_ASSERT(isxdigit(*p));
333 /* Skip past offset field, which we ignore. */
334 while (!isspace(*p))
335 ++p;
336 while (isspace(*p))
337 p++;
338 maj_dev_start = p;
339 GC_ASSERT(isxdigit(*maj_dev_start));
340 *p_maj_dev = strtoul((const char *)maj_dev_start, NULL, 16);
341
342 if (p_mapping_name != NULL) {
343 while (*p && *p != '\n' && *p != '/' && *p != '[')
344 p++;
345 *p_mapping_name = (const char *)p;
346 }
347 while (*p && *p++ != '\n') {
348 /* Empty. */
349 }
350 return (const char *)p;
351 }
352 # endif /* REDIRECT_MALLOC || DYNAMIC_LOADING || IA64 || ... */
353
354 # if defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) \
355 || (defined(CHECK_SOFT_VDB) && defined(MPROTECT_VDB))
356 GC_INNER GC_bool
357 GC_enclosing_writable_mapping(ptr_t addr, ptr_t *startp, ptr_t *endp)
358 {
359 const char *prot;
360 ptr_t my_start, my_end;
361 const char *maps_ptr;
362 unsigned maj_dev;
363
364 GC_ASSERT(I_HOLD_LOCK());
365 maps_ptr = GC_get_maps();
366 for (;;) {
367 maps_ptr = GC_parse_map_entry(maps_ptr, &my_start, &my_end, &prot,
368 &maj_dev, NULL);
369 if (NULL == maps_ptr)
370 break;
371
372 if (ADDR_INSIDE(addr, my_start, my_end)) {
373 if (prot[1] != 'w' || maj_dev != 0)
374 break;
375 *startp = my_start;
376 *endp = my_end;
377 return TRUE;
378 }
379 }
380 return FALSE;
381 }
382 # endif /* IA64 || INCLUDE_LINUX_THREAD_DESCR */
383
384 # ifdef REDIR_MALLOC_AND_LINUXTHREADS
385 GC_INNER GC_bool
386 GC_text_mapping(const char *nm, ptr_t *startp, ptr_t *endp)
387 {
388 size_t nm_len;
389 const char *prot, *map_path;
390 ptr_t my_start, my_end;
391 unsigned int maj_dev;
392 const char *maps_ptr;
393
394 GC_ASSERT(I_HOLD_LOCK());
395 maps_ptr = GC_get_maps();
396 nm_len = strlen(nm);
397 for (;;) {
398 maps_ptr = GC_parse_map_entry(maps_ptr, &my_start, &my_end, &prot,
399 &maj_dev, &map_path);
400 if (NULL == maps_ptr)
401 break;
402
403 if (prot[0] == 'r' && prot[1] == '-' && prot[2] == 'x') {
404 const char *p = map_path;
405
406 /* Set `p` to point just past last slash, if any. */
407 while (*p != '\0' && *p != '\n' && *p != ' ' && *p != '\t') {
408 ++p;
409 }
410 while (ADDR_GE((ptr_t)p, (ptr_t)map_path) && *p != '/') {
411 --p;
412 }
413 ++p;
414
415 if (strncmp(nm, p, nm_len) == 0) {
416 *startp = my_start;
417 *endp = my_end;
418 return TRUE;
419 }
420 }
421 }
422 return FALSE;
423 }
424 # endif /* REDIR_MALLOC_AND_LINUXTHREADS */
425
426 # ifdef IA64
427 static ptr_t
428 backing_store_base_from_proc(void)
429 {
430 ptr_t my_start, my_end;
431
432 GC_ASSERT(I_HOLD_LOCK());
433 if (!GC_enclosing_writable_mapping(GC_save_regs_in_stack(), &my_start,
434 &my_end)) {
435 GC_COND_LOG_PRINTF("Failed to find backing store base from /proc\n");
436 return 0;
437 }
438 return my_start;
439 }
440 # endif
441
442 #endif /* NEED_PROC_MAPS */
443
444 #if defined(SEARCH_FOR_DATA_START)
445 /*
446 * The i686 case can be handled without a search. The Alpha case used to
447 * be handled differently as well, but the rules changed for recent Linux
448 * versions. This seems to be the easiest way to cover all versions.
449 */
450
451 # if defined(LINUX) || defined(HURD)
452 /*
453 * Some Linux distributions arrange to define `__data_start`.
454 * Some define `data_start` as a weak symbol. The latter is technically
455 * broken, since the user program may define `data_start`, in which
456 * case we lose. Nonetheless, we try both, preferring `__data_start`.
457 * We assume gcc-compatible pragmas.
458 */
459 EXTERN_C_BEGIN
460 # pragma weak __data_start
461 # pragma weak data_start
462 extern int __data_start[], data_start[];
463 EXTERN_C_END
464 # elif defined(NETBSD)
465 EXTERN_C_BEGIN
466 extern char **environ;
467 EXTERN_C_END
468 # endif
469
470 ptr_t GC_data_start = NULL;
471
472 GC_INNER void
473 GC_init_linux_data_start(void)
474 {
475 ptr_t data_end = DATAEND;
476
477 # if (defined(LINUX) || defined(HURD)) && defined(USE_PROG_DATA_START)
478 /*
479 * Try the easy approaches first. However, this may lead to wrong
480 * data start value if the collector code is put into a shared library
481 * (directly or indirectly) which is linked with `-Bsymbolic-functions`
482 * option. Thus, the following is not used by default.
483 */
484 if (COVERT_DATAFLOW(ADDR(__data_start)) != 0) {
485 GC_data_start = (ptr_t)(__data_start);
486 } else {
487 GC_data_start = (ptr_t)(data_start);
488 }
489 if (COVERT_DATAFLOW(ADDR(GC_data_start)) != 0) {
490 if (ADDR_LT(data_end, GC_data_start))
491 ABORT_ARG2("Wrong __data_start/_end pair", ": %p .. %p",
492 (void *)GC_data_start, (void *)data_end);
493 return;
494 }
495 # ifdef DEBUG_ADD_DEL_ROOTS
496 GC_log_printf("__data_start not provided\n");
497 # endif
498 # endif /* LINUX */
499
500 if (GC_no_dls) {
501 /*
502 * Not needed, avoids the `SIGSEGV` caused by `GC_find_limit` which
503 * complicates debugging.
504 */
505 GC_data_start = data_end; /*< set data root size to 0 */
506 return;
507 }
508
509 # ifdef NETBSD
510 /*
511 * This may need to be `environ`, without the underscore, for
512 * some versions.
513 */
514 GC_data_start = (ptr_t)GC_find_limit(&environ, FALSE);
515 # else
516 GC_data_start = (ptr_t)GC_find_limit(data_end, FALSE);
517 # endif
518 }
519 #endif /* SEARCH_FOR_DATA_START */
520
521 #ifdef ECOS
522
523 # ifndef ECOS_GC_MEMORY_SIZE
524 # define ECOS_GC_MEMORY_SIZE (448 * 1024)
525 # endif /* ECOS_GC_MEMORY_SIZE */
526
527 /*
528 * TODO: This is a simple way of allocating memory which is
529 * compatible with ECOS early releases. Later releases use a more
530 * sophisticated means of allocating memory than this simple static
531 * allocator, but this method is at least bound to work.
532 */
533 static char ecos_gc_memory[ECOS_GC_MEMORY_SIZE];
534 static ptr_t ecos_gc_brk = ecos_gc_memory;
535
536 static void *
537 tiny_sbrk(ptrdiff_t increment)
538 {
539 void *p = ecos_gc_brk;
540
541 if (ADDR_LT((ptr_t)ecos_gc_memory + sizeof(ecos_gc_memory),
542 (ptr_t)p + increment))
543 return NULL;
544 ecos_gc_brk += increment;
545 return p;
546 }
547 # define sbrk tiny_sbrk
548 #endif /* ECOS */
549
550 #if defined(ADDRESS_SANITIZER) \
551 && (defined(UNIX_LIKE) || defined(NEED_FIND_LIMIT) \
552 || defined(MPROTECT_VDB)) \
553 && !defined(CUSTOM_ASAN_DEF_OPTIONS)
554 EXTERN_C_BEGIN
555 GC_API const char *__asan_default_options(void);
556 EXTERN_C_END
557
558 /*
559 * To tell ASan to allow the collector to use its own `SIGBUS` and `SIGSEGV`
560 * handlers. The function is exported just to be visible to ASan library.
561 */
562 GC_API const char *
563 __asan_default_options(void)
564 {
565 return "allow_user_segv_handler=1";
566 }
567 #endif
568
569 #ifdef OPENBSD
570 static struct sigaction old_segv_act;
571 STATIC JMP_BUF GC_jmp_buf_openbsd;
572
573 STATIC void
574 GC_fault_handler_openbsd(int sig)
575 {
576 UNUSED_ARG(sig);
577 LONGJMP(GC_jmp_buf_openbsd, 1);
578 }
579
580 static volatile int firstpass;
581
582 /*
583 * Return first addressable location that is greater than `p` or return
584 * `bound`.
585 */
586 STATIC ptr_t
587 GC_skip_hole_openbsd(ptr_t p, ptr_t bound)
588 {
589 static volatile ptr_t result;
590 struct sigaction act;
591 size_t pgsz;
592
593 GC_ASSERT(I_HOLD_LOCK());
594 pgsz = (size_t)sysconf(_SC_PAGESIZE);
595 GC_ASSERT(ADDR(bound) >= (word)pgsz);
596
597 act.sa_handler = GC_fault_handler_openbsd;
598 sigemptyset(&act.sa_mask);
599 act.sa_flags = SA_NODEFER | SA_RESTART;
600 /* `act.sa_restorer` is deprecated and should not be initialized. */
601 sigaction(SIGSEGV, &act, &old_segv_act);
602
603 firstpass = 1;
604 result = PTR_ALIGN_DOWN(p, pgsz);
605 if (SETJMP(GC_jmp_buf_openbsd) != 0 || firstpass) {
606 firstpass = 0;
607 if (ADDR_GE(result, bound - pgsz)) {
608 result = bound;
609 } else {
610 /*
611 * Notes: no overflow is expected; do not use compound assignment
612 * with `volatile`-qualified left operand.
613 */
614 result = result + pgsz;
615 GC_noop1((word)(unsigned char)(*result));
616 }
617 }
618
619 sigaction(SIGSEGV, &old_segv_act, 0);
620 return result;
621 }
622 #endif /* OPENBSD */
623
624 #ifdef OS2
625
626 # include <stddef.h>
627
628 # if !defined(__IBMC__) && !defined(__WATCOMC__) /*< e.g. EMX */
629
630 struct exe_hdr {
631 unsigned short magic_number;
632 unsigned short padding[29];
633 long new_exe_offset;
634 };
635
636 # define E_MAGIC(x) (x).magic_number
637 # define EMAGIC 0x5A4D
638 # define E_LFANEW(x) (x).new_exe_offset
639
640 struct e32_exe {
641 unsigned char magic_number[2];
642 unsigned char byte_order;
643 unsigned char word_order;
644 unsigned long exe_format_level;
645 unsigned short cpu;
646 unsigned short os;
647 unsigned long padding1[13];
648 unsigned long object_table_offset;
649 unsigned long object_count;
650 unsigned long padding2[31];
651 };
652
653 # define E32_MAGIC1(x) (x).magic_number[0]
654 # define E32MAGIC1 'L'
655 # define E32_MAGIC2(x) (x).magic_number[1]
656 # define E32MAGIC2 'X'
657 # define E32_BORDER(x) (x).byte_order
658 # define E32LEBO 0
659 # define E32_WORDER(x) (x).word_order
660 # define E32LEWO 0
661 # define E32_CPU(x) (x).cpu
662 # define E32CPU286 1
663 # define E32_OBJTAB(x) (x).object_table_offset
664 # define E32_OBJCNT(x) (x).object_count
665
666 struct o32_obj {
667 unsigned long size;
668 unsigned long base;
669 unsigned long flags;
670 unsigned long pagemap;
671 unsigned long mapsize;
672 unsigned long reserved;
673 };
674
675 # define O32_FLAGS(x) (x).flags
676 # define OBJREAD 0x0001L
677 # define OBJWRITE 0x0002L
678 # define OBJINVALID 0x0080L
679 # define O32_SIZE(x) (x).size
680 # define O32_BASE(x) (x).base
681
682 # else /* IBM's compiler */
683
684 /* A kludge to get around what appears to be a header file bug. */
685 # ifndef WORD
686 # define WORD unsigned short
687 # endif
688 # ifndef DWORD
689 # define DWORD unsigned long
690 # endif
691
692 # define EXE386 1
693 # include <exe386.h>
694 # include <newexe.h>
695
696 # endif /* __IBMC__ */
697
698 # define INCL_DOSERRORS
699 # define INCL_DOSEXCEPTIONS
700 # define INCL_DOSFILEMGR
701 # define INCL_DOSMEMMGR
702 # define INCL_DOSMISC
703 # define INCL_DOSMODULEMGR
704 # define INCL_DOSPROCESS
705 # include <os2.h>
706
707 #endif /* OS2 */
708
709 GC_INNER size_t GC_page_size = 0;
710 #ifdef REAL_PAGESIZE_NEEDED
711 GC_INNER size_t GC_real_page_size = 0;
712 #endif
713
714 #ifdef SOFT_VDB
715 STATIC unsigned GC_log_pagesize = 0;
716 #endif
717
718 #ifdef ANY_MSWIN
719
720 # ifndef VER_PLATFORM_WIN32_CE
721 # define VER_PLATFORM_WIN32_CE 3
722 # endif
723
724 # if defined(MSWINCE) && defined(THREADS)
725 GC_INNER GC_bool GC_dont_query_stack_min = FALSE;
726 # endif
727
728 GC_INNER SYSTEM_INFO GC_sysinfo;
729
730 # ifndef CYGWIN32
731 # define is_writable(prot) \
732 ((prot) == PAGE_READWRITE || (prot) == PAGE_WRITECOPY \
733 || (prot) == PAGE_EXECUTE_READWRITE \
734 || (prot) == PAGE_EXECUTE_WRITECOPY)
735 /*
736 * Return the number of bytes that are writable starting at `p`.
737 * The pointer `p` is assumed to be page-aligned. If `base` is not `NULL`,
738 * then `*base` becomes the beginning of the allocation region containing `p`.
739 */
740 STATIC word
741 GC_get_writable_length(ptr_t p, ptr_t *base)
742 {
743 MEMORY_BASIC_INFORMATION buf;
744 word result;
745 word protect;
746
747 result = VirtualQuery(p, &buf, sizeof(buf));
748 if (result != sizeof(buf))
749 ABORT("Weird VirtualQuery result");
750 if (base != 0)
751 *base = (ptr_t)(buf.AllocationBase);
752 protect = buf.Protect & ~(word)(PAGE_GUARD | PAGE_NOCACHE);
753 if (!is_writable(protect) || buf.State != MEM_COMMIT)
754 return 0;
755 return buf.RegionSize;
756 }
757
758 GC_API int GC_CALL
759 GC_get_stack_base(struct GC_stack_base *sb)
760 {
761 /*
762 * Note: this function should not acquire the allocator lock as it is
763 * used by `GC_DllMain`.
764 */
765 ptr_t trunc_sp;
766 word size;
767
768 /*
769 * Set page size if it is not ready (so client can use this function even
770 * before the collector is initialized).
771 */
772 if (!GC_page_size)
773 GC_setpagesize();
774
775 trunc_sp = PTR_ALIGN_DOWN(GC_approx_sp(), GC_page_size);
776 /*
777 * FIXME: This will not work if called from a deeply recursive
778 * client code (and the committed stack space has grown).
779 */
780 size = GC_get_writable_length(trunc_sp, 0);
781 GC_ASSERT(size != 0);
782 sb->mem_base = trunc_sp + size;
783 return GC_SUCCESS;
784 }
785 # else /* CYGWIN32 */
786 GC_API int GC_CALL
787 GC_get_stack_base(struct GC_stack_base *sb)
788 {
789 /*
790 * An alternate variant for Cygwin (adapted from Dave Korn's gcc version
791 * of boehm-gc).
792 */
793 # ifdef X86_64
794 sb->mem_base = ((NT_TIB *)NtCurrentTeb())->StackBase;
795 # else
796 void *_tlsbase;
797
798 __asm__("movl %%fs:4, %0" : "=r"(_tlsbase));
799 sb->mem_base = _tlsbase;
800 # endif
801 return GC_SUCCESS;
802 }
803 # endif /* CYGWIN32 */
804 # define HAVE_GET_STACK_BASE
805
806 #elif defined(OS2)
807
808 static int
809 os2_getpagesize(void)
810 {
811 ULONG result[1];
812
813 if (DosQuerySysInfo(QSV_PAGE_SIZE, QSV_PAGE_SIZE, (void *)result,
814 sizeof(ULONG))
815 != NO_ERROR) {
816 WARN("DosQuerySysInfo failed\n", 0);
817 result[0] = 4096;
818 }
819 return (int)result[0];
820 }
821
822 #endif /* !ANY_MSWIN && OS2 */
823
824 GC_INNER void
825 GC_setpagesize(void)
826 {
827 #ifdef ANY_MSWIN
828 GetSystemInfo(&GC_sysinfo);
829 # ifdef ALT_PAGESIZE_USED
830 /*
831 * Allocations made with `mmap()` are aligned to the allocation
832 * granularity, which (at least on Win64) is not the same as the
833 * page size. Probably we could distinguish the allocation
834 * granularity from the actual page size, but in practice there
835 * is no good reason to make allocations smaller than
836 * `dwAllocationGranularity`, so we just use it instead of the
837 * actual page size here (as Cygwin itself does in many cases).
838 */
839 GC_page_size = (size_t)GC_sysinfo.dwAllocationGranularity;
840 # ifdef REAL_PAGESIZE_NEEDED
841 GC_real_page_size = (size_t)GC_sysinfo.dwPageSize;
842 GC_ASSERT(GC_page_size >= GC_real_page_size);
843 # endif
844 # else
845 GC_page_size = (size_t)GC_sysinfo.dwPageSize;
846 # endif
847 # if defined(MSWINCE) && !defined(_WIN32_WCE_EMULATION)
848 {
849 OSVERSIONINFO verInfo;
850 /* Check the current WinCE version. */
851 verInfo.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
852 if (!GetVersionEx(&verInfo))
853 ABORT("GetVersionEx failed");
854 if (verInfo.dwPlatformId == VER_PLATFORM_WIN32_CE
855 && verInfo.dwMajorVersion < 6) {
856 /*
857 * Only the first 32 MB of address space belongs to the
858 * current process (unless WinCE 6.0+ or emulation).
859 */
860 GC_sysinfo.lpMaximumApplicationAddress = (LPVOID)((word)32 << 20);
861 # ifdef THREADS
862 /*
863 * On some old WinCE versions, it is observed that
864 * `VirtualQuery()` calls do not work properly when used to
865 * get thread current stack committed minimum.
866 */
867 if (verInfo.dwMajorVersion < 5)
868 GC_dont_query_stack_min = TRUE;
869 # endif
870 }
871 }
872 # endif
873 #else
874 # ifdef ALT_PAGESIZE_USED
875 # ifdef REAL_PAGESIZE_NEEDED
876 GC_real_page_size = (size_t)GETPAGESIZE();
877 # endif
878 /* It is acceptable to fake it. */
879 GC_page_size = HBLKSIZE;
880 # else
881 GC_page_size = (size_t)GETPAGESIZE();
882 # if !defined(CPPCHECK)
883 if (0 == GC_page_size)
884 ABORT("getpagesize failed");
885 # endif
886 # endif
887 #endif /* !ANY_MSWIN */
888 #ifdef SOFT_VDB
889 {
890 size_t pgsize;
891 unsigned log_pgsize = 0;
892
893 # if !defined(CPPCHECK)
894 if (((GC_page_size - 1) & GC_page_size) != 0) {
895 /* Not a power of two. */
896 ABORT("Invalid page size");
897 }
898 # endif
899 for (pgsize = GC_page_size; pgsize > 1; pgsize >>= 1)
900 log_pgsize++;
901 GC_log_pagesize = log_pgsize;
902 }
903 #endif
904 }
905
906 #ifdef EMBOX
907 # include <kernel/thread/thread_stack.h>
908 # include <pthread.h>
909
910 GC_API int GC_CALL
911 GC_get_stack_base(struct GC_stack_base *sb)
912 {
913 pthread_t self = pthread_self();
914 void *stack_addr = thread_stack_get(self);
915
916 /* TODO: Use `pthread_getattr_np`, `pthread_attr_getstack` alternatively. */
917 # ifdef STACK_GROWS_UP
918 sb->mem_base = stack_addr;
919 # else
920 sb->mem_base = (ptr_t)stack_addr + thread_stack_get_size(self);
921 # endif
922 return GC_SUCCESS;
923 }
924 # define HAVE_GET_STACK_BASE
925 #endif /* EMBOX */
926
927 #ifdef OS2
928 GC_API int GC_CALL
929 GC_get_stack_base(struct GC_stack_base *sb)
930 {
931 PTIB ptib; /*< thread information block */
932 PPIB ppib;
933
934 if (DosGetInfoBlocks(&ptib, &ppib) != NO_ERROR) {
935 WARN("DosGetInfoBlocks failed\n", 0);
936 return GC_UNIMPLEMENTED;
937 }
938 sb->mem_base = ptib->tib_pstacklimit;
939 return GC_SUCCESS;
940 }
941 # define HAVE_GET_STACK_BASE
942 #endif /* OS2 */
943
944 #ifdef SERENITY
945 # include <serenity.h>
946
947 GC_API int GC_CALL
948 GC_get_stack_base(struct GC_stack_base *sb)
949 {
950 uintptr_t base;
951 size_t size;
952
953 if (get_stack_bounds(&base, &size) < 0) {
954 WARN("get_stack_bounds failed\n", 0);
955 return GC_UNIMPLEMENTED;
956 }
957 sb->mem_base = base + size;
958 return GC_SUCCESS;
959 }
960 # define HAVE_GET_STACK_BASE
961 #endif /* SERENITY */
962
963 #if defined(NEED_FIND_LIMIT) \
964 || (defined(UNIX_LIKE) && !defined(NO_DEBUGGING)) \
965 || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) \
966 || (defined(WRAP_MARK_SOME) && defined(NO_SEH_AVAILABLE))
967
968 # include <signal.h>
969
970 # ifdef USE_SEGV_SIGACT
971 # ifndef OPENBSD
972 static struct sigaction old_segv_act;
973 # endif
974 # ifdef USE_BUS_SIGACT
975 static struct sigaction old_bus_act;
976 # endif
977 # else
978 static GC_fault_handler_t old_segv_hand;
979 # ifdef HAVE_SIGBUS
980 static GC_fault_handler_t old_bus_hand;
981 # endif
982 # endif /* !USE_SEGV_SIGACT */
983
984 GC_INNER void
985 GC_set_and_save_fault_handler(GC_fault_handler_t h)
986 {
987 # ifdef USE_SEGV_SIGACT
988 struct sigaction act;
989
990 act.sa_handler = h;
991 # ifdef SIGACTION_FLAGS_NODEFER_HACK
992 /* Was necessary for Solaris 2.3 and very temporary NetBSD bugs. */
993 act.sa_flags = SA_RESTART | SA_NODEFER;
994 # else
995 act.sa_flags = SA_RESTART;
996 # endif
997
998 (void)sigemptyset(&act.sa_mask);
999 /* `act.sa_restorer` is deprecated and should not be initialized. */
1000 # if defined(IRIX5) && defined(THREADS)
1001 /*
1002 * Older versions have a bug related to retrieving and setting
1003 * a handler at the same time.
1004 */
1005 (void)sigaction(SIGSEGV, 0, &old_segv_act);
1006 (void)sigaction(SIGSEGV, &act, 0);
1007 # else
1008 (void)sigaction(SIGSEGV, &act, &old_segv_act);
1009 # ifdef USE_BUS_SIGACT
1010 /*
1011 * `pthreads` library does not exist under Irix 5.x, so we do not have
1012 * to worry of the multi-threaded case.
1013 */
1014 (void)sigaction(SIGBUS, &act, &old_bus_act);
1015 # endif
1016 # endif /* !IRIX5 || !THREADS */
1017 # else
1018 old_segv_hand = signal(SIGSEGV, h);
1019 # ifdef HAVE_SIGBUS
1020 old_bus_hand = signal(SIGBUS, h);
1021 # endif
1022 # endif /* !USE_SEGV_SIGACT */
1023 # if defined(CPPCHECK) && defined(ADDRESS_SANITIZER)
1024 GC_noop1((word)(GC_funcptr_uint)(&__asan_default_options));
1025 # endif
1026 }
1027 #endif /* NEED_FIND_LIMIT || UNIX_LIKE || WRAP_MARK_SOME */
1028
1029 #if defined(NEED_FIND_LIMIT) \
1030 || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) \
1031 || (defined(WRAP_MARK_SOME) && defined(NO_SEH_AVAILABLE))
1032 GC_INNER JMP_BUF GC_jmp_buf;
1033
1034 STATIC void
1035 GC_fault_handler(int sig)
1036 {
1037 UNUSED_ARG(sig);
1038 LONGJMP(GC_jmp_buf, 1);
1039 }
1040
1041 GC_INNER void
1042 GC_setup_temporary_fault_handler(void)
1043 {
1044 /*
1045 * Handler is process-wide, so this should only happen in one thread
1046 * at a time.
1047 */
1048 GC_ASSERT(I_HOLD_LOCK());
1049 GC_set_and_save_fault_handler(GC_fault_handler);
1050 }
1051
1052 GC_INNER void
1053 GC_reset_fault_handler(void)
1054 {
1055 # ifdef USE_SEGV_SIGACT
1056 (void)sigaction(SIGSEGV, &old_segv_act, 0);
1057 # ifdef USE_BUS_SIGACT
1058 (void)sigaction(SIGBUS, &old_bus_act, 0);
1059 # endif
1060 # else
1061 (void)signal(SIGSEGV, old_segv_hand);
1062 # ifdef HAVE_SIGBUS
1063 (void)signal(SIGBUS, old_bus_hand);
1064 # endif
1065 # endif
1066 }
1067 #endif /* NEED_FIND_LIMIT || USE_PROC_FOR_LIBRARIES || WRAP_MARK_SOME */
1068
1069 #if defined(NEED_FIND_LIMIT) \
1070 || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS))
1071 # define MIN_PAGE_SIZE 256 /*< smallest conceivable page size, in bytes */
1072
1073 /*
1074 * Return the first non-addressable location greater than `p` (if `up`)
1075 * or the smallest location `q` such that [`q`,`p`) is addressable (if
1076 * not `up`). We assume that `p` (if `up`) or `p - 1` (if not `up`) is
1077 * addressable.
1078 */
1079 GC_ATTR_NO_SANITIZE_ADDR
1080 STATIC ptr_t
1081 GC_find_limit_with_bound(ptr_t p, GC_bool up, ptr_t bound)
1082 {
1083 /*
1084 * This is safer if `static`, since otherwise it may not be preserved
1085 * across the `longjmp`. Can safely be `static` since it is only called
1086 * with the allocator lock held.
1087 */
1088 static volatile ptr_t result;
1089
1090 GC_ASSERT(up ? ADDR(bound) >= MIN_PAGE_SIZE
1091 : ADDR(bound) <= ~(word)MIN_PAGE_SIZE);
1092 GC_ASSERT(I_HOLD_LOCK());
1093 result = PTR_ALIGN_DOWN(p, MIN_PAGE_SIZE);
1094 GC_setup_temporary_fault_handler();
1095 if (SETJMP(GC_jmp_buf) == 0) {
1096 for (;;) {
1097 if (up) {
1098 if (ADDR_GE(result, bound - MIN_PAGE_SIZE)) {
1099 result = bound;
1100 break;
1101 }
1102 /*
1103 * Notes: no overflow is expected; do not use compound assignment
1104 * with `volatile`-qualified left operand.
1105 */
1106 result = result + MIN_PAGE_SIZE;
1107 } else {
1108 if (ADDR_GE(bound + MIN_PAGE_SIZE, result)) {
1109 /*
1110 * This is to compensate further result increment (we do not
1111 * modify `up` variable since it might be clobbered by `setjmp()`
1112 * otherwise).
1113 */
1114 result = bound - MIN_PAGE_SIZE;
1115 break;
1116 }
1117 /* See the notes for the case when `up` is `TRUE`. */
1118 result = result - MIN_PAGE_SIZE;
1119 }
1120 GC_noop1((word)(unsigned char)(*result));
1121 }
1122 }
1123 GC_reset_fault_handler();
1124 return up ? result : result + MIN_PAGE_SIZE;
1125 }
1126
1127 void *
1128 GC_find_limit(void *p, int up)
1129 {
1130 ptr_t bound;
1131
1132 # ifdef CHERI_PURECAP
1133 bound = (ptr_t)cheri_address_set(p, cheri_base_get(p)
1134 + (up ? cheri_length_get(p) : 0));
1135 # else
1136 bound = up ? MAKE_CPTR(GC_WORD_MAX) : NULL;
1137 # endif
1138 return GC_find_limit_with_bound((ptr_t)p, (GC_bool)up, bound);
1139 }
1140 #endif /* NEED_FIND_LIMIT || USE_PROC_FOR_LIBRARIES */
1141
1142 #if defined(HPUX) && defined(IA64)
1143 # include <sys/param.h>
1144 # include <sys/pstat.h>
1145
1146 GC_INNER ptr_t
1147 GC_get_register_stack_base(void)
1148 {
1149 struct pst_vm_status vm_status;
1150
1151 int i = 0;
1152 while (pstat_getprocvm(&vm_status, sizeof(vm_status), 0, i++) == 1) {
1153 if (vm_status.pst_type == PS_RSESTACK) {
1154 return (ptr_t)vm_status.pst_vaddr;
1155 }
1156 }
1157
1158 /* Old way to get the register stack bottom. */
1159 GC_ASSERT(GC_stackbottom != NULL);
1160 return PTR_ALIGN_DOWN(GC_stackbottom - BACKING_STORE_DISPLACEMENT - 1,
1161 BACKING_STORE_ALIGNMENT);
1162 }
1163 #endif /* HPUX && IA64 */
1164
1165 #if defined(LINUX) && defined(IA64)
1166 # ifdef USE_LIBC_PRIVATES
1167 EXTERN_C_BEGIN
1168 # pragma weak __libc_ia64_register_backing_store_base
1169 extern ptr_t __libc_ia64_register_backing_store_base;
1170 EXTERN_C_END
1171 # endif
1172
1173 GC_INNER ptr_t
1174 GC_get_register_stack_base(void)
1175 {
1176 ptr_t result;
1177
1178 GC_ASSERT(I_HOLD_LOCK());
1179 # ifdef USE_LIBC_PRIVATES
1180 {
1181 ptr_t *p_libc_ia64_register_backing_store_base
1182 = &__libc_ia64_register_backing_store_base;
1183
1184 # ifdef CPPCHECK
1185 /*
1186 * Workaround a warning that the address of the global symbol
1187 * (which is a weak one) cannot be null.
1188 */
1189 GC_noop1_ptr(&p_libc_ia64_register_backing_store_base);
1190 # endif
1191 if (p_libc_ia64_register_backing_store_base != NULL
1192 && __libc_ia64_register_backing_store_base != NULL) {
1193 /*
1194 * `glibc` 2.2.4 has a bug such that for dynamically linked
1195 * executables `__libc_ia64_register_backing_store_base` is
1196 * defined but uninitialized during constructor calls.
1197 * Hence we check for both nonzero address and value.
1198 */
1199 return __libc_ia64_register_backing_store_base;
1200 }
1201 }
1202 # endif
1203 result = backing_store_base_from_proc();
1204 if (0 == result) {
1205 /* This works better than a constant displacement heuristic. */
1206 result = (ptr_t)GC_find_limit(GC_save_regs_in_stack(), FALSE);
1207 }
1208 return result;
1209 }
1210 #endif /* LINUX && IA64 */
1211
1212 #ifdef SPECIFIC_MAIN_STACKBOTTOM
1213
1214 # ifdef HPUX
1215 # include <sys/param.h>
1216 # include <sys/pstat.h>
1217
1218 static ptr_t
1219 os_main_stackbottom(void)
1220 {
1221 struct pst_vm_status vm_status;
1222 int i = 0;
1223
1224 while (pstat_getprocvm(&vm_status, sizeof(vm_status), 0, i++) == 1) {
1225 if (vm_status.pst_type == PS_STACK)
1226 return (ptr_t)vm_status.pst_vaddr;
1227 }
1228
1229 /* Old way to get the stack bottom. */
1230 # ifdef STACK_GROWS_UP
1231 return (ptr_t)GC_find_limit(GC_approx_sp(), FALSE);
1232 # else
1233 return (ptr_t)GC_find_limit(GC_approx_sp(), TRUE /* `up` */);
1234 # endif
1235 }
1236
1237 # elif defined(LINUX)
1238 # include <sys/stat.h>
1239
1240 /* Number of fields preceding `startstack` one in `/proc/self/stat` file. */
1241 # define STAT_SKIP 27
1242
1243 # ifdef USE_LIBC_PRIVATES
1244 EXTERN_C_BEGIN
1245 # pragma weak __libc_stack_end
1246 extern ptr_t __libc_stack_end;
1247 EXTERN_C_END
1248 # endif
1249
1250 static ptr_t
1251 os_main_stackbottom(void)
1252 {
1253 /*
1254 * We read the stack bottom value from `/proc/self/stat` file.
1255 * We do this using direct I/O system calls in order to avoid
1256 * calling `malloc` in case `REDIRECT_MALLOC` is defined.
1257 */
1258 # define STAT_BUF_SIZE 4096
1259 unsigned char stat_buf[STAT_BUF_SIZE];
1260 int f;
1261 word addr;
1262 ssize_t i, buf_offset = 0, len;
1263
1264 /*
1265 * First try the easy way. This should work for `glibc` 2.2.
1266 * This fails in a prelinked (`prelink` command) executable
1267 * since the correct value of `__libc_stack_end` never becomes
1268 * visible to us. The second test is a workaround for this.
1269 */
1270 # ifdef USE_LIBC_PRIVATES
1271 ptr_t *p_libc_stack_end = &__libc_stack_end;
1272
1273 # ifdef CPPCHECK
1274 GC_noop1_ptr(&p_libc_stack_end);
1275 # endif
1276 if (p_libc_stack_end != NULL && __libc_stack_end != NULL) {
1277 # ifdef IA64
1278 /*
1279 * Some versions of `glibc` set the address 16 bytes too low
1280 * while the initialization code is running.
1281 */
1282 if ((ADDR(__libc_stack_end) & 0xfff) + 0x10 < 0x1000) {
1283 return __libc_stack_end + 0x10;
1284 } else {
1285 /* It is not safe to add 16 bytes. Thus, fall back to using `/proc`. */
1286 }
1287 # elif defined(SPARC)
1288 /*
1289 * Older versions of `glibc` for 64-bit SPARC do not set this
1290 * variable correctly, it gets set to either zero or one.
1291 */
1292 if (ADDR(__libc_stack_end) != 1)
1293 return __libc_stack_end;
1294 # else
1295 return __libc_stack_end;
1296 # endif
1297 }
1298 # endif
1299
1300 f = open("/proc/self/stat", O_RDONLY);
1301 if (-1 == f)
1302 ABORT_ARG1("Could not open /proc/self/stat", ": errno= %d", errno);
1303 len = GC_repeat_read(f, (char *)stat_buf, sizeof(stat_buf));
1304 if (len < 0)
1305 ABORT_ARG1("Failed to read /proc/self/stat", ": errno= %d", errno);
1306 close(f);
1307
1308 /*
1309 * Skip the required number of fields. This number is hopefully constant
1310 * across all Linux implementations.
1311 */
1312 for (i = 0; i < STAT_SKIP; ++i) {
1313 while (buf_offset < len && isspace(stat_buf[buf_offset++])) {
1314 /* Empty. */
1315 }
1316 while (buf_offset < len && !isspace(stat_buf[buf_offset++])) {
1317 /* Empty. */
1318 }
1319 }
1320 /* Skip spaces. */
1321 while (buf_offset < len && isspace(stat_buf[buf_offset])) {
1322 buf_offset++;
1323 }
1324 /* Find the end of the number and cut the buffer there. */
1325 for (i = 0; buf_offset + i < len; i++) {
1326 if (!isdigit(stat_buf[buf_offset + i]))
1327 break;
1328 }
1329 if (buf_offset + i >= len)
1330 ABORT("Could not parse /proc/self/stat");
1331 stat_buf[buf_offset + i] = '\0';
1332
1333 addr = (word)STRTOULL((char *)stat_buf + buf_offset, NULL, 10);
1334 if (addr < 0x100000 || addr % ALIGNMENT != 0)
1335 ABORT_ARG1("Absurd stack bottom value", ": 0x%lx", (unsigned long)addr);
1336 return MAKE_CPTR(addr);
1337 }
1338
1339 # elif defined(QNX)
1340 static ptr_t
1341 os_main_stackbottom(void)
1342 {
1343 /*
1344 * TODO: This approach is not very exact but it works for the tests,
1345 * at least, unlike other available heuristics.
1346 */
1347 return (ptr_t)__builtin_frame_address(0);
1348 }
1349
1350 # elif defined(FREEBSD)
1351 # include <sys/sysctl.h>
1352
1353 /*
1354 * This uses an undocumented `sysctl` call, but at least one expert
1355 * believes it will stay.
1356 */
1357 static ptr_t
1358 os_main_stackbottom(void)
1359 {
1360 int nm[2] = { CTL_KERN, KERN_USRSTACK };
1361 ptr_t base;
1362 size_t len = sizeof(ptr_t);
1363 int r = sysctl(nm, 2, &base, &len, NULL, 0);
1364
1365 if (r != 0)
1366 ABORT("Error getting main stack base");
1367 return base;
1368 }
1369 # endif
1370
1371 #endif /* SPECIFIC_MAIN_STACKBOTTOM */
1372
1373 #if defined(ECOS) || defined(NOSYS)
1374 GC_INNER ptr_t
1375 GC_get_main_stack_base(void)
1376 {
1377 return STACKBOTTOM;
1378 }
1379 # define GET_MAIN_STACKBASE_SPECIAL
1380
1381 #elif defined(SYMBIAN)
1382 EXTERN_C_BEGIN
1383 extern int GC_get_main_symbian_stack_base(void);
1384 EXTERN_C_END
1385
1386 GC_INNER ptr_t
1387 GC_get_main_stack_base(void)
1388 {
1389 return (ptr_t)GC_get_main_symbian_stack_base();
1390 }
1391 # define GET_MAIN_STACKBASE_SPECIAL
1392
1393 #elif defined(EMSCRIPTEN)
1394 # include <emscripten/stack.h>
1395
1396 GC_INNER ptr_t
1397 GC_get_main_stack_base(void)
1398 {
1399 return (ptr_t)emscripten_stack_get_base();
1400 }
1401 # define GET_MAIN_STACKBASE_SPECIAL
1402
1403 #elif !defined(ANY_MSWIN) && !defined(EMBOX) && !defined(OS2) \
1404 && !(defined(OPENBSD) && defined(THREADS)) && !defined(SERENITY) \
1405 && (!(defined(SOLARIS) && defined(THREADS)) || defined(_STRICT_STDC))
1406
1407 # if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
1408 && (defined(THREADS) || defined(USE_GET_STACKBASE_FOR_MAIN))
1409 # include <pthread.h>
1410 # ifdef HAVE_PTHREAD_NP_H
1411 # include <pthread_np.h> /*< for `pthread_attr_get_np()` */
1412 # endif
1413 # elif defined(DARWIN) && !defined(NO_PTHREAD_GET_STACKADDR_NP)
1414 /*
1415 * We could use `pthread_get_stackaddr_np` even in case of a single-threaded
1416 * collector build (there is no `-lpthread` option on Darwin).
1417 */
1418 # include <pthread.h>
1419 # undef STACKBOTTOM
1420 # define STACKBOTTOM (ptr_t) pthread_get_stackaddr_np(pthread_self())
1421 # endif
1422
1423 GC_INNER ptr_t
1424 GC_get_main_stack_base(void)
1425 {
1426 ptr_t result;
1427 # if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
1428 && (defined(USE_GET_STACKBASE_FOR_MAIN) \
1429 || (defined(THREADS) && !defined(REDIRECT_MALLOC)))
1430 pthread_attr_t attr;
1431 void *stackaddr;
1432 size_t size;
1433
1434 # ifdef HAVE_PTHREAD_ATTR_GET_NP
1435 if (pthread_attr_init(&attr) == 0
1436 && (pthread_attr_get_np(pthread_self(), &attr) == 0
1437 ? TRUE
1438 : (pthread_attr_destroy(&attr), FALSE)))
1439 # else /* HAVE_PTHREAD_GETATTR_NP */
1440 if (pthread_getattr_np(pthread_self(), &attr) == 0)
1441 # endif
1442 {
1443 if (pthread_attr_getstack(&attr, &stackaddr, &size) == 0
1444 && stackaddr != NULL) {
1445 (void)pthread_attr_destroy(&attr);
1446 # ifndef STACK_GROWS_UP
1447 stackaddr = (char *)stackaddr + size;
1448 # endif
1449 return (ptr_t)stackaddr;
1450 }
1451 (void)pthread_attr_destroy(&attr);
1452 }
1453 WARN("pthread_getattr_np or pthread_attr_getstack failed"
1454 " for main thread\n",
1455 0);
1456 # endif
1457 # ifdef STACKBOTTOM
1458 result = STACKBOTTOM;
1459 # else
1460 # ifdef HEURISTIC1
1461 # define STACKBOTTOM_ALIGNMENT_M1 ((word)STACK_GRAN - 1)
1462 # ifdef STACK_GROWS_UP
1463 result = PTR_ALIGN_DOWN(GC_approx_sp(), STACKBOTTOM_ALIGNMENT_M1 + 1);
1464 # else
1465 result = PTR_ALIGN_UP(GC_approx_sp(), STACKBOTTOM_ALIGNMENT_M1 + 1);
1466 # endif
1467 # elif defined(SPECIFIC_MAIN_STACKBOTTOM)
1468 result = os_main_stackbottom();
1469 # elif defined(HEURISTIC2)
1470 {
1471 ptr_t sp = GC_approx_sp();
1472
1473 # ifdef STACK_GROWS_UP
1474 result = (ptr_t)GC_find_limit(sp, FALSE);
1475 # else
1476 result = (ptr_t)GC_find_limit(sp, TRUE /* `up` */);
1477 # endif
1478 # if defined(HEURISTIC2_LIMIT) && !defined(CPPCHECK)
1479 if (HOTTER_THAN(HEURISTIC2_LIMIT, result)
1480 && HOTTER_THAN(sp, HEURISTIC2_LIMIT))
1481 result = HEURISTIC2_LIMIT;
1482 # endif
1483 }
1484 # elif defined(STACK_NOT_SCANNED) || defined(CPPCHECK)
1485 result = NULL;
1486 # else
1487 # error None of HEURISTIC* and *STACKBOTTOM defined!
1488 # endif
1489 # if !defined(STACK_GROWS_UP) && !defined(CPPCHECK)
1490 if (NULL == result)
1491 result = MAKE_CPTR((GC_signed_word)(-sizeof(ptr_t)));
1492 # endif
1493 # endif
1494 # if !defined(CPPCHECK)
1495 GC_ASSERT(HOTTER_THAN(GC_approx_sp(), result));
1496 # endif
1497 return result;
1498 }
1499 # define GET_MAIN_STACKBASE_SPECIAL
1500 #endif /* !ANY_MSWIN && !EMBOX && !OS2 && !SERENITY */
1501
1502 #if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) \
1503 && defined(THREADS) && !defined(HAVE_GET_STACK_BASE)
1504 # include <pthread.h>
1505 # ifdef HAVE_PTHREAD_NP_H
1506 # include <pthread_np.h>
1507 # endif
1508
1509 GC_API int GC_CALL
1510 GC_get_stack_base(struct GC_stack_base *b)
1511 {
1512 pthread_attr_t attr;
1513 size_t size;
1514
1515 # ifdef HAVE_PTHREAD_ATTR_GET_NP
1516 if (pthread_attr_init(&attr) != 0)
1517 ABORT("pthread_attr_init failed");
1518 if (pthread_attr_get_np(pthread_self(), &attr) != 0) {
1519 WARN("pthread_attr_get_np failed\n", 0);
1520 (void)pthread_attr_destroy(&attr);
1521 return GC_UNIMPLEMENTED;
1522 }
1523 # else /* HAVE_PTHREAD_GETATTR_NP */
1524 if (pthread_getattr_np(pthread_self(), &attr) != 0) {
1525 WARN("pthread_getattr_np failed\n", 0);
1526 return GC_UNIMPLEMENTED;
1527 }
1528 # endif
1529 if (pthread_attr_getstack(&attr, &b->mem_base, &size) != 0) {
1530 ABORT("pthread_attr_getstack failed");
1531 }
1532 (void)pthread_attr_destroy(&attr);
1533 # ifndef STACK_GROWS_UP
1534 b->mem_base = (char *)b->mem_base + size;
1535 # endif
1536 # ifdef IA64
1537 /*
1538 * We could try `backing_store_base_from_proc`, but that is safe only
1539 * if no mappings are being asynchronously created. Subtracting the size
1540 * from the stack base does not work for at least the main thread.
1541 */
1542 LOCK();
1543 {
1544 IF_CANCEL(int cancel_state;)
1545 ptr_t bsp;
1546 ptr_t next_stack;
1547
1548 DISABLE_CANCEL(cancel_state);
1549 bsp = GC_save_regs_in_stack();
1550 next_stack = GC_greatest_stack_base_below(bsp);
1551 if (NULL == next_stack) {
1552 b->reg_base = GC_find_limit(bsp, FALSE);
1553 } else {
1554 /*
1555 * Avoid walking backwards into preceding memory stack and
1556 * growing it.
1557 */
1558 b->reg_base = GC_find_limit_with_bound(bsp, FALSE, next_stack);
1559 }
1560 RESTORE_CANCEL(cancel_state);
1561 }
1562 UNLOCK();
1563 # elif defined(E2K)
1564 b->reg_base = NULL;
1565 # endif
1566 return GC_SUCCESS;
1567 }
1568 # define HAVE_GET_STACK_BASE
1569 #endif /* THREADS && (HAVE_PTHREAD_ATTR_GET_NP || HAVE_PTHREAD_GETATTR_NP) */
1570
1571 #if defined(DARWIN) && defined(THREADS) \
1572 && !defined(NO_PTHREAD_GET_STACKADDR_NP)
1573 # include <pthread.h>
1574
1575 GC_API int GC_CALL
1576 GC_get_stack_base(struct GC_stack_base *b)
1577 {
1578 /*
1579 * `pthread_get_stackaddr_np()` should return stack bottom (highest
1580 * stack address plus 1).
1581 */
1582 b->mem_base = pthread_get_stackaddr_np(pthread_self());
1583 GC_ASSERT(HOTTER_THAN(GC_approx_sp(), (ptr_t)b->mem_base));
1584 return GC_SUCCESS;
1585 }
1586 # define HAVE_GET_STACK_BASE
1587 #endif /* DARWIN && THREADS && !NO_PTHREAD_GET_STACKADDR_NP */
1588
1589 #if defined(OPENBSD) && defined(THREADS)
1590 # include <pthread.h>
1591 # include <pthread_np.h>
1592 # include <sys/signal.h>
1593
1594 GC_API int GC_CALL
1595 GC_get_stack_base(struct GC_stack_base *sb)
1596 {
1597 stack_t stack;
1598
1599 /* Find the stack using `pthread_stackseg_np()`. */
1600 if (pthread_stackseg_np(pthread_self(), &stack))
1601 ABORT("pthread_stackseg_np(self) failed");
1602 sb->mem_base = stack.ss_sp;
1603 return GC_SUCCESS;
1604 }
1605 # define HAVE_GET_STACK_BASE
1606 #endif /* OPENBSD && THREADS */
1607
1608 #if defined(SOLARIS) && defined(THREADS) && !defined(_STRICT_STDC)
1609
1610 # include <pthread.h>
1611 # include <thread.h>
1612
1613 /*
1614 * These variables are used to cache `ss_sp` value for the primordial
1615 * thread (it is better not to call `thr_stksegment()` twice for this
1616 * thread - see JDK bug #4352906).
1617 * Note: `stackbase_main_self` set to zero means `stackbase_main_ss_sp`
1618 * value is unset.
1619 */
1620 static pthread_t stackbase_main_self = 0;
1621 static void *stackbase_main_ss_sp = NULL;
1622
1623 # ifdef CAN_HANDLE_FORK
1624 GC_INNER void
1625 GC_stackbase_info_update_after_fork(void)
1626 {
1627 if (stackbase_main_self == GC_parent_pthread_self) {
1628 /* The primordial thread has forked the process. */
1629 stackbase_main_self = pthread_self();
1630 } else {
1631 stackbase_main_self = 0;
1632 }
1633 }
1634 # endif
1635
1636 GC_API int GC_CALL
1637 GC_get_stack_base(struct GC_stack_base *b)
1638 {
1639 stack_t s;
1640 pthread_t self = pthread_self();
1641
1642 if (self == stackbase_main_self) {
1643 /*
1644 * If the client calls `GC_get_stack_base()` from the main thread,
1645 * then just return the cached value.
1646 */
1647 b->mem_base = stackbase_main_ss_sp;
1648 GC_ASSERT(b->mem_base != NULL);
1649 return GC_SUCCESS;
1650 }
1651
1652 if (thr_stksegment(&s)) {
1653 /*
1654 * According to the manual, the only failure error code returned is
1655 * `EAGAIN` meaning "the information is not available due to the thread
1656 * is not yet completely initialized or it is an internal thread" - this
1657 * should not happen here.
1658 */
1659 ABORT("thr_stksegment failed");
1660 }
1661 /* `s.ss_sp` holds the pointer to the stack bottom. */
1662 GC_ASSERT(HOTTER_THAN(GC_approx_sp(), (ptr_t)s.ss_sp));
1663
1664 if (!stackbase_main_self && thr_main() != 0) {
1665 /*
1666 * Cache the stack bottom pointer for the primordial thread
1667 * (this is done during `GC_init`, so there is no race).
1668 */
1669 stackbase_main_ss_sp = s.ss_sp;
1670 stackbase_main_self = self;
1671 }
1672
1673 b->mem_base = s.ss_sp;
1674 return GC_SUCCESS;
1675 }
1676 # define HAVE_GET_STACK_BASE
1677 #endif /* SOLARIS && THREADS */
1678
1679 #if defined(RTEMS) && defined(THREADS)
1680 GC_API int GC_CALL
1681 GC_get_stack_base(struct GC_stack_base *sb)
1682 {
1683 sb->mem_base = rtems_get_stack_bottom();
1684 return GC_SUCCESS;
1685 }
1686 # define HAVE_GET_STACK_BASE
1687 #endif /* RTEMS && THREADS */
1688
1689 #ifndef HAVE_GET_STACK_BASE
1690
1691 # ifdef NEED_FIND_LIMIT
1692 GC_API int GC_CALL
1693 GC_get_stack_base(struct GC_stack_base *b)
1694 {
1695 IF_CANCEL(int cancel_state;)
1696
1697 /*
1698 * Note: using the `GC_find_limit` variant is risky; in the IA-64 case,
1699 * e.g., there is no guard page between the stack of one thread and the
1700 * register backing store of the next; thus this is likely to identify way
1701 * too large a "stack" and thus at least result in disastrous performance.
1702 */
1703 /* TODO: Implement better strategies here. */
1704 LOCK();
1705 /* TODO: `DISABLE_CANCEL` may be unnecessary? */
1706 DISABLE_CANCEL(cancel_state);
1707 # ifdef STACK_GROWS_UP
1708 b->mem_base = GC_find_limit(GC_approx_sp(), FALSE);
1709 # else
1710 b->mem_base = GC_find_limit(GC_approx_sp(), TRUE /* `up` */);
1711 # endif
1712 # ifdef IA64
1713 b->reg_base = GC_find_limit(GC_save_regs_in_stack(), FALSE);
1714 # elif defined(E2K)
1715 b->reg_base = NULL;
1716 # endif
1717 RESTORE_CANCEL(cancel_state);
1718 UNLOCK();
1719 return GC_SUCCESS;
1720 }
1721 # else /* !NEED_FIND_LIMIT */
1722 GC_API int GC_CALL
1723 GC_get_stack_base(struct GC_stack_base *b)
1724 {
1725 # if defined(GET_MAIN_STACKBASE_SPECIAL) && !defined(THREADS) \
1726 && !defined(IA64)
1727 b->mem_base = GC_get_main_stack_base();
1728 return GC_SUCCESS;
1729 # else
1730 UNUSED_ARG(b);
1731 return GC_UNIMPLEMENTED;
1732 # endif
1733 }
1734 # endif
1735
1736 #endif /* !HAVE_GET_STACK_BASE */
1737
1738 #ifndef GET_MAIN_STACKBASE_SPECIAL
1739 GC_INNER ptr_t
1740 GC_get_main_stack_base(void)
1741 {
1742 /* Default implementation. */
1743 struct GC_stack_base sb;
1744
1745 if (GC_get_stack_base(&sb) != GC_SUCCESS)
1746 ABORT("GC_get_stack_base failed");
1747 GC_ASSERT(HOTTER_THAN(GC_approx_sp(), (ptr_t)sb.mem_base));
1748 return (ptr_t)sb.mem_base;
1749 }
1750 #endif /* !GET_MAIN_STACKBASE_SPECIAL */
1751
1752 /*
1753 * Register static data segment(s) as roots. If more data segments are
1754 * added later, then they need to be registered at that point (as we do
1755 * with SunOS dynamic loading), or `GC_mark_roots` needs to check for them.
1756 */
1757
1758 #ifdef ANY_MSWIN
1759
1760 # if defined(GWW_VDB)
1761 # ifndef MEM_WRITE_WATCH
1762 # define MEM_WRITE_WATCH 0x200000
1763 # endif
1764 # ifndef WRITE_WATCH_FLAG_RESET
1765 # define WRITE_WATCH_FLAG_RESET 1
1766 # endif
1767
1768 /*
1769 * Since we cannot easily check whether `ULONG_PTR` and `SIZE_T` are
1770 * defined in Win32 `basetsd.h` file, we define own `ULONG_PTR`.
1771 */
1772 # define GC_ULONG_PTR word
1773
1774 typedef UINT(WINAPI *GetWriteWatch_type)(DWORD, PVOID,
1775 GC_ULONG_PTR /* `SIZE_T` */, PVOID *,
1776 GC_ULONG_PTR *, PULONG);
1777 static FARPROC GetWriteWatch_func;
1778 static DWORD GetWriteWatch_alloc_flag;
1779
1780 # define GC_GWW_AVAILABLE() (GetWriteWatch_func != 0)
1781
1782 static void
1783 detect_GetWriteWatch(void)
1784 {
1785 static GC_bool done;
1786 HMODULE hK32;
1787 if (done)
1788 return;
1789
1790 # if defined(MPROTECT_VDB)
1791 {
1792 char *str = GETENV("GC_USE_GETWRITEWATCH");
1793 # if defined(GC_PREFER_MPROTECT_VDB)
1794 if (NULL == str || (*str == '0' && *(str + 1) == '\0')) {
1795 /*
1796 * `GC_USE_GETWRITEWATCH` environment variable is unset or set to "0".
1797 * Falling back to `MPROTECT_VDB` strategy.
1798 */
1799 done = TRUE;
1800 /* This should work as if `GWW_VDB` macro is not defined. */
1801 return;
1802 }
1803 # else
1804 if (str != NULL && *str == '0' && *(str + 1) == '\0') {
1805 /*
1806 * `GC_USE_GETWRITEWATCH` environment variable is set "0".
1807 * Falling back to `MPROTECT_VDB` strategy.
1808 */
1809 done = TRUE;
1810 return;
1811 }
1812 # endif
1813 }
1814 # endif
1815
1816 # if defined(MSWINRT_FLAVOR) && defined(FUNCPTR_IS_DATAPTR)
1817 {
1818 MEMORY_BASIC_INFORMATION memInfo;
1819 SIZE_T result = VirtualQuery(CAST_THRU_UINTPTR(void *, GetProcAddress),
1820 &memInfo, sizeof(memInfo));
1821
1822 if (result != sizeof(memInfo))
1823 ABORT("Weird VirtualQuery result");
1824 hK32 = (HMODULE)memInfo.AllocationBase;
1825 }
1826 # else
1827 hK32 = GetModuleHandle(TEXT("kernel32.dll"));
1828 # endif
1829 if (hK32 != (HMODULE)0
1830 && (GetWriteWatch_func = GetProcAddress(hK32, "GetWriteWatch")) != 0) {
1831 void *page;
1832
1833 GC_ASSERT(GC_page_size != 0);
1834 /*
1835 * Also check whether `VirtualAlloc()` accepts `MEM_WRITE_WATCH`,
1836 * as some versions of `kernel32.dll` library have one but not the other,
1837 * making the feature completely broken.
1838 */
1839 page = VirtualAlloc(NULL, GC_page_size, MEM_WRITE_WATCH | MEM_RESERVE,
1840 PAGE_READWRITE);
1841 if (page != NULL) {
1842 PVOID pages[16];
1843 GC_ULONG_PTR count = sizeof(pages) / sizeof(PVOID);
1844 DWORD page_size;
1845 /*
1846 * Check that it actually works. In spite of some documentation
1847 * it actually seems to exist on Win2K.
1848 * This test may be unnecessary, but...
1849 */
1850 if ((*(GetWriteWatch_type)(GC_funcptr_uint)GetWriteWatch_func)(
1851 WRITE_WATCH_FLAG_RESET, page, GC_page_size, pages, &count,
1852 &page_size)
1853 != 0) {
1854 /* `GetWriteWatch()` always fails. */
1855 GetWriteWatch_func = 0;
1856 } else {
1857 GetWriteWatch_alloc_flag = MEM_WRITE_WATCH;
1858 }
1859 VirtualFree(page, 0 /* `dwSize` */, MEM_RELEASE);
1860 } else {
1861 /* `GetWriteWatch` will be useless. */
1862 GetWriteWatch_func = 0;
1863 }
1864 }
1865 done = TRUE;
1866 }
1867
1868 # else
1869 # define GetWriteWatch_alloc_flag 0
1870 # endif /* !GWW_VDB */
1871
1872 # ifdef MSWIN32
1873 /*
1874 * Unfortunately, we have to handle win32s very differently from Windows NT,
1875 * since `VirtualQuery()` has very different semantics. In particular,
1876 * under win32s a `VirtualQuery()` call on an unmapped page returns
1877 * an invalid result. Under Windows NT, `GC_register_data_segments()` is
1878 * a no-op and all real work is done by `GC_register_dynamic_libraries()`.
1879 * Under win32s, we cannot find the data segments associated with DLL files.
1880 * We register the main data segment here.
1881 */
1882
1883 GC_INNER GC_bool GC_no_win32_dlls = FALSE;
1884
1885 GC_INNER GC_bool GC_wnt = FALSE;
1886
1887 GC_INNER void
1888 GC_init_win32(void)
1889 {
1890 # if defined(_WIN64) || (defined(_MSC_VER) && _MSC_VER >= 1800)
1891 /*
1892 * MS Visual Studio 2013 deprecates `GetVersion`, but on the other hand
1893 * it cannot be used to target pre-Win2K.
1894 */
1895 GC_wnt = TRUE;
1896 # else
1897 /*
1898 * Set `GC_wnt`. If we are running under win32s, assume that no DLL file
1899 * will be loaded. I doubt anyone still runs win32s, but...
1900 */
1901 DWORD v = GetVersion();
1902
1903 GC_wnt = !(v & (DWORD)0x80000000UL);
1904 GC_no_win32_dlls |= ((!GC_wnt) && (v & 0xff) <= 3);
1905 # endif
1906 # ifdef USE_MUNMAP
1907 if (GC_no_win32_dlls) {
1908 /*
1909 * Turn off unmapping for safety (since may not work well with
1910 * `GlobalAlloc()`).
1911 */
1912 GC_unmap_threshold = 0;
1913 }
1914 # endif
1915 }
1916
1917 /*
1918 * Return the smallest address `p` such that `VirtualQuery()` returns
1919 * correct results for all addresses between `p` and `start`.
1920 * Assumes `VirtualQuery()` returns correct information for `start`.
1921 */
1922 STATIC ptr_t
1923 GC_least_described_address(ptr_t start)
1924 {
1925 ptr_t limit = (ptr_t)GC_sysinfo.lpMinimumApplicationAddress;
1926 ptr_t p = PTR_ALIGN_DOWN(start, GC_page_size);
1927
1928 GC_ASSERT(GC_page_size != 0);
1929 for (;;) {
1930 MEMORY_BASIC_INFORMATION buf;
1931 size_t result;
1932 ptr_t q;
1933
1934 if (UNLIKELY(ADDR(p) <= (word)GC_page_size))
1935 break; /*< avoid underflow */
1936 q = p - GC_page_size;
1937 if (ADDR_LT(q, limit))
1938 break;
1939
1940 result = VirtualQuery((LPVOID)q, &buf, sizeof(buf));
1941 if (result != sizeof(buf) || 0 == buf.AllocationBase)
1942 break;
1943 p = (ptr_t)buf.AllocationBase;
1944 }
1945 return p;
1946 }
1947
1948 STATIC void
1949 GC_register_root_section(ptr_t static_root)
1950 {
1951 ptr_t p, base, limit;
1952
1953 GC_ASSERT(I_HOLD_LOCK());
1954 if (!GC_no_win32_dlls)
1955 return;
1956
1957 p = GC_least_described_address(static_root);
1958 base = limit = p;
1959 while (ADDR_LT(p, (ptr_t)GC_sysinfo.lpMaximumApplicationAddress)) {
1960 MEMORY_BASIC_INFORMATION buf;
1961 size_t result = VirtualQuery((LPVOID)p, &buf, sizeof(buf));
1962
1963 if (result != sizeof(buf) || 0 == buf.AllocationBase
1964 || GC_is_heap_base(buf.AllocationBase))
1965 break;
1966 if (ADDR(p) > GC_WORD_MAX - buf.RegionSize) {
1967 /* Avoid overflow. */
1968 break;
1969 }
1970 if (buf.State == MEM_COMMIT && is_writable(buf.Protect)) {
1971 if (p != limit) {
1972 if (base != limit)
1973 GC_add_roots_inner(base, limit, FALSE);
1974 base = p;
1975 }
1976 limit = p + buf.RegionSize;
1977 }
1978 p += buf.RegionSize;
1979 }
1980 if (base != limit)
1981 GC_add_roots_inner(base, limit, FALSE);
1982 }
1983 # endif /* MSWIN32 */
1984
1985 # if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
1986 /*
1987 * We maintain a linked list of `AllocationBase` values (that we know)
1988 * correspond to `malloc` heap sections. Currently this is only called
1989 * during a collection. But there is some hope that for long-running
1990 * programs we will eventually see most heap sections.
1991 *
1992 * In the long run, it would be more reliable to occasionally walk
1993 * the `malloc` heap with `HeapWalk()` on the default heap.
1994 * But that apparently works only for NT-based Windows.
1995 */
1996
1997 /* Note: initialized to approximate largest root size. */
1998 STATIC size_t GC_max_root_size = 100000;
1999
2000 /* In the long run, a better data structure would also be nice... */
2001 STATIC struct GC_malloc_heap_list {
2002 void *allocation_base;
2003 struct GC_malloc_heap_list *next;
2004 } *GC_malloc_heap_l = 0;
2005
2006 /*
2007 * Is `p` the base of one of the `malloc` heap sections we already
2008 * know about?
2009 */
2010 STATIC GC_bool
2011 GC_is_malloc_heap_base(const void *p)
2012 {
2013 struct GC_malloc_heap_list *q;
2014
2015 for (q = GC_malloc_heap_l; q != NULL; q = q->next) {
2016 if (q->allocation_base == p)
2017 return TRUE;
2018 }
2019 return FALSE;
2020 }
2021
2022 STATIC void *
2023 GC_get_allocation_base(void *p)
2024 {
2025 MEMORY_BASIC_INFORMATION buf;
2026 size_t result = VirtualQuery(p, &buf, sizeof(buf));
2027
2028 if (result != sizeof(buf)) {
2029 ABORT("Weird VirtualQuery result");
2030 }
2031 return buf.AllocationBase;
2032 }
2033
2034 GC_INNER void
2035 GC_add_current_malloc_heap(void)
2036 {
2037 struct GC_malloc_heap_list *new_l = (struct GC_malloc_heap_list *)malloc(
2038 sizeof(struct GC_malloc_heap_list));
2039 void *candidate;
2040
2041 if (NULL == new_l)
2042 return;
2043 /* Explicitly set to suppress "maybe-uninitialized" gcc warning. */
2044 new_l->allocation_base = NULL;
2045
2046 candidate = GC_get_allocation_base(new_l);
2047 if (GC_is_malloc_heap_base(candidate)) {
2048 /* Try a little harder to find `malloc` heap. */
2049 size_t req_size = 10000;
2050
2051 do {
2052 void *p = malloc(req_size);
2053
2054 if (NULL == p) {
2055 free(new_l);
2056 return;
2057 }
2058 candidate = GC_get_allocation_base(p);
2059 free(p);
2060 req_size *= 2;
2061 } while (GC_is_malloc_heap_base(candidate)
2062 && req_size < GC_max_root_size / 10 && req_size < 500000);
2063 if (GC_is_malloc_heap_base(candidate)) {
2064 free(new_l);
2065 return;
2066 }
2067 }
2068 GC_COND_LOG_PRINTF("Found new system malloc AllocationBase at %p\n",
2069 candidate);
2070 new_l->allocation_base = candidate;
2071 new_l->next = GC_malloc_heap_l;
2072 GC_malloc_heap_l = new_l;
2073 }
2074
2075 /*
2076 * Free all the linked list nodes. Could be invoked at process exit
2077 * to avoid memory leak complains of a dynamic code analysis tool.
2078 */
2079 STATIC void
2080 GC_free_malloc_heap_list(void)
2081 {
2082 struct GC_malloc_heap_list *q = GC_malloc_heap_l;
2083
2084 GC_malloc_heap_l = NULL;
2085 while (q != NULL) {
2086 struct GC_malloc_heap_list *next = q->next;
2087
2088 free(q);
2089 q = next;
2090 }
2091 }
2092 # endif /* USE_WINALLOC && !REDIRECT_MALLOC */
2093
2094 GC_INNER GC_bool
2095 GC_is_heap_base(const void *p)
2096 {
2097 size_t i;
2098
2099 # if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
2100 if (GC_root_size > GC_max_root_size)
2101 GC_max_root_size = GC_root_size;
2102 if (GC_is_malloc_heap_base(p))
2103 return TRUE;
2104 # endif
2105 for (i = 0; i < GC_n_heap_bases; i++) {
2106 if (GC_heap_bases[i] == p)
2107 return TRUE;
2108 }
2109 return FALSE;
2110 }
2111
2112 GC_INNER void
2113 GC_register_data_segments(void)
2114 {
2115 # ifdef MSWIN32
2116 /* Note: any other GC global variable would fit too. */
2117 GC_register_root_section((ptr_t)&GC_pages_executable);
2118 # endif
2119 }
2120
2121 #endif /* ANY_MSWIN */
2122
2123 #ifdef DATASTART_USES_XGETDATASTART
2124 # ifdef CHERI_PURECAP
2125 # include <link.h>
2126
2127 /*
2128 * The CheriBSD LLVM compiler declares `etext`, `edata` and `end` as
2129 * typeless variables. If the collector library is statically linked
2130 * with the executable, these capabilities are compiled with the
2131 * read-only permissions and bounds that span the `.data` and `.bss`
2132 * sections. If the collector is compiled as a shared library, these
2133 * symbols are compiled with zero bounds and cannot be dereferenced;
2134 * instead, the read-only capability returned by the loader is used.
2135 */
2136
2137 struct scan_bounds_s {
2138 word start_addr;
2139 word end_addr;
2140 ptr_t ld_cap;
2141 };
2142
2143 static int
2144 ld_cap_search(struct dl_phdr_info *info, size_t size, void *cd)
2145 {
2146 struct scan_bounds_s *region = (struct scan_bounds_s *)cd;
2147 ptr_t load_ptr = (ptr_t)info->dlpi_addr;
2148
2149 UNUSED_ARG(size);
2150 if (!SPANNING_CAPABILITY(load_ptr, region->start_addr, region->end_addr))
2151 return 0;
2152
2153 region->ld_cap = (ptr_t)cheri_bounds_set(
2154 cheri_address_set(load_ptr, region->start_addr),
2155 region->end_addr - region->start_addr);
2156 return 1; /*< stop */
2157 }
2158
2159 static ptr_t
2160 derive_cap_from_ldr(ptr_t range_start, ptr_t range_end)
2161 {
2162 word scan_start = ADDR(range_start);
2163 word scan_end = ADDR(range_end);
2164 struct scan_bounds_s region;
2165
2166 /* If symbols already span the required range, return one of them. */
2167 if (SPANNING_CAPABILITY(range_start, scan_start, scan_end))
2168 return range_start;
2169 if (SPANNING_CAPABILITY(range_end, scan_start, scan_end))
2170 return range_end;
2171
2172 /*
2173 * Fall-back option: derive `.data` plus `.bss` end pointer from the
2174 * read-only capability provided by loader.
2175 */
2176 region.start_addr = scan_start;
2177 region.end_addr = scan_end;
2178 region.ld_cap = NULL; /*< prevent a compiler warning */
2179 if (!dl_iterate_phdr(ld_cap_search, ®ion))
2180 ABORT("Cannot find static roots for capability system");
2181 GC_ASSERT(region.ld_cap != NULL);
2182 return region.ld_cap;
2183 }
2184 # endif /* CHERI_PURECAP */
2185
2186 GC_INNER ptr_t
2187 GC_SysVGetDataStart(size_t max_page_size, ptr_t etext_ptr)
2188 {
2189 volatile ptr_t result;
2190
2191 GC_ASSERT(max_page_size % ALIGNMENT == 0);
2192 result = PTR_ALIGN_UP(etext_ptr, ALIGNMENT);
2193 # ifdef CHERI_PURECAP
2194 result = derive_cap_from_ldr(result, DATAEND);
2195 # endif
2196
2197 GC_setup_temporary_fault_handler();
2198 if (SETJMP(GC_jmp_buf) == 0) {
2199 /*
2200 * Note that this is not equivalent to just adding `max_page_size` to
2201 * `etext_ptr` because the latter is not guaranteed to be multiple of
2202 * the page size.
2203 */
2204 ptr_t next_page = PTR_ALIGN_UP(result, max_page_size);
2205
2206 # ifdef FREEBSD
2207 /*
2208 * It is unclear whether this should be identical to the below, or
2209 * whether it should apply to non-x86 architectures. For now we
2210 * do not assume that there is always an empty page after `etext`.
2211 * But in some cases there actually seems to be slightly more.
2212 * It also deals with holes between read-only and writable data.
2213 *
2214 * Try reading at the address. This should happen before there is
2215 * another thread.
2216 */
2217 for (; ADDR_LT(next_page, DATAEND); next_page += max_page_size) {
2218 GC_noop1((word)(*(volatile unsigned char *)next_page));
2219 }
2220 # else
2221 result = next_page + (ADDR(result) & ((word)max_page_size - 1));
2222 /* Try writing to the address. */
2223 {
2224 # ifdef AO_HAVE_fetch_and_add
2225 volatile AO_t zero = 0;
2226
2227 (void)AO_fetch_and_add((volatile AO_t *)result, zero);
2228 # else
2229 /* Fall back to non-atomic fetch-and-store. */
2230 char v = *result;
2231
2232 # ifdef CPPCHECK
2233 GC_noop1_ptr(&v);
2234 # endif
2235 *result = v;
2236 # endif
2237 }
2238 # endif
2239 GC_reset_fault_handler();
2240 } else {
2241 GC_reset_fault_handler();
2242 /*
2243 * We got here via a `longjmp`. The address is not readable.
2244 * This is known to happen under Solaris 2.4 with gcc, which places
2245 * string constants in the `text` segment, but after `etext`.
2246 * Use plan B. Note that we now know there is a gap between `text`
2247 * and `data` segments, so plan A brought us something.
2248 */
2249 # ifdef CHERI_PURECAP
2250 result = (ptr_t)GC_find_limit(cheri_address_set(result, ADDR(DATAEND)),
2251 FALSE);
2252 # else
2253 result = (ptr_t)GC_find_limit(DATAEND, FALSE);
2254 # endif
2255 }
2256 return (ptr_t)CAST_AWAY_VOLATILE_PVOID(result);
2257 }
2258 #endif /* DATASTART_USES_XGETDATASTART */
2259
2260 #if defined(OS2)
2261 GC_INNER void
2262 GC_register_data_segments(void)
2263 {
2264 PTIB ptib;
2265 PPIB ppib;
2266 HMODULE module_handle;
2267 # define PBUFSIZ 512
2268 UCHAR path[PBUFSIZ];
2269 FILE *myexefile;
2270 struct exe_hdr hdrdos; /*< MSDOS header */
2271 struct e32_exe hdr386; /*< real header for my executable */
2272 struct o32_obj seg; /*< current segment */
2273 int nsegs;
2274
2275 # if defined(CPPCHECK)
2276 hdrdos.padding[0] = 0; /*< to prevent "field unused" warnings */
2277 hdr386.exe_format_level = 0;
2278 hdr386.os = 0;
2279 hdr386.padding1[0] = 0;
2280 hdr386.padding2[0] = 0;
2281 seg.pagemap = 0;
2282 seg.mapsize = 0;
2283 seg.reserved = 0;
2284 # endif
2285 if (DosGetInfoBlocks(&ptib, &ppib) != NO_ERROR) {
2286 ABORT("DosGetInfoBlocks failed");
2287 }
2288 module_handle = ppib->pib_hmte;
2289 if (DosQueryModuleName(module_handle, PBUFSIZ, path) != NO_ERROR) {
2290 ABORT("DosQueryModuleName failed");
2291 }
2292 myexefile = fopen(path, "rb");
2293 if (myexefile == 0) {
2294 ABORT_ARG1("Failed to open executable", ": %s", path);
2295 }
2296 if (fread((char *)&hdrdos, 1, sizeof(hdrdos), myexefile) < sizeof(hdrdos)) {
2297 ABORT_ARG1("Could not read MSDOS header", " from: %s", path);
2298 }
2299 if (E_MAGIC(hdrdos) != EMAGIC) {
2300 ABORT_ARG1("Bad DOS magic number", " in file: %s", path);
2301 }
2302 if (fseek(myexefile, E_LFANEW(hdrdos), SEEK_SET) != 0) {
2303 ABORT_ARG1("Bad DOS magic number", " in file: %s", path);
2304 }
2305 if (fread((char *)&hdr386, 1, sizeof(hdr386), myexefile) < sizeof(hdr386)) {
2306 ABORT_ARG1("Could not read OS/2 header", " from: %s", path);
2307 }
2308 if (E32_MAGIC1(hdr386) != E32MAGIC1 || E32_MAGIC2(hdr386) != E32MAGIC2) {
2309 ABORT_ARG1("Bad OS/2 magic number", " in file: %s", path);
2310 }
2311 if (E32_BORDER(hdr386) != E32LEBO || E32_WORDER(hdr386) != E32LEWO) {
2312 ABORT_ARG1("Bad byte order in executable", " file: %s", path);
2313 }
2314 if (E32_CPU(hdr386) == E32CPU286) {
2315 ABORT_ARG1("GC cannot handle 80286 executables", ": %s", path);
2316 }
2317 if (fseek(myexefile, E_LFANEW(hdrdos) + E32_OBJTAB(hdr386), SEEK_SET) != 0) {
2318 ABORT_ARG1("Seek to object table failed", " in file: %s", path);
2319 }
2320 for (nsegs = E32_OBJCNT(hdr386); nsegs > 0; nsegs--) {
2321 int flags;
2322 if (fread((char *)&seg, 1, sizeof(seg), myexefile) < sizeof(seg)) {
2323 ABORT_ARG1("Could not read obj table entry", " from file: %s", path);
2324 }
2325 flags = O32_FLAGS(seg);
2326 if (!(flags & OBJWRITE))
2327 continue;
2328 if (!(flags & OBJREAD))
2329 continue;
2330 if (flags & OBJINVALID) {
2331 GC_err_printf("Object with invalid pages?\n");
2332 continue;
2333 }
2334 GC_add_roots_inner((ptr_t)O32_BASE(seg),
2335 (ptr_t)(O32_BASE(seg) + O32_SIZE(seg)), FALSE);
2336 }
2337 (void)fclose(myexefile);
2338 }
2339
2340 #elif defined(OPENBSD)
2341 GC_INNER void
2342 GC_register_data_segments(void)
2343 {
2344 /*
2345 * Depending on arch alignment, there can be multiple holes between
2346 * `DATASTART` and `DATAEND`. Scan in `DATASTART` .. `DATAEND` and
2347 * register each region.
2348 */
2349 ptr_t region_start = DATASTART;
2350
2351 GC_ASSERT(I_HOLD_LOCK());
2352 if (ADDR(region_start) - 1U >= ADDR(DATAEND))
2353 ABORT_ARG2("Wrong DATASTART/END pair", ": %p .. %p", (void *)region_start,
2354 (void *)DATAEND);
2355 for (;;) {
2356 ptr_t region_end = GC_find_limit_with_bound(region_start, TRUE, DATAEND);
2357
2358 GC_add_roots_inner(region_start, region_end, FALSE);
2359 if (ADDR_GE(region_end, DATAEND))
2360 break;
2361 region_start = GC_skip_hole_openbsd(region_end, DATAEND);
2362 }
2363 }
2364
2365 #elif !defined(ANY_MSWIN)
2366 GC_INNER void
2367 GC_register_data_segments(void)
2368 {
2369 GC_ASSERT(I_HOLD_LOCK());
2370 # if !defined(DYNAMIC_LOADING) && defined(GC_DONT_REGISTER_MAIN_STATIC_DATA)
2371 /*
2372 * Avoid even referencing `DATASTART` and `DATAEND` as they are
2373 * unnecessary and cause linker errors when bitcode is enabled.
2374 * `GC_register_data_segments` is not called anyway.
2375 */
2376 # elif defined(DYNAMIC_LOADING) && (defined(DARWIN) || defined(HAIKU))
2377 /* No-op. `GC_register_main_static_data()` always returns `FALSE`. */
2378 # elif defined(REDIRECT_MALLOC) && defined(SOLARIS) && defined(THREADS)
2379 /*
2380 * As of Solaris 2.3, the Solaris threads implementation allocates
2381 * the data structure for the initial thread with `sbrk` at the
2382 * process startup. It needs to be scanned, so that we do not lose
2383 * some `malloc`-allocated data structures hanging from it.
2384 * We are on thin ice here...
2385 */
2386 GC_ASSERT(DATASTART);
2387 {
2388 ptr_t p = (ptr_t)sbrk(0);
2389
2390 if (ADDR_LT(DATASTART, p))
2391 GC_add_roots_inner(DATASTART, p, FALSE);
2392 }
2393 # else
2394 /*
2395 * Note: subtract one is to also check for `NULL` without a compiler
2396 * warning.
2397 */
2398 if (ADDR(DATASTART) - 1U >= ADDR(DATAEND)) {
2399 ABORT_ARG2("Wrong DATASTART/END pair", ": %p .. %p", (void *)DATASTART,
2400 (void *)DATAEND);
2401 }
2402 GC_add_roots_inner(DATASTART, DATAEND, FALSE);
2403 # ifdef GC_HAVE_DATAREGION2
2404 if (ADDR(DATASTART2) - 1U >= ADDR(DATAEND2))
2405 ABORT_ARG2("Wrong DATASTART/END2 pair", ": %p .. %p", (void *)DATASTART2,
2406 (void *)DATAEND2);
2407 GC_add_roots_inner(DATASTART2, DATAEND2, FALSE);
2408 # endif
2409 # endif
2410 /*
2411 * Dynamic libraries are added at every collection, since they
2412 * may change.
2413 */
2414 }
2415 #endif /* !ANY_MSWIN && !OPENBSD && !OS2 */
2416
2417 /* Auxiliary routines for obtaining memory from OS. */
2418
2419 #ifdef NEED_UNIX_GET_MEM
2420
2421 # define SBRK_ARG_T ptrdiff_t
2422
2423 # if defined(MMAP_SUPPORTED)
2424
2425 # ifdef USE_MMAP_FIXED
2426 /*
2427 * Seems to yield better performance on Solaris 2, but can be unreliable
2428 * if something is already mapped at the address.
2429 */
2430 # define GC_MMAP_FLAGS MAP_FIXED | MAP_PRIVATE
2431 # else
2432 # define GC_MMAP_FLAGS MAP_PRIVATE
2433 # endif
2434
2435 # ifdef USE_MMAP_ANON
2436 # define zero_fd -1
2437 # if defined(MAP_ANONYMOUS) && !defined(CPPCHECK)
2438 # define OPT_MAP_ANON MAP_ANONYMOUS
2439 # else
2440 # define OPT_MAP_ANON MAP_ANON
2441 # endif
2442 # else
2443 static int zero_fd = -1;
2444 # define OPT_MAP_ANON 0
2445 # endif
2446
2447 # ifndef MSWIN_XBOX1
2448 # if defined(SYMBIAN) && !defined(USE_MMAP_ANON)
2449 EXTERN_C_BEGIN
2450 extern char *GC_get_private_path_and_zero_file(void);
2451 EXTERN_C_END
2452 # endif
2453
2454 STATIC void *
2455 GC_unix_mmap_get_mem(size_t bytes)
2456 {
2457 void *result;
2458 static word last_addr = HEAP_START;
2459
2460 # ifndef USE_MMAP_ANON
2461 static GC_bool initialized = FALSE;
2462
2463 if (UNLIKELY(!initialized)) {
2464 # ifdef SYMBIAN
2465 char *path = GC_get_private_path_and_zero_file();
2466 if (path != NULL) {
2467 zero_fd = open(path, O_RDWR | O_CREAT, 0644);
2468 free(path);
2469 }
2470 # else
2471 zero_fd = open("/dev/zero", O_RDONLY);
2472 # endif
2473 if (zero_fd == -1)
2474 ABORT("Could not open /dev/zero");
2475 if (fcntl(zero_fd, F_SETFD, FD_CLOEXEC) == -1)
2476 WARN("Could not set FD_CLOEXEC for /dev/zero\n", 0);
2477
2478 initialized = TRUE;
2479 }
2480 # endif
2481
2482 GC_ASSERT(GC_page_size != 0);
2483 if (bytes & (GC_page_size - 1))
2484 ABORT("Bad GET_MEM arg");
2485 /*
2486 * Note: it is essential for CHERI to have only address part in
2487 * `last_addr` without metadata (thus the variable is of `word` type
2488 * intentionally), otherwise `mmap()` fails setting `errno` to `EPROT`.
2489 */
2490 result
2491 = mmap(MAKE_CPTR(last_addr), bytes,
2492 (PROT_READ | PROT_WRITE) | (GC_pages_executable ? PROT_EXEC : 0),
2493 GC_MMAP_FLAGS | OPT_MAP_ANON, zero_fd, 0 /* `offset` */);
2494 # undef IGNORE_PAGES_EXECUTABLE
2495
2496 if (UNLIKELY(MAP_FAILED == result)) {
2497 if (HEAP_START == last_addr && GC_pages_executable
2498 && (EACCES == errno || EPERM == errno))
2499 ABORT("Cannot allocate executable pages");
2500 return NULL;
2501 }
2502 # ifdef LINUX
2503 GC_ASSERT(ADDR(result) <= ~(word)(GC_page_size - 1) - bytes);
2504 /* The following `PTR_ALIGN_UP()` cannot overflow. */
2505 # else
2506 if (UNLIKELY(ADDR(result) > ~(word)(GC_page_size - 1) - bytes)) {
2507 /*
2508 * Oops. We got the end of the address space. This is not usable
2509 * by arbitrary C code, since one-past-end pointers do not work,
2510 * so we discard it and try again. Leave the last page mapped,
2511 * so we cannot repeat.
2512 */
2513 (void)munmap(result, ~(GC_page_size - 1) - (size_t)ADDR(result));
2514 return GC_unix_mmap_get_mem(bytes);
2515 }
2516 # endif
2517 if ((ADDR(result) % HBLKSIZE) != 0)
2518 ABORT("Memory returned by mmap is not aligned to HBLKSIZE");
2519 last_addr = ADDR(result) + bytes;
2520 GC_ASSERT((last_addr & (GC_page_size - 1)) == 0);
2521 return result;
2522 }
2523 # endif /* !MSWIN_XBOX1 */
2524
2525 # endif /* MMAP_SUPPORTED */
2526
2527 # if defined(USE_MMAP)
2528
2529 GC_INNER void *
2530 GC_unix_get_mem(size_t bytes)
2531 {
2532 return GC_unix_mmap_get_mem(bytes);
2533 }
2534
2535 # else /* !USE_MMAP */
2536
2537 STATIC void *
2538 GC_unix_sbrk_get_mem(size_t bytes)
2539 {
2540 void *result;
2541
2542 # ifdef IRIX5
2543 /*
2544 * Bare `sbrk()` is not thread-safe. Play by `malloc` rules.
2545 * The equivalent may be needed on other systems as well.
2546 */
2547 __LOCK_MALLOC();
2548 # endif
2549 {
2550 ptr_t cur_brk = (ptr_t)sbrk(0);
2551 SBRK_ARG_T lsbs = ADDR(cur_brk) & (GC_page_size - 1);
2552
2553 GC_ASSERT(GC_page_size != 0);
2554 if (UNLIKELY((SBRK_ARG_T)bytes < 0)) {
2555 /* Value of `bytes` is too big. */
2556 result = NULL;
2557 goto out;
2558 }
2559 if (lsbs != 0) {
2560 if ((ptr_t)sbrk((SBRK_ARG_T)GC_page_size - lsbs) == (ptr_t)(-1)) {
2561 result = NULL;
2562 goto out;
2563 }
2564 }
2565 # ifdef ADD_HEAP_GUARD_PAGES
2566 /*
2567 * This is useful for catching severe memory overwrite problems
2568 * that span heap sections. It should not otherwise be turned on.
2569 */
2570 {
2571 ptr_t guard = (ptr_t)sbrk((SBRK_ARG_T)GC_page_size);
2572 if (mprotect(guard, GC_page_size, PROT_NONE) != 0)
2573 ABORT("ADD_HEAP_GUARD_PAGES: mprotect failed");
2574 }
2575 # endif
2576 result = sbrk((SBRK_ARG_T)bytes);
2577 if (UNLIKELY(ADDR(result) == GC_WORD_MAX))
2578 result = NULL;
2579 }
2580 out:
2581 # ifdef IRIX5
2582 __UNLOCK_MALLOC();
2583 # endif
2584 return result;
2585 }
2586
2587 GC_INNER void *
2588 GC_unix_get_mem(size_t bytes)
2589 {
2590 # if defined(MMAP_SUPPORTED)
2591 /* By default, we try both `sbrk` and `mmap`, in that order. */
2592 static GC_bool sbrk_failed = FALSE;
2593 void *result = NULL;
2594
2595 if (GC_pages_executable) {
2596 /*
2597 * If the allocated memory should have the execute permission,
2598 * then `sbrk()` cannot be used.
2599 */
2600 return GC_unix_mmap_get_mem(bytes);
2601 }
2602 if (!sbrk_failed)
2603 result = GC_unix_sbrk_get_mem(bytes);
2604 if (NULL == result) {
2605 sbrk_failed = TRUE;
2606 result = GC_unix_mmap_get_mem(bytes);
2607 if (NULL == result) {
2608 /* Try `sbrk()` again, in case `sbrk` memory became available. */
2609 result = GC_unix_sbrk_get_mem(bytes);
2610 }
2611 }
2612 return result;
2613 # else /* !MMAP_SUPPORTED */
2614 return GC_unix_sbrk_get_mem(bytes);
2615 # endif
2616 }
2617
2618 # endif /* !USE_MMAP */
2619
2620 #endif /* NEED_UNIX_GET_MEM */
2621
2622 #if defined(OS2)
2623 GC_INNER void *
2624 GC_get_mem(size_t bytes)
2625 {
2626 void *result = NULL;
2627 int retry;
2628
2629 GC_ASSERT(GC_page_size != 0);
2630 bytes = SIZET_SAT_ADD(bytes, GC_page_size);
2631 for (retry = 0;; retry++) {
2632 if (DosAllocMem(&result, bytes,
2633 (PAG_READ | PAG_WRITE | PAG_COMMIT)
2634 | (GC_pages_executable ? PAG_EXECUTE : 0))
2635 == NO_ERROR
2636 && LIKELY(result != NULL))
2637 break;
2638 /*
2639 * TODO: Unclear the purpose of the retry. (Probably, if `DosAllocMem`
2640 * returns memory at address zero, then just retry once.)
2641 */
2642 if (retry >= 1)
2643 return NULL;
2644 }
2645 return HBLKPTR((ptr_t)result + GC_page_size - 1);
2646 }
2647
2648 #elif defined(MSWIN_XBOX1)
2649 GC_INNER void *
2650 GC_get_mem(size_t bytes)
2651 {
2652 if (UNLIKELY(0 == bytes))
2653 return NULL;
2654 return VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_TOP_DOWN, PAGE_READWRITE);
2655 }
2656
2657 #elif defined(MSWINCE)
2658 GC_INNER void *
2659 GC_get_mem(size_t bytes)
2660 {
2661 void *result = NULL; /*< initialized to prevent a compiler warning */
2662 size_t i;
2663
2664 GC_ASSERT(GC_page_size != 0);
2665 bytes = ROUNDUP_PAGESIZE(bytes);
2666
2667 /* Try to find reserved, uncommitted pages. */
2668 for (i = 0; i < GC_n_heap_bases; i++) {
2669 if (((word)(-(GC_signed_word)GC_heap_lengths[i])
2670 & (GC_sysinfo.dwAllocationGranularity - 1))
2671 >= bytes) {
2672 result = GC_heap_bases[i] + GC_heap_lengths[i];
2673 break;
2674 }
2675 }
2676
2677 if (i == GC_n_heap_bases) {
2678 /* Reserve more pages. */
2679 size_t res_bytes
2680 = SIZET_SAT_ADD(bytes, (size_t)GC_sysinfo.dwAllocationGranularity - 1)
2681 & ~((size_t)GC_sysinfo.dwAllocationGranularity - 1);
2682 /*
2683 * If we ever support `MPROTECT_VDB` here, we will probably need
2684 * to ensure that `res_bytes` is greater (strictly) than `bytes`,
2685 * so that `VirtualProtect()` never spans regions. It seems to be
2686 * fine for a `VirtualFree()` argument to span regions, so we
2687 * should be OK for now.
2688 */
2689 result = VirtualAlloc(NULL, res_bytes, MEM_RESERVE | MEM_TOP_DOWN,
2690 GC_pages_executable ? PAGE_EXECUTE_READWRITE
2691 : PAGE_READWRITE);
2692 if (HBLKDISPL(result) != 0) {
2693 /*
2694 * If I read the documentation correctly, this can only happen
2695 * if `HBLKSIZE` is greater than 64 KB or not a power of 2.
2696 */
2697 ABORT("Bad VirtualAlloc result");
2698 }
2699 if (GC_n_heap_bases >= MAX_HEAP_SECTS)
2700 ABORT("Too many heap sections");
2701 if (UNLIKELY(NULL == result))
2702 return NULL;
2703 GC_heap_bases[GC_n_heap_bases] = (ptr_t)result;
2704 GC_heap_lengths[GC_n_heap_bases] = 0;
2705 GC_n_heap_bases++;
2706 }
2707
2708 /* Commit pages. */
2709 result = VirtualAlloc(result, bytes, MEM_COMMIT,
2710 GC_pages_executable ? PAGE_EXECUTE_READWRITE
2711 : PAGE_READWRITE);
2712 # undef IGNORE_PAGES_EXECUTABLE
2713
2714 if (HBLKDISPL(result) != 0)
2715 ABORT("Bad VirtualAlloc result");
2716 if (LIKELY(result != NULL))
2717 GC_heap_lengths[i] += bytes;
2718 return result;
2719 }
2720
2721 #elif defined(CYGWIN32) || defined(MSWIN32)
2722 # ifdef USE_GLOBAL_ALLOC
2723 # define GLOBAL_ALLOC_TEST 1
2724 # else
2725 # define GLOBAL_ALLOC_TEST GC_no_win32_dlls
2726 # endif
2727
2728 # if (defined(GC_USE_MEM_TOP_DOWN) && defined(USE_WINALLOC)) \
2729 || defined(CPPCHECK)
2730 /*
2731 * Use `GC_USE_MEM_TOP_DOWN` for better 64-bit testing.
2732 * Otherwise all addresses tend to end up in the first 4 GB, hiding bugs.
2733 */
2734 DWORD GC_mem_top_down = MEM_TOP_DOWN;
2735 # else
2736 # define GC_mem_top_down 0
2737 # endif /* !GC_USE_MEM_TOP_DOWN */
2738
2739 GC_INNER void *
2740 GC_get_mem(size_t bytes)
2741 {
2742 void *result;
2743
2744 # ifndef USE_WINALLOC
2745 result = GC_unix_get_mem(bytes);
2746 # else
2747 # if defined(MSWIN32) && !defined(MSWINRT_FLAVOR)
2748 if (GLOBAL_ALLOC_TEST) {
2749 /*
2750 * `VirtualAlloc()` does not like `PAGE_EXECUTE_READWRITE`.
2751 * There are also unconfirmed rumors of other problems, so we
2752 * dodge the issue.
2753 */
2754 result = GlobalAlloc(0, SIZET_SAT_ADD(bytes, HBLKSIZE));
2755 /* Align it at `HBLKSIZE` boundary (`NULL` value remains unchanged). */
2756 result = PTR_ALIGN_UP((ptr_t)result, HBLKSIZE);
2757 } else
2758 # endif
2759 /* else */ {
2760 /*
2761 * `VirtualProtect()` only works on regions returned by a single
2762 * `VirtualAlloc()` call. Thus we allocate one extra page, which will
2763 * prevent merging of blocks in separate regions, and eliminate any
2764 * temptation to call `VirtualProtect()` on a range spanning regions.
2765 * This wastes a small amount of memory, and risks increased
2766 * fragmentation. But better alternatives would require effort.
2767 */
2768 # ifdef MPROTECT_VDB
2769 /*
2770 * We cannot check for `GC_incremental` here (because
2771 * `GC_enable_incremental()` might be called some time later after
2772 * the collector initialization).
2773 */
2774 # ifdef GWW_VDB
2775 # define VIRTUAL_ALLOC_PAD (GC_GWW_AVAILABLE() ? 0 : 1)
2776 # else
2777 # define VIRTUAL_ALLOC_PAD 1
2778 # endif
2779 # else
2780 # define VIRTUAL_ALLOC_PAD 0
2781 # endif
2782 /*
2783 * Pass `MEM_WRITE_WATCH` only if `GetWriteWatch`-based VDB is
2784 * enabled and `GetWriteWatch()` is available. Otherwise we waste
2785 * resources or possibly cause `VirtualAlloc()` to fail (observed
2786 * in Windows 2000 SP2).
2787 */
2788 result = VirtualAlloc(
2789 NULL, SIZET_SAT_ADD(bytes, VIRTUAL_ALLOC_PAD),
2790 MEM_COMMIT | MEM_RESERVE | GetWriteWatch_alloc_flag | GC_mem_top_down,
2791 GC_pages_executable ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE);
2792 # undef IGNORE_PAGES_EXECUTABLE
2793 }
2794 # endif
2795 if (HBLKDISPL(result) != 0)
2796 ABORT("Bad VirtualAlloc result");
2797 if (GC_n_heap_bases >= MAX_HEAP_SECTS)
2798 ABORT("Too many heap sections");
2799 if (LIKELY(result != NULL))
2800 GC_heap_bases[GC_n_heap_bases++] = (ptr_t)result;
2801 return result;
2802 }
2803 #endif /* CYGWIN32 || MSWIN32 */
2804
2805 #if defined(ANY_MSWIN) || defined(MSWIN_XBOX1)
2806 GC_API void GC_CALL
2807 GC_win32_free_heap(void)
2808 {
2809 # if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC)
2810 GC_free_malloc_heap_list();
2811 # endif
2812 # if defined(CYGWIN32) || defined(MSWIN32)
2813 # ifndef MSWINRT_FLAVOR
2814 # ifdef MSWIN32
2815 if (GLOBAL_ALLOC_TEST)
2816 # endif
2817 {
2818 while (GC_n_heap_bases > 0) {
2819 GC_n_heap_bases--;
2820 # ifdef CYGWIN32
2821 /* FIXME: Is it OK to use non-GC `free()` here? */
2822 # else
2823 GlobalFree(GC_heap_bases[GC_n_heap_bases]);
2824 # endif
2825 GC_heap_bases[GC_n_heap_bases] = 0;
2826 }
2827 return;
2828 }
2829 # endif /* !MSWINRT_FLAVOR */
2830 # ifndef CYGWIN32
2831 /* Avoiding `VirtualAlloc` leak. */
2832 while (GC_n_heap_bases > 0) {
2833 VirtualFree(GC_heap_bases[--GC_n_heap_bases], 0, MEM_RELEASE);
2834 GC_heap_bases[GC_n_heap_bases] = 0;
2835 }
2836 # endif
2837 # endif
2838 }
2839 #endif /* ANY_MSWIN || MSWIN_XBOX1 */
2840
2841 #if (defined(USE_MUNMAP) || defined(MPROTECT_VDB)) && !defined(USE_WINALLOC)
2842 # define ABORT_ON_REMAP_FAIL(C_msg_prefix, start_addr, len) \
2843 ABORT_ARG3(C_msg_prefix " failed", " at %p (length %lu), errno= %d", \
2844 (void *)(start_addr), (unsigned long)(len), errno)
2845 #endif
2846
2847 #ifdef USE_MUNMAP
2848
2849 # if !defined(NN_PLATFORM_CTR) && !defined(MSWIN32) && !defined(MSWINCE) \
2850 && !defined(MSWIN_XBOX1)
2851 # ifdef SN_TARGET_PS3
2852 # include <sys/memory.h>
2853 # else
2854 # include <sys/mman.h>
2855 # endif
2856 # include <sys/stat.h>
2857 # endif
2858
2859 /*
2860 * Compute a page-aligned starting address for the memory unmap
2861 * operation on a block of size `bytes` starting at `start`.
2862 * Return `NULL` if the block is too small to make this feasible.
2863 */
2864 STATIC ptr_t
2865 GC_unmap_start(ptr_t start, size_t bytes)
2866 {
2867 ptr_t result;
2868
2869 GC_ASSERT(GC_page_size != 0);
2870 result = PTR_ALIGN_UP(start, GC_page_size);
2871 if (ADDR_LT(start + bytes, result + GC_page_size))
2872 return NULL;
2873
2874 return result;
2875 }
2876
2877 /*
2878 * We assume that `GC_remap` is called on exactly the same range as
2879 * the previous call to `GC_unmap`. It is safe to consistently round
2880 * the endpoints in both places.
2881 */
2882
2883 static void
2884 block_unmap_inner(ptr_t start_addr, size_t len)
2885 {
2886 if (0 == start_addr)
2887 return;
2888
2889 # ifdef USE_WINALLOC
2890 /*
2891 * Under Win32/WinCE we commit (map) and decommit (unmap) memory
2892 * using `VirtualAlloc()` and `VirtualFree()`. These functions
2893 * work on individual allocations of virtual memory, made
2894 * previously using `VirtualAlloc()` with the `MEM_RESERVE` flag.
2895 * The ranges we need to (de)commit may span several of these
2896 * allocations; therefore we use `VirtualQuery()` to check
2897 * allocation lengths, and split up the range as necessary.
2898 */
2899 while (len != 0) {
2900 MEMORY_BASIC_INFORMATION mem_info;
2901 word free_len;
2902
2903 if (VirtualQuery(start_addr, &mem_info, sizeof(mem_info))
2904 != sizeof(mem_info))
2905 ABORT("Weird VirtualQuery result");
2906 free_len = (len < mem_info.RegionSize) ? len : mem_info.RegionSize;
2907 if (!VirtualFree(start_addr, free_len, MEM_DECOMMIT))
2908 ABORT("VirtualFree failed");
2909 GC_unmapped_bytes += free_len;
2910 start_addr += free_len;
2911 len -= free_len;
2912 }
2913 # else
2914 if (len != 0) {
2915 # ifdef SN_TARGET_PS3
2916 ps3_free_mem(start_addr, len);
2917 # elif defined(AIX) || defined(COSMO) || defined(CYGWIN32) \
2918 || defined(HPUX) \
2919 || (defined(LINUX) && !defined(PREFER_MMAP_PROT_NONE))
2920 /*
2921 * On AIX, `mmap(PROT_NONE)` fails with `ENOMEM` unless the
2922 * environment variable `XPG_SUS_ENV` is set to `ON`.
2923 * On Cygwin, calling `mmap()` with the new protection flags on
2924 * an existing memory map with `MAP_FIXED` is broken.
2925 * However, calling `mprotect()` on the given address range
2926 * with `PROT_NONE` seems to work fine. On Linux, low `RLIMIT_AS`
2927 * value may lead to `mmap()` failure.
2928 */
2929 # if (defined(COSMO) || defined(LINUX)) \
2930 && !defined(FORCE_MPROTECT_BEFORE_MADVISE)
2931 /* On Linux, at least, `madvise()` should be sufficient. */
2932 # else
2933 if (mprotect(start_addr, len, PROT_NONE))
2934 ABORT_ON_REMAP_FAIL("unmap: mprotect", start_addr, len);
2935 # endif
2936 # if !defined(CYGWIN32)
2937 /*
2938 * On Linux (and some other platforms probably), `mprotect(PROT_NONE)`
2939 * is just disabling access to the pages but not returning them to OS.
2940 */
2941 if (madvise(start_addr, len, MADV_DONTNEED) == -1)
2942 ABORT_ON_REMAP_FAIL("unmap: madvise", start_addr, len);
2943 # endif
2944 # else
2945 /*
2946 * We immediately remap it to prevent an intervening `mmap()` from
2947 * accidentally grabbing the same address space.
2948 */
2949 void *result = mmap(start_addr, len, PROT_NONE,
2950 MAP_PRIVATE | MAP_FIXED | OPT_MAP_ANON, zero_fd,
2951 0 /* `offset` */);
2952
2953 if (UNLIKELY(MAP_FAILED == result))
2954 ABORT_ON_REMAP_FAIL("unmap: mmap", start_addr, len);
2955 if (result != start_addr)
2956 ABORT("unmap: mmap() result differs from start_addr");
2957 # if defined(CPPCHECK) || defined(LINT2)
2958 /* Explicitly store the resource handle to a global variable. */
2959 GC_noop1_ptr(result);
2960 # endif
2961 # endif
2962 GC_unmapped_bytes += len;
2963 }
2964 # endif
2965 }
2966
2967 /* Compute end address for an unmap operation on the indicated block. */
2968 GC_INLINE ptr_t
2969 GC_unmap_end(ptr_t start, size_t bytes)
2970 {
2971 return (ptr_t)HBLK_PAGE_ALIGNED(start + bytes);
2972 }
2973
2974 GC_INNER void
2975 GC_unmap(ptr_t start, size_t bytes)
2976 {
2977 ptr_t start_addr = GC_unmap_start(start, bytes);
2978 ptr_t end_addr = GC_unmap_end(start, bytes);
2979
2980 block_unmap_inner(start_addr, (size_t)(end_addr - start_addr));
2981 }
2982
2983 GC_INNER void
2984 GC_remap(ptr_t start, size_t bytes)
2985 {
2986 ptr_t start_addr = GC_unmap_start(start, bytes);
2987 ptr_t end_addr = GC_unmap_end(start, bytes);
2988 word len = (word)(end_addr - start_addr);
2989 if (0 == start_addr) {
2990 return;
2991 }
2992
2993 /* FIXME: Handle out-of-memory correctly (at least for Win32). */
2994 # ifdef USE_WINALLOC
2995 while (len != 0) {
2996 MEMORY_BASIC_INFORMATION mem_info;
2997 word alloc_len;
2998 ptr_t result;
2999
3000 if (VirtualQuery(start_addr, &mem_info, sizeof(mem_info))
3001 != sizeof(mem_info))
3002 ABORT("Weird VirtualQuery result");
3003 alloc_len = (len < mem_info.RegionSize) ? len : mem_info.RegionSize;
3004 result = (ptr_t)VirtualAlloc(start_addr, alloc_len, MEM_COMMIT,
3005 GC_pages_executable ? PAGE_EXECUTE_READWRITE
3006 : PAGE_READWRITE);
3007 if (result != start_addr) {
3008 if (GetLastError() == ERROR_NOT_ENOUGH_MEMORY
3009 || GetLastError() == ERROR_OUTOFMEMORY) {
3010 ABORT("Not enough memory to process remapping");
3011 } else {
3012 ABORT("VirtualAlloc remapping failed");
3013 }
3014 }
3015 # ifdef LINT2
3016 GC_noop1_ptr(result);
3017 # endif
3018 GC_ASSERT(GC_unmapped_bytes >= alloc_len);
3019 GC_unmapped_bytes -= alloc_len;
3020 start_addr += alloc_len;
3021 len -= alloc_len;
3022 }
3023 # undef IGNORE_PAGES_EXECUTABLE
3024 # else
3025 /* It was already remapped with `PROT_NONE`. */
3026 {
3027 # if !defined(SN_TARGET_PS3) && !defined(FORCE_MPROTECT_BEFORE_MADVISE) \
3028 && (defined(LINUX) && !defined(PREFER_MMAP_PROT_NONE) \
3029 || defined(COSMO))
3030 /* Nothing to unprotect as `madvise()` is just a hint. */
3031 # elif defined(COSMO) || defined(NACL) || defined(NETBSD)
3032 /*
3033 * NaCl does not expose `mprotect`, but `mmap` should work fine.
3034 * In case of NetBSD, `mprotect` fails (unlike `mmap`) even without
3035 * `PROT_EXEC` if PaX `MPROTECT` feature is enabled.
3036 */
3037 void *result = mmap(
3038 start_addr, len,
3039 (PROT_READ | PROT_WRITE) | (GC_pages_executable ? PROT_EXEC : 0),
3040 MAP_PRIVATE | MAP_FIXED | OPT_MAP_ANON, zero_fd, 0 /* `offset` */);
3041 if (UNLIKELY(MAP_FAILED == result))
3042 ABORT_ON_REMAP_FAIL("remap: mmap", start_addr, len);
3043 if (result != start_addr)
3044 ABORT("remap: mmap() result differs from start_addr");
3045 # if defined(CPPCHECK) || defined(LINT2)
3046 GC_noop1_ptr(result);
3047 # endif
3048 # undef IGNORE_PAGES_EXECUTABLE
3049 # else
3050 if (mprotect(start_addr, len,
3051 (PROT_READ | PROT_WRITE)
3052 | (GC_pages_executable ? PROT_EXEC : 0)))
3053 ABORT_ON_REMAP_FAIL("remap: mprotect", start_addr, len);
3054 # undef IGNORE_PAGES_EXECUTABLE
3055 # endif /* !NACL */
3056 }
3057 GC_ASSERT(GC_unmapped_bytes >= len);
3058 GC_unmapped_bytes -= len;
3059 # endif
3060 }
3061
3062 GC_INNER void
3063 GC_unmap_gap(ptr_t start1, size_t bytes1, ptr_t start2, size_t bytes2)
3064 {
3065 ptr_t start1_addr = GC_unmap_start(start1, bytes1);
3066 ptr_t end1_addr = GC_unmap_end(start1, bytes1);
3067 ptr_t start2_addr = GC_unmap_start(start2, bytes2);
3068 ptr_t start_addr = end1_addr;
3069 ptr_t end_addr = start2_addr;
3070
3071 GC_ASSERT(start1 + bytes1 == start2);
3072 if (0 == start1_addr)
3073 start_addr = GC_unmap_start(start1, bytes1 + bytes2);
3074 if (0 == start2_addr)
3075 end_addr = GC_unmap_end(start1, bytes1 + bytes2);
3076 block_unmap_inner(start_addr, (size_t)(end_addr - start_addr));
3077 }
3078
3079 #endif /* USE_MUNMAP */
3080
3081 /*
3082 * Routine for pushing any additional roots. In the multi-threaded
3083 * environment, this is also responsible for marking from thread stacks.
3084 */
3085 #ifndef THREADS
3086
3087 # if defined(EMSCRIPTEN) && defined(EMSCRIPTEN_ASYNCIFY)
3088 # include <emscripten.h>
3089
3090 static void
3091 scan_regs_cb(void *begin, void *finish)
3092 {
3093 GC_push_all_stack((ptr_t)begin, (ptr_t)finish);
3094 }
3095
3096 STATIC void GC_CALLBACK
3097 GC_default_push_other_roots(void)
3098 {
3099 /* Note: this needs `-sASYNCIFY` linker flag. */
3100 emscripten_scan_registers(scan_regs_cb);
3101 }
3102
3103 # else
3104 # define GC_default_push_other_roots 0
3105 # endif
3106
3107 #else /* THREADS */
3108
3109 # if defined(SN_TARGET_PS3)
3110 STATIC void GC_CALLBACK
3111 GC_default_push_other_roots(void)
3112 {
3113 ABORT("GC_default_push_other_roots is not implemented");
3114 }
3115
3116 GC_INNER void
3117 GC_push_thread_structures(void)
3118 {
3119 ABORT("GC_push_thread_structures is not implemented");
3120 }
3121
3122 # else /* GC_PTHREADS, etc. */
3123 STATIC void GC_CALLBACK
3124 GC_default_push_other_roots(void)
3125 {
3126 GC_push_all_stacks();
3127 }
3128 # endif
3129
3130 #endif /* THREADS */
3131
3132 GC_push_other_roots_proc GC_push_other_roots = GC_default_push_other_roots;
3133
3134 GC_API void GC_CALL
3135 GC_set_push_other_roots(GC_push_other_roots_proc fn)
3136 {
3137 GC_push_other_roots = fn;
3138 }
3139
3140 GC_API GC_push_other_roots_proc GC_CALL
3141 GC_get_push_other_roots(void)
3142 {
3143 return GC_push_other_roots;
3144 }
3145
3146 #if defined(SOFT_VDB) && !defined(NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK) \
3147 || (defined(GLIBC_2_19_TSX_BUG) && defined(GC_PTHREADS_PARAMARK))
3148 GC_INNER int
3149 GC_parse_version(int *pminor, const char *pverstr)
3150 {
3151 char *endp;
3152 unsigned long value = strtoul(pverstr, &endp, 10);
3153 int major = (int)value;
3154
3155 if (major < 0 || (char *)pverstr == endp || (unsigned)major != value) {
3156 /* Parse error. */
3157 return -1;
3158 }
3159 if (*endp != '.') {
3160 /* No minor part. */
3161 *pminor = -1;
3162 } else {
3163 value = strtoul(endp + 1, &endp, 10);
3164 *pminor = (int)value;
3165 if (*pminor < 0 || (unsigned)(*pminor) != value) {
3166 return -1;
3167 }
3168 }
3169 return major;
3170 }
3171 #endif
3172
3173 /*
3174 * Routines for accessing dirty bits on virtual pages. There are 6 ways to
3175 * maintain this information, as of now:
3176 *
3177 * - `DEFAULT_VDB`: A simple dummy implementation that treats every page
3178 * as possibly dirty. This makes incremental collection useless, but
3179 * the implementation is still correct.
3180 *
3181 * - `MANUAL_VDB`: Stacks and static data are always considered dirty.
3182 * Heap pages are considered dirty if `GC_dirty(p)` has been called on
3183 * some `p` pointing to somewhere inside an object on that page.
3184 * A `GC_dirty()` call on a large object directly dirties only a single
3185 * page, but for the manual VDB we are careful to treat an object with
3186 * a dirty page as completely dirty. In order to avoid races, an object
3187 * must be marked dirty after it is written, and a reference to the
3188 * object must be kept on a stack or in a register in the interim.
3189 * With threads enabled, an object directly reachable from the stack at
3190 * the time of a collection is treated as dirty. In the single-threaded
3191 * mode, it suffices to ensure that no collection can take place between
3192 * the pointer assignment and the `GC_dirty()` call.
3193 *
3194 * - `PROC_VDB`: Use the `/proc` facility for reading dirty bits.
3195 * Only works under some SVR4 variants. Even then, it may be too slow
3196 * to be entirely satisfactory. Requires reading dirty bits for entire
3197 * address space. Implementations tend to assume that the client is
3198 * a (slow) debugger.
3199 *
3200 * - `SOFT_VDB`: Use the `/proc` facility for reading soft-dirty PTEs
3201 * (page table entries). Works on Linux 3.18+ if the kernel is
3202 * properly configured. The proposed implementation iterates over
3203 * `GC_heap_sects` and `GC_static_roots` examining the soft-dirty bit
3204 * of the `word` elements in `/proc/self/pagemap` file corresponding to
3205 * the pages of the sections; finally all soft-dirty bits of the process
3206 * are cleared (by writing some special value to `/proc/self/clear_refs`
3207 * file). In case the soft-dirty bit is not supported by the kernel,
3208 * `MPROTECT_VDB` may be defined as a fall back strategy.
3209 *
3210 * - `MPROTECT_VDB`: Protect pages and then catch the faults to keep
3211 * track of dirtied pages. The implementation (and implementability)
3212 * is highly system-dependent. This usually fails when system calls
3213 * write to a protected page. We prevent the `read` system call from
3214 * doing so. It is the clients responsibility to make sure that other
3215 * system calls are similarly protected or write only to the stack.
3216 *
3217 * - `GWW_VDB`: Use the Win32 `GetWriteWatch` function, if available, to
3218 * read dirty bits. In case it is not available (because we are
3219 * running on Windows 95, Windows 2000 or earlier), `MPROTECT_VDB` may
3220 * be defined as a fall back strategy.
3221 */
3222
3223 #if (defined(CHECKSUMS) && defined(GWW_VDB)) || defined(PROC_VDB)
3224 /* Add all pages in `pht2` to `pht1`. */
3225 STATIC void
3226 GC_or_pages(page_hash_table pht1, const word *pht2)
3227 {
3228 size_t i;
3229
3230 for (i = 0; i < PHT_SIZE; i++)
3231 pht1[i] |= pht2[i];
3232 }
3233 #endif /* CHECKSUMS && GWW_VDB || PROC_VDB */
3234
3235 #ifdef GWW_VDB
3236
3237 /*
3238 * Note: this is still susceptible to overflow, if there are very large
3239 * allocations, and everything is dirty.
3240 */
3241 # define GC_GWW_BUF_LEN (MAXHINCR * HBLKSIZE / 4096 /* x86 page size */)
3242 static PVOID gww_buf[GC_GWW_BUF_LEN];
3243
3244 # ifndef MPROTECT_VDB
3245 # define GC_gww_dirty_init GC_dirty_init
3246 # endif
3247
3248 GC_INNER GC_bool
3249 GC_gww_dirty_init(void)
3250 {
3251 /* No assumption about the allocator lock. */
3252 detect_GetWriteWatch();
3253 return GC_GWW_AVAILABLE();
3254 }
3255
3256 GC_INLINE void
3257 GC_gww_read_dirty(GC_bool output_unneeded)
3258 {
3259 size_t i;
3260
3261 GC_ASSERT(I_HOLD_LOCK());
3262 if (!output_unneeded)
3263 BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
3264
3265 for (i = 0; i < GC_n_heap_sects; ++i) {
3266 GC_ULONG_PTR count;
3267
3268 do {
3269 PVOID *pages = gww_buf;
3270 DWORD page_size;
3271
3272 count = GC_GWW_BUF_LEN;
3273 /*
3274 * `GetWriteWatch()` is documented as returning nonzero when
3275 * it fails, but the documentation does not explicitly say why
3276 * it would fail or what its behavior will be if it fails.
3277 * It does appear to fail, at least on recent Win2K instances,
3278 * if the underlying memory was not allocated with the appropriate
3279 * flag. This is common if `GC_enable_incremental` is called
3280 * shortly after the collector initialization. To avoid modifying
3281 * the interface, we silently work around such a failure, it only
3282 * affects the initial (small) heap allocation. If there are
3283 * more dirty pages than will fit in the buffer, this is not
3284 * treated as a failure; we must check the page count in the
3285 * loop condition. Since each partial call will reset the
3286 * status of some pages, this should eventually terminate even
3287 * in the overflow case.
3288 */
3289 if ((*(GetWriteWatch_type)(GC_funcptr_uint)GetWriteWatch_func)(
3290 WRITE_WATCH_FLAG_RESET, GC_heap_sects[i].hs_start,
3291 GC_heap_sects[i].hs_bytes, pages, &count, &page_size)
3292 != 0) {
3293 static int warn_count = 0;
3294 struct hblk *start = (struct hblk *)GC_heap_sects[i].hs_start;
3295 static const struct hblk *last_warned = NULL;
3296 size_t nblocks = divHBLKSZ(GC_heap_sects[i].hs_bytes);
3297
3298 if (i != 0 && last_warned != start && warn_count++ < 5) {
3299 last_warned = start;
3300 WARN("GC_gww_read_dirty unexpectedly failed at %p:"
3301 " Falling back to marking all pages dirty\n",
3302 start);
3303 }
3304 if (!output_unneeded) {
3305 size_t j;
3306
3307 for (j = 0; j < nblocks; ++j) {
3308 size_t index = PHT_HASH(start + j);
3309
3310 set_pht_entry_from_index(GC_grungy_pages, index);
3311 }
3312 }
3313 /* Done with this section. */
3314 count = 1;
3315 } else if (!output_unneeded) { /*< succeeded */
3316 const PVOID *pages_end = pages + count;
3317
3318 while (pages != pages_end) {
3319 struct hblk *h = (struct hblk *)(*pages++);
3320 ptr_t h_end = (ptr_t)h + page_size;
3321
3322 do {
3323 set_pht_entry_from_index(GC_grungy_pages, PHT_HASH(h));
3324 h++;
3325 } while (ADDR_LT((ptr_t)h, h_end));
3326 }
3327 }
3328 } while (count == GC_GWW_BUF_LEN);
3329 /*
3330 * FIXME: It is unclear from Microsoft's documentation if this loop
3331 * is useful. We suspect the call just fails if the buffer fills up.
3332 * But that should still be handled correctly.
3333 */
3334 }
3335
3336 # ifdef CHECKSUMS
3337 GC_ASSERT(!output_unneeded);
3338 GC_or_pages(GC_written_pages, GC_grungy_pages);
3339 # endif
3340 }
3341
3342 #elif defined(SOFT_VDB)
3343 static int clear_refs_fd = -1;
3344 # define GC_GWW_AVAILABLE() (clear_refs_fd != -1)
3345 #else
3346 # define GC_GWW_AVAILABLE() FALSE
3347 #endif /* !GWW_VDB && !SOFT_VDB */
3348
3349 #ifdef DEFAULT_VDB
3350 /*
3351 * The client asserts that unallocated pages in the heap are never
3352 * written.
3353 */
3354
3355 GC_INNER GC_bool
3356 GC_dirty_init(void)
3357 {
3358 GC_VERBOSE_LOG_PRINTF("Initializing DEFAULT_VDB...\n");
3359 /* `GC_dirty_pages` and `GC_grungy_pages` are already cleared. */
3360 return TRUE;
3361 }
3362 #endif /* DEFAULT_VDB */
3363
3364 #if !defined(NO_MANUAL_VDB) || defined(MPROTECT_VDB)
3365 # if !defined(THREADS) || defined(HAVE_LOCKFREE_AO_OR)
3366 # ifdef MPROTECT_VDB
3367 # define async_set_pht_entry_from_index(db, index) \
3368 set_pht_entry_from_index_concurrent_volatile(db, index)
3369 # else
3370 # define async_set_pht_entry_from_index(db, index) \
3371 set_pht_entry_from_index_concurrent(db, index)
3372 # endif
3373 # elif defined(NEED_FAULT_HANDLER_LOCK)
3374 /*
3375 * We need to lock around the bitmap update (in the write fault
3376 * handler or `GC_dirty`) in order to avoid the risk of losing a bit.
3377 * We do this with a test-and-set spin lock if possible.
3378 */
3379 static void
3380 async_set_pht_entry_from_index(volatile page_hash_table db, size_t index)
3381 {
3382 GC_acquire_dirty_lock();
3383 set_pht_entry_from_index(db, index);
3384 GC_release_dirty_lock();
3385 }
3386 # else /* THREADS && !NEED_FAULT_HANDLER_LOCK */
3387 # error No test_and_set operation: Introduces a race.
3388 # endif
3389 #endif /* !NO_MANUAL_VDB || MPROTECT_VDB */
3390
3391 #ifdef MPROTECT_VDB
3392 /*
3393 * This implementation maintains dirty bits itself by catching write
3394 * faults and keeping track of them. We assume nobody else catches
3395 * `SIGBUS` or `SIGSEGV`. We assume no write faults occur in system
3396 * calls. This means that clients must ensure that system calls do
3397 * not write to the write-protected heap. Probably the best way to
3398 * do this is to ensure that system calls write at most to
3399 * pointer-free objects in the heap, and do even that only if we are
3400 * on a platform on which those are not protected (or the collector
3401 * is built with `DONT_PROTECT_PTRFREE` defined). We assume the page
3402 * size is a multiple of `HBLKSIZE`.
3403 */
3404
3405 # ifdef DARWIN
3406 /* `#define BROKEN_EXCEPTION_HANDLING` */
3407
3408 /*
3409 * Using `vm_protect` (a `mach` `syscall`) over `mprotect` (a BSD `syscall`)
3410 * seems to decrease the likelihood of some of the problems described below.
3411 */
3412 # include <mach/vm_map.h>
3413 STATIC mach_port_t GC_task_self = 0;
3414 # define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
3415 if (vm_protect(GC_task_self, (vm_address_t)(addr), (vm_size_t)(len), \
3416 FALSE, \
3417 VM_PROT_READ | ((allow_write) ? VM_PROT_WRITE : 0) \
3418 | (GC_pages_executable ? VM_PROT_EXECUTE : 0)) \
3419 == KERN_SUCCESS) { \
3420 } else \
3421 ABORT(C_msg_prefix "vm_protect() failed")
3422
3423 # elif !defined(USE_WINALLOC)
3424 # include <sys/mman.h>
3425 # if !defined(AIX) && !defined(CYGWIN32) && !defined(HAIKU)
3426 # include <sys/syscall.h>
3427 # endif
3428
3429 # define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
3430 if (mprotect((caddr_t)(addr), (size_t)(len), \
3431 PROT_READ | ((allow_write) ? PROT_WRITE : 0) \
3432 | (GC_pages_executable ? PROT_EXEC : 0)) \
3433 >= 0) { \
3434 } else if (GC_pages_executable) { \
3435 ABORT_ON_REMAP_FAIL(C_msg_prefix "mprotect vdb executable pages", \
3436 addr, len); \
3437 } else \
3438 ABORT_ON_REMAP_FAIL(C_msg_prefix "mprotect vdb", addr, len)
3439 # undef IGNORE_PAGES_EXECUTABLE
3440
3441 # else /* USE_WINALLOC */
3442 static DWORD protect_junk;
3443 # define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \
3444 if (VirtualProtect(addr, len, \
3445 GC_pages_executable \
3446 ? ((allow_write) ? PAGE_EXECUTE_READWRITE \
3447 : PAGE_EXECUTE_READ) \
3448 : (allow_write) ? PAGE_READWRITE \
3449 : PAGE_READONLY, \
3450 &protect_junk)) { \
3451 } else \
3452 ABORT_ARG1(C_msg_prefix "VirtualProtect failed", ": errcode= 0x%X", \
3453 (unsigned)GetLastError())
3454 # endif /* USE_WINALLOC */
3455
3456 # define PROTECT(addr, len) PROTECT_INNER(addr, len, FALSE, "")
3457 # define UNPROTECT(addr, len) PROTECT_INNER(addr, len, TRUE, "un-")
3458
3459 # if defined(MSWIN32)
3460 typedef LPTOP_LEVEL_EXCEPTION_FILTER SIG_HNDLR_PTR;
3461 # undef SIG_DFL
3462 # define SIG_DFL ((LPTOP_LEVEL_EXCEPTION_FILTER)(~(GC_funcptr_uint)0))
3463 # elif defined(MSWINCE)
3464 typedef LONG(WINAPI *SIG_HNDLR_PTR)(struct _EXCEPTION_POINTERS *);
3465 # undef SIG_DFL
3466 # define SIG_DFL ((SIG_HNDLR_PTR)(~(GC_funcptr_uint)0))
3467 # elif defined(DARWIN)
3468 # ifdef BROKEN_EXCEPTION_HANDLING
3469 typedef void (*SIG_HNDLR_PTR)();
3470 # endif
3471 # else
3472 typedef void (*SIG_HNDLR_PTR)(int, siginfo_t *, void *);
3473 typedef void (*PLAIN_HNDLR_PTR)(int);
3474 # endif /* !DARWIN && !MSWIN32 && !MSWINCE */
3475
3476 # ifndef DARWIN
3477 /* Also old `MSWIN32` `ACCESS_VIOLATION` filter. */
3478 STATIC SIG_HNDLR_PTR GC_old_segv_handler = 0;
3479 # ifdef USE_BUS_SIGACT
3480 STATIC SIG_HNDLR_PTR GC_old_bus_handler = 0;
3481 STATIC GC_bool GC_old_bus_handler_used_si = FALSE;
3482 # endif
3483 # if !defined(MSWIN32) && !defined(MSWINCE)
3484 STATIC GC_bool GC_old_segv_handler_used_si = FALSE;
3485 # endif
3486 # endif /* !DARWIN */
3487
3488 # ifdef THREADS
3489 /*
3490 * This function is used only by the fault handler. Potential data
3491 * race between this function and `GC_install_header`, `GC_remove_header`
3492 * should not be harmful because the added or removed header should be
3493 * already unprotected.
3494 */
3495 GC_ATTR_NO_SANITIZE_THREAD
3496 static GC_bool
3497 is_header_found_async(const void *p)
3498 {
3499 # ifdef HASH_TL
3500 hdr *result;
3501
3502 GET_HDR(p, result);
3503 return result != NULL;
3504 # else
3505 return HDR_INNER(p) != NULL;
3506 # endif
3507 }
3508 # else
3509 # define is_header_found_async(p) (HDR(p) != NULL)
3510 # endif /* !THREADS */
3511
3512 # ifndef DARWIN
3513
3514 # if !defined(MSWIN32) && !defined(MSWINCE)
3515 # include <errno.h>
3516 # ifdef USE_BUS_SIGACT
3517 # define SIG_OK (sig == SIGBUS || sig == SIGSEGV)
3518 # else
3519 /* Catch `SIGSEGV` but ignore `SIGBUS`. */
3520 # define SIG_OK (sig == SIGSEGV)
3521 # endif
3522 # if defined(FREEBSD) || defined(OPENBSD)
3523 # ifndef SEGV_ACCERR
3524 # define SEGV_ACCERR 2
3525 # endif
3526 # if defined(AARCH64) || defined(ARM32) || defined(MIPS) \
3527 || (__FreeBSD__ >= 7 || defined(OPENBSD))
3528 # define CODE_OK (si->si_code == SEGV_ACCERR)
3529 # elif defined(POWERPC)
3530 /* Pretend that we are AIM. */
3531 # define AIM
3532 # include <machine/trap.h>
3533 # define CODE_OK \
3534 (si->si_code == EXC_DSI || si->si_code == SEGV_ACCERR)
3535 # else
3536 # define CODE_OK \
3537 (si->si_code == BUS_PAGE_FAULT || si->si_code == SEGV_ACCERR)
3538 # endif
3539 # elif defined(OSF1)
3540 # define CODE_OK (si->si_code == 2) /*< experimentally determined */
3541 # elif defined(IRIX5)
3542 # define CODE_OK (si->si_code == EACCES)
3543 # elif defined(AIX) || defined(COSMO) || defined(CYGWIN32) \
3544 || defined(HAIKU) || defined(HURD) || defined(LINUX) \
3545 || defined(NETBSD)
3546 /*
3547 * Linux/i686: Empirically `c.trapno == 14`, but is that useful?
3548 * Should probably consider alignment issues on other architectures.
3549 */
3550 # define CODE_OK TRUE
3551 # elif defined(HPUX)
3552 # define CODE_OK \
3553 (si->si_code == SEGV_ACCERR || si->si_code == BUS_ADRERR \
3554 || si->si_code == BUS_UNKNOWN || si->si_code == SEGV_UNKNOWN \
3555 || si->si_code == BUS_OBJERR)
3556 # elif defined(SUNOS5SIGS)
3557 # define CODE_OK (si->si_code == SEGV_ACCERR)
3558 # endif
3559 # ifndef NO_GETCONTEXT
3560 # include <ucontext.h>
3561 # endif
3562 STATIC void
3563 GC_write_fault_handler(int sig, siginfo_t *si, void *raw_sc)
3564 # else /* MSWIN32 || MSWINCE */
3565 # define SIG_OK \
3566 (exc_info->ExceptionRecord->ExceptionCode == STATUS_ACCESS_VIOLATION)
3567 # define CODE_OK \
3568 (exc_info->ExceptionRecord->ExceptionInformation[0] \
3569 == 1) /*< write fault */
3570 STATIC LONG WINAPI
3571 GC_write_fault_handler(struct _EXCEPTION_POINTERS *exc_info)
3572 # endif
3573 {
3574 # if !defined(MSWIN32) && !defined(MSWINCE)
3575 char *addr = (char *)si->si_addr;
3576 # else
3577 char *addr = (char *)exc_info->ExceptionRecord->ExceptionInformation[1];
3578 # endif
3579
3580 if (SIG_OK && CODE_OK) {
3581 struct hblk *h = HBLK_PAGE_ALIGNED(addr);
3582 GC_bool in_allocd_block;
3583 size_t i;
3584
3585 GC_ASSERT(GC_page_size != 0);
3586 # ifdef CHECKSUMS
3587 GC_record_fault(h);
3588 # endif
3589 # ifdef SUNOS5SIGS
3590 /* Address is only within the correct physical page. */
3591 in_allocd_block = FALSE;
3592 for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
3593 if (is_header_found_async(&h[i])) {
3594 in_allocd_block = TRUE;
3595 break;
3596 }
3597 }
3598 # else
3599 in_allocd_block = is_header_found_async(addr);
3600 # endif
3601 if (!in_allocd_block) {
3602 /*
3603 * FIXME: We should make sure that we invoke the old handler with the
3604 * appropriate calling sequence, which often depends on `SA_SIGINFO`.
3605 */
3606
3607 /* Heap blocks now begin and end on page boundaries. */
3608 SIG_HNDLR_PTR old_handler;
3609
3610 # if defined(MSWIN32) || defined(MSWINCE)
3611 old_handler = GC_old_segv_handler;
3612 # else
3613 GC_bool used_si;
3614
3615 # ifdef USE_BUS_SIGACT
3616 if (sig == SIGBUS) {
3617 old_handler = GC_old_bus_handler;
3618 used_si = GC_old_bus_handler_used_si;
3619 } else
3620 # endif
3621 /* else */ {
3622 old_handler = GC_old_segv_handler;
3623 used_si = GC_old_segv_handler_used_si;
3624 }
3625 # endif
3626
3627 if ((GC_funcptr_uint)old_handler == (GC_funcptr_uint)SIG_DFL) {
3628 # if !defined(MSWIN32) && !defined(MSWINCE)
3629 ABORT_ARG1("Unexpected segmentation fault outside heap", " at %p",
3630 (void *)addr);
3631 # else
3632 return EXCEPTION_CONTINUE_SEARCH;
3633 # endif
3634 } else {
3635 /*
3636 * FIXME: This code should probably check if the old signal handler
3637 * used the traditional style and if so, call it using that style.
3638 */
3639 # if defined(MSWIN32) || defined(MSWINCE)
3640 return (*old_handler)(exc_info);
3641 # else
3642 if (used_si)
3643 ((SIG_HNDLR_PTR)old_handler)(sig, si, raw_sc);
3644 else
3645 /* FIXME: Should pass nonstandard arguments as well. */
3646 ((PLAIN_HNDLR_PTR)(GC_funcptr_uint)old_handler)(sig);
3647 return;
3648 # endif
3649 }
3650 }
3651 UNPROTECT(h, GC_page_size);
3652 /*
3653 * We need to make sure that no collection occurs between the
3654 * `UNPROTECT()` call and the setting of the dirty bit.
3655 * Otherwise a write by a third thread might go unnoticed.
3656 * Reversing the order is just as bad, since we would end up
3657 * unprotecting a page in a collection cycle during which it is not
3658 * marked. Currently we do this by disabling the thread stopping
3659 * signals while this handler is running. An alternative might be
3660 * to record the fact that we are about to unprotect, or have just
3661 * unprotected a page in the collector's thread structure, and then
3662 * to have the thread stopping code set the dirty flag, if necessary.
3663 */
3664 for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
3665 size_t index = PHT_HASH(h + i);
3666
3667 async_set_pht_entry_from_index(GC_dirty_pages, index);
3668 }
3669 /*
3670 * The `write()` may not take place before dirty bits are read.
3671 * But then we will fault again...
3672 */
3673 # if defined(MSWIN32) || defined(MSWINCE)
3674 return EXCEPTION_CONTINUE_EXECUTION;
3675 # else
3676 return;
3677 # endif
3678 }
3679 # if defined(MSWIN32) || defined(MSWINCE)
3680 return EXCEPTION_CONTINUE_SEARCH;
3681 # else
3682 ABORT_ARG1("Unexpected bus error or segmentation fault", " at %p",
3683 (void *)addr);
3684 # endif
3685 }
3686
3687 # if defined(GC_WIN32_THREADS) && !defined(CYGWIN32)
3688 GC_INNER void
3689 GC_set_write_fault_handler(void)
3690 {
3691 SetUnhandledExceptionFilter(GC_write_fault_handler);
3692 }
3693 # endif
3694
3695 # ifdef SOFT_VDB
3696 static GC_bool soft_dirty_init(void);
3697 # endif
3698
3699 GC_INNER GC_bool
3700 GC_dirty_init(void)
3701 {
3702 # if !defined(MSWIN32) && !defined(MSWINCE)
3703 struct sigaction act, oldact;
3704 # endif
3705
3706 GC_ASSERT(I_HOLD_LOCK());
3707 # ifdef COUNT_PROTECTED_REGIONS
3708 GC_ASSERT(GC_page_size != 0);
3709 if ((GC_signed_word)(GC_heapsize / (word)GC_page_size)
3710 >= ((GC_signed_word)GC_UNMAPPED_REGIONS_SOFT_LIMIT
3711 - GC_num_unmapped_regions)
3712 * 2) {
3713 GC_COND_LOG_PRINTF("Cannot turn on GC incremental mode"
3714 " as heap contains too many pages\n");
3715 return FALSE;
3716 }
3717 # endif
3718 # if !defined(MSWIN32) && !defined(MSWINCE)
3719 act.sa_flags = SA_RESTART | SA_SIGINFO;
3720 act.sa_sigaction = GC_write_fault_handler;
3721 (void)sigemptyset(&act.sa_mask);
3722 # ifdef SIGNAL_BASED_STOP_WORLD
3723 /*
3724 * Arrange to postpone the signal while we are in a write fault handler.
3725 * This effectively makes the handler atomic w.r.t. stopping the world
3726 * for the collection.
3727 */
3728 (void)sigaddset(&act.sa_mask, GC_get_suspend_signal());
3729 # endif
3730 # endif /* !MSWIN32 */
3731 GC_VERBOSE_LOG_PRINTF(
3732 "Initializing mprotect virtual dirty bit implementation\n");
3733 if (GC_page_size % HBLKSIZE != 0) {
3734 ABORT("Page size not multiple of HBLKSIZE");
3735 }
3736 # ifdef GWW_VDB
3737 if (GC_gww_dirty_init()) {
3738 GC_COND_LOG_PRINTF("Using GetWriteWatch()\n");
3739 return TRUE;
3740 }
3741 # elif defined(SOFT_VDB)
3742 # ifdef CHECK_SOFT_VDB
3743 if (!soft_dirty_init())
3744 ABORT("Soft-dirty bit support is missing");
3745 # else
3746 if (soft_dirty_init()) {
3747 GC_COND_LOG_PRINTF("Using soft-dirty bit feature\n");
3748 return TRUE;
3749 }
3750 # endif
3751 # endif
3752 # ifdef MSWIN32
3753 GC_old_segv_handler = SetUnhandledExceptionFilter(GC_write_fault_handler);
3754 if (GC_old_segv_handler != NULL) {
3755 GC_COND_LOG_PRINTF("Replaced other UnhandledExceptionFilter\n");
3756 } else {
3757 GC_old_segv_handler = SIG_DFL;
3758 }
3759 # elif defined(MSWINCE)
3760 {
3761 /* `MPROTECT_VDB` is unsupported for WinCE at present. */
3762 /* FIXME: Implement (if possible). */
3763 }
3764 # else
3765 /* `act.sa_restorer` is deprecated and should not be initialized. */
3766 # if defined(IRIX5) && defined(THREADS)
3767 sigaction(SIGSEGV, 0, &oldact);
3768 sigaction(SIGSEGV, &act, 0);
3769 # else
3770 {
3771 int res = sigaction(SIGSEGV, &act, &oldact);
3772 if (res != 0)
3773 ABORT("Sigaction failed");
3774 }
3775 # endif
3776 if (oldact.sa_flags & SA_SIGINFO) {
3777 GC_old_segv_handler = oldact.sa_sigaction;
3778 GC_old_segv_handler_used_si = TRUE;
3779 } else {
3780 GC_old_segv_handler = (SIG_HNDLR_PTR)(GC_funcptr_uint)oldact.sa_handler;
3781 GC_old_segv_handler_used_si = FALSE;
3782 }
3783 if ((GC_funcptr_uint)GC_old_segv_handler == (GC_funcptr_uint)SIG_IGN) {
3784 WARN("Previously ignored segmentation violation!?\n", 0);
3785 GC_old_segv_handler = (SIG_HNDLR_PTR)(GC_funcptr_uint)SIG_DFL;
3786 }
3787 if ((GC_funcptr_uint)GC_old_segv_handler != (GC_funcptr_uint)SIG_DFL) {
3788 GC_VERBOSE_LOG_PRINTF("Replaced other SIGSEGV handler\n");
3789 }
3790 # ifdef USE_BUS_SIGACT
3791 sigaction(SIGBUS, &act, &oldact);
3792 if ((oldact.sa_flags & SA_SIGINFO) != 0) {
3793 GC_old_bus_handler = oldact.sa_sigaction;
3794 GC_old_bus_handler_used_si = TRUE;
3795 } else {
3796 GC_old_bus_handler = (SIG_HNDLR_PTR)(GC_funcptr_uint)oldact.sa_handler;
3797 }
3798 if ((GC_funcptr_uint)GC_old_bus_handler == (GC_funcptr_uint)SIG_IGN) {
3799 WARN("Previously ignored bus error!?\n", 0);
3800 GC_old_bus_handler = (SIG_HNDLR_PTR)(GC_funcptr_uint)SIG_DFL;
3801 } else if ((GC_funcptr_uint)GC_old_bus_handler != (GC_funcptr_uint)SIG_DFL) {
3802 GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n");
3803 }
3804 # endif
3805 # endif /* !MSWIN32 && !MSWINCE */
3806 # if defined(CPPCHECK) && defined(ADDRESS_SANITIZER)
3807 GC_noop1((word)(GC_funcptr_uint)(&__asan_default_options));
3808 # endif
3809 return TRUE;
3810 }
3811 # endif /* !DARWIN */
3812
3813 STATIC void
3814 GC_protect_heap(void)
3815 {
3816 size_t i;
3817
3818 GC_ASSERT(GC_page_size != 0);
3819 for (i = 0; i < GC_n_heap_sects; i++) {
3820 ptr_t start = GC_heap_sects[i].hs_start;
3821 size_t len = GC_heap_sects[i].hs_bytes;
3822 struct hblk *current;
3823 struct hblk *current_start; /*< start of block to be protected */
3824 ptr_t limit;
3825
3826 GC_ASSERT((ADDR(start) & (GC_page_size - 1)) == 0);
3827 GC_ASSERT((len & (GC_page_size - 1)) == 0);
3828 # ifndef DONT_PROTECT_PTRFREE
3829 /*
3830 * We avoid protecting pointer-free objects unless the page size
3831 * differs from `HBLKSIZE`.
3832 */
3833 if (GC_page_size != HBLKSIZE) {
3834 PROTECT(start, len);
3835 continue;
3836 }
3837 # endif
3838
3839 current_start = (struct hblk *)start;
3840 limit = start + len;
3841 for (current = current_start;;) {
3842 size_t nblocks = 0;
3843 GC_bool is_ptrfree = TRUE;
3844
3845 if (ADDR_LT((ptr_t)current, limit)) {
3846 hdr *hhdr;
3847
3848 GET_HDR(current, hhdr);
3849 if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) {
3850 /*
3851 * This can happen only if we are at the beginning of a heap
3852 * segment, and a block spans heap segments. We will handle
3853 * that block as part of the preceding segment.
3854 */
3855 GC_ASSERT(current_start == current);
3856
3857 current_start = ++current;
3858 continue;
3859 }
3860 if (HBLK_IS_FREE(hhdr)) {
3861 GC_ASSERT(modHBLKSZ(hhdr->hb_sz) == 0);
3862 nblocks = divHBLKSZ(hhdr->hb_sz);
3863 } else {
3864 nblocks = OBJ_SZ_TO_BLOCKS(hhdr->hb_sz);
3865 is_ptrfree = IS_PTRFREE(hhdr);
3866 }
3867 }
3868 if (is_ptrfree) {
3869 if (ADDR_LT((ptr_t)current_start, (ptr_t)current)) {
3870 # ifdef DONT_PROTECT_PTRFREE
3871 ptr_t cur_aligned = PTR_ALIGN_UP((ptr_t)current, GC_page_size);
3872
3873 current_start = HBLK_PAGE_ALIGNED(current_start);
3874 /*
3875 * Adjacent free blocks might be protected too because
3876 * of the alignment by the page size.
3877 */
3878 PROTECT(current_start, cur_aligned - (ptr_t)current_start);
3879 # else
3880 PROTECT(current_start, (ptr_t)current - (ptr_t)current_start);
3881 # endif
3882 }
3883 if (ADDR_GE((ptr_t)current, limit))
3884 break;
3885 }
3886 current += nblocks;
3887 if (is_ptrfree)
3888 current_start = current;
3889 }
3890 }
3891 }
3892
3893 # if defined(CAN_HANDLE_FORK) && defined(DARWIN) && defined(THREADS) \
3894 || defined(COUNT_PROTECTED_REGIONS)
3895 /* Remove protection for the entire heap not updating `GC_dirty_pages`. */
3896 STATIC void
3897 GC_unprotect_all_heap(void)
3898 {
3899 size_t i;
3900
3901 GC_ASSERT(I_HOLD_LOCK());
3902 GC_ASSERT(GC_auto_incremental);
3903 for (i = 0; i < GC_n_heap_sects; i++) {
3904 UNPROTECT(GC_heap_sects[i].hs_start, GC_heap_sects[i].hs_bytes);
3905 }
3906 }
3907 # endif
3908
3909 # ifdef COUNT_PROTECTED_REGIONS
3910 GC_INNER void
3911 GC_handle_protected_regions_limit(void)
3912 {
3913 GC_ASSERT(GC_page_size != 0);
3914 /*
3915 * To prevent exceeding the limit of `vm.max_map_count`, the most
3916 * trivial (though highly restrictive) way is to turn off the
3917 * incremental collection mode (based on `mprotect`) once the number
3918 * of pages in the heap reaches that limit.
3919 */
3920 if (GC_auto_incremental && !GC_GWW_AVAILABLE()
3921 && (GC_signed_word)(GC_heapsize / (word)GC_page_size)
3922 >= ((GC_signed_word)GC_UNMAPPED_REGIONS_SOFT_LIMIT
3923 - GC_num_unmapped_regions)
3924 * 2) {
3925 GC_unprotect_all_heap();
3926 # ifdef DARWIN
3927 GC_task_self = 0;
3928 # endif
3929 GC_incremental = FALSE;
3930 WARN("GC incremental mode is turned off"
3931 " to prevent hitting VM maps limit\n",
3932 0);
3933 }
3934 }
3935 # endif /* COUNT_PROTECTED_REGIONS */
3936
3937 #endif /* MPROTECT_VDB */
3938
3939 #if !defined(THREADS) && (defined(PROC_VDB) || defined(SOFT_VDB))
3940 static pid_t saved_proc_pid; /*< `pid` used to compose `/proc` file names */
3941 #endif
3942
3943 #ifdef PROC_VDB
3944 /*
3945 * This implementation assumes the Solaris new structured `/proc`
3946 * pseudo-file-system from which we can read page modified bits.
3947 * This facility is far from optimal (e.g. we would like to get the
3948 * info for only some of the address space), but it avoids intercepting
3949 * system calls.
3950 */
3951
3952 # include <errno.h>
3953 # include <sys/signal.h>
3954 # include <sys/stat.h>
3955 # include <sys/syscall.h>
3956
3957 # ifdef GC_NO_SYS_FAULT_H
3958 /* This exists only to check `PROC_VDB` code compilation (on Linux). */
3959 # define PG_MODIFIED 1
3960 struct prpageheader {
3961 long dummy[2]; /*< `pr_tstamp` */
3962 long pr_nmap;
3963 long pr_npage;
3964 };
3965 struct prasmap {
3966 GC_uintptr_t pr_vaddr;
3967 size_t pr_npage;
3968 char dummy1[64 + 8]; /*< `pr_mapname`, `pr_offset` */
3969 int pr_mflags;
3970 int pr_pagesize;
3971 int dummy2[2]; /*< `pr_shmid`, `pr_filler` */
3972 };
3973 # else
3974 /* Use the new structured `/proc` definitions. */
3975 # include <procfs.h>
3976 # endif
3977
3978 # define INITIAL_BUF_SZ 8192
3979 STATIC size_t GC_proc_buf_size = INITIAL_BUF_SZ;
3980 STATIC char *GC_proc_buf = NULL;
3981 STATIC int GC_proc_fd = -1;
3982
3983 static GC_bool
3984 proc_dirty_open_files(void)
3985 {
3986 char buf[6 + 20 + 9 + 1];
3987 pid_t pid = getpid();
3988
3989 GC_snprintf_s_ld_s(buf, sizeof(buf), "/proc/", (long)pid, "/pagedata");
3990 GC_proc_fd = open(buf, O_RDONLY);
3991 if (-1 == GC_proc_fd) {
3992 WARN("/proc open failed; cannot enable GC incremental mode\n", 0);
3993 return FALSE;
3994 }
3995 if (syscall(SYS_fcntl, GC_proc_fd, F_SETFD, FD_CLOEXEC) == -1)
3996 WARN("Could not set FD_CLOEXEC for /proc\n", 0);
3997 # ifndef THREADS
3998 /* Updated on success only. */
3999 saved_proc_pid = pid;
4000 # endif
4001 return TRUE;
4002 }
4003
4004 # ifdef CAN_HANDLE_FORK
4005 GC_INNER void
4006 GC_dirty_update_child(void)
4007 {
4008 GC_ASSERT(I_HOLD_LOCK());
4009 if (-1 == GC_proc_fd) {
4010 /* The GC incremental mode is off. */
4011 return;
4012 }
4013 close(GC_proc_fd);
4014 if (!proc_dirty_open_files()) {
4015 /* Should be safe to turn it off. */
4016 GC_incremental = FALSE;
4017 }
4018 }
4019 # endif /* CAN_HANDLE_FORK */
4020
4021 GC_INNER GC_bool
4022 GC_dirty_init(void)
4023 {
4024 GC_ASSERT(I_HOLD_LOCK());
4025 if (GC_bytes_allocd != 0 || GC_bytes_allocd_before_gc != 0) {
4026 memset(GC_written_pages, 0xff, sizeof(page_hash_table));
4027 GC_VERBOSE_LOG_PRINTF(
4028 "Allocated %lu bytes: all pages may have been written\n",
4029 (unsigned long)(GC_bytes_allocd + GC_bytes_allocd_before_gc));
4030 }
4031 if (!proc_dirty_open_files())
4032 return FALSE;
4033 GC_proc_buf = GC_scratch_alloc(GC_proc_buf_size);
4034 if (GC_proc_buf == NULL)
4035 ABORT("Insufficient space for /proc read");
4036 return TRUE;
4037 }
4038
4039 GC_INLINE void
4040 GC_proc_read_dirty(GC_bool output_unneeded)
4041 {
4042 size_t i, nmaps;
4043 ssize_t pagedata_len;
4044 char *bufp = GC_proc_buf;
4045
4046 GC_ASSERT(I_HOLD_LOCK());
4047 # ifndef THREADS
4048 /*
4049 * If the current `pid` differs from the saved one, then we are in
4050 * the forked (child) process, the current `/proc` file should be
4051 * closed, the new one should be opened with the updated path.
4052 * Note, this is not needed for the multi-threaded case because
4053 * `fork_child_proc()` reopens the file right after `fork()` call.
4054 */
4055 if (getpid() != saved_proc_pid
4056 && (-1 == GC_proc_fd /*< no need to retry */
4057 || (close(GC_proc_fd), !proc_dirty_open_files()))) {
4058 /* Failed to reopen the file. Punt! */
4059 if (!output_unneeded)
4060 memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
4061 memset(GC_written_pages, 0xff, sizeof(page_hash_table));
4062 return;
4063 }
4064 # endif
4065
4066 for (;;) {
4067 char *new_buf;
4068 size_t new_size;
4069
4070 pagedata_len = PROC_READ(GC_proc_fd, bufp, GC_proc_buf_size);
4071 if (LIKELY(pagedata_len != -1))
4072 break;
4073 if (errno != E2BIG) {
4074 WARN("read /proc failed, errno= %" WARN_PRIdPTR "\n",
4075 (GC_signed_word)errno);
4076 /* Punt. */
4077 if (!output_unneeded)
4078 memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
4079 memset(GC_written_pages, 0xff, sizeof(page_hash_table));
4080 return;
4081 }
4082 /* Retry with larger buffer. */
4083 new_size = 2 * GC_proc_buf_size;
4084 /*
4085 * Alternatively, we could use `fstat()` to determine the required
4086 * buffer size.
4087 */
4088 # ifdef DEBUG_DIRTY_BITS
4089 GC_log_printf("Growing proc buf to %lu bytes at collection #%lu\n",
4090 (unsigned long)new_size, (unsigned long)GC_gc_no + 1);
4091 # endif
4092 new_buf = GC_scratch_alloc(new_size);
4093 if (new_buf != NULL) {
4094 GC_scratch_recycle_no_gww(bufp, GC_proc_buf_size);
4095 GC_proc_buf = bufp = new_buf;
4096 GC_proc_buf_size = new_size;
4097 }
4098 }
4099 GC_ASSERT((size_t)pagedata_len <= GC_proc_buf_size);
4100
4101 /* Copy dirty bits into `GC_grungy_pages`. */
4102 BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
4103 nmaps = (size_t)(((struct prpageheader *)bufp)->pr_nmap);
4104 # ifdef DEBUG_DIRTY_BITS
4105 GC_log_printf("Proc VDB read: pr_nmap= %u, pr_npage= %ld\n", (unsigned)nmaps,
4106 ((struct prpageheader *)bufp)->pr_npage);
4107 # endif
4108 # if defined(GC_NO_SYS_FAULT_H) && defined(CPPCHECK)
4109 GC_noop1(((struct prpageheader *)bufp)->dummy[0]);
4110 # endif
4111 bufp += sizeof(struct prpageheader);
4112 for (i = 0; i < nmaps; i++) {
4113 struct prasmap *map = (struct prasmap *)bufp;
4114 ptr_t vaddr, limit;
4115 unsigned long npages = 0;
4116 unsigned pagesize;
4117
4118 bufp += sizeof(struct prasmap);
4119 /* Ensure no buffer overrun. */
4120 if (bufp - GC_proc_buf < pagedata_len)
4121 npages = (unsigned long)map->pr_npage;
4122 if (bufp - GC_proc_buf > pagedata_len - (ssize_t)npages)
4123 ABORT("Wrong pr_nmap or pr_npage read from /proc");
4124
4125 vaddr = (ptr_t)map->pr_vaddr;
4126 pagesize = (unsigned)map->pr_pagesize;
4127 # if defined(GC_NO_SYS_FAULT_H) && defined(CPPCHECK)
4128 GC_noop1(map->dummy1[0] + map->dummy2[0]);
4129 # endif
4130 # ifdef DEBUG_DIRTY_BITS
4131 GC_log_printf("pr_vaddr= %p, npage= %lu, mflags= 0x%x, pagesize= 0x%x\n",
4132 (void *)vaddr, npages, map->pr_mflags, pagesize);
4133 # endif
4134 if (0 == pagesize || ((pagesize - 1) & pagesize) != 0)
4135 ABORT("Wrong pagesize read from /proc");
4136
4137 limit = vaddr + pagesize * npages;
4138 for (; ADDR_LT(vaddr, limit); vaddr += pagesize) {
4139 if ((*bufp++) & PG_MODIFIED) {
4140 struct hblk *h;
4141 ptr_t next_vaddr = vaddr + pagesize;
4142
4143 # ifdef DEBUG_DIRTY_BITS
4144 GC_log_printf("dirty page at: %p\n", (void *)vaddr);
4145 # endif
4146 for (h = (struct hblk *)vaddr; ADDR_LT((ptr_t)h, next_vaddr); h++) {
4147 size_t index = PHT_HASH(h);
4148
4149 set_pht_entry_from_index(GC_grungy_pages, index);
4150 }
4151 }
4152 }
4153 /*
4154 * According to the new structured `pagedata` file format, an 8-byte
4155 * alignment is enforced (preceding the next `struct prasmap`)
4156 * regardless of the pointer size.
4157 */
4158 bufp = PTR_ALIGN_UP(bufp, 8);
4159 }
4160 # ifdef DEBUG_DIRTY_BITS
4161 GC_log_printf("Proc VDB read done\n");
4162 # endif
4163
4164 /* Update `GC_written_pages` (even if `output_unneeded`). */
4165 GC_or_pages(GC_written_pages, GC_grungy_pages);
4166 }
4167
4168 #endif /* PROC_VDB */
4169
4170 #ifdef SOFT_VDB
4171 # ifndef VDB_BUF_SZ
4172 # define VDB_BUF_SZ 16384
4173 # endif
4174
4175 static int
4176 open_proc_fd(pid_t pid, const char *slash_filename, int mode)
4177 {
4178 int f;
4179 char buf[6 + 20 + 11 + 1];
4180
4181 GC_snprintf_s_ld_s(buf, sizeof(buf), "/proc/", (long)pid, slash_filename);
4182 f = open(buf, mode);
4183 if (-1 == f) {
4184 WARN("/proc/self%s open failed; cannot enable GC incremental mode\n",
4185 slash_filename);
4186 } else if (fcntl(f, F_SETFD, FD_CLOEXEC) == -1) {
4187 WARN("Could not set FD_CLOEXEC for /proc\n", 0);
4188 }
4189 return f;
4190 }
4191
4192 # include <stdint.h> /*< for `uint64_t` */
4193
4194 typedef uint64_t pagemap_elem_t;
4195
4196 static pagemap_elem_t *soft_vdb_buf;
4197 static int pagemap_fd;
4198
4199 static GC_bool
4200 soft_dirty_open_files(void)
4201 {
4202 pid_t pid = getpid();
4203
4204 clear_refs_fd = open_proc_fd(pid, "/clear_refs", O_WRONLY);
4205 if (-1 == clear_refs_fd)
4206 return FALSE;
4207 pagemap_fd = open_proc_fd(pid, "/pagemap", O_RDONLY);
4208 if (-1 == pagemap_fd) {
4209 close(clear_refs_fd);
4210 clear_refs_fd = -1;
4211 return FALSE;
4212 }
4213 # ifndef THREADS
4214 /* Updated on success only. */
4215 saved_proc_pid = pid;
4216 # endif
4217 return TRUE;
4218 }
4219
4220 # ifdef CAN_HANDLE_FORK
4221 GC_INNER void
4222 GC_dirty_update_child(void)
4223 {
4224 GC_ASSERT(I_HOLD_LOCK());
4225 if (-1 == clear_refs_fd) {
4226 /* The GC incremental mode is off. */
4227 return;
4228 }
4229 close(clear_refs_fd);
4230 close(pagemap_fd);
4231 if (!soft_dirty_open_files())
4232 GC_incremental = FALSE;
4233 }
4234 # endif /* CAN_HANDLE_FORK */
4235
4236 /* Clear soft-dirty bits from the task's PTEs. */
4237 static void
4238 clear_soft_dirty_bits(void)
4239 {
4240 ssize_t res = write(clear_refs_fd, "4\n", 2);
4241
4242 if (res != 2)
4243 ABORT_ARG1("Failed to write to /proc/self/clear_refs", ": errno= %d",
4244 res < 0 ? errno : 0);
4245 }
4246
4247 /* The bit 55 of the 64-bit `qword` of `pagemap` file is the soft-dirty one. */
4248 # define PM_SOFTDIRTY_MASK ((pagemap_elem_t)1 << 55)
4249
4250 static GC_bool
4251 detect_soft_dirty_supported(ptr_t vaddr)
4252 {
4253 off_t fpos;
4254 pagemap_elem_t buf[1];
4255
4256 GC_ASSERT(GC_log_pagesize != 0);
4257 /* Make it dirty. */
4258 *vaddr = 1;
4259 fpos = (off_t)((ADDR(vaddr) >> GC_log_pagesize) * sizeof(pagemap_elem_t));
4260
4261 for (;;) {
4262 /* Read the relevant PTE from the `pagemap` file. */
4263 if (lseek(pagemap_fd, fpos, SEEK_SET) == (off_t)(-1))
4264 return FALSE;
4265 if (PROC_READ(pagemap_fd, buf, sizeof(buf)) != (int)sizeof(buf))
4266 return FALSE;
4267
4268 /* Is the soft-dirty bit unset? */
4269 if ((buf[0] & PM_SOFTDIRTY_MASK) == 0)
4270 return FALSE;
4271
4272 if (0 == *vaddr)
4273 break;
4274 /*
4275 * Retry to check that writing to `clear_refs` works as expected.
4276 * This malfunction of the soft-dirty bits implementation is
4277 * observed on some Linux kernels on Power9 (e.g. in Fedora 36).
4278 */
4279 clear_soft_dirty_bits();
4280 *vaddr = 0;
4281 }
4282 return TRUE; /*< success */
4283 }
4284
4285 # ifndef NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK
4286 # include <string.h> /*< for strcmp() */
4287 # include <sys/utsname.h>
4288
4289 /* Ensure the linux (kernel) major/minor version is as given or higher. */
4290 static GC_bool
4291 ensure_min_linux_ver(int major, int minor)
4292 {
4293 struct utsname info;
4294 int actual_major;
4295 int actual_minor = -1;
4296
4297 if (uname(&info) == -1) {
4298 /* `uname()` has failed, should not happen actually. */
4299 return FALSE;
4300 }
4301 if (strcmp(info.sysname, "Linux")) {
4302 WARN("Cannot ensure Linux version as running on other OS: %s\n",
4303 info.sysname);
4304 return FALSE;
4305 }
4306 actual_major = GC_parse_version(&actual_minor, info.release);
4307 return actual_major > major
4308 || (actual_major == major && actual_minor >= minor);
4309 }
4310 # endif
4311
4312 # ifdef MPROTECT_VDB
4313 static GC_bool
4314 soft_dirty_init(void)
4315 # else
4316 GC_INNER GC_bool
4317 GC_dirty_init(void)
4318 # endif
4319 {
4320 # if defined(MPROTECT_VDB) && !defined(CHECK_SOFT_VDB)
4321 char *str = GETENV("GC_USE_GETWRITEWATCH");
4322 # ifdef GC_PREFER_MPROTECT_VDB
4323 if (NULL == str || (*str == '0' && *(str + 1) == '\0')) {
4324 /* The environment variable is unset or set to "0". */
4325 return FALSE;
4326 }
4327 # else
4328 if (str != NULL && *str == '0' && *(str + 1) == '\0') {
4329 /* The environment variable is set "0". */
4330 return FALSE;
4331 }
4332 # endif
4333 # endif
4334 GC_ASSERT(I_HOLD_LOCK());
4335 GC_ASSERT(NULL == soft_vdb_buf);
4336 # ifndef NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK
4337 if (!ensure_min_linux_ver(3, 18)) {
4338 GC_COND_LOG_PRINTF(
4339 "Running on old kernel lacking correct soft-dirty bit support\n");
4340 return FALSE;
4341 }
4342 # endif
4343 if (!soft_dirty_open_files())
4344 return FALSE;
4345 soft_vdb_buf = (pagemap_elem_t *)GC_scratch_alloc(VDB_BUF_SZ);
4346 if (NULL == soft_vdb_buf)
4347 ABORT("Insufficient space for /proc pagemap buffer");
4348 if (!detect_soft_dirty_supported((ptr_t)soft_vdb_buf)) {
4349 GC_COND_LOG_PRINTF("Soft-dirty bit is not supported by kernel\n");
4350 /* Release the resources. */
4351 GC_scratch_recycle_no_gww(soft_vdb_buf, VDB_BUF_SZ);
4352 soft_vdb_buf = NULL;
4353 close(clear_refs_fd);
4354 clear_refs_fd = -1;
4355 close(pagemap_fd);
4356 return FALSE;
4357 }
4358 return TRUE;
4359 }
4360
4361 static off_t pagemap_buf_fpos; /*< valid only if `pagemap_buf_len > 0` */
4362
4363 static size_t pagemap_buf_len;
4364
4365 /*
4366 * Read bytes from `/proc/self/pagemap` file at given file position.
4367 * `len` is the maximum number of bytes to read; `*pres` is the amount
4368 * of bytes actually read (always bigger than 0 but never exceeds `len`);
4369 * `next_fpos_hint` is the file position of the next bytes block to read
4370 * ahead if possible (0 means no information provided).
4371 */
4372 static const pagemap_elem_t *
4373 pagemap_buffered_read(size_t *pres, off_t fpos, size_t len,
4374 off_t next_fpos_hint)
4375 {
4376 ssize_t res;
4377 size_t ofs;
4378
4379 GC_ASSERT(GC_page_size != 0);
4380 GC_ASSERT(len > 0);
4381 if (pagemap_buf_fpos <= fpos
4382 && fpos < pagemap_buf_fpos + (off_t)pagemap_buf_len) {
4383 /* The requested data is already in the buffer. */
4384 ofs = (size_t)(fpos - pagemap_buf_fpos);
4385 res = (ssize_t)(pagemap_buf_fpos + pagemap_buf_len - fpos);
4386 } else {
4387 off_t aligned_pos = fpos
4388 & ~(off_t)(GC_page_size < VDB_BUF_SZ ? GC_page_size - 1
4389 : VDB_BUF_SZ - 1);
4390
4391 for (;;) {
4392 size_t count;
4393
4394 if ((0 == pagemap_buf_len
4395 || pagemap_buf_fpos + (off_t)pagemap_buf_len != aligned_pos)
4396 && lseek(pagemap_fd, aligned_pos, SEEK_SET) == (off_t)(-1))
4397 ABORT_ARG2("Failed to lseek /proc/self/pagemap",
4398 ": offset= %lu, errno= %d", (unsigned long)fpos, errno);
4399
4400 /* How much to read at once? */
4401 ofs = (size_t)(fpos - aligned_pos);
4402 GC_ASSERT(ofs < VDB_BUF_SZ);
4403 if (next_fpos_hint > aligned_pos
4404 && next_fpos_hint - aligned_pos < VDB_BUF_SZ) {
4405 count = VDB_BUF_SZ;
4406 } else {
4407 count = len + ofs;
4408 if (count > VDB_BUF_SZ)
4409 count = VDB_BUF_SZ;
4410 }
4411
4412 GC_ASSERT(count % sizeof(pagemap_elem_t) == 0);
4413 res = PROC_READ(pagemap_fd, soft_vdb_buf, count);
4414 if (res > (ssize_t)ofs)
4415 break;
4416 if (res <= 0)
4417 ABORT_ARG1("Failed to read /proc/self/pagemap", ": errno= %d",
4418 res < 0 ? errno : 0);
4419 /* Retry (once) w/o page-alignment. */
4420 aligned_pos = fpos;
4421 }
4422
4423 /* Save the buffer (file window) position and size. */
4424 pagemap_buf_fpos = aligned_pos;
4425 pagemap_buf_len = (size_t)res;
4426 res -= (ssize_t)ofs;
4427 }
4428
4429 GC_ASSERT(ofs % sizeof(pagemap_elem_t) == 0);
4430 *pres = (size_t)res < len ? (size_t)res : len;
4431 return &soft_vdb_buf[ofs / sizeof(pagemap_elem_t)];
4432 }
4433
4434 static void
4435 soft_set_grungy_pages(ptr_t start, ptr_t limit, ptr_t next_start_hint,
4436 GC_bool is_static_root)
4437 {
4438 ptr_t vaddr = (ptr_t)HBLK_PAGE_ALIGNED(start);
4439 off_t next_fpos_hint = (off_t)((ADDR(next_start_hint) >> GC_log_pagesize)
4440 * sizeof(pagemap_elem_t));
4441
4442 GC_ASSERT(I_HOLD_LOCK());
4443 GC_ASSERT(modHBLKSZ(ADDR(start)) == 0);
4444 GC_ASSERT(GC_log_pagesize != 0);
4445 while (ADDR_LT(vaddr, limit)) {
4446 size_t res;
4447 ptr_t limit_buf;
4448 word vlen_p = ADDR(limit) - ADDR(vaddr) + GC_page_size - 1;
4449 const pagemap_elem_t *bufp = pagemap_buffered_read(
4450 &res,
4451 (off_t)((ADDR(vaddr) >> GC_log_pagesize) * sizeof(pagemap_elem_t)),
4452 (size_t)((vlen_p >> GC_log_pagesize) * sizeof(pagemap_elem_t)),
4453 next_fpos_hint);
4454
4455 if (res % sizeof(pagemap_elem_t) != 0) {
4456 /* Punt. */
4457 memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
4458 WARN("Incomplete read of pagemap, not multiple of entry size\n", 0);
4459 break;
4460 }
4461
4462 limit_buf = vaddr + ((res / sizeof(pagemap_elem_t)) << GC_log_pagesize);
4463 for (; ADDR_LT(vaddr, limit_buf); vaddr += GC_page_size, bufp++) {
4464 if ((*bufp & PM_SOFTDIRTY_MASK) != 0) {
4465 struct hblk *h;
4466 ptr_t next_vaddr = vaddr + GC_page_size;
4467
4468 if (UNLIKELY(ADDR_LT(limit, next_vaddr))) {
4469 next_vaddr = limit;
4470 }
4471
4472 /*
4473 * If the bit is set, the respective PTE was written to
4474 * since clearing the soft-dirty bits.
4475 */
4476 # ifdef DEBUG_DIRTY_BITS
4477 if (is_static_root)
4478 GC_log_printf("static root dirty page at: %p\n", (void *)vaddr);
4479 # endif
4480 h = (struct hblk *)vaddr;
4481 if (UNLIKELY(ADDR_LT(vaddr, start))) {
4482 h = (struct hblk *)start;
4483 }
4484 for (; ADDR_LT((ptr_t)h, next_vaddr); h++) {
4485 size_t index = PHT_HASH(h);
4486
4487 /*
4488 * Filter out the blocks without pointers. It might worth for
4489 * the case when the heap is large enough for the hash collisions
4490 * to occur frequently. Thus, off by default.
4491 */
4492 # if defined(FILTER_PTRFREE_HBLKS_IN_SOFT_VDB) || defined(CHECKSUMS) \
4493 || defined(DEBUG_DIRTY_BITS)
4494 if (!is_static_root) {
4495 hdr *hhdr;
4496
4497 # ifdef CHECKSUMS
4498 set_pht_entry_from_index(GC_written_pages, index);
4499 # endif
4500 GET_HDR(h, hhdr);
4501 if (NULL == hhdr)
4502 continue;
4503
4504 (void)GC_find_starting_hblk(h, &hhdr);
4505 if (HBLK_IS_FREE(hhdr) || IS_PTRFREE(hhdr))
4506 continue;
4507 # ifdef DEBUG_DIRTY_BITS
4508 GC_log_printf("dirty page (hblk) at: %p\n", (void *)h);
4509 # endif
4510 }
4511 # else
4512 UNUSED_ARG(is_static_root);
4513 # endif
4514 set_pht_entry_from_index(GC_grungy_pages, index);
4515 }
4516 } else {
4517 # if defined(CHECK_SOFT_VDB) /* `&& defined(MPROTECT_VDB)` */
4518 /*
4519 * Ensure that each clean page according to the soft-dirty VDB is
4520 * also identified such by the `mprotect`-based one.
4521 */
4522 if (!is_static_root
4523 && get_pht_entry_from_index(GC_dirty_pages, PHT_HASH(vaddr))) {
4524 ptr_t my_start, my_end; /*< the values are not used */
4525
4526 /*
4527 * There could be a hash collision, thus we need to verify the
4528 * page is clean using slow `GC_get_maps()`.
4529 */
4530 if (GC_enclosing_writable_mapping(vaddr, &my_start, &my_end)) {
4531 ABORT("Inconsistent soft-dirty against mprotect dirty bits");
4532 }
4533 }
4534 # endif
4535 }
4536 }
4537 /* Read the next portion of `pagemap` file if incomplete. */
4538 }
4539 }
4540
4541 GC_INLINE void
4542 GC_soft_read_dirty(GC_bool output_unneeded)
4543 {
4544 GC_ASSERT(I_HOLD_LOCK());
4545 # ifndef THREADS
4546 /* Similar as for `GC_proc_read_dirty`. */
4547 if (getpid() != saved_proc_pid
4548 && (-1 == clear_refs_fd /*< no need to retry */
4549 || (close(clear_refs_fd), close(pagemap_fd),
4550 !soft_dirty_open_files()))) {
4551 /* Failed to reopen the files. */
4552 if (!output_unneeded) {
4553 /* Punt. */
4554 memset(GC_grungy_pages, 0xff, sizeof(page_hash_table));
4555 # ifdef CHECKSUMS
4556 memset(GC_written_pages, 0xff, sizeof(page_hash_table));
4557 # endif
4558 }
4559 return;
4560 }
4561 # endif
4562
4563 if (!output_unneeded) {
4564 size_t i;
4565
4566 BZERO(GC_grungy_pages, sizeof(GC_grungy_pages));
4567 pagemap_buf_len = 0; /*< invalidate `soft_vdb_buf` */
4568
4569 for (i = 0; i < GC_n_heap_sects; ++i) {
4570 ptr_t start = GC_heap_sects[i].hs_start;
4571
4572 soft_set_grungy_pages(
4573 start, start + GC_heap_sects[i].hs_bytes,
4574 i + 1 < GC_n_heap_sects ? GC_heap_sects[i + 1].hs_start : NULL,
4575 FALSE);
4576 }
4577
4578 # ifndef NO_VDB_FOR_STATIC_ROOTS
4579 for (i = 0; i < n_root_sets; ++i) {
4580 soft_set_grungy_pages(
4581 (ptr_t)HBLKPTR(GC_static_roots[i].r_start), GC_static_roots[i].r_end,
4582 i + 1 < n_root_sets ? GC_static_roots[i + 1].r_start : NULL, TRUE);
4583 }
4584 # endif
4585 }
4586
4587 clear_soft_dirty_bits();
4588 }
4589 #endif /* SOFT_VDB */
4590
4591 #ifndef NO_MANUAL_VDB
4592 GC_INNER GC_bool GC_manual_vdb = FALSE;
4593
4594 void
4595 GC_dirty_inner(const void *p)
4596 {
4597 size_t index = PHT_HASH(p);
4598
4599 # if defined(MPROTECT_VDB)
4600 /*
4601 * Do not update `GC_dirty_pages` if it should be followed by the
4602 * page unprotection.
4603 */
4604 GC_ASSERT(GC_manual_vdb);
4605 # endif
4606 async_set_pht_entry_from_index(GC_dirty_pages, index);
4607 }
4608 #endif /* !NO_MANUAL_VDB */
4609
4610 #ifndef GC_DISABLE_INCREMENTAL
4611 GC_INNER void
4612 GC_read_dirty(GC_bool output_unneeded)
4613 {
4614 GC_ASSERT(I_HOLD_LOCK());
4615 # ifdef DEBUG_DIRTY_BITS
4616 GC_log_printf("read dirty begin\n");
4617 # endif
4618 if (GC_manual_vdb
4619 # if defined(MPROTECT_VDB)
4620 || !GC_GWW_AVAILABLE()
4621 # endif
4622 ) {
4623 if (!output_unneeded)
4624 BCOPY(CAST_AWAY_VOLATILE_PVOID(GC_dirty_pages), GC_grungy_pages,
4625 sizeof(GC_dirty_pages));
4626 BZERO(CAST_AWAY_VOLATILE_PVOID(GC_dirty_pages), sizeof(GC_dirty_pages));
4627 # ifdef MPROTECT_VDB
4628 if (!GC_manual_vdb)
4629 GC_protect_heap();
4630 # endif
4631 return;
4632 }
4633
4634 # ifdef GWW_VDB
4635 GC_gww_read_dirty(output_unneeded);
4636 # elif defined(PROC_VDB)
4637 GC_proc_read_dirty(output_unneeded);
4638 # elif defined(SOFT_VDB)
4639 GC_soft_read_dirty(output_unneeded);
4640 # endif
4641 # if defined(CHECK_SOFT_VDB) /* `&& defined(MPROTECT_VDB)` */
4642 BZERO(CAST_AWAY_VOLATILE_PVOID(GC_dirty_pages), sizeof(GC_dirty_pages));
4643 GC_protect_heap();
4644 # endif
4645 }
4646
4647 # if !defined(NO_VDB_FOR_STATIC_ROOTS) && !defined(PROC_VDB)
4648 GC_INNER GC_bool
4649 GC_is_vdb_for_static_roots(void)
4650 {
4651 if (GC_manual_vdb)
4652 return FALSE;
4653 # if defined(MPROTECT_VDB)
4654 /* Currently used only in conjunction with `SOFT_VDB`. */
4655 return GC_GWW_AVAILABLE();
4656 # else
4657 # ifndef LINT2
4658 GC_ASSERT(GC_incremental);
4659 # endif
4660 return TRUE;
4661 # endif
4662 }
4663 # endif
4664
4665 GC_INNER GC_bool
4666 GC_page_was_dirty(struct hblk *h)
4667 {
4668 size_t index;
4669
4670 # ifdef DEFAULT_VDB
4671 if (!GC_manual_vdb)
4672 return TRUE;
4673 # elif defined(PROC_VDB)
4674 /* Unless manual VDB is on, the bitmap covers all process memory. */
4675 if (GC_manual_vdb)
4676 # endif
4677 {
4678 if (NULL == HDR(h))
4679 return TRUE;
4680 }
4681 index = PHT_HASH(h);
4682 return get_pht_entry_from_index(GC_grungy_pages, index);
4683 }
4684
4685 # if defined(CHECKSUMS) || defined(PROC_VDB)
4686 GC_INNER GC_bool
4687 GC_page_was_ever_dirty(struct hblk *h)
4688 {
4689 # if defined(GWW_VDB) || defined(PROC_VDB) || defined(SOFT_VDB)
4690 size_t index;
4691
4692 # ifdef MPROTECT_VDB
4693 if (!GC_GWW_AVAILABLE())
4694 return TRUE;
4695 # endif
4696 # if defined(PROC_VDB)
4697 if (GC_manual_vdb)
4698 # endif
4699 {
4700 if (NULL == HDR(h))
4701 return TRUE;
4702 }
4703 index = PHT_HASH(h);
4704 return get_pht_entry_from_index(GC_written_pages, index);
4705 # else
4706 /* TODO: Implement for `MANUAL_VDB` case. */
4707 UNUSED_ARG(h);
4708 return TRUE;
4709 # endif
4710 }
4711 # endif /* CHECKSUMS || PROC_VDB */
4712
4713 GC_INNER void
4714 GC_remove_protection(struct hblk *h, size_t nblocks, GC_bool is_ptrfree)
4715 {
4716 # ifdef MPROTECT_VDB
4717 struct hblk *current;
4718 struct hblk *h_trunc; /*< truncated to page boundary */
4719 ptr_t h_end; /*< page boundary following the block end */
4720 # endif
4721
4722 # ifndef PARALLEL_MARK
4723 GC_ASSERT(I_HOLD_LOCK());
4724 # endif
4725 # ifdef MPROTECT_VDB
4726 /*
4727 * Note it is not allowed to call `GC_printf` (and the friends)
4728 * in this function, see Win32 `GC_stop_world` for the details.
4729 */
4730 # ifdef DONT_PROTECT_PTRFREE
4731 if (is_ptrfree)
4732 return;
4733 # endif
4734 if (!GC_auto_incremental || GC_GWW_AVAILABLE())
4735 return;
4736 GC_ASSERT(GC_page_size != 0);
4737 h_trunc = HBLK_PAGE_ALIGNED(h);
4738 h_end = PTR_ALIGN_UP((ptr_t)(h + nblocks), GC_page_size);
4739 /*
4740 * Note that we cannot examine `GC_dirty_pages` to check whether the
4741 * page at `h_trunc` has already been marked dirty as there could be
4742 * a hash collision.
4743 */
4744 for (current = h_trunc; ADDR_LT((ptr_t)current, h_end); ++current) {
4745 size_t index = PHT_HASH(current);
4746
4747 # ifndef DONT_PROTECT_PTRFREE
4748 if (!is_ptrfree
4749 || !ADDR_INSIDE((ptr_t)current, (ptr_t)h, (ptr_t)(h + nblocks)))
4750 # endif
4751 {
4752 async_set_pht_entry_from_index(GC_dirty_pages, index);
4753 }
4754 }
4755 UNPROTECT(h_trunc, h_end - (ptr_t)h_trunc);
4756 # else
4757 /* Ignore write hints. They do not help us here. */
4758 UNUSED_ARG(h);
4759 UNUSED_ARG(nblocks);
4760 UNUSED_ARG(is_ptrfree);
4761 # endif
4762 }
4763 #endif /* !GC_DISABLE_INCREMENTAL */
4764
4765 #if defined(MPROTECT_VDB) && defined(DARWIN)
4766 /*
4767 * The following sources were used as a "reference" for this exception
4768 * handling code:
4769 * - Apple's mach/xnu documentation;
4770 * - Timothy J. Wood's "Mach Exception Handlers 101" post to the omnigroup's
4771 * macosx-dev list;
4772 * - macosx-nat.c from Apple's GDB source code.
4773 */
4774
4775 /*
4776 * The bug that caused all this trouble should now be fixed.
4777 * This should eventually be removed if all goes well.
4778 */
4779
4780 # include <mach/exception.h>
4781 # include <mach/mach.h>
4782 # include <mach/mach_error.h>
4783 # include <mach/task.h>
4784
4785 EXTERN_C_BEGIN
4786
4787 /*
4788 * Some of the following prototypes are missing in any header, although
4789 * they are documented. Some are in platform `mach/exc.h` file.
4790 */
4791 extern boolean_t exc_server(mach_msg_header_t *, mach_msg_header_t *);
4792
4793 extern kern_return_t exception_raise(mach_port_t, mach_port_t, mach_port_t,
4794 exception_type_t, exception_data_t,
4795 mach_msg_type_number_t);
4796
4797 extern kern_return_t exception_raise_state(
4798 mach_port_t, mach_port_t, mach_port_t, exception_type_t, exception_data_t,
4799 mach_msg_type_number_t, thread_state_flavor_t *, thread_state_t,
4800 mach_msg_type_number_t, thread_state_t, mach_msg_type_number_t *);
4801
4802 extern kern_return_t exception_raise_state_identity(
4803 mach_port_t, mach_port_t, mach_port_t, exception_type_t, exception_data_t,
4804 mach_msg_type_number_t, thread_state_flavor_t *, thread_state_t,
4805 mach_msg_type_number_t, thread_state_t, mach_msg_type_number_t *);
4806
4807 GC_API_OSCALL kern_return_t catch_exception_raise(
4808 mach_port_t exception_port, mach_port_t thread, mach_port_t task,
4809 exception_type_t exception, exception_data_t code,
4810 mach_msg_type_number_t code_count);
4811
4812 GC_API_OSCALL kern_return_t catch_exception_raise_state(
4813 mach_port_name_t exception_port, int exception, exception_data_t code,
4814 mach_msg_type_number_t codeCnt, int flavor, thread_state_t old_state,
4815 int old_stateCnt, thread_state_t new_state, int new_stateCnt);
4816
4817 GC_API_OSCALL kern_return_t catch_exception_raise_state_identity(
4818 mach_port_name_t exception_port, mach_port_t thread, mach_port_t task,
4819 int exception, exception_data_t code, mach_msg_type_number_t codeCnt,
4820 int flavor, thread_state_t old_state, int old_stateCnt,
4821 thread_state_t new_state, int new_stateCnt);
4822
4823 EXTERN_C_END
4824
4825 /* These should never be called, but just in case... */
4826 GC_API_OSCALL kern_return_t
4827 catch_exception_raise_state(mach_port_name_t exception_port, int exception,
4828 exception_data_t code,
4829 mach_msg_type_number_t codeCnt, int flavor,
4830 thread_state_t old_state, int old_stateCnt,
4831 thread_state_t new_state, int new_stateCnt)
4832 {
4833 UNUSED_ARG(exception_port);
4834 UNUSED_ARG(exception);
4835 UNUSED_ARG(code);
4836 UNUSED_ARG(codeCnt);
4837 UNUSED_ARG(flavor);
4838 UNUSED_ARG(old_state);
4839 UNUSED_ARG(old_stateCnt);
4840 UNUSED_ARG(new_state);
4841 UNUSED_ARG(new_stateCnt);
4842 ABORT_RET("Unexpected catch_exception_raise_state invocation");
4843 return KERN_INVALID_ARGUMENT;
4844 }
4845
4846 GC_API_OSCALL kern_return_t
4847 catch_exception_raise_state_identity(
4848 mach_port_name_t exception_port, mach_port_t thread, mach_port_t task,
4849 int exception, exception_data_t code, mach_msg_type_number_t codeCnt,
4850 int flavor, thread_state_t old_state, int old_stateCnt,
4851 thread_state_t new_state, int new_stateCnt)
4852 {
4853 UNUSED_ARG(exception_port);
4854 UNUSED_ARG(thread);
4855 UNUSED_ARG(task);
4856 UNUSED_ARG(exception);
4857 UNUSED_ARG(code);
4858 UNUSED_ARG(codeCnt);
4859 UNUSED_ARG(flavor);
4860 UNUSED_ARG(old_state);
4861 UNUSED_ARG(old_stateCnt);
4862 UNUSED_ARG(new_state);
4863 UNUSED_ARG(new_stateCnt);
4864 ABORT_RET("Unexpected catch_exception_raise_state_identity invocation");
4865 return KERN_INVALID_ARGUMENT;
4866 }
4867
4868 # define MAX_EXCEPTION_PORTS 16
4869
4870 static struct {
4871 mach_msg_type_number_t count;
4872 exception_mask_t masks[MAX_EXCEPTION_PORTS];
4873 exception_handler_t ports[MAX_EXCEPTION_PORTS];
4874 exception_behavior_t behaviors[MAX_EXCEPTION_PORTS];
4875 thread_state_flavor_t flavors[MAX_EXCEPTION_PORTS];
4876 } GC_old_exc_ports;
4877
4878 STATIC struct ports_s {
4879 void (*volatile os_callback[3])(void);
4880 mach_port_t exception;
4881 # if defined(THREADS)
4882 mach_port_t reply;
4883 # endif
4884 } GC_ports = { { /*< this is to prevent stripping these routines as dead */
4885 (void (*)(void))catch_exception_raise,
4886 (void (*)(void))catch_exception_raise_state,
4887 (void (*)(void))catch_exception_raise_state_identity },
4888 # ifdef THREADS
4889 0 /* `exception` */,
4890 # endif
4891 0 };
4892
4893 typedef struct {
4894 mach_msg_header_t head;
4895 } GC_msg_t;
4896
4897 typedef enum {
4898 GC_MP_NORMAL,
4899 GC_MP_DISCARDING,
4900 GC_MP_STOPPED
4901 } GC_mprotect_state_t;
4902
4903 # ifdef THREADS
4904 /*
4905 * FIXME: 1 and 2 seem to be safe to use in the `msgh_id` field, but it
4906 * is not documented. Use the source and see if they should be OK.
4907 */
4908 # define ID_STOP 1
4909 # define ID_RESUME 2
4910
4911 /* This value is only used on the reply port. */
4912 # define ID_ACK 3
4913
4914 STATIC GC_mprotect_state_t GC_mprotect_state = GC_MP_NORMAL;
4915
4916 /* The following should *only* be called when the world is stopped. */
4917 STATIC void
4918 GC_mprotect_thread_notify(mach_msg_id_t id)
4919 {
4920 struct buf_s {
4921 GC_msg_t msg;
4922 mach_msg_trailer_t trailer;
4923 } buf;
4924 mach_msg_return_t r;
4925
4926 /* remote, local */
4927 buf.msg.head.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND, 0);
4928 buf.msg.head.msgh_size = sizeof(buf.msg);
4929 buf.msg.head.msgh_remote_port = GC_ports.exception;
4930 buf.msg.head.msgh_local_port = MACH_PORT_NULL;
4931 buf.msg.head.msgh_id = id;
4932
4933 r = mach_msg(&buf.msg.head, MACH_SEND_MSG | MACH_RCV_MSG | MACH_RCV_LARGE,
4934 sizeof(buf.msg), sizeof(buf), GC_ports.reply,
4935 MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
4936 if (r != MACH_MSG_SUCCESS)
4937 ABORT("mach_msg failed in GC_mprotect_thread_notify");
4938 if (buf.msg.head.msgh_id != ID_ACK)
4939 ABORT("Invalid ack in GC_mprotect_thread_notify");
4940 }
4941
4942 /* Should only be called by the `mprotect` thread. */
4943 STATIC void
4944 GC_mprotect_thread_reply(void)
4945 {
4946 GC_msg_t msg;
4947 mach_msg_return_t r;
4948
4949 /* remote, local */
4950 msg.head.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND, 0);
4951 msg.head.msgh_size = sizeof(msg);
4952 msg.head.msgh_remote_port = GC_ports.reply;
4953 msg.head.msgh_local_port = MACH_PORT_NULL;
4954 msg.head.msgh_id = ID_ACK;
4955
4956 r = mach_msg(&msg.head, MACH_SEND_MSG, sizeof(msg), 0, MACH_PORT_NULL,
4957 MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
4958 if (r != MACH_MSG_SUCCESS)
4959 ABORT("mach_msg failed in GC_mprotect_thread_reply");
4960 }
4961
4962 GC_INNER void
4963 GC_mprotect_stop(void)
4964 {
4965 GC_mprotect_thread_notify(ID_STOP);
4966 }
4967
4968 GC_INNER void
4969 GC_mprotect_resume(void)
4970 {
4971 GC_mprotect_thread_notify(ID_RESUME);
4972 }
4973
4974 # ifdef CAN_HANDLE_FORK
4975 GC_INNER void
4976 GC_dirty_update_child(void)
4977 {
4978 GC_ASSERT(I_HOLD_LOCK());
4979 if (0 == GC_task_self) {
4980 /* The GC incremental mode is off. */
4981 return;
4982 }
4983
4984 GC_ASSERT(GC_mprotect_state == GC_MP_NORMAL);
4985 GC_task_self = mach_task_self(); /*< needed by `UNPROTECT()` */
4986 GC_unprotect_all_heap();
4987
4988 /* Restore the old task exception ports. */
4989 /* TODO: Should we do it in `fork_prepare_proc`/`fork_parent_proc`? */
4990 if (GC_old_exc_ports.count > 0) {
4991 /* TODO: Should we check `GC_old_exc_ports.count <= 1`? */
4992 if (task_set_exception_ports(
4993 GC_task_self, GC_old_exc_ports.masks[0], GC_old_exc_ports.ports[0],
4994 GC_old_exc_ports.behaviors[0], GC_old_exc_ports.flavors[0])
4995 != KERN_SUCCESS)
4996 ABORT("task_set_exception_ports failed (in child)");
4997 }
4998
4999 /* TODO: Re-enable incremental mode in child. */
5000 GC_task_self = 0;
5001 GC_incremental = FALSE;
5002 }
5003 # endif /* CAN_HANDLE_FORK */
5004
5005 # else
5006 /* The compiler should optimize away any `GC_mprotect_state` computations. */
5007 # define GC_mprotect_state GC_MP_NORMAL
5008 # endif /* !THREADS */
5009
5010 struct mp_reply_s {
5011 mach_msg_header_t head;
5012 char data[256];
5013 };
5014
5015 struct mp_msg_s {
5016 mach_msg_header_t head;
5017 mach_msg_body_t msgh_body;
5018 char data[1024];
5019 };
5020
5021 STATIC void *
5022 GC_mprotect_thread(void *arg)
5023 {
5024 mach_msg_return_t r;
5025 /*
5026 * These two structures contain some private kernel data. We do not need
5027 * to access any of it so we do not bother defining a proper structure.
5028 * The correct definitions are in the `xnu` source code.
5029 */
5030 struct mp_reply_s reply;
5031 struct mp_msg_s msg;
5032 mach_msg_id_t id;
5033
5034 if (ADDR(arg) == GC_WORD_MAX)
5035 return 0; /*< to prevent a compiler warning */
5036 # if defined(CPPCHECK)
5037 reply.data[0] = 0; /*< to prevent "field unused" warnings */
5038 msg.data[0] = 0;
5039 # endif
5040
5041 # if defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID)
5042 (void)pthread_setname_np("GC-mprotect");
5043 # endif
5044 # if defined(THREADS) && !defined(GC_NO_THREADS_DISCOVERY)
5045 GC_darwin_register_self_mach_handler();
5046 # endif
5047
5048 for (;;) {
5049 r = mach_msg(
5050 &msg.head,
5051 MACH_RCV_MSG | MACH_RCV_LARGE
5052 | (GC_mprotect_state == GC_MP_DISCARDING ? MACH_RCV_TIMEOUT : 0),
5053 0, sizeof(msg), GC_ports.exception,
5054 GC_mprotect_state == GC_MP_DISCARDING ? 0 : MACH_MSG_TIMEOUT_NONE,
5055 MACH_PORT_NULL);
5056 id = r == MACH_MSG_SUCCESS ? msg.head.msgh_id : -1;
5057
5058 # if defined(THREADS)
5059 if (GC_mprotect_state == GC_MP_DISCARDING) {
5060 if (r == MACH_RCV_TIMED_OUT) {
5061 GC_mprotect_state = GC_MP_STOPPED;
5062 GC_mprotect_thread_reply();
5063 continue;
5064 }
5065 if (r == MACH_MSG_SUCCESS && (id == ID_STOP || id == ID_RESUME))
5066 ABORT("Out of order mprotect thread request");
5067 }
5068 # endif /* THREADS */
5069
5070 if (r != MACH_MSG_SUCCESS) {
5071 ABORT_ARG2("mach_msg failed", ": errcode= %d (%s)", (int)r,
5072 mach_error_string(r));
5073 }
5074
5075 switch (id) {
5076 # if defined(THREADS)
5077 case ID_STOP:
5078 if (GC_mprotect_state != GC_MP_NORMAL)
5079 ABORT("Called mprotect_stop when state wasn't normal");
5080 GC_mprotect_state = GC_MP_DISCARDING;
5081 break;
5082 case ID_RESUME:
5083 if (GC_mprotect_state != GC_MP_STOPPED)
5084 ABORT("Called mprotect_resume when state wasn't stopped");
5085 GC_mprotect_state = GC_MP_NORMAL;
5086 GC_mprotect_thread_reply();
5087 break;
5088 # endif /* THREADS */
5089 default:
5090 /* Handle the message (it calls `catch_exception_raise`). */
5091 if (!exc_server(&msg.head, &reply.head))
5092 ABORT("exc_server failed");
5093 /* Send the reply. */
5094 r = mach_msg(&reply.head, MACH_SEND_MSG, reply.head.msgh_size, 0,
5095 MACH_PORT_NULL, MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
5096 if (r != MACH_MSG_SUCCESS) {
5097 /*
5098 * This will fail if the thread dies, but the thread should
5099 * not die...
5100 */
5101 # ifdef BROKEN_EXCEPTION_HANDLING
5102 GC_err_printf("mach_msg failed with %d %s while sending "
5103 "exc reply\n",
5104 (int)r, mach_error_string(r));
5105 # else
5106 ABORT("mach_msg failed while sending exception reply");
5107 # endif
5108 }
5109 }
5110 }
5111 }
5112
5113 /*
5114 * All this `SIGBUS` code should not be necessary. All protection
5115 * faults should be going through the `mach` exception handler.
5116 * However, it seems a `SIGBUS` is occasionally sent for some unknown
5117 * reason; even more odd, it seems to be meaningless and safe to ignore.
5118 */
5119 # ifdef BROKEN_EXCEPTION_HANDLING
5120
5121 /*
5122 * Updates to this are not atomic, but the `SIGBUS` signals seem pretty rare.
5123 * Even if this does not get updated property, it is not really a problem.
5124 */
5125 STATIC int GC_sigbus_count = 0;
5126
5127 STATIC void
5128 GC_darwin_sigbus(int num, siginfo_t *sip, void *context)
5129 {
5130 if (num != SIGBUS)
5131 ABORT("Got a non-sigbus signal in the sigbus handler");
5132
5133 /*
5134 * Ugh... some seem safe to ignore, but too many in a row probably means
5135 * trouble. `GC_sigbus_count` is reset for each `mach` exception that
5136 * is handled.
5137 */
5138 if (GC_sigbus_count >= 8)
5139 ABORT("Got many SIGBUS signals in a row!");
5140 GC_sigbus_count++;
5141 WARN("Ignoring SIGBUS\n", 0);
5142 }
5143 # endif /* BROKEN_EXCEPTION_HANDLING */
5144
5145 GC_INNER GC_bool
5146 GC_dirty_init(void)
5147 {
5148 kern_return_t r;
5149 mach_port_t me;
5150 pthread_t thread;
5151 pthread_attr_t attr;
5152 exception_mask_t mask;
5153
5154 GC_ASSERT(I_HOLD_LOCK());
5155 # if defined(CAN_HANDLE_FORK) && !defined(THREADS)
5156 if (GC_handle_fork) {
5157 /*
5158 * To both support GC incremental mode and GC functions usage in
5159 * the forked child process, `pthread_atfork` should be used to
5160 * install handlers that switch off `GC_incremental` in the child
5161 * gracefully (unprotecting all pages and clearing
5162 * `GC_mach_handler_thread`). For now, we just disable incremental
5163 * mode if `fork()` handling is requested by the client.
5164 */
5165 WARN("Can't turn on GC incremental mode as fork()"
5166 " handling requested\n",
5167 0);
5168 return FALSE;
5169 }
5170 # endif
5171
5172 GC_VERBOSE_LOG_PRINTF("Initializing mach/darwin mprotect"
5173 " virtual dirty bit implementation\n");
5174 # ifdef BROKEN_EXCEPTION_HANDLING
5175 WARN("Enabling workarounds for various darwin exception handling bugs\n", 0);
5176 # endif
5177 if (GC_page_size % HBLKSIZE != 0) {
5178 ABORT("Page size not multiple of HBLKSIZE");
5179 }
5180
5181 GC_task_self = me = mach_task_self();
5182 GC_ASSERT(me != 0);
5183
5184 r = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, &GC_ports.exception);
5185 /* TODO: Call `WARN()` and return `FALSE` in case of a failure. */
5186 if (r != KERN_SUCCESS)
5187 ABORT("mach_port_allocate failed (exception port)");
5188
5189 r = mach_port_insert_right(me, GC_ports.exception, GC_ports.exception,
5190 MACH_MSG_TYPE_MAKE_SEND);
5191 if (r != KERN_SUCCESS)
5192 ABORT("mach_port_insert_right failed (exception port)");
5193
5194 # if defined(THREADS)
5195 r = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, &GC_ports.reply);
5196 if (r != KERN_SUCCESS)
5197 ABORT("mach_port_allocate failed (reply port)");
5198 # endif
5199
5200 /* The exceptions we want to catch. */
5201 mask = EXC_MASK_BAD_ACCESS;
5202 r = task_get_exception_ports(me, mask, GC_old_exc_ports.masks,
5203 &GC_old_exc_ports.count, GC_old_exc_ports.ports,
5204 GC_old_exc_ports.behaviors,
5205 GC_old_exc_ports.flavors);
5206 if (r != KERN_SUCCESS)
5207 ABORT("task_get_exception_ports failed");
5208
5209 r = task_set_exception_ports(me, mask, GC_ports.exception, EXCEPTION_DEFAULT,
5210 GC_MACH_THREAD_STATE);
5211 if (r != KERN_SUCCESS)
5212 ABORT("task_set_exception_ports failed");
5213
5214 if (pthread_attr_init(&attr) != 0)
5215 ABORT("pthread_attr_init failed");
5216 if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0)
5217 ABORT("pthread_attr_setdetachedstate failed");
5218 /* This will call the real `pthreads` routine, not our wrapper. */
5219 if (GC_inner_pthread_create(&thread, &attr, GC_mprotect_thread, NULL) != 0)
5220 ABORT("pthread_create failed");
5221 (void)pthread_attr_destroy(&attr);
5222
5223 /* Setup the handler for ignoring the meaningless `SIGBUS` signals. */
5224 # ifdef BROKEN_EXCEPTION_HANDLING
5225 {
5226 struct sigaction sa, oldsa;
5227 sa.sa_handler = (SIG_HNDLR_PTR)GC_darwin_sigbus;
5228 sigemptyset(&sa.sa_mask);
5229 sa.sa_flags = SA_RESTART | SA_SIGINFO;
5230 /* `sa.sa_restorer` is deprecated and should not be initialized. */
5231 if (sigaction(SIGBUS, &sa, &oldsa) < 0)
5232 ABORT("sigaction failed");
5233 if ((GC_funcptr_uint)oldsa.sa_handler != (GC_funcptr_uint)SIG_DFL) {
5234 GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n");
5235 }
5236 }
5237 # endif /* BROKEN_EXCEPTION_HANDLING */
5238 # if defined(CPPCHECK)
5239 GC_noop1((word)(GC_funcptr_uint)GC_ports.os_callback[0]);
5240 # endif
5241 return TRUE;
5242 }
5243
5244 /*
5245 * The source code for Apple's GDB was used as a reference for the
5246 * exception forwarding code. This code is similar to be GDB code only
5247 * because there is only one way to do it.
5248 */
5249 STATIC kern_return_t
5250 GC_forward_exception(mach_port_t thread, mach_port_t task,
5251 exception_type_t exception, exception_data_t data,
5252 mach_msg_type_number_t data_count)
5253 {
5254 size_t i;
5255 kern_return_t r;
5256 mach_port_t port;
5257 exception_behavior_t behavior;
5258 thread_state_flavor_t flavor;
5259
5260 thread_state_data_t thread_state;
5261 mach_msg_type_number_t thread_state_count = THREAD_STATE_MAX;
5262
5263 for (i = 0; i < (size_t)GC_old_exc_ports.count; i++) {
5264 if ((GC_old_exc_ports.masks[i] & ((exception_mask_t)1 << exception)) != 0)
5265 break;
5266 }
5267 if (i == (size_t)GC_old_exc_ports.count)
5268 ABORT("No handler for exception!");
5269
5270 port = GC_old_exc_ports.ports[i];
5271 behavior = GC_old_exc_ports.behaviors[i];
5272 flavor = GC_old_exc_ports.flavors[i];
5273
5274 if (behavior == EXCEPTION_STATE || behavior == EXCEPTION_STATE_IDENTITY) {
5275 r = thread_get_state(thread, flavor, thread_state, &thread_state_count);
5276 if (r != KERN_SUCCESS)
5277 ABORT("thread_get_state failed in forward_exception");
5278 }
5279
5280 switch (behavior) {
5281 case EXCEPTION_STATE:
5282 r = exception_raise_state(port, thread, task, exception, data, data_count,
5283 &flavor, thread_state, thread_state_count,
5284 thread_state, &thread_state_count);
5285 break;
5286 case EXCEPTION_STATE_IDENTITY:
5287 r = exception_raise_state_identity(
5288 port, thread, task, exception, data, data_count, &flavor, thread_state,
5289 thread_state_count, thread_state, &thread_state_count);
5290 break;
5291 /* `case EXCEPTION_DEFAULT:` - the default signal handlers. */
5292 default:
5293 /* The user-supplied signal handlers. */
5294 r = exception_raise(port, thread, task, exception, data, data_count);
5295 }
5296
5297 if (behavior == EXCEPTION_STATE || behavior == EXCEPTION_STATE_IDENTITY) {
5298 r = thread_set_state(thread, flavor, thread_state, thread_state_count);
5299 if (r != KERN_SUCCESS)
5300 ABORT("thread_set_state failed in forward_exception");
5301 }
5302 return r;
5303 }
5304
5305 # define FWD() GC_forward_exception(thread, task, exception, code, code_count)
5306
5307 # ifdef ARM32
5308 # define DARWIN_EXC_STATE ARM_EXCEPTION_STATE
5309 # define DARWIN_EXC_STATE_COUNT ARM_EXCEPTION_STATE_COUNT
5310 # define DARWIN_EXC_STATE_T arm_exception_state_t
5311 # define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(far)
5312 # elif defined(AARCH64)
5313 # define DARWIN_EXC_STATE ARM_EXCEPTION_STATE64
5314 # define DARWIN_EXC_STATE_COUNT ARM_EXCEPTION_STATE64_COUNT
5315 # define DARWIN_EXC_STATE_T arm_exception_state64_t
5316 # define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(far)
5317 # elif defined(POWERPC)
5318 # if CPP_WORDSZ == 32
5319 # define DARWIN_EXC_STATE PPC_EXCEPTION_STATE
5320 # define DARWIN_EXC_STATE_COUNT PPC_EXCEPTION_STATE_COUNT
5321 # define DARWIN_EXC_STATE_T ppc_exception_state_t
5322 # else
5323 # define DARWIN_EXC_STATE PPC_EXCEPTION_STATE64
5324 # define DARWIN_EXC_STATE_COUNT PPC_EXCEPTION_STATE64_COUNT
5325 # define DARWIN_EXC_STATE_T ppc_exception_state64_t
5326 # endif
5327 # define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(dar)
5328 # elif defined(I386) || defined(X86_64)
5329 # if CPP_WORDSZ == 32
5330 # if defined(i386_EXCEPTION_STATE_COUNT) \
5331 && !defined(x86_EXCEPTION_STATE32_COUNT)
5332 /* Use old naming convention for i686. */
5333 # define DARWIN_EXC_STATE i386_EXCEPTION_STATE
5334 # define DARWIN_EXC_STATE_COUNT i386_EXCEPTION_STATE_COUNT
5335 # define DARWIN_EXC_STATE_T i386_exception_state_t
5336 # else
5337 # define DARWIN_EXC_STATE x86_EXCEPTION_STATE32
5338 # define DARWIN_EXC_STATE_COUNT x86_EXCEPTION_STATE32_COUNT
5339 # define DARWIN_EXC_STATE_T x86_exception_state32_t
5340 # endif
5341 # else
5342 # define DARWIN_EXC_STATE x86_EXCEPTION_STATE64
5343 # define DARWIN_EXC_STATE_COUNT x86_EXCEPTION_STATE64_COUNT
5344 # define DARWIN_EXC_STATE_T x86_exception_state64_t
5345 # endif
5346 # define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(faultvaddr)
5347 # elif !defined(CPPCHECK)
5348 # error FIXME for non-arm/ppc/x86 darwin
5349 # endif
5350
5351 /*
5352 * This violates the namespace rules but there is not anything that can
5353 * be done about it. The exception handling stuff is hard-coded to
5354 * call this. `catch_exception_raise`, `catch_exception_raise_state`
5355 * and `catch_exception_raise_state_identity` are called from OS.
5356 */
5357 GC_API_OSCALL kern_return_t
5358 catch_exception_raise(mach_port_t exception_port, mach_port_t thread,
5359 mach_port_t task, exception_type_t exception,
5360 exception_data_t code, mach_msg_type_number_t code_count)
5361 {
5362 kern_return_t r;
5363 char *addr;
5364 thread_state_flavor_t flavor = DARWIN_EXC_STATE;
5365 mach_msg_type_number_t exc_state_count = DARWIN_EXC_STATE_COUNT;
5366 DARWIN_EXC_STATE_T exc_state;
5367
5368 UNUSED_ARG(exception_port);
5369 UNUSED_ARG(task);
5370 if (exception != EXC_BAD_ACCESS || code[0] != KERN_PROTECTION_FAILURE) {
5371 # ifdef DEBUG_EXCEPTION_HANDLING
5372 /* We are not interested in, pass it on to the old handler. */
5373 GC_log_printf("Exception: 0x%x Code: 0x%x 0x%x in catch...\n", exception,
5374 code_count > 0 ? code[0] : -1,
5375 code_count > 1 ? code[1] : -1);
5376 # else
5377 UNUSED_ARG(code_count);
5378 # endif
5379 return FWD();
5380 }
5381
5382 r = thread_get_state(thread, flavor, (natural_t *)&exc_state,
5383 &exc_state_count);
5384 if (r != KERN_SUCCESS) {
5385 /*
5386 * The thread is supposed to be suspended while the exception
5387 * handler is called. This should not fail.
5388 */
5389 # ifdef BROKEN_EXCEPTION_HANDLING
5390 GC_err_printf("thread_get_state failed in catch_exception_raise\n");
5391 return KERN_SUCCESS;
5392 # else
5393 ABORT("thread_get_state failed in catch_exception_raise");
5394 # endif
5395 }
5396
5397 /* This is the address that caused the fault. */
5398 addr = (char *)exc_state.DARWIN_EXC_STATE_DAR;
5399 if (!is_header_found_async(addr)) {
5400 /*
5401 * Ugh... just like the `SIGBUS` problem above, it seems we get
5402 * a bogus `KERN_PROTECTION_FAILURE` every once and a while.
5403 * We wait till we get a bunch in a row before doing anything
5404 * about it. If a "real" fault ever occurs, it will just keep
5405 * faulting over and over, and we will hit the limit pretty quickly.
5406 */
5407 # ifdef BROKEN_EXCEPTION_HANDLING
5408 static const char *last_fault;
5409 static int last_fault_count;
5410
5411 if (addr != last_fault) {
5412 last_fault = addr;
5413 last_fault_count = 0;
5414 }
5415 if (++last_fault_count < 32) {
5416 if (last_fault_count == 1)
5417 WARN("Ignoring KERN_PROTECTION_FAILURE at %p\n", addr);
5418 return KERN_SUCCESS;
5419 }
5420
5421 GC_err_printf("Unexpected KERN_PROTECTION_FAILURE at %p; aborting...\n",
5422 (void *)addr);
5423 /*
5424 * Cannot pass it along to the signal handler because that is ignoring
5425 * `SIGBUS` signals. We also should not call `ABORT()` here as signals
5426 * do not always work too well from the exception handler.
5427 */
5428 EXIT();
5429 # else
5430 /*
5431 * Pass it along to the next exception handler (which should call
5432 * `SIGBUS`/`SIGSEGV`).
5433 */
5434 return FWD();
5435 # endif /* !BROKEN_EXCEPTION_HANDLING */
5436 }
5437
5438 # ifdef BROKEN_EXCEPTION_HANDLING
5439 /* Reset the number of consecutive `SIGBUS` signals. */
5440 GC_sigbus_count = 0;
5441 # endif
5442
5443 GC_ASSERT(GC_page_size != 0);
5444 if (GC_mprotect_state == GC_MP_NORMAL) {
5445 /* The common case. */
5446 struct hblk *h = HBLK_PAGE_ALIGNED(addr);
5447 size_t i;
5448
5449 # ifdef CHECKSUMS
5450 GC_record_fault(h);
5451 # endif
5452 UNPROTECT(h, GC_page_size);
5453 for (i = 0; i < divHBLKSZ(GC_page_size); i++) {
5454 size_t index = PHT_HASH(h + i);
5455
5456 async_set_pht_entry_from_index(GC_dirty_pages, index);
5457 }
5458 } else if (GC_mprotect_state == GC_MP_DISCARDING) {
5459 /*
5460 * Lie to the thread for now. No sense `UNPROTECT`'ing the memory
5461 * when we are just going to `PROTECT()` it again later.
5462 * The thread will just fault again once it resumes.
5463 */
5464 } else {
5465 /* Should not happen, I do not think. */
5466 GC_err_printf("KERN_PROTECTION_FAILURE while world is stopped\n");
5467 return FWD();
5468 }
5469 return KERN_SUCCESS;
5470 }
5471 # undef FWD
5472
5473 # ifndef NO_DESC_CATCH_EXCEPTION_RAISE
5474 /*
5475 * These symbols should have `REFERENCED_DYNAMICALLY` (0x10) bit set to
5476 * let strip know they are not to be stripped.
5477 */
5478 __asm__(".desc _catch_exception_raise, 0x10");
5479 __asm__(".desc _catch_exception_raise_state, 0x10");
5480 __asm__(".desc _catch_exception_raise_state_identity, 0x10");
5481 # endif
5482
5483 #endif /* DARWIN && MPROTECT_VDB */
5484
5485 GC_API int GC_CALL
5486 GC_incremental_protection_needs(void)
5487 {
5488 GC_ASSERT(GC_is_initialized);
5489 #ifdef MPROTECT_VDB
5490 # if defined(GWW_VDB) || (defined(SOFT_VDB) && !defined(CHECK_SOFT_VDB))
5491 /* Only if the incremental mode is already switched on. */
5492 if (GC_GWW_AVAILABLE())
5493 return GC_PROTECTS_NONE;
5494 # endif
5495 # ifndef DONT_PROTECT_PTRFREE
5496 if (GC_page_size != HBLKSIZE)
5497 return GC_PROTECTS_POINTER_HEAP | GC_PROTECTS_PTRFREE_HEAP;
5498 # endif
5499 return GC_PROTECTS_POINTER_HEAP;
5500 #else
5501 return GC_PROTECTS_NONE;
5502 #endif
5503 }
5504
5505 GC_API unsigned GC_CALL
5506 GC_get_actual_vdb(void)
5507 {
5508 #ifndef GC_DISABLE_INCREMENTAL
5509 if (GC_incremental) {
5510 # ifndef NO_MANUAL_VDB
5511 if (GC_manual_vdb)
5512 return GC_VDB_MANUAL;
5513 # endif
5514 # ifdef MPROTECT_VDB
5515 # ifdef GWW_VDB
5516 if (GC_GWW_AVAILABLE())
5517 return GC_VDB_GWW;
5518 # endif
5519 # ifdef SOFT_VDB
5520 if (GC_GWW_AVAILABLE())
5521 return GC_VDB_SOFT;
5522 # endif
5523 return GC_VDB_MPROTECT;
5524 # elif defined(GWW_VDB)
5525 return GC_VDB_GWW;
5526 # elif defined(SOFT_VDB)
5527 return GC_VDB_SOFT;
5528 # elif defined(PROC_VDB)
5529 return GC_VDB_PROC;
5530 # else /* DEFAULT_VDB */
5531 return GC_VDB_DEFAULT;
5532 # endif
5533 }
5534 #endif
5535 return GC_VDB_NONE;
5536 }
5537
5538 #ifdef ECOS
5539 /* Undo `sbrk()` redirection. */
5540 # undef sbrk
5541 #endif
5542
5543 GC_API void GC_CALL
5544 GC_set_pages_executable(int value)
5545 {
5546 GC_ASSERT(!GC_is_initialized);
5547 /*
5548 * Even if `IGNORE_PAGES_EXECUTABLE` macro is defined,
5549 * `GC_pages_executable` is touched here to prevent a compiler warning.
5550 */
5551 GC_pages_executable = (GC_bool)(value != 0);
5552 }
5553
5554 GC_API int GC_CALL
5555 GC_get_pages_executable(void)
5556 {
5557 /*
5558 * `GC_get_pages_executable` is defined after all the places
5559 * where `GC_get_pages_executable` is undefined.
5560 */
5561 #ifdef IGNORE_PAGES_EXECUTABLE
5562 /* Always allocate executable memory. */
5563 return 1;
5564 #else
5565 return (int)GC_pages_executable;
5566 #endif
5567 }
5568
5569 /*
5570 * Call stack save code for debugging. Should probably be in
5571 * `mach_dep.c` file, but that requires reorganization.
5572 */
5573 #ifdef NEED_CALLINFO
5574
5575 /*
5576 * I suspect the following works for most Un*x i686 variants, so long as
5577 * the frame pointer is explicitly stored. In the case of gcc, the client
5578 * code should not be compiled with `-fomit-frame-pointer` option.
5579 */
5580 # if defined(I386) && defined(LINUX) && defined(SAVE_CALL_CHAIN)
5581 struct frame {
5582 struct frame *fr_savfp;
5583 long fr_savpc;
5584 # if NARGS > 0
5585 /* All the arguments go here. */
5586 long fr_arg[NARGS];
5587 # endif
5588 };
5589 # endif
5590
5591 # if defined(SPARC)
5592 # if defined(LINUX)
5593 # if defined(SAVE_CALL_CHAIN)
5594 struct frame {
5595 long fr_local[8];
5596 long fr_arg[6];
5597 struct frame *fr_savfp;
5598 long fr_savpc;
5599 # ifndef __arch64__
5600 char *fr_stret;
5601 # endif
5602 long fr_argd[6];
5603 long fr_argx[0];
5604 };
5605 # endif
5606 # elif defined(DRSNX)
5607 # include <sys/sparc/frame.h>
5608 # elif defined(OPENBSD)
5609 # include <frame.h>
5610 # elif defined(FREEBSD) || defined(NETBSD)
5611 # include <machine/frame.h>
5612 # else
5613 # include <sys/frame.h>
5614 # endif
5615 # if NARGS > 6
5616 # error We only know how to get the first 6 arguments
5617 # endif
5618 # endif /* SPARC */
5619
5620 /*
5621 * Fill in the `pc` and `argument` information for up to `NFRAMES` of
5622 * my callers. Ignore my frame and my callers frame.
5623 */
5624
5625 # if defined(GC_HAVE_BUILTIN_BACKTRACE)
5626 # ifdef _MSC_VER
5627 EXTERN_C_BEGIN
5628 int backtrace(void *addresses[], int count);
5629 char **backtrace_symbols(void *const addresses[], int count);
5630 EXTERN_C_END
5631 # else
5632 # include <execinfo.h>
5633 # endif
5634 # endif /* GC_HAVE_BUILTIN_BACKTRACE */
5635
5636 # ifdef SAVE_CALL_CHAIN
5637
5638 # if NARGS == 0 && NFRAMES % 2 == 0 /*< no padding */ \
5639 && defined(GC_HAVE_BUILTIN_BACKTRACE)
5640
5641 # ifdef REDIRECT_MALLOC
5642 /*
5643 * Deal with possible `malloc()` calls in `backtrace()` by omitting
5644 * the infinitely recursing backtrace.
5645 */
5646 STATIC GC_bool GC_in_save_callers = FALSE;
5647
5648 # if defined(THREADS) && defined(DBG_HDRS_ALL)
5649 # include "private/dbg_mlc.h"
5650
5651 GC_INNER void
5652 GC_save_callers_no_unlock(struct callinfo info[NFRAMES])
5653 {
5654 GC_ASSERT(I_HOLD_LOCK());
5655 info[0].ci_pc
5656 = CAST_THRU_UINTPTR(GC_return_addr_t, GC_save_callers_no_unlock);
5657 BZERO(&info[1], sizeof(void *) * (NFRAMES - 1));
5658 }
5659 # endif
5660 # endif /* REDIRECT_MALLOC */
5661
5662 GC_INNER void
5663 GC_save_callers(struct callinfo info[NFRAMES])
5664 {
5665 void *tmp_info[NFRAMES + 1];
5666 int npcs, i;
5667
5668 /*
5669 * `backtrace()` may call `dl_iterate_phdr` which is also used by
5670 * `GC_register_dynamic_libraries()`, and `dl_iterate_phdr` is not
5671 * guaranteed to be reentrant.
5672 */
5673 GC_ASSERT(I_HOLD_LOCK());
5674
5675 GC_STATIC_ASSERT(sizeof(struct callinfo) == sizeof(void *));
5676 # ifdef REDIRECT_MALLOC
5677 if (GC_in_save_callers) {
5678 info[0].ci_pc = CAST_THRU_UINTPTR(GC_return_addr_t, GC_save_callers);
5679 BZERO(&info[1], sizeof(void *) * (NFRAMES - 1));
5680 return;
5681 }
5682 GC_in_save_callers = TRUE;
5683 /* `backtrace()` might call a redirected `malloc`. */
5684 UNLOCK();
5685 npcs = backtrace((void **)tmp_info, NFRAMES + 1);
5686 LOCK();
5687 # else
5688 npcs = backtrace((void **)tmp_info, NFRAMES + 1);
5689 # endif
5690 /*
5691 * We retrieve `NFRAMES + 1` `pc` values, but discard the first one,
5692 * since it points to our own frame.
5693 */
5694 i = 0;
5695 if (npcs > 1) {
5696 i = npcs - 1;
5697 BCOPY(&tmp_info[1], info, (unsigned)i * sizeof(void *));
5698 }
5699 BZERO(&info[i], sizeof(void *) * (unsigned)(NFRAMES - i));
5700 # ifdef REDIRECT_MALLOC
5701 GC_in_save_callers = FALSE;
5702 # endif
5703 }
5704
5705 # elif defined(I386) || defined(SPARC)
5706
5707 # if defined(ANY_BSD) && defined(SPARC)
5708 # define FR_SAVFP fr_fp
5709 # define FR_SAVPC fr_pc
5710 # else
5711 # define FR_SAVFP fr_savfp
5712 # define FR_SAVPC fr_savpc
5713 # endif
5714
5715 # if defined(SPARC) && (defined(__arch64__) || defined(__sparcv9))
5716 # define BIAS 2047
5717 # else
5718 # define BIAS 0
5719 # endif
5720
5721 GC_INNER void
5722 GC_save_callers(struct callinfo info[NFRAMES])
5723 {
5724 struct frame *frame;
5725 struct frame *fp;
5726 int nframes = 0;
5727 # ifdef I386
5728 /* We assume this is turned on only with gcc as the compiler. */
5729 asm("movl %%ebp,%0" : "=r"(frame));
5730 fp = frame;
5731 # else /* SPARC */
5732 frame = (struct frame *)GC_save_regs_in_stack();
5733 fp = (struct frame *)((ptr_t)frame->FR_SAVFP + BIAS);
5734 # endif
5735
5736 for (; !HOTTER_THAN((ptr_t)fp, (ptr_t)frame)
5737 # ifndef THREADS
5738 && !HOTTER_THAN(GC_stackbottom, (ptr_t)fp)
5739 # elif defined(STACK_GROWS_UP)
5740 && fp != NULL
5741 # endif
5742 && nframes < NFRAMES;
5743 fp = (struct frame *)((ptr_t)fp->FR_SAVFP + BIAS), nframes++) {
5744 # if NARGS > 0
5745 int i;
5746 # endif
5747
5748 info[nframes].ci_pc = (GC_return_addr_t)fp->FR_SAVPC;
5749 # if NARGS > 0
5750 for (i = 0; i < NARGS; i++) {
5751 info[nframes].ci_arg[i] = GC_HIDE_NZ_POINTER(MAKE_CPTR(fp->fr_arg[i]));
5752 }
5753 # endif
5754 }
5755 if (nframes < NFRAMES)
5756 info[nframes].ci_pc = 0;
5757 }
5758
5759 # endif /* !GC_HAVE_BUILTIN_BACKTRACE */
5760
5761 # endif /* SAVE_CALL_CHAIN */
5762
5763 GC_INNER void
5764 GC_print_callers(struct callinfo info[NFRAMES])
5765 {
5766 int i, reent_cnt;
5767 # if defined(AO_HAVE_fetch_and_add1) && defined(AO_HAVE_fetch_and_sub1)
5768 static volatile AO_t reentry_count = 0;
5769
5770 /*
5771 * Note: alternatively, if available, we may use a thread-local storage,
5772 * thus, enabling concurrent usage of `GC_print_callers()`;
5773 * but practically this has little sense because printing is done into
5774 * a single output stream.
5775 */
5776 GC_ASSERT(I_DONT_HOLD_LOCK());
5777 reent_cnt = (int)(GC_signed_word)AO_fetch_and_add1(&reentry_count);
5778 # else
5779 static int reentry_count = 0;
5780
5781 /* Note: this could use a different lock. */
5782 LOCK();
5783 reent_cnt = reentry_count++;
5784 UNLOCK();
5785 # endif
5786 # if NFRAMES == 1
5787 GC_err_printf("\tCaller at allocation:\n");
5788 # else
5789 GC_err_printf("\tCall chain at allocation:\n");
5790 # endif
5791 for (i = 0; i < NFRAMES; i++) {
5792 # if defined(LINUX) && !defined(SMALL_CONFIG)
5793 GC_bool stop = FALSE;
5794 # endif
5795
5796 if (0 == info[i].ci_pc)
5797 break;
5798 # if NARGS > 0
5799 {
5800 int j;
5801
5802 GC_err_printf("\t\targs: ");
5803 for (j = 0; j < NARGS; j++) {
5804 void *p = GC_REVEAL_NZ_POINTER(info[i].ci_arg[j]);
5805
5806 if (j != 0)
5807 GC_err_printf(", ");
5808 GC_err_printf("%ld (%p)", (long)(GC_signed_word)ADDR(p), p);
5809 }
5810 GC_err_printf("\n");
5811 }
5812 # endif
5813 if (reent_cnt > 0) {
5814 /*
5815 * We were called either concurrently or during an allocation
5816 * by `backtrace_symbols()` called from `GC_print_callers`; punt.
5817 */
5818 GC_err_printf("\t\t##PC##= 0x%lx\n", (unsigned long)ADDR(info[i].ci_pc));
5819 continue;
5820 }
5821
5822 {
5823 char buf[40];
5824 char *name;
5825 # if defined(GC_HAVE_BUILTIN_BACKTRACE) \
5826 && !defined(GC_BACKTRACE_SYMBOLS_BROKEN) && defined(FUNCPTR_IS_DATAPTR)
5827 char **sym_name = backtrace_symbols((void **)&info[i].ci_pc, 1);
5828 if (sym_name != NULL) {
5829 name = sym_name[0];
5830 } else
5831 # endif
5832 /* else */ {
5833 (void)snprintf(buf, sizeof(buf), "##PC##= 0x%lx",
5834 (unsigned long)ADDR(info[i].ci_pc));
5835 buf[sizeof(buf) - 1] = '\0';
5836 name = buf;
5837 }
5838 # if defined(LINUX) && !defined(SMALL_CONFIG)
5839 /* Try for a line number. */
5840 do {
5841 FILE *pipe;
5842 # define EXE_SZ 100
5843 static char exe_name[EXE_SZ];
5844 # define CMD_SZ 200
5845 char cmd_buf[CMD_SZ];
5846 # define RESULT_SZ 200
5847 static char result_buf[RESULT_SZ];
5848 size_t result_len;
5849 const char *old_preload;
5850 # define PRELOAD_SZ 200
5851 char preload_buf[PRELOAD_SZ];
5852 static GC_bool found_exe_name = FALSE;
5853 static GC_bool will_fail = FALSE;
5854
5855 /*
5856 * Try to get it via a hairy and expensive scheme.
5857 * First we get the name of the executable.
5858 */
5859 if (will_fail)
5860 break;
5861 if (!found_exe_name) {
5862 int ret_code = readlink("/proc/self/exe", exe_name, EXE_SZ);
5863
5864 if (ret_code < 0 || ret_code >= EXE_SZ || exe_name[0] != '/') {
5865 /* Do not try again. */
5866 will_fail = TRUE;
5867 break;
5868 }
5869 exe_name[ret_code] = '\0';
5870 found_exe_name = TRUE;
5871 }
5872 /*
5873 * Then we use `popen()` to start `addr2line -e <exe> <addr>`.
5874 * There are faster ways to do this, but hopefully this is
5875 * not time critical.
5876 */
5877 (void)snprintf(cmd_buf, sizeof(cmd_buf),
5878 "/usr/bin/addr2line -f -e %s 0x%lx", exe_name,
5879 (unsigned long)ADDR(info[i].ci_pc));
5880 cmd_buf[sizeof(cmd_buf) - 1] = '\0';
5881 old_preload = GETENV("LD_PRELOAD");
5882 if (old_preload != NULL) {
5883 size_t old_len = strlen(old_preload);
5884 if (old_len >= PRELOAD_SZ) {
5885 will_fail = TRUE;
5886 break;
5887 }
5888 BCOPY(old_preload, preload_buf, old_len + 1);
5889 unsetenv("LD_PRELOAD");
5890 }
5891 pipe = popen(cmd_buf, "r");
5892 if (old_preload != NULL
5893 && setenv("LD_PRELOAD", preload_buf, 0 /* `overwrite` */) == -1) {
5894 WARN("Failed to reset LD_PRELOAD\n", 0);
5895 }
5896 if (NULL == pipe) {
5897 will_fail = TRUE;
5898 break;
5899 }
5900 result_len = fread(result_buf, 1, RESULT_SZ - 1, pipe);
5901 (void)pclose(pipe);
5902 if (0 == result_len) {
5903 will_fail = TRUE;
5904 break;
5905 }
5906 if (result_buf[result_len - 1] == '\n')
5907 --result_len;
5908 result_buf[result_len] = 0;
5909 if (result_buf[0] == '?'
5910 || (result_buf[result_len - 2] == ':'
5911 && result_buf[result_len - 1] == '0'))
5912 break;
5913 /* Get rid of embedded newline, if any. Test for "main". */
5914 {
5915 char *nl = strchr(result_buf, '\n');
5916 if (nl != NULL && ADDR_LT(nl, result_buf + result_len)) {
5917 *nl = ':';
5918 }
5919 if (strncmp(result_buf, "main",
5920 nl != NULL
5921 ? (size_t)(ADDR(nl) /*< CPPCHECK */
5922 - COVERT_DATAFLOW(ADDR(result_buf)))
5923 : result_len)
5924 == 0) {
5925 stop = TRUE;
5926 }
5927 }
5928 if (result_len < RESULT_SZ - 25) {
5929 /* Add address in the hex format. */
5930 (void)snprintf(&result_buf[result_len],
5931 sizeof(result_buf) - result_len, " [0x%lx]",
5932 (unsigned long)ADDR(info[i].ci_pc));
5933 result_buf[sizeof(result_buf) - 1] = '\0';
5934 }
5935 # if defined(CPPCHECK)
5936 GC_noop1((unsigned char)name[0]);
5937 /* The value of name computed previously is discarded. */
5938 # endif
5939 name = result_buf;
5940 } while (0);
5941 # endif /* LINUX */
5942 GC_err_printf("\t\t%s\n", name);
5943 # if defined(GC_HAVE_BUILTIN_BACKTRACE) \
5944 && !defined(GC_BACKTRACE_SYMBOLS_BROKEN) && defined(FUNCPTR_IS_DATAPTR)
5945 if (sym_name != NULL) {
5946 /* May call `GC_free()`, `GC_debug_free()`; that is OK. */
5947 free(sym_name);
5948 }
5949 # endif
5950 }
5951 # if defined(LINUX) && !defined(SMALL_CONFIG)
5952 if (stop)
5953 break;
5954 # endif
5955 }
5956 # if defined(AO_HAVE_fetch_and_add1) && defined(AO_HAVE_fetch_and_sub1)
5957 (void)AO_fetch_and_sub1(&reentry_count);
5958 # else
5959 LOCK();
5960 --reentry_count;
5961 UNLOCK();
5962 # endif
5963 }
5964
5965 #endif /* NEED_CALLINFO */
5966
5967 #if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG)
5968 /*
5969 * Dump `/proc/self/maps` file to `GC_stderr`, to enable looking up names
5970 * for addresses in `FIND_LEAK` output.
5971 */
5972 void
5973 GC_print_address_map(void)
5974 {
5975 const char *maps_ptr;
5976
5977 GC_ASSERT(I_HOLD_LOCK());
5978 maps_ptr = GC_get_maps();
5979 GC_err_printf("---------- Begin address map ----------\n");
5980 GC_err_puts(maps_ptr);
5981 GC_err_printf("---------- End address map ----------\n");
5982 }
5983 #endif /* LINUX && ELF */
5984