dyn_load.c raw
1 /*
2 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
3 * Copyright (c) 1997 by Silicon Graphics. All rights reserved.
4 * Copyright (c) 2009-2022 Ivan Maidanski
5 *
6 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
7 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
8 *
9 * Permission is hereby granted to use or copy this program
10 * for any purpose, provided the above notices are retained on all copies.
11 * Permission to modify the code and to distribute modified code is granted,
12 * provided the above notices are retained, and a notice that the code was
13 * modified is included with the above copyright notice.
14 */
15
16 #include "private/gc_priv.h"
17
18 /*
19 * This is incredibly OS specific code for tracking down data sections in
20 * dynamic libraries. There appears to be no way of doing this quickly
21 * without groveling through undocumented data structures. We would argue
22 * that this is a bug in the design of the `dlopen` interface. This code
23 * may break in future OS releases. If this matters to you, do not hesitate
24 * to let your vendor know...
25 *
26 * None of this is safe with `dlclose` and incremental collection.
27 * But, then not much of anything is safe in the presence of `dlclose`.
28 */
29
30 /*
31 * BTL: avoid circular redefinition of `dlopen` if `SOLARIS` and `THREADS`
32 * are both defined.
33 */
34 #undef GC_MUST_RESTORE_REDEFINED_DLOPEN
35 #if defined(GC_PTHREADS) && !defined(GC_NO_DLOPEN) \
36 && !defined(GC_NO_THREAD_REDIRECTS) && !defined(GC_USE_LD_WRAP)
37 /*
38 * To support threads in Solaris, `gc.h` file interposes on `dlopen` by
39 * defining `dlopen` to be `GC_dlopen`. `GC_FirstDLOpenedLinkMap()` needs
40 * to use the real system `dlopen()`, thus we first undo `dlopen` definition
41 * by `gc.h` file and restore it at the end of the current file.
42 */
43 # undef dlopen
44 # define GC_MUST_RESTORE_REDEFINED_DLOPEN
45 #endif /* !GC_NO_DLOPEN */
46
47 #if defined(SOLARISDL) && defined(THREADS) && !defined(SOLARIS) \
48 && !defined(CPPCHECK)
49 # error Fix mutual exclusion with dlopen
50 #endif
51
52 /*
53 * A user-supplied routine (custom filter) that might be called to
54 * determine whether a DSO really needs to be scanned by the collector.
55 * Zero means no filter installed. May be unused on some platforms.
56 */
57 /* FIXME: Add filter support for more platforms. */
58 STATIC GC_has_static_roots_func GC_has_static_roots = 0;
59
60 #ifdef ANY_MSWIN
61 /*
62 * We traverse the entire address space and register all segments that
63 * could possibly have been written to.
64 */
65 STATIC void
66 GC_cond_add_roots(ptr_t base, ptr_t limit)
67 {
68 # ifdef THREADS
69 ptr_t curr_base = base;
70 ptr_t next_stack_lo, next_stack_hi;
71 # else
72 ptr_t stack_top;
73 # endif
74
75 GC_ASSERT(I_HOLD_LOCK());
76 if (base == limit)
77 return;
78 # ifdef THREADS
79 for (;;) {
80 GC_get_next_stack(curr_base, limit, &next_stack_lo, &next_stack_hi);
81 if (ADDR_GE(next_stack_lo, limit))
82 break;
83 if (ADDR_LT(curr_base, next_stack_lo))
84 GC_add_roots_inner(curr_base, next_stack_lo, TRUE);
85 curr_base = next_stack_hi;
86 }
87 if (ADDR_LT(curr_base, limit))
88 GC_add_roots_inner(curr_base, limit, TRUE);
89 # else
90 stack_top
91 = PTR_ALIGN_DOWN(GC_approx_sp(), GC_sysinfo.dwAllocationGranularity);
92 if (ADDR_LT(stack_top, limit) && ADDR_LT(base, GC_stackbottom)) {
93 /* Part of the stack; ignore it. */
94 return;
95 }
96 GC_add_roots_inner(base, limit, TRUE);
97 # endif
98 }
99
100 # ifdef DYNAMIC_LOADING
101 GC_INNER GC_bool
102 GC_register_main_static_data(void)
103 {
104 # if defined(MSWINCE) || defined(CYGWIN32)
105 return FALSE;
106 # else
107 return GC_no_win32_dlls;
108 # endif
109 }
110 # define HAVE_REGISTER_MAIN_STATIC_DATA
111 # endif /* DYNAMIC_LOADING */
112
113 # ifdef DEBUG_VIRTUALQUERY
114 void
115 GC_dump_meminfo(MEMORY_BASIC_INFORMATION *buf)
116 {
117 GC_printf("BaseAddress= 0x%lx, AllocationBase= 0x%lx,"
118 " RegionSize= 0x%lx(%lu)\n",
119 buf->BaseAddress, buf->AllocationBase, buf->RegionSize,
120 buf->RegionSize);
121 GC_printf("\tAllocationProtect= 0x%lx, State= 0x%lx, Protect= 0x%lx, "
122 "Type= 0x%lx\n",
123 buf->AllocationProtect, buf->State, buf->Protect, buf->Type);
124 }
125 # endif /* DEBUG_VIRTUALQUERY */
126
127 # if defined(MSWINCE) || defined(CYGWIN32)
128 /* FIXME: Should we really need to scan `MEM_PRIVATE` sections? */
129 /*
130 * For now, we do not add `MEM_PRIVATE` sections to the data roots for
131 * WinCE because otherwise `SIGSEGV` fault sometimes happens to occur
132 * in `GC_mark_from()` (and, even if we use `WRAP_MARK_SOME`, WinCE
133 * prints a "Data Abort" message to the debugging console).
134 * To workaround that, use `-D GC_REGISTER_MEM_PRIVATE`.
135 */
136 # define GC_wnt TRUE
137 # endif
138
139 GC_INNER void
140 GC_register_dynamic_libraries(void)
141 {
142 ptr_t p, base, limit;
143
144 GC_ASSERT(I_HOLD_LOCK());
145 # ifdef MSWIN32
146 if (GC_no_win32_dlls)
147 return;
148 # endif
149 p = (ptr_t)GC_sysinfo.lpMinimumApplicationAddress;
150 base = limit = p;
151 while (ADDR_LT(p, (ptr_t)GC_sysinfo.lpMaximumApplicationAddress)) {
152 MEMORY_BASIC_INFORMATION buf;
153 size_t result = VirtualQuery((LPVOID)p, &buf, sizeof(buf));
154
155 # ifdef MSWINCE
156 if (0 == result) {
157 if (ADDR(p) > GC_WORD_MAX - GC_sysinfo.dwAllocationGranularity) {
158 /* An overflow. */
159 break;
160 }
161 /* Page is free; advance to the next possible allocation base. */
162 p = PTR_ALIGN_UP(p + 1, GC_sysinfo.dwAllocationGranularity);
163 } else
164 # endif
165 /* else */ {
166 DWORD protect;
167
168 if (result != sizeof(buf))
169 ABORT("Weird VirtualQuery result");
170 if (ADDR(p) > GC_WORD_MAX - buf.RegionSize) {
171 /* An overflow. */
172 break;
173 }
174
175 protect = buf.Protect;
176 if (buf.State == MEM_COMMIT
177 && (protect == PAGE_EXECUTE_READWRITE
178 || protect == PAGE_EXECUTE_WRITECOPY || protect == PAGE_READWRITE
179 || protect == PAGE_WRITECOPY)
180 && (buf.Type == MEM_IMAGE
181 # ifdef GC_REGISTER_MEM_PRIVATE
182 || (protect == PAGE_READWRITE && buf.Type == MEM_PRIVATE)
183 # else
184 /*
185 * There is some evidence that we cannot always ignore
186 * `MEM_PRIVATE` sections under Windows ME and predecessors.
187 * Hence we now also check for that case.
188 */
189 || (!GC_wnt && buf.Type == MEM_PRIVATE)
190 # endif
191 )
192 && !GC_is_heap_base(buf.AllocationBase)) {
193 # ifdef DEBUG_VIRTUALQUERY
194 GC_dump_meminfo(&buf);
195 # endif
196 if (p != limit) {
197 GC_cond_add_roots(base, limit);
198 base = p;
199 }
200 limit = p + buf.RegionSize;
201 }
202 p += buf.RegionSize;
203 }
204 }
205 GC_cond_add_roots(base, limit);
206 }
207
208 #elif defined(DYNAMIC_LOADING) /* `&& !ANY_MSWIN` */
209
210 # if !(defined(CPPCHECK) || defined(AIX) || defined(DARWIN) || defined(DGUX) \
211 || defined(IRIX5) || defined(HAIKU) || defined(HPUX) || defined(HURD) \
212 || defined(NACL) || defined(OSF1) || defined(SCO_ELF) \
213 || defined(SERENITY) || defined(SOLARISDL) \
214 || ((defined(ANY_BSD) || defined(LINUX)) && defined(__ELF__)) \
215 || (defined(OPENBSD) && defined(M68K)))
216 # error Finding data segments of dynamic libraries is unsupported on target
217 # endif
218
219 # if defined(DARWIN) && !defined(USE_DYLD_TO_BIND) \
220 && !defined(NO_DYLD_BIND_FULLY_IMAGE)
221 # include <dlfcn.h>
222 # endif
223
224 # ifdef SOLARISDL
225 # include <dlfcn.h>
226 # include <link.h>
227 # include <sys/elf.h>
228 # endif
229
230 # if defined(NETBSD)
231 # include <dlfcn.h>
232 # include <machine/elf_machdep.h>
233 # include <sys/param.h>
234 # define ELFSIZE ARCH_ELFSIZE
235 # endif
236
237 # if defined(OPENBSD)
238 # include <sys/param.h>
239 # if (OpenBSD >= 200519) && !defined(HAVE_DL_ITERATE_PHDR)
240 # define HAVE_DL_ITERATE_PHDR
241 # endif
242 # endif /* OPENBSD */
243
244 # if defined(DGUX) || defined(HURD) || defined(NACL) || defined(SCO_ELF) \
245 || defined(SERENITY) \
246 || ((defined(ANY_BSD) || defined(LINUX)) && defined(__ELF__))
247 # include <stddef.h>
248 # if !defined(OPENBSD) && !defined(HOST_ANDROID)
249 /*
250 * OpenBSD does not have platform `elf.h` file; include `link.h` file below
251 * is sufficient. Exclude this on Android because platform `linker.h` file
252 * below includes its own variant.
253 */
254 # include <elf.h>
255 # endif
256 # ifdef HOST_ANDROID
257 /*
258 * If you do not need the "dynamic loading" feature, you may build the
259 * collector with `-D IGNORE_DYNAMIC_LOADING`.
260 */
261 # ifdef BIONIC_ELFDATA_REDEF_BUG
262 /*
263 * Workaround a problem in Bionic (as of Android 4.2) which has mismatching
264 * `ELF_DATA` definitions in platform `sys/exec_elf.h` and `asm/elf.h` files
265 * included from `linker.h` file (similar to `EM_ALPHA`).
266 */
267 # include <asm/elf.h>
268 # include <linux/elf-em.h>
269 # undef ELF_DATA
270 # undef EM_ALPHA
271 # endif
272 # include <link.h>
273 # endif /* HOST_ANDROID */
274 # if ((defined(HOST_ANDROID) && !defined(GC_DONT_DEFINE_LINK_MAP) \
275 && !(__ANDROID_API__ >= 21)) \
276 || defined(SERENITY)) \
277 && !defined(USE_PROC_FOR_LIBRARIES)
278 /*
279 * `link_map` and `r_debug` are defined in `link.h` file of NDK r10+.
280 * `bionic/linker/linker.h` file defines them too but the header itself
281 * is a C++ one starting from Android 4.3.
282 */
283 struct link_map {
284 uintptr_t l_addr;
285 char *l_name;
286 uintptr_t l_ld;
287 struct link_map *l_next;
288 struct link_map *l_prev;
289 };
290 struct r_debug {
291 int32_t r_version;
292 struct link_map *r_map;
293 /* `void (*r_brk)(void);` */
294 /* `int32_t r_state;` */
295 /* `uintptr_t r_ldbase;` */
296 };
297 # define LINK_MAP_R_DEBUG_DEFINED
298 # endif /* __ANDROID_API__ >= 21 || SERENITY */
299 # ifndef HOST_ANDROID
300 /*
301 * Workaround missing `extern "C"` around `_DYNAMIC` symbol in platform
302 * `link.h` file of some Linux hosts.
303 */
304 EXTERN_C_BEGIN
305 # include <link.h>
306 EXTERN_C_END
307 # endif /* !HOST_ANDROID */
308 # endif
309
310 /*
311 * Newer versions of GNU/Linux define this macro. We define it similarly
312 * for all ELF systems that do not.
313 */
314 # ifndef ElfW
315 # if defined(FREEBSD)
316 # if __ELF_WORD_SIZE == 32
317 # define ElfW(type) Elf32_##type
318 # else
319 # define ElfW(type) Elf64_##type
320 # endif
321 # elif defined(NETBSD) || defined(OPENBSD)
322 # if ELFSIZE == 32
323 # define ElfW(type) Elf32_##type
324 # elif ELFSIZE == 64
325 # define ElfW(type) Elf64_##type
326 # else
327 # error Missing ELFSIZE define
328 # endif
329 # else
330 # if !defined(ELF_CLASS) || ELF_CLASS == ELFCLASS32
331 # define ElfW(type) Elf32_##type
332 # else
333 # define ElfW(type) Elf64_##type
334 # endif
335 # endif
336 # endif
337
338 # if defined(DGUX) || defined(HURD) || defined(NACL) || defined(SCO_ELF) \
339 || defined(SERENITY) \
340 || ((defined(ANY_BSD) || defined(LINUX)) && defined(__ELF__))
341
342 # ifdef USE_PROC_FOR_LIBRARIES
343
344 # include <fcntl.h>
345 # include <string.h>
346 # include <sys/stat.h>
347
348 # define MAPS_BUF_SIZE (32 * 1024)
349
350 /*
351 * Sort an array of `HeapSects` elements by `hs_start` value.
352 * Unfortunately, at least some versions of Linux, `qsort` end up calling
353 * `malloc` by way of `sysconf`, and hence cannot be used in the collector.
354 * Hence we roll our own. Should be reasonably fast if the array is
355 * already mostly sorted, as we expect it to be.
356 */
357 static void
358 sort_heap_sects(struct HeapSect *base, size_t number_of_elements)
359 {
360 GC_signed_word n = (GC_signed_word)number_of_elements;
361 GC_signed_word nsorted = 1;
362
363 while (nsorted < n) {
364 GC_signed_word i;
365
366 while (nsorted < n
367 && ADDR_LT(base[nsorted - 1].hs_start, base[nsorted].hs_start)) {
368 ++nsorted;
369 }
370 if (nsorted == n)
371 break;
372 GC_ASSERT(ADDR_LT(base[nsorted].hs_start, base[nsorted - 1].hs_start));
373 for (i = nsorted - 1;
374 i >= 0 && ADDR_LT(base[i + 1].hs_start, base[i].hs_start); --i) {
375 struct HeapSect tmp = base[i];
376
377 base[i] = base[i + 1];
378 base[i + 1] = tmp;
379 }
380 GC_ASSERT(ADDR_LT(base[nsorted - 1].hs_start, base[nsorted].hs_start));
381 ++nsorted;
382 }
383 }
384
385 STATIC void
386 GC_register_map_entries(const char *maps)
387 {
388 const char *prot, *path;
389 ptr_t my_start, my_end;
390 ptr_t least_ha, greatest_ha;
391 unsigned maj_dev;
392 unsigned i;
393
394 GC_ASSERT(I_HOLD_LOCK());
395 sort_heap_sects(GC_our_memory, GC_n_memory);
396 least_ha = GC_our_memory[0].hs_start;
397 greatest_ha = GC_our_memory[GC_n_memory - 1].hs_start
398 + GC_our_memory[GC_n_memory - 1].hs_bytes;
399
400 for (;;) {
401 maps
402 = GC_parse_map_entry(maps, &my_start, &my_end, &prot, &maj_dev, &path);
403 if (NULL == maps)
404 break;
405
406 if (prot[1] == 'w') {
407 /*
408 * This is a writable mapping. Add it to the root set unless
409 * it is already otherwise accounted for.
410 */
411 # ifndef THREADS
412 if (ADDR_GE(GC_stackbottom, my_start)
413 && ADDR_GE(my_end, GC_stackbottom)) {
414 /* Stack mapping; discard it. */
415 continue;
416 }
417 # endif
418 # if defined(E2K) && defined(__ptr64__)
419 /* TODO: Avoid hard-coded addresses. */
420 if (ADDR(my_start) == 0xc2fffffff000UL
421 && ADDR(my_end) == 0xc30000000000UL && path[0] == '\n') {
422 /* Discard some special mapping. */
423 continue;
424 }
425 # endif
426 if (path[0] == '[' && strncmp(path + 1, "heap]", 5) != 0) {
427 /* Discard a pseudo-file path except for "[heap]". */
428 continue;
429 }
430
431 # ifdef THREADS
432 /*
433 * This may fail, since a thread may already be unregistered, but
434 * its thread stack may still be there. That can fail because the
435 * stack may disappear while we are marking. Thus the marker is,
436 * and has to be prepared to recover from segmentation faults.
437 */
438
439 if (GC_segment_is_thread_stack(my_start, my_end)) {
440 continue;
441 /*
442 * FIXME: NPTL squirrels away pointers in pieces of the stack
443 * segment that we do not scan. We work around this by treating
444 * anything allocated by `libpthread` as uncollectible, as we do
445 * in some other cases. A specifically identified problem is that
446 * thread stacks contain pointers to dynamic thread vectors, that
447 * may be reused due to thread caching. They may not be marked
448 * if the thread is still live. This specific instance should be
449 * addressed by `INCLUDE_LINUX_THREAD_DESCR`, but that does not
450 * quite seem to suffice. We currently trace entire thread stacks,
451 * if they are are currently cached but unused. This is very
452 * suboptimal for performance reasons.
453 */
454 }
455 # endif
456 /* We no longer exclude the main data segment. */
457 if (ADDR_GE(least_ha, my_end) || ADDR_GE(my_start, greatest_ha)) {
458 /* The easy case; just trace the entire segment. */
459 GC_add_roots_inner(my_start, my_end, TRUE);
460 continue;
461 }
462 /* Add sections that do not belong to us. */
463 i = 0;
464 while (ADDR_LT(GC_our_memory[i].hs_start + GC_our_memory[i].hs_bytes,
465 my_start)) {
466 ++i;
467 }
468 GC_ASSERT(i < GC_n_memory);
469 if (ADDR_GE(my_start, GC_our_memory[i].hs_start)) {
470 my_start = GC_our_memory[i].hs_start + GC_our_memory[i].hs_bytes;
471 ++i;
472 }
473 for (; i < GC_n_memory && ADDR_LT(my_start, my_end)
474 && ADDR_LT(GC_our_memory[i].hs_start, my_end);
475 ++i) {
476 if (ADDR_LT(my_start, GC_our_memory[i].hs_start))
477 GC_add_roots_inner(my_start, GC_our_memory[i].hs_start, TRUE);
478 my_start = GC_our_memory[i].hs_start + GC_our_memory[i].hs_bytes;
479 }
480 if (ADDR_LT(my_start, my_end))
481 GC_add_roots_inner(my_start, my_end, TRUE);
482 } else if (prot[0] == '-' && prot[1] == '-' && prot[2] == '-') {
483 /*
484 * Even roots added statically might disappear partially
485 * (e.g. the roots added by `INCLUDE_LINUX_THREAD_DESCR`).
486 */
487 GC_remove_roots_subregion(my_start, my_end);
488 }
489 }
490 }
491
492 GC_INNER void
493 GC_register_dynamic_libraries(void)
494 {
495 GC_register_map_entries(GC_get_maps());
496 }
497
498 GC_INNER GC_bool
499 GC_register_main_static_data(void)
500 {
501 /* We now take care of the main data segment ourselves. */
502 return FALSE;
503 }
504 # define HAVE_REGISTER_MAIN_STATIC_DATA
505
506 # else /* !USE_PROC_FOR_LIBRARIES */
507
508 /*
509 * The following is the preferred way to walk dynamic libraries for
510 * `glibc` 2.2.4+. Unfortunately, it does not work for older versions.
511 */
512
513 # if GC_GLIBC_PREREQ(2, 3) || defined(HOST_ANDROID)
514 /* Are others OK here, too? */
515 # ifndef HAVE_DL_ITERATE_PHDR
516 # define HAVE_DL_ITERATE_PHDR
517 # endif
518 # ifdef HOST_ANDROID
519 /* Android headers might have no such definition for some targets. */
520 EXTERN_C_BEGIN
521 extern int dl_iterate_phdr(int (*cb)(struct dl_phdr_info *, size_t, void *),
522 void *data);
523 EXTERN_C_END
524 # endif
525 # endif /* __GLIBC__ >= 2 || HOST_ANDROID */
526
527 # if defined(__DragonFly__) || defined(__FreeBSD_kernel__) \
528 || (defined(FREEBSD) && __FreeBSD__ >= 7)
529 /*
530 * On the FreeBSD system, any target system at major version 7 shall have
531 * `dl_iterate_phdr`; therefore, we need not make it weak as below.
532 */
533 # ifndef HAVE_DL_ITERATE_PHDR
534 # define HAVE_DL_ITERATE_PHDR
535 # endif
536 # define DL_ITERATE_PHDR_STRONG
537 # elif defined(HAVE_DL_ITERATE_PHDR)
538 /*
539 * We have the header files for a `glibc` that includes `dl_iterate_phdr`.
540 * It may still not be available in the library on the target system.
541 * Thus we also treat it as a weak symbol.
542 */
543 EXTERN_C_BEGIN
544 # pragma weak dl_iterate_phdr
545 EXTERN_C_END
546 # endif
547
548 # ifdef HAVE_DL_ITERATE_PHDR
549
550 # ifdef PT_GNU_RELRO
551 /*
552 * Instead of registering `PT_LOAD` sections directly, we keep them
553 * in a temporary list, and filter them by excluding `PT_GNU_RELRO`
554 * segments. Processing `PT_GNU_RELRO` sections with
555 * `GC_exclude_static_roots` instead would be superficially cleaner.
556 * But it runs into trouble if a client registers an overlapping segment,
557 * which unfortunately seems quite possible.
558 */
559
560 # define MAX_LOAD_SEGS MAX_ROOT_SETS
561
562 static struct load_segment {
563 ptr_t start;
564 ptr_t end;
565 /*
566 * The room for the second segment if we remove a `PT_GNU_RELRO` segment
567 * from the middle.
568 */
569 ptr_t start2;
570 ptr_t end2;
571 } load_segs[MAX_LOAD_SEGS];
572
573 static int n_load_segs;
574 static GC_bool load_segs_overflow;
575 # endif /* PT_GNU_RELRO */
576
577 STATIC int
578 GC_register_dynlib_callback(struct dl_phdr_info *info, size_t size, void *ptr)
579 {
580 const ElfW(Phdr) * p;
581 ptr_t load_ptr, my_start, my_end;
582 int i;
583
584 GC_ASSERT(I_HOLD_LOCK());
585 /* Make sure `dl_phdr_info` structure is at least as big as we need. */
586 if (size
587 < offsetof(struct dl_phdr_info, dlpi_phnum) + sizeof(info->dlpi_phnum))
588 return 1; /*< stop */
589
590 load_ptr = (ptr_t)info->dlpi_addr;
591 p = info->dlpi_phdr;
592 for (i = 0; i < (int)info->dlpi_phnum; i++, p++) {
593 if (p->p_type == PT_LOAD) {
594 GC_has_static_roots_func callback = GC_has_static_roots;
595 if ((p->p_flags & PF_W) == 0)
596 continue;
597
598 # ifdef CHERI_PURECAP
599 my_start = load_ptr + p->p_vaddr;
600 # else
601 /*
602 * Prevent "applying nonzero offset to null pointer" compiler warning
603 * as `load_ptr` could be `NULL`.
604 */
605 my_start = (ptr_t)((GC_uintptr_t)load_ptr + p->p_vaddr);
606 # endif
607 my_end = my_start + p->p_memsz;
608 # ifdef CHERI_PURECAP
609 my_start = PTR_ALIGN_UP(my_start, ALIGNMENT);
610 my_end = PTR_ALIGN_DOWN(my_end, ALIGNMENT);
611 if (!SPANNING_CAPABILITY(load_ptr, ADDR(my_start), ADDR(my_end)))
612 continue;
613 my_start = cheri_bounds_set(my_start, (word)(my_end - my_start));
614 # endif
615
616 if (callback != 0 && !callback(info->dlpi_name, my_start, p->p_memsz))
617 continue;
618 # ifdef PT_GNU_RELRO
619 # if CPP_PTRSZ >= 64 && !defined(CHERI_PURECAP)
620 /*
621 * TODO: `GC_push_all` eventually does the correct rounding to the
622 * next multiple of `ALIGNMENT`, so, most probably, we should remove
623 * the corresponding assertion check in `GC_add_roots_inner` along
624 * with this code line. `my_start` value may require aligning.
625 */
626 my_start = PTR_ALIGN_DOWN(my_start, ALIGNMENT);
627 # endif
628 if (n_load_segs >= MAX_LOAD_SEGS) {
629 if (!load_segs_overflow) {
630 WARN("Too many PT_LOAD segments;"
631 " registering as roots directly...\n",
632 0);
633 load_segs_overflow = TRUE;
634 }
635 GC_add_roots_inner(my_start, my_end, TRUE);
636 } else {
637 load_segs[n_load_segs].start = my_start;
638 load_segs[n_load_segs].end = my_end;
639 load_segs[n_load_segs].start2 = NULL;
640 load_segs[n_load_segs].end2 = NULL;
641 ++n_load_segs;
642 }
643 # else
644 GC_add_roots_inner(my_start, my_end, TRUE);
645 # endif /* !PT_GNU_RELRO */
646 }
647 }
648
649 # ifdef PT_GNU_RELRO
650 p = info->dlpi_phdr;
651 for (i = 0; i < (int)info->dlpi_phnum; i++, p++) {
652 if (p->p_type == PT_GNU_RELRO) {
653 /*
654 * This entry is known to be constant and will eventually be
655 * remapped as read-only. However, the address range covered
656 * by this entry is typically a subset of a previously
657 * encountered `PT_LOAD` segment, so we need to exclude it.
658 */
659 int j;
660
661 # ifdef CHERI_PURECAP
662 my_start = load_ptr + p->p_vaddr;
663 # else
664 my_start = (ptr_t)((GC_uintptr_t)load_ptr + p->p_vaddr);
665 # endif
666 my_end = my_start + p->p_memsz;
667 for (j = n_load_segs; --j >= 0;) {
668 if (ADDR_INSIDE(my_start, load_segs[j].start, load_segs[j].end)) {
669 if (load_segs[j].start2 != NULL) {
670 WARN("More than one GNU_RELRO segment per load one\n", 0);
671 } else {
672 GC_ASSERT(
673 ADDR_GE(PTR_ALIGN_UP(load_segs[j].end, GC_page_size), my_end));
674 /* Remove it from the existing load segment. */
675 load_segs[j].end2 = load_segs[j].end;
676 load_segs[j].end = my_start;
677 load_segs[j].start2 = my_end;
678 /*
679 * Note that `start2` may be greater than `end2` because of
680 * `p->p_memsz` value is multiple of page size.
681 */
682 }
683 break;
684 }
685 if (0 == j && 0 == GC_has_static_roots)
686 WARN("Failed to find PT_GNU_RELRO segment"
687 " inside PT_LOAD region\n",
688 0);
689 /*
690 * No warning reported in case of the callback is present because
691 * most likely the segment has been excluded.
692 */
693 }
694 }
695 }
696 # endif
697
698 /* Signal that we were called. */
699 *(int *)ptr = 1;
700 return 0;
701 }
702
703 GC_INNER GC_bool
704 GC_register_main_static_data(void)
705 {
706 # if defined(DL_ITERATE_PHDR_STRONG) && !defined(CPPCHECK)
707 /*
708 * If `dl_iterate_phdr` is not a weak symbol, then do not test against
709 * zero (otherwise a compiler might issue a warning).
710 */
711 return FALSE;
712 # else
713 return 0 == COVERT_DATAFLOW(ADDR(dl_iterate_phdr));
714 # endif
715 }
716 # define HAVE_REGISTER_MAIN_STATIC_DATA
717
718 /*
719 * Return `TRUE` if we succeed; return `FALSE` if `dl_iterate_phdr` was
720 * not there.
721 */
722 STATIC GC_bool
723 GC_register_dynamic_libraries_dl_iterate_phdr(void)
724 {
725 int did_something;
726
727 GC_ASSERT(I_HOLD_LOCK());
728 if (GC_register_main_static_data())
729 return FALSE;
730
731 # ifdef PT_GNU_RELRO
732 {
733 static GC_bool excluded_segs = FALSE;
734 n_load_segs = 0;
735 load_segs_overflow = FALSE;
736 if (UNLIKELY(!excluded_segs)) {
737 GC_exclude_static_roots_inner((ptr_t)load_segs,
738 (ptr_t)load_segs + sizeof(load_segs));
739 excluded_segs = TRUE;
740 }
741 }
742 # endif
743
744 did_something = 0;
745 dl_iterate_phdr(GC_register_dynlib_callback, &did_something);
746 if (did_something) {
747 # ifdef PT_GNU_RELRO
748 int i;
749
750 for (i = 0; i < n_load_segs; ++i) {
751 if (ADDR_LT(load_segs[i].start, load_segs[i].end))
752 GC_add_roots_inner(load_segs[i].start, load_segs[i].end, TRUE);
753 if (ADDR_LT(load_segs[i].start2, load_segs[i].end2))
754 GC_add_roots_inner(load_segs[i].start2, load_segs[i].end2, TRUE);
755 }
756 # endif
757 } else {
758 ptr_t datastart, dataend;
759 # ifdef DATASTART_USES_XGETDATASTART
760 static ptr_t datastart_cached = MAKE_CPTR(GC_WORD_MAX);
761
762 /* Evaluate `DATASTART` only once. */
763 if (ADDR(datastart_cached) == GC_WORD_MAX) {
764 datastart_cached = DATASTART;
765 }
766 datastart = datastart_cached;
767 # else
768 datastart = DATASTART;
769 # endif
770 # ifdef DATAEND_IS_FUNC
771 {
772 static ptr_t dataend_cached = 0;
773 /* Evaluate `DATAEND` only once. */
774 if (dataend_cached == 0) {
775 dataend_cached = DATAEND;
776 }
777 dataend = dataend_cached;
778 }
779 # else
780 dataend = DATAEND;
781 # endif
782 if (NULL == *(char *volatile *)&datastart || ADDR_LT(dataend, datastart))
783 ABORT_ARG2("Wrong DATASTART/END pair", ": %p .. %p", (void *)datastart,
784 (void *)dataend);
785
786 /*
787 * `dl_iterate_phdr` may forget the static data segment in
788 * statically linked executables.
789 */
790 GC_add_roots_inner(datastart, dataend, TRUE);
791 # ifdef GC_HAVE_DATAREGION2
792 /* Subtract one to check also for `NULL` without a compiler warning. */
793 if (ADDR(DATASTART2) - 1U >= ADDR(DATAEND2)) {
794 ABORT_ARG2("Wrong DATASTART/END2 pair", ": %p .. %p", (void *)DATASTART2,
795 (void *)DATAEND2);
796 }
797 GC_add_roots_inner(DATASTART2, DATAEND2, TRUE);
798 # endif
799 }
800 return TRUE;
801 }
802
803 # else /* !HAVE_DL_ITERATE_PHDR */
804
805 # if defined(NETBSD) || defined(OPENBSD)
806 # include <sys/exec_elf.h>
807 /* For compatibility with 1.4.x. */
808 # ifndef DT_DEBUG
809 # define DT_DEBUG 21
810 # endif
811 # ifndef PT_LOAD
812 # define PT_LOAD 1
813 # endif
814 # ifndef PF_W
815 # define PF_W 2
816 # endif
817 # elif !defined(HOST_ANDROID)
818 # include <elf.h>
819 # endif
820
821 # ifndef HOST_ANDROID
822 # include <link.h>
823 # endif
824
825 # endif /* !HAVE_DL_ITERATE_PHDR */
826
827 # endif /* !USE_PROC_FOR_LIBRARIES */
828
829 # endif /* DGUX || HURD || NACL || (ANY_BSD || LINUX) && __ELF__ */
830
831 # if (defined(DGUX) || defined(HURD) || defined(NACL) || defined(SCO_ELF) \
832 || defined(SERENITY) || defined(SOLARISDL) \
833 || ((defined(ANY_BSD) || defined(LINUX)) && defined(__ELF__))) \
834 && !defined(USE_PROC_FOR_LIBRARIES)
835
836 /* Dynamic loading code for Linux, Solaris or similar OS running ELF. */
837
838 /*
839 * This does not necessarily work in all cases, e.g. with preloaded
840 * dynamic libraries.
841 */
842
843 EXTERN_C_BEGIN
844 # if defined(__GNUC__) && !defined(SOLARISDL)
845 # pragma weak _DYNAMIC
846 # endif
847 extern ElfW(Dyn) _DYNAMIC[];
848 EXTERN_C_END
849
850 STATIC struct link_map *
851 GC_FirstDLOpenedLinkMap(void)
852 {
853 static struct link_map *cachedResult = NULL;
854
855 # ifdef SUNOS53_SHARED_LIB
856 /*
857 * BTL: Avoid the Solaris 5.3 `ld.so` bug that `_DYNAMIC` is not being
858 * setup properly in dynamically linked library file. This means we
859 * have to use its value in the set of original object files loaded at
860 * the program startup.
861 */
862 static ElfW(Dyn) *dynStructureAddr = NULL;
863
864 if (NULL == dynStructureAddr) {
865 void *startupSyms = dlopen(NULL, RTLD_LAZY);
866
867 dynStructureAddr = (ElfW(Dyn) *)dlsym(startupSyms, "_DYNAMIC");
868 /* Note: `dlclose()` is not called intentionally. */
869 if (NULL == dynStructureAddr) {
870 /* `_DYNAMIC` symbol is not resolved. */
871 return NULL;
872 }
873 }
874 # else
875 if (0 == COVERT_DATAFLOW(ADDR(_DYNAMIC))) {
876 /* `_DYNAMIC` symbol is not resolved. */
877 return NULL;
878 }
879 # endif
880
881 if (NULL == cachedResult) {
882 # if defined(NETBSD) && defined(RTLD_DI_LINKMAP)
883 # if defined(CPPCHECK)
884 # define GC_RTLD_DI_LINKMAP 2
885 # else
886 # define GC_RTLD_DI_LINKMAP RTLD_DI_LINKMAP
887 # endif
888 struct link_map *lm = NULL;
889
890 if (!dlinfo(RTLD_SELF, GC_RTLD_DI_LINKMAP, &lm) && lm != NULL) {
891 /*
892 * Now `lm` points `link_map` object of the collector.
893 * Since it might not be the first dynamically linked object,
894 * try to find it (object next to the main object).
895 */
896 while (lm->l_prev != NULL) {
897 lm = lm->l_prev;
898 }
899 cachedResult = lm->l_next;
900 }
901 # else
902 ElfW(Dyn) * dp;
903 int tag;
904
905 for (dp = _DYNAMIC; (tag = dp->d_tag) != 0; dp++) {
906 if (tag == DT_DEBUG) {
907 const struct r_debug *rd
908 = CAST_THRU_UINTPTR(struct r_debug *, dp->d_un.d_ptr);
909
910 /* `d_ptr` could be `NULL` if libs are linked statically. */
911 if (rd != NULL) {
912 const struct link_map *lm = rd->r_map;
913
914 # if defined(CPPCHECK) && defined(LINK_MAP_R_DEBUG_DEFINED)
915 GC_noop1((word)rd->r_version);
916 # endif
917 if (lm != NULL) {
918 cachedResult = lm->l_next;
919 /* Might be `NULL`. */
920 }
921 }
922 break;
923 }
924 }
925 # endif
926 }
927 return cachedResult;
928 }
929
930 GC_INNER void
931 GC_register_dynamic_libraries(void)
932 {
933 struct link_map *lm;
934
935 GC_ASSERT(I_HOLD_LOCK());
936 # ifdef HAVE_DL_ITERATE_PHDR
937 if (GC_register_dynamic_libraries_dl_iterate_phdr()) {
938 return;
939 }
940 # endif
941 for (lm = GC_FirstDLOpenedLinkMap(); lm != NULL; lm = lm->l_next) {
942 const ElfW(Ehdr) * e;
943 const ElfW(Phdr) * p;
944 ptr_t load_ptr = (ptr_t)lm->l_addr;
945 size_t i;
946
947 # ifdef HOST_ANDROID
948 if (NULL == load_ptr)
949 continue;
950 # endif
951 e = (ElfW(Ehdr) *)load_ptr;
952 p = (ElfW(Phdr) *)(load_ptr + e->e_phoff);
953 # ifdef LINT2
954 /* Workaround tainted e->e_phnum usage as a loop boundary. */
955 GC_noop1_ptr((/* no const */ void *)&e->e_phnum);
956 # endif
957 for (i = 0; i < (size_t)e->e_phnum; i++, p++) {
958 ptr_t start;
959
960 switch (p->p_type) {
961 case PT_LOAD:
962 if ((p->p_flags & PF_W) == 0)
963 break;
964 start = load_ptr + p->p_vaddr;
965 GC_add_roots_inner(start, start + p->p_memsz, TRUE);
966 break;
967 default:
968 break;
969 }
970 }
971 # if defined(CPPCHECK) && defined(LINK_MAP_R_DEBUG_DEFINED)
972 GC_noop1_ptr(lm->l_name);
973 GC_noop1((word)lm->l_ld);
974 GC_noop1_ptr(lm->l_prev);
975 # endif
976 }
977 }
978
979 # endif /* (SOLARISDL || LINUX && __ELF__) && !USE_PROC_FOR_LIBRARIES */
980
981 # if defined(USE_PROC_FOR_LIBRARIES) && !defined(LINUX) || defined(IRIX5)
982
983 # include <elf.h>
984 # include <errno.h>
985 # include <fcntl.h>
986 # include <sys/procfs.h>
987 # include <sys/stat.h>
988
989 /* This is included only for the following test. */
990 # include <signal.h>
991 # ifndef _sigargs
992 # define IRIX6
993 # endif
994
995 GC_INNER void
996 GC_register_dynamic_libraries(void)
997 {
998 /*
999 * We use `/proc` to track down all parts of the address space that
1000 * are mapped by the process, and throw out regions we know we should
1001 * not worry about. This may also work under other SVR4 variants.
1002 */
1003 static int fd = -1;
1004 static prmap_t *addr_map = 0;
1005 /* Number of records currently in `addr_map`. */
1006 static int current_sz = 0;
1007 char buf[6 + 20 + 1];
1008 /* Required size of `addr_map`. */
1009 int needed_sz = 0;
1010 int i;
1011 long flags;
1012 ptr_t start;
1013 ptr_t limit;
1014 word heap_start = HEAP_START;
1015 word heap_end = heap_start;
1016 # ifdef SOLARISDL
1017 # define MA_PHYS 0
1018 # endif
1019
1020 GC_ASSERT(I_HOLD_LOCK());
1021 if (fd < 0) {
1022 GC_snprintf_s_ld_s(buf, sizeof(buf), "/proc/", (long)getpid(), "");
1023 fd = open(buf, O_RDONLY);
1024 if (fd < 0) {
1025 ABORT("/proc open failed");
1026 }
1027 }
1028 if (ioctl(fd, PIOCNMAP, &needed_sz) < 0) {
1029 ABORT_ARG2("/proc PIOCNMAP ioctl failed", ": fd= %d, errno= %d", fd,
1030 errno);
1031 }
1032 if (needed_sz >= current_sz) {
1033 GC_scratch_recycle_no_gww(addr_map, (size_t)current_sz * sizeof(prmap_t));
1034 /* Expansion, plus room for record 0. */
1035 current_sz = needed_sz * 2 + 1;
1036 addr_map
1037 = (prmap_t *)GC_scratch_alloc((size_t)current_sz * sizeof(prmap_t));
1038 if (NULL == addr_map)
1039 ABORT("Insufficient memory for address map");
1040 }
1041 if (ioctl(fd, PIOCMAP, addr_map) < 0) {
1042 ABORT_ARG3("/proc PIOCMAP ioctl failed",
1043 ": errcode= %d, needed_sz= %d, addr_map= %p", errno, needed_sz,
1044 (void *)addr_map);
1045 }
1046 if (GC_n_heap_sects > 0) {
1047 heap_end = ADDR(GC_heap_sects[GC_n_heap_sects - 1].hs_start)
1048 + GC_heap_sects[GC_n_heap_sects - 1].hs_bytes;
1049 if (heap_end < GC_scratch_last_end_addr)
1050 heap_end = GC_scratch_last_end_addr;
1051 }
1052 for (i = 0; i < needed_sz; i++) {
1053 flags = addr_map[i].pr_mflags;
1054 if ((flags & (MA_BREAK | MA_STACK | MA_PHYS | MA_FETCHOP | MA_NOTCACHED))
1055 != 0)
1056 goto irrelevant;
1057 if ((flags & (MA_READ | MA_WRITE)) != (MA_READ | MA_WRITE))
1058 goto irrelevant;
1059 /*
1060 * The latter test is empirically useless in very old Irix versions.
1061 * Other than the main data and stack segments, everything appears
1062 * to be mapped readable, writable, executable, and shared(!!).
1063 * Probably this makes no sense.
1064 */
1065 start = (ptr_t)addr_map[i].pr_vaddr;
1066 if (GC_roots_present(start)
1067 || (ADDR(start) >= heap_start && ADDR(start) < heap_end))
1068 goto irrelevant;
1069
1070 limit = start + addr_map[i].pr_size;
1071 /*
1072 * The following seemed to be necessary for very old versions of Irix,
1073 * but it has been reported to discard relevant segments under Irix 6.5.
1074 */
1075 # ifndef IRIX6
1076 if (addr_map[i].pr_off == 0 && strncmp(start, ELFMAG, 4) == 0) {
1077 /*
1078 * Discard text segments, i.e. zero-offset mappings against
1079 * executable files that appear to have ELF headers.
1080 */
1081 caddr_t arg;
1082 int obj;
1083 # define MAP_IRR_SZ 10
1084 /* Known irrelevant map entries. */
1085 static ptr_t map_irr[MAP_IRR_SZ];
1086 static int n_irr = 0;
1087 struct stat buf;
1088 int j;
1089
1090 for (j = 0; j < n_irr; j++) {
1091 if (map_irr[j] == start)
1092 goto irrelevant;
1093 }
1094 arg = (caddr_t)start;
1095 obj = ioctl(fd, PIOCOPENM, &arg);
1096 if (obj >= 0) {
1097 fstat(obj, &buf);
1098 close(obj);
1099 if ((buf.st_mode & 0111) != 0) {
1100 if (n_irr < MAP_IRR_SZ) {
1101 map_irr[n_irr++] = start;
1102 }
1103 goto irrelevant;
1104 }
1105 }
1106 }
1107 # endif /* !IRIX6 */
1108 GC_add_roots_inner(start, limit, TRUE);
1109 irrelevant:;
1110 }
1111 /*
1112 * Do not keep cached descriptor, for now. Some kernels do not like us
1113 * to keep a `/proc` file descriptor around during `kill -9`. Otherwise,
1114 * it should also require `FD_CLOEXEC` and proper handling at process fork
1115 * (i.e. close the file descriptor because of the `pid` change).
1116 */
1117 if (close(fd) < 0)
1118 ABORT("Couldn't close /proc file");
1119 fd = -1;
1120 }
1121
1122 # endif /* USE_PROC_FOR_LIBRARIES && !LINUX || IRIX5 */
1123
1124 # ifdef AIX
1125 # include <alloca.h>
1126 # include <sys/errno.h>
1127 # include <sys/ldr.h>
1128
1129 GC_INNER void
1130 GC_register_dynamic_libraries(void)
1131 {
1132 int ldibuflen = 8192;
1133
1134 GC_ASSERT(I_HOLD_LOCK());
1135 for (;;) {
1136 int len;
1137 struct ld_info *ldi;
1138 # if defined(CPPCHECK)
1139 char ldibuf[ldibuflen];
1140 # else
1141 char *ldibuf = alloca(ldibuflen);
1142 # endif
1143
1144 len = loadquery(L_GETINFO, ldibuf, ldibuflen);
1145 if (len < 0) {
1146 if (errno != ENOMEM) {
1147 ABORT("loadquery failed");
1148 }
1149 ldibuflen *= 2;
1150 continue;
1151 }
1152
1153 ldi = (struct ld_info *)ldibuf;
1154 for (;;) {
1155 len = ldi->ldinfo_next;
1156 GC_add_roots_inner((ptr_t)ldi->ldinfo_dataorg,
1157 (ptr_t)ldi->ldinfo_dataorg + ldi->ldinfo_datasize,
1158 TRUE);
1159 if (0 == len)
1160 break;
1161 ldi = (struct ld_info *)((ptr_t)ldi + len);
1162 }
1163 break;
1164 }
1165 }
1166 # endif /* AIX */
1167
1168 # ifdef DARWIN
1169
1170 /* `__private_extern__` hack required for gcc-3.3 and earlier. */
1171 # ifndef __private_extern__
1172 # define __private_extern__ extern
1173 # include <mach-o/dyld.h>
1174 # undef __private_extern__
1175 # else
1176 # include <mach-o/dyld.h>
1177 # endif
1178
1179 # if CPP_WORDSZ == 64
1180 # define GC_MACH_HEADER mach_header_64
1181 # else
1182 # define GC_MACH_HEADER mach_header
1183 # endif
1184
1185 # ifdef MISSING_MACH_O_GETSECT_H
1186 EXTERN_C_BEGIN
1187 extern uint8_t *getsectiondata(const struct GC_MACH_HEADER *, const char *seg,
1188 const char *sect, unsigned long *psz);
1189 EXTERN_C_END
1190 # else
1191 # include <mach-o/getsect.h>
1192 # endif
1193
1194 /* Writable sections generally available on Darwin. */
1195 STATIC const struct dyld_sections_s {
1196 const char *seg;
1197 const char *sect;
1198 } GC_dyld_sections[]
1199 = { { SEG_DATA, SECT_DATA },
1200 /* Used by FSF gcc, but not by OS X system tools, so far. */
1201 { SEG_DATA, "__static_data" },
1202 { SEG_DATA, SECT_BSS },
1203 { SEG_DATA, SECT_COMMON },
1204 /*
1205 * FSF gcc - zero-sized object sections for targets supporting
1206 * section anchors.
1207 */
1208 { SEG_DATA, "__zobj_data" },
1209 { SEG_DATA, "__zobj_bss" } };
1210
1211 /*
1212 * Additional writable sections: gcc on Darwin constructs aligned sections
1213 * "on demand", where the alignment size is embedded in the section name.
1214 * Furthermore, there are distinctions between sections containing private
1215 * vs. public symbols. It also constructs sections specifically for
1216 * zero-sized objects, when the target supports section anchors.
1217 */
1218 STATIC const char *const GC_dyld_bss_prefixes[]
1219 = { "__bss", "__pu_bss", "__zo_bss", "__zo_pu_bss" };
1220
1221 /*
1222 * Currently, `mach-o` will allow up to the max of `2**15` alignment
1223 * in an object file.
1224 */
1225 # ifndef L2_MAX_OFILE_ALIGNMENT
1226 # define L2_MAX_OFILE_ALIGNMENT 15
1227 # endif
1228
1229 STATIC const char *
1230 GC_dyld_name_for_hdr(const struct GC_MACH_HEADER *phdr)
1231 {
1232 unsigned long i, count = _dyld_image_count();
1233
1234 for (i = 0; i < count; i++) {
1235 if ((const struct GC_MACH_HEADER *)_dyld_get_image_header(i) == phdr)
1236 return _dyld_get_image_name(i);
1237 }
1238 /* TODO: Probably `ABORT()` in this case? */
1239 return NULL; /*< not found */
1240 }
1241
1242 /*
1243 * `getsectbynamefromheader` is deprecated (first time in macOS 13.0),
1244 * `getsectiondata` (introduced in macOS 10.7) is used instead, if exists.
1245 * Define `USE_GETSECTBYNAME` macro to use the deprecated symbol, if needed.
1246 */
1247 # if !defined(USE_GETSECTBYNAME) \
1248 && (MAC_OS_X_VERSION_MIN_REQUIRED \
1249 < 1070 /* `MAC_OS_X_VERSION_10_7` */)
1250 # define USE_GETSECTBYNAME
1251 # endif
1252
1253 static void
1254 dyld_section_add_del(const struct GC_MACH_HEADER *phdr, intptr_t slide,
1255 const char *dlpi_name, GC_has_static_roots_func callback,
1256 const char *seg, const char *secnam, GC_bool is_add)
1257 {
1258 ptr_t start, finish;
1259 unsigned long sec_size;
1260 # ifdef USE_GETSECTBYNAME
1261 # if CPP_WORDSZ == 64
1262 const struct section_64 *sec = getsectbynamefromheader_64(phdr, seg, secnam);
1263 # else
1264 const struct section *sec = getsectbynamefromheader(phdr, seg, secnam);
1265 # endif
1266
1267 if (NULL == sec)
1268 return;
1269 sec_size = sec->size;
1270 start = (ptr_t)slide + sec->addr;
1271 # else
1272
1273 UNUSED_ARG(slide);
1274 sec_size = 0;
1275 start = (ptr_t)getsectiondata(phdr, seg, secnam, &sec_size);
1276 if (NULL == start)
1277 return;
1278 # endif
1279 if (sec_size < sizeof(ptr_t))
1280 return;
1281 finish = start + sec_size;
1282 if (is_add) {
1283 LOCK();
1284 /* The user callback is invoked holding the allocator lock. */
1285 if (UNLIKELY(callback != 0)
1286 && !callback(dlpi_name, start, (size_t)sec_size)) {
1287 UNLOCK();
1288 return; /*< skip section */
1289 }
1290 GC_add_roots_inner(start, finish, FALSE);
1291 UNLOCK();
1292 } else {
1293 GC_remove_roots(start, finish);
1294 }
1295 # ifdef DARWIN_DEBUG
1296 GC_log_printf("%s section __DATA,%s at %p-%p (%lu bytes) from image %s\n",
1297 is_add ? "Added" : "Removed", secnam, (void *)start,
1298 (void *)finish, sec_size, dlpi_name);
1299 # endif
1300 }
1301
1302 static void
1303 dyld_image_add_del(const struct GC_MACH_HEADER *phdr, intptr_t slide,
1304 GC_has_static_roots_func callback, GC_bool is_add)
1305 {
1306 unsigned i, j;
1307 const char *dlpi_name;
1308
1309 GC_ASSERT(I_DONT_HOLD_LOCK());
1310 # ifndef DARWIN_DEBUG
1311 if (0 == callback) {
1312 dlpi_name = NULL;
1313 } else
1314 # endif
1315 /* else */ {
1316 dlpi_name = GC_dyld_name_for_hdr(phdr);
1317 }
1318 for (i = 0; i < sizeof(GC_dyld_sections) / sizeof(GC_dyld_sections[0]);
1319 i++) {
1320 dyld_section_add_del(phdr, slide, dlpi_name, callback,
1321 GC_dyld_sections[i].seg, GC_dyld_sections[i].sect,
1322 is_add);
1323 }
1324
1325 /* Sections constructed on demand. */
1326 for (j = 0; j < sizeof(GC_dyld_bss_prefixes) / sizeof(char *); j++) {
1327 /* Our manufactured aligned BSS sections. */
1328 for (i = 0; i <= L2_MAX_OFILE_ALIGNMENT; i++) {
1329 char secnam[11 + 20 + 1];
1330
1331 GC_snprintf_s_ld_s(secnam, sizeof(secnam), GC_dyld_bss_prefixes[j],
1332 (long)i, "");
1333 dyld_section_add_del(phdr, slide, dlpi_name, 0 /* `callback` */,
1334 SEG_DATA, secnam, is_add);
1335 }
1336 }
1337
1338 # if defined(DARWIN_DEBUG) && !defined(NO_DEBUGGING)
1339 READER_LOCK();
1340 GC_print_static_roots();
1341 READER_UNLOCK();
1342 # endif
1343 }
1344
1345 STATIC void
1346 GC_dyld_image_add(const struct GC_MACH_HEADER *phdr, intptr_t slide)
1347 {
1348 if (!GC_no_dls)
1349 dyld_image_add_del(phdr, slide, GC_has_static_roots, TRUE);
1350 }
1351
1352 STATIC void
1353 GC_dyld_image_remove(const struct GC_MACH_HEADER *phdr, intptr_t slide)
1354 {
1355 dyld_image_add_del(phdr, slide, 0 /* `callback` */, FALSE);
1356 }
1357
1358 GC_INNER void
1359 GC_register_dynamic_libraries(void)
1360 {
1361 /*
1362 * Currently does nothing. The callbacks are setup by `GC_init_dyld()`.
1363 * The `dyld` library takes it from there.
1364 */
1365 }
1366
1367 /*
1368 * The `_dyld_*` functions have an internal lock, so none of them can
1369 * be called while the world is stopped without the risk of a deadlock.
1370 * Because of this we must setup callbacks before we ever stop the world.
1371 * This should be called before any thread is created and without the
1372 * allocator lock held.
1373 */
1374
1375 /*
1376 * `_dyld_bind_fully_image_containing_address` is deprecated, so use
1377 * `dlopen(NULL, RTLD_NOW)` instead; define `USE_DYLD_TO_BIND` macro
1378 * to override this, if needed.
1379 */
1380
1381 GC_INNER void
1382 GC_init_dyld(void)
1383 {
1384 static GC_bool initialized = FALSE;
1385
1386 GC_ASSERT(I_DONT_HOLD_LOCK());
1387 if (initialized)
1388 return;
1389
1390 # ifdef DARWIN_DEBUG
1391 GC_log_printf("Registering dyld callbacks...\n");
1392 # endif
1393
1394 /*
1395 * From Apple's documentation:
1396 * When you call `_dyld_register_func_for_add_image`, the dynamic linker
1397 * runtime calls the specified callback (func) once for each of the images
1398 * that is currently loaded into the program. When a new image is added to
1399 * the program, your callback is invoked again with the `mach_header` for
1400 * the new image, and the virtual memory slide amount of the new image.
1401 *
1402 * This *will* properly register already linked libraries and libraries
1403 * linked in the future.
1404 */
1405
1406 /*
1407 * `struct mach_header_64` has the same fields as `struct mach_header`
1408 * except for the reserved one at the end, so these casts are OK.
1409 */
1410 _dyld_register_func_for_add_image(
1411 (void (*)(const struct mach_header *, intptr_t))GC_dyld_image_add);
1412 _dyld_register_func_for_remove_image(
1413 (void (*)(const struct mach_header *, intptr_t))GC_dyld_image_remove);
1414
1415 /* Set this early to avoid reentrancy issues. */
1416 initialized = TRUE;
1417
1418 # ifndef NO_DYLD_BIND_FULLY_IMAGE
1419 if (GC_no_dls) {
1420 /* Skip main data segment registration. */
1421 return;
1422 }
1423
1424 /*
1425 * When the environment variable is set, the dynamic linker binds
1426 * all undefined symbols the application needs at launch time.
1427 * This includes function symbols that are normally bound lazily at
1428 * the time of their first invocation.
1429 */
1430 if (GETENV("DYLD_BIND_AT_LAUNCH") != NULL) {
1431 return;
1432 }
1433 /* Else we should bind manually. */
1434 # ifdef DARWIN_DEBUG
1435 GC_log_printf("Forcing full bind of GC code...\n");
1436 # endif
1437 # ifndef USE_DYLD_TO_BIND
1438 {
1439 void *dl_handle = dlopen(NULL, RTLD_NOW);
1440
1441 if (!dl_handle) {
1442 ABORT("dlopen failed (to bind fully image)");
1443 }
1444 /* Note that the handle is never closed. */
1445 # if defined(CPPCHECK) || defined(LINT2)
1446 GC_noop1_ptr(dl_handle);
1447 # endif
1448 }
1449 # else
1450 /* Note: `_dyld_bind_fully_image_containing_address` is deprecated. */
1451 if (!_dyld_bind_fully_image_containing_address((unsigned long *)GC_malloc))
1452 ABORT("_dyld_bind_fully_image_containing_address failed");
1453 # endif
1454 # endif
1455 }
1456
1457 GC_INNER GC_bool
1458 GC_register_main_static_data(void)
1459 {
1460 /* Already done through `dyld` callbacks. */
1461 return FALSE;
1462 }
1463 # define HAVE_REGISTER_MAIN_STATIC_DATA
1464
1465 # endif /* DARWIN */
1466
1467 # if defined(HAIKU)
1468 # include <kernel/image.h>
1469
1470 GC_INNER void
1471 GC_register_dynamic_libraries(void)
1472 {
1473 image_info info;
1474 int32 cookie = 0;
1475
1476 GC_ASSERT(I_HOLD_LOCK());
1477 while (get_next_image_info(B_CURRENT_TEAM, &cookie, &info) == B_OK) {
1478 ptr_t data = (ptr_t)info.data;
1479
1480 GC_add_roots_inner(data, data + info.data_size, TRUE);
1481 }
1482 }
1483
1484 GC_INNER GC_bool
1485 GC_register_main_static_data(void)
1486 {
1487 /*
1488 * On Haiku, the main application binary is also a "shared image" and will
1489 * be reported in an `image_info` same as for dynamically-loaded libraries.
1490 */
1491 return FALSE;
1492 }
1493 # define HAVE_REGISTER_MAIN_STATIC_DATA
1494 # endif /* HAIKU */
1495
1496 # ifdef HPUX
1497 # include <dl.h>
1498 # include <errno.h>
1499
1500 EXTERN_C_BEGIN
1501 extern char *sys_errlist[];
1502 extern int sys_nerr;
1503 EXTERN_C_END
1504
1505 GC_INNER void
1506 GC_register_dynamic_libraries(void)
1507 {
1508 /* Ordinal position in shared library search list. */
1509 int index = 1;
1510
1511 GC_ASSERT(I_HOLD_LOCK());
1512 /* For each dynamic library loaded. */
1513 for (;;) {
1514 /* Shared library info, see platform `dl.h` file. */
1515 struct shl_descriptor *shl_desc;
1516 /* Get info about next shared library. */
1517 int status = shl_get(index, &shl_desc);
1518
1519 /* Check if this is the end of the list or if some error occurred. */
1520 if (status != 0) {
1521 # ifdef THREADS
1522 /*
1523 * I have seen `errno` values of 0. The man page is not clear as
1524 * to whether `errno` should get set when -1 is returned.
1525 */
1526 break;
1527 # else
1528 if (errno == EINVAL) {
1529 /* Moved past end of shared library list. Finish. */
1530 break;
1531 } else {
1532 ABORT_ARG3("shl_get failed", ": status= %d, errcode= %d (%s)", status,
1533 errno, errno < sys_nerr ? sys_errlist[errno] : "");
1534 }
1535 # endif
1536 }
1537
1538 # ifdef DL_VERBOSE
1539 GC_log_printf("---Shared library---\n");
1540 GC_log_printf("filename= \"%s\"\n", shl_desc->filename);
1541 GC_log_printf("index= %d\n", index);
1542 GC_log_printf("handle= %08x\n", (unsigned long)shl_desc->handle);
1543 GC_log_printf("text seg.start= %08x\n", shl_desc->tstart);
1544 GC_log_printf("text seg.end= %08x\n", shl_desc->tend);
1545 GC_log_printf("data seg.start= %08x\n", shl_desc->dstart);
1546 GC_log_printf("data seg.end= %08x\n", shl_desc->dend);
1547 GC_log_printf("ref.count= %lu\n", shl_desc->ref_count);
1548 # endif
1549
1550 /*
1551 * Register shared library's data segment as a garbage collection
1552 * root.
1553 */
1554 GC_add_roots_inner((char *)shl_desc->dstart, (char *)shl_desc->dend, TRUE);
1555
1556 index++;
1557 }
1558 }
1559 # endif /* HPUX */
1560
1561 # ifdef OSF1
1562 # include <loader.h>
1563
1564 EXTERN_C_BEGIN
1565 extern char *sys_errlist[];
1566 extern int sys_nerr, errno;
1567 EXTERN_C_END
1568
1569 GC_INNER void
1570 GC_register_dynamic_libraries(void)
1571 {
1572 ldr_module_t moduleid = LDR_NULL_MODULE;
1573 ldr_process_t mypid;
1574
1575 GC_ASSERT(I_HOLD_LOCK());
1576 /* Obtain id of this process. */
1577 mypid = ldr_my_process();
1578
1579 /* For each module. */
1580 for (;;) {
1581 ldr_module_info_t moduleinfo;
1582 size_t modulereturnsize;
1583 ldr_region_t region;
1584 ldr_region_info_t regioninfo;
1585 size_t regionreturnsize;
1586 /* Get the next (first) module. */
1587 int status = ldr_next_module(mypid, &moduleid);
1588
1589 if (moduleid == LDR_NULL_MODULE) {
1590 /* No more modules. */
1591 break;
1592 }
1593
1594 /*
1595 * Check status *after* checking `moduleid` because of a bug
1596 * in the non-shared `ldr_next_module` stub.
1597 */
1598 if (status != 0) {
1599 ABORT_ARG3("ldr_next_module failed", ": status= %d, errcode= %d (%s)",
1600 status, errno, errno < sys_nerr ? sys_errlist[errno] : "");
1601 }
1602
1603 /* Get the module information. */
1604 status = ldr_inq_module(mypid, moduleid, &moduleinfo, sizeof(moduleinfo),
1605 &modulereturnsize);
1606 if (status != 0)
1607 ABORT("ldr_inq_module failed");
1608
1609 /* Is module for the main program (i.e. nonshared portion)? */
1610 if ((moduleinfo.lmi_flags & LDR_MAIN) != 0) {
1611 /* Skip the main module. */
1612 continue;
1613 }
1614
1615 # ifdef DL_VERBOSE
1616 GC_log_printf("---Module---\n");
1617 GC_log_printf("Module ID: %ld\n", moduleinfo.lmi_modid);
1618 GC_log_printf("Count of regions: %d\n", moduleinfo.lmi_nregion);
1619 GC_log_printf("Flags for module: %016lx\n", moduleinfo.lmi_flags);
1620 GC_log_printf("Module pathname: \"%s\"\n", moduleinfo.lmi_name);
1621 # endif
1622
1623 /* For each region in this module. */
1624 for (region = 0; region < moduleinfo.lmi_nregion; region++) {
1625 /* Get the region information. */
1626 status = ldr_inq_region(mypid, moduleid, region, ®ioninfo,
1627 sizeof(regioninfo), ®ionreturnsize);
1628 if (status != 0)
1629 ABORT("ldr_inq_region failed");
1630
1631 /* Only process writable (data) regions. */
1632 if ((regioninfo.lri_prot & LDR_W) == 0)
1633 continue;
1634
1635 # ifdef DL_VERBOSE
1636 GC_log_printf("--- Region ---\n");
1637 GC_log_printf("Region number: %ld\n", regioninfo.lri_region_no);
1638 GC_log_printf("Protection flags: %016x\n", regioninfo.lri_prot);
1639 GC_log_printf("Virtual address: %p\n", regioninfo.lri_vaddr);
1640 GC_log_printf("Mapped address: %p\n", regioninfo.lri_mapaddr);
1641 GC_log_printf("Region size: %ld\n", regioninfo.lri_size);
1642 GC_log_printf("Region name: \"%s\"\n", regioninfo.lri_name);
1643 # endif
1644
1645 /* Register region as a garbage collection root. */
1646 GC_add_roots_inner((char *)regioninfo.lri_mapaddr,
1647 (char *)regioninfo.lri_mapaddr + regioninfo.lri_size,
1648 TRUE);
1649 }
1650 }
1651 }
1652 # endif /* OSF1 */
1653
1654 #endif /* DYNAMIC_LOADING */
1655
1656 #ifdef GC_MUST_RESTORE_REDEFINED_DLOPEN
1657 # define dlopen GC_dlopen
1658 #endif
1659
1660 #if !defined(HAVE_REGISTER_MAIN_STATIC_DATA) && defined(DYNAMIC_LOADING)
1661 GC_INNER GC_bool
1662 GC_register_main_static_data(void)
1663 {
1664 return TRUE;
1665 }
1666 #endif /* HAVE_REGISTER_MAIN_STATIC_DATA */
1667
1668 GC_API void GC_CALL
1669 GC_register_has_static_roots_callback(GC_has_static_roots_func callback)
1670 {
1671 GC_has_static_roots = callback;
1672 }
1673