relocate.mx raw
1 // Mark-compact arena relocator. Three passes:
2 //
3 // Pass 1 - Mark: walk from roots, bump-allocate one RelocEntry per
4 // reachable object in fn_arena.
5 // Pass 2 - Forward: walk entries, allocate destinations in caller's arena.
6 // Pass 3 - Compact + Fixup: copy objects, rewrite pointer fields.
7 //
8 // Region locality is determined by scanning the arena's flat region
9 // table - no linked-list traversal, no single-point-of-failure chains.
10
11 package runtime
12
13 import "unsafe"
14
15 // RelocEntry: one forwarding record. Linked list for traversal order.
16 type RelocEntry struct {
17 old uintptr
18 new uintptr
19 size uintptr
20 next *RelocEntry
21 fixup uintptr // function pointer for per-instance field fixup (0 = none)
22 }
23
24 // RelocState: header on the stack.
25 type RelocState struct {
26 arena *Arena // fn_arena - source arena with region table
27 mark uintptr // opaque mark from ArenaMark at function entry
28 first *RelocEntry // linked list head
29 count int32 // number of entries
30 }
31
32 // RelocInit captures the source arena and entry point.
33 func RelocInit(rs *RelocState, fnArena *Arena, mark uintptr) {
34 rs.arena = fnArena
35 rs.mark = mark
36 rs.first = nil
37 rs.count = 0
38 }
39
40 // relocIsLocal checks if p was allocated by this function in any arena region.
41 // Scans the flat region table - bounded by arenaMaxRegions.
42 func relocIsLocal(rs *RelocState, p uintptr) (ok bool) {
43 a := rs.arena
44 markRegion := int32(rs.mark >> 32)
45 markOff := rs.mark & 0xFFFFFFFF
46 for i := markRegion; i < a.nregions; i++ {
47 r := a.regions[i]
48 start := r.base
49 if i == markRegion {
50 start += markOff
51 }
52 if p >= start && p < r.base+r.off {
53 return true
54 }
55 }
56 return false
57 }
58
59 // --- Pass 1: Mark ---
60
61 // RelocMark records a reachable object. Allocates entry from fn_arena.
62 func RelocMark(rs *RelocState, ptr unsafe.Pointer, size uintptr, fixup uintptr) (isNew bool) {
63 if ptr == nil {
64 return false
65 }
66 p := uintptr(ptr)
67 if !relocIsLocal(rs, p) {
68 return false
69 }
70 // Already marked?
71 e := rs.first
72 for e != nil {
73 if e.old == p {
74 return false
75 }
76 e = e.next
77 }
78 // Allocate entry from fn_arena.
79 ne := (*RelocEntry)(ArenaAlloc(rs.arena, unsafe.Sizeof(RelocEntry{})))
80 ne.old = p
81 ne.new = 0
82 ne.size = size
83 ne.next = rs.first
84 ne.fixup = fixup
85 rs.first = ne
86 rs.count++
87 return true
88 }
89
90 // --- Pass 2: Forward ---
91
92 // RelocForward assigns destinations in the current (caller's) arena.
93 func RelocForward(rs *RelocState) {
94 e := rs.first
95 for e != nil {
96 e.new = uintptr(alloc(e.size, nil))
97 e = e.next
98 }
99 }
100
101 // --- Pass 3: Compact + Fixup ---
102
103 // RelocCompact copies all marked objects to their destinations.
104 func RelocCompact(rs *RelocState) {
105 e := rs.first
106 for e != nil {
107 memcpy(unsafe.Pointer(e.new), unsafe.Pointer(e.old), e.size)
108 e = e.next
109 }
110 }
111
112 // RelocFixupAll iterates forwarding entries and calls each entry's fixup
113 // function on the copied object. Per-instance execution of per-type codegen.
114 func RelocFixupAll(rs *RelocState) {
115 e := rs.first
116 for e != nil {
117 if e.fixup != 0 {
118 relocCallFixup(e.fixup, unsafe.Pointer(e.new), unsafe.Pointer(rs))
119 }
120 e = e.next
121 }
122 }
123
124 // relocCallFixup: indirect call trampoline. Implemented in initall.ll.
125 //
126 //:linkname relocCallFixup runtime.relocCallFixup
127 func relocCallFixup(fn uintptr, obj unsafe.Pointer, rs unsafe.Pointer)
128
129 // RelocLookup translates a pointer through the forwarding table.
130 func RelocLookup(rs *RelocState, old unsafe.Pointer) (new unsafe.Pointer) {
131 if old == nil {
132 return nil
133 }
134 p := uintptr(old)
135 if !relocIsLocal(rs, p) {
136 return old
137 }
138 e := rs.first
139 for e != nil {
140 if p >= e.old && p < e.old+e.size {
141 return unsafe.Pointer(e.new + (p - e.old))
142 }
143 e = e.next
144 }
145 return old
146 }
147
148 // RelocFixupPtr rewrites a pointer field at fieldAddr.
149 func RelocFixupPtr(rs *RelocState, fieldAddr unsafe.Pointer) {
150 old := *(*unsafe.Pointer)(fieldAddr)
151 if old == nil {
152 return
153 }
154 *(*unsafe.Pointer)(fieldAddr) = RelocLookup(rs, old)
155 }
156
157 // RelocFixupSlice rewrites a slice/string data pointer at sliceAddr.
158 func RelocFixupSlice(rs *RelocState, sliceAddr unsafe.Pointer) {
159 old := *(*unsafe.Pointer)(sliceAddr)
160 if old == nil {
161 return
162 }
163 *(*unsafe.Pointer)(sliceAddr) = RelocLookup(rs, old)
164 }
165