// Mark-compact arena relocator. Three passes: // // Pass 1 - Mark: walk from roots, bump-allocate one RelocEntry per // reachable object in fn_arena. // Pass 2 - Forward: walk entries, allocate destinations in caller's arena. // Pass 3 - Compact + Fixup: copy objects, rewrite pointer fields. // // Region locality is determined by scanning the arena's flat region // table - no linked-list traversal, no single-point-of-failure chains. package runtime import "unsafe" // RelocEntry: one forwarding record. Linked list for traversal order. type RelocEntry struct { old uintptr new uintptr size uintptr next *RelocEntry fixup uintptr // function pointer for per-instance field fixup (0 = none) } // RelocState: header on the stack. type RelocState struct { arena *Arena // fn_arena - source arena with region table mark uintptr // opaque mark from ArenaMark at function entry first *RelocEntry // linked list head count int32 // number of entries } // RelocInit captures the source arena and entry point. func RelocInit(rs *RelocState, fnArena *Arena, mark uintptr) { rs.arena = fnArena rs.mark = mark rs.first = nil rs.count = 0 } // relocIsLocal checks if p was allocated by this function in any arena region. // Scans the flat region table - bounded by arenaMaxRegions. func relocIsLocal(rs *RelocState, p uintptr) (ok bool) { a := rs.arena markRegion := int32(rs.mark >> 32) markOff := rs.mark & 0xFFFFFFFF for i := markRegion; i < a.nregions; i++ { r := a.regions[i] start := r.base if i == markRegion { start += markOff } if p >= start && p < r.base+r.off { return true } } return false } // --- Pass 1: Mark --- // RelocMark records a reachable object. Allocates entry from fn_arena. func RelocMark(rs *RelocState, ptr unsafe.Pointer, size uintptr, fixup uintptr) (isNew bool) { if ptr == nil { return false } p := uintptr(ptr) if !relocIsLocal(rs, p) { return false } // Already marked? e := rs.first for e != nil { if e.old == p { return false } e = e.next } // Allocate entry from fn_arena. ne := (*RelocEntry)(ArenaAlloc(rs.arena, unsafe.Sizeof(RelocEntry{}))) ne.old = p ne.new = 0 ne.size = size ne.next = rs.first ne.fixup = fixup rs.first = ne rs.count++ return true } // --- Pass 2: Forward --- // RelocForward assigns destinations in the current (caller's) arena. func RelocForward(rs *RelocState) { e := rs.first for e != nil { e.new = uintptr(alloc(e.size, nil)) e = e.next } } // --- Pass 3: Compact + Fixup --- // RelocCompact copies all marked objects to their destinations. func RelocCompact(rs *RelocState) { e := rs.first for e != nil { memcpy(unsafe.Pointer(e.new), unsafe.Pointer(e.old), e.size) e = e.next } } // RelocFixupAll iterates forwarding entries and calls each entry's fixup // function on the copied object. Per-instance execution of per-type codegen. func RelocFixupAll(rs *RelocState) { e := rs.first for e != nil { if e.fixup != 0 { relocCallFixup(e.fixup, unsafe.Pointer(e.new), unsafe.Pointer(rs)) } e = e.next } } // relocCallFixup: indirect call trampoline. Implemented in initall.ll. // //:linkname relocCallFixup runtime.relocCallFixup func relocCallFixup(fn uintptr, obj unsafe.Pointer, rs unsafe.Pointer) // RelocLookup translates a pointer through the forwarding table. func RelocLookup(rs *RelocState, old unsafe.Pointer) (new unsafe.Pointer) { if old == nil { return nil } p := uintptr(old) if !relocIsLocal(rs, p) { return old } e := rs.first for e != nil { if p >= e.old && p < e.old+e.size { return unsafe.Pointer(e.new + (p - e.old)) } e = e.next } return old } // RelocFixupPtr rewrites a pointer field at fieldAddr. func RelocFixupPtr(rs *RelocState, fieldAddr unsafe.Pointer) { old := *(*unsafe.Pointer)(fieldAddr) if old == nil { return } *(*unsafe.Pointer)(fieldAddr) = RelocLookup(rs, old) } // RelocFixupSlice rewrites a slice/string data pointer at sliceAddr. func RelocFixupSlice(rs *RelocState, sliceAddr unsafe.Pointer) { old := *(*unsafe.Pointer)(sliceAddr) if old == nil { return } *(*unsafe.Pointer)(sliceAddr) = RelocLookup(rs, old) }