// Arena allocator with domain root and function arena stack. // // One TLS slot: domainRoot. Written once at domain creation, never mutated. // Current arena = domainRoot.fnStack[domainRoot.fnTop]. // Function entry pushes, function return pops. LIFO matches lifetime. // // Arena structs are self-contained in their own mmap region (offset 0). // No linked lists. No heap allocation for metadata. Region ownership // is a bounded table scan of the arena's flat region registry. package runtime import ( "unsafe" ) const needsStaticHeap = true var heapptr uintptr var totalAlloc uint64 var mallocs uint64 // domainRoot: one TLS slot per domain. Written once, never mutated. var domainRoot *DomainRoot // fnStackMax is the maximum function arena nesting depth. // 512 entries = 4096 bytes = one page. More call depth than any // compiler pass will hit. const fnStackMax = int32(512) // domStackMax is the maximum child domain count per parent. const domStackMax = int32(64) // DomainRoot lives at offset 0 of the domain's root mmap region. // Contains the function arena stack and child domain stack. // The root arena is embedded inline - fnStack[0] points to it. type DomainRoot struct { fnStack [512]*Arena fnTop int32 _pad1 int32 domStack [64]uintptr // child domain root addresses domTop int32 _pad2 int32 root Arena // inline root arena, fnStack[0] = &root } // ArenaRegion is one mmap'd region owned by an Arena. type ArenaRegion struct { base uintptr off uintptr // bytes allocated cap uintptr } // arenaMaxRegions is the maximum number of regions per arena. const arenaMaxRegions = int32(256) // Arena is a bump-pointer allocator with a flat region registry. // For function arenas, the Arena struct lives at offset 0 of its // primary mmap region. No external allocation needed. type Arena struct { nregions int32 _pad int32 regions [256]ArenaRegion } // Arena pool: bounded freelist of reset arenas for reuse. // Eliminates mmap/munmap syscalls for short-lived function arenas. const arenaPoolMax = int32(8) var arenaPool [8]*Arena var arenaPoolN int32 //:noinline func alloc(size uintptr, _ unsafe.Pointer) unsafe.Pointer { if size == 0 { size = 1 } if domainRoot != nil { a := domainRoot.fnStack[domainRoot.fnTop] return ArenaAlloc(a, align(size)) } // Startup bump allocator: used before domain root exists. size = align(size) addr := heapptr heapptr += size for heapptr >= heapEnd { if growHeap() { continue } runtimePanic("out of memory") } mallocs++ totalAlloc += uint64(size) ptr := unsafe.Pointer(addr) memzero(ptr, size) return ptr } func ReadMemStats(m *MemStats) { m.ArenaIdle = 0 m.ArenaInuse = totalAlloc m.ArenaFreed = 0 m.ArenaSys = m.ArenaInuse + m.ArenaIdle m.TotalAlloc = totalAlloc m.Allocs = mallocs m.Frees = 0 m.Sys = uint64(heapEnd - heapStart) m.ArenaAlloc = totalAlloc m.Alloc = m.ArenaAlloc } func initHeap() { heapptr = heapStart initRootArena() } // --- Domain lifecycle --- // initDomain creates the domain root in a new mmap region. // Sets the one TLS slot. Called once per domain lifetime. func initDomain(capacity uintptr) { capacity = (capacity + 4095) &^ 4095 headerSz := align(unsafe.Sizeof(DomainRoot{})) if capacity < headerSz+4096 { capacity = (headerSz + 4096 + 4095) &^ 4095 } printstring("DOMAIN_INIT cap=") printuint64(uint64(capacity)) printstring(" hdr=") printuint64(uint64(headerSz)) printstring("\n") addr := mmap(nil, capacity, flag_PROT_READ|flag_PROT_WRITE, flag_MAP_PRIVATE|flag_MAP_ANONYMOUS, -1, 0) if addr == unsafe.Pointer(^uintptr(0)) { runtimePanic("domain: mmap failed") } memzero(addr, unsafe.Sizeof(DomainRoot{})) dr := (*DomainRoot)(addr) dr.root.nregions = 1 dr.root.regions[0].base = uintptr(addr) dr.root.regions[0].off = headerSz dr.root.regions[0].cap = capacity dr.fnStack[0] = &dr.root dr.fnTop = 0 dr.domTop = 0 domainRoot = dr // write TLS slot once } // --- Function arena stack --- // FnArenaPush creates a new function arena and pushes it onto the stack. // Returns the arena pointer (used by codegen for RelocInit). func FnArenaPush(capacity uintptr) (a *Arena) { domainRoot.fnTop++ if domainRoot.fnTop >= fnStackMax { runtimePanic("function arena stack overflow") } a = ArenaAcquire(capacity) domainRoot.fnStack[domainRoot.fnTop] = a return a } // FnArenaPop removes the top function arena from the stack and returns it. // The caller is responsible for calling ArenaFree on the returned arena. func FnArenaPop() (a *Arena) { a = domainRoot.fnStack[domainRoot.fnTop] domainRoot.fnStack[domainRoot.fnTop] = nil domainRoot.fnTop-- return a } // FnArenaPopFree pops the top function arena and frees it. // Used by the no-copy return path. func FnArenaPopFree() { a := FnArenaPop() ArenaRelease(a) } // --- Backward-compatible arena API --- // RootArena returns the domain root arena (fnStack[0]). func RootArena() (a *Arena) { if domainRoot == nil { return nil } return domainRoot.fnStack[0] } // CurrentArena returns the active arena (top of function stack). func CurrentArena() (a *Arena) { if domainRoot == nil { return nil } return domainRoot.fnStack[domainRoot.fnTop] } // SetCurrentArena overwrites the top of the function arena stack. // Used by hashmap owner-arena switching. Caller must save/restore. func SetCurrentArena(a *Arena) { if domainRoot == nil { return } domainRoot.fnStack[domainRoot.fnTop] = a } // ArenaMark returns an opaque mark encoding the current allocation point. // High 32 bits: active region index. Low 32 bits: offset in that region. func ArenaMark() (mark uintptr) { if domainRoot == nil { return 0 } a := domainRoot.fnStack[domainRoot.fnTop] if a == nil { return 0 } idx := a.nregions - 1 return uintptr(idx)<<32 | a.regions[idx].off } // ArenaRestore resets the current arena to a previously saved mark. func ArenaRestore(mark uintptr) { if domainRoot == nil { return } a := domainRoot.fnStack[domainRoot.fnTop] if a == nil { return } regionIdx := int32(mark >> 32) off := mark & 0xFFFFFFFF for i := a.nregions - 1; i > regionIdx; i-- { munmap(unsafe.Pointer(a.regions[i].base), a.regions[i].cap) } a.nregions = regionIdx + 1 a.regions[regionIdx].off = off } // --- Arena lifecycle --- const arenaDefaultRegion = uintptr(1024) * 1024 // 1MB chain regions // ArenaNew creates an arena with the given initial capacity via mmap. // The Arena struct is embedded at offset 0 of the mmap'd region. // No external allocation needed. func ArenaNew(capacity uintptr) (a *Arena) { capacity = (capacity + 4095) &^ 4095 headerSz := align(unsafe.Sizeof(Arena{})) if capacity < headerSz+4096 { capacity = (headerSz + 4096 + 4095) &^ 4095 } addr := mmap(nil, capacity, flag_PROT_READ|flag_PROT_WRITE, flag_MAP_PRIVATE|flag_MAP_ANONYMOUS, -1, 0) if addr == unsafe.Pointer(^uintptr(0)) { runtimePanic("arena: mmap failed") } a = (*Arena)(addr) a.nregions = 1 a.regions[0].base = uintptr(addr) a.regions[0].off = headerSz // skip Arena header a.regions[0].cap = capacity return a } // ArenaAlloc allocates size bytes from the arena, 16-byte aligned, zeroed. func ArenaAlloc(a *Arena, size uintptr) unsafe.Pointer { if a == nil { runtimePanic("ArenaAlloc: nil arena") } size = align(size) active := a.nregions - 1 if a.regions[active].off+size > a.regions[active].cap { if a.nregions >= arenaMaxRegions { printstring("OVERFLOW nregions=") printuint64(uint64(a.nregions)) printstring(" size=") printuint64(uint64(size)) printstring(" r0.cap=") printuint64(uint64(a.regions[0].cap)) printstring(" r0.off=") printuint64(uint64(a.regions[0].off)) printstring(" active.cap=") printuint64(uint64(a.regions[active].cap)) printstring(" active.off=") printuint64(uint64(a.regions[active].off)) printstring(" arena=") printuint64(uint64(uintptr(unsafe.Pointer(a)))) printstring("\n") runtimePanic("arena: region table full") } newCap := arenaDefaultRegion if size > newCap { newCap = (size + 4095) &^ 4095 } addr := mmap(nil, newCap, flag_PROT_READ|flag_PROT_WRITE, flag_MAP_PRIVATE|flag_MAP_ANONYMOUS, -1, 0) if addr == unsafe.Pointer(^uintptr(0)) { runtimePanic("arena: region mmap failed") } a.regions[a.nregions].base = uintptr(addr) a.regions[a.nregions].off = 0 a.regions[a.nregions].cap = newCap a.nregions++ active = a.nregions - 1 } ptr := unsafe.Pointer(a.regions[active].base + a.regions[active].off) a.regions[active].off += size memzero(ptr, size) return ptr } // ArenaFree unmaps all regions in the arena. // The primary region contains the Arena struct itself, so read // base/cap into locals before the final munmap. func ArenaFree(a *Arena) { if a == nil { return } for i := a.nregions - 1; i > 0; i-- { munmap(unsafe.Pointer(a.regions[i].base), a.regions[i].cap) } b := a.regions[0].base c := a.regions[0].cap munmap(unsafe.Pointer(b), c) } // ArenaReset resets the arena for reuse. Frees all regions except // the first, resets the first region's offset. func ArenaReset(a *Arena) { if a == nil { return } for i := a.nregions - 1; i > 0; i-- { munmap(unsafe.Pointer(a.regions[i].base), a.regions[i].cap) } a.nregions = 1 a.regions[0].off = align(unsafe.Sizeof(Arena{})) } // ArenaAcquire returns a pooled arena with cap >= capacity, or // creates a new one via ArenaNew. First-fit scan of bounded pool. func ArenaAcquire(capacity uintptr) (a *Arena) { capacity = (capacity + 4095) &^ 4095 headerSz := align(unsafe.Sizeof(Arena{})) if capacity < headerSz+4096 { capacity = (headerSz + 4096 + 4095) &^ 4095 } for i := int32(0); i < arenaPoolN; i++ { if arenaPool[i].regions[0].cap >= capacity { a = arenaPool[i] arenaPoolN-- arenaPool[i] = arenaPool[arenaPoolN] arenaPool[arenaPoolN] = nil return a } } return ArenaNew(capacity) } // ArenaRelease resets an arena and returns it to the pool. // If the pool is full, frees the arena entirely via munmap. func ArenaRelease(a *Arena) { if a == nil { return } if arenaPoolN >= arenaPoolMax { ArenaFree(a) return } ArenaReset(a) arenaPool[arenaPoolN] = a arenaPoolN++ } // heapAlloc: direct startup bump allocation. Only used before // domain root exists. Does not touch any TLS or arena state. func heapAlloc(size uintptr) unsafe.Pointer { size = align(size) addr := heapptr heapptr += size for heapptr >= heapEnd { if growHeap() { continue } runtimePanic("out of memory") } ptr := unsafe.Pointer(addr) memzero(ptr, size) return ptr } // --- Region locality queries --- // arenaContains checks if ptr is in any region of the arena above mark. func arenaContains(a *Arena, ptr uintptr, mark uintptr) (ok bool) { regionIdx := int32(mark >> 32) regionOff := mark & 0xFFFFFFFF for i := regionIdx; i < a.nregions; i++ { r := a.regions[i] start := r.base if i == regionIdx { start += regionOff } if ptr >= start && ptr < r.base+r.off { return true } } return false } // arenaCopySliceIfLocal copies a slice's backing array to the current arena // position only if its data pointer is in the source arena region. func arenaCopySliceIfLocal(s string, mark uintptr) (r string) { if len(s) == 0 || domainRoot == nil { return s } a := domainRoot.fnStack[domainRoot.fnTop] p := uintptr(unsafe.Pointer(unsafe.SliceData([]byte(s)))) if !arenaContains(a, p, mark) { return s } buf := []byte{:len(s)} copy(buf, s) return string(buf) } // arenaCopyPtrIfLocal copies size bytes at ptr only if in source region. func arenaCopyPtrIfLocal(ptr unsafe.Pointer, mark uintptr, size uintptr) unsafe.Pointer { if ptr == nil || domainRoot == nil { return ptr } a := domainRoot.fnStack[domainRoot.fnTop] p := uintptr(ptr) if !arenaContains(a, p, mark) { return ptr } dst := alloc(size, nil) memcpy(dst, ptr, size) return dst } func setHeapEnd(newHeapEnd uintptr) { heapEnd = newHeapEnd } //:export moxie_scanstack func scanstack(sp uintptr) {}