alloc.mx raw
1 // Arena allocator with domain root and function arena stack.
2 //
3 // One TLS slot: domainRoot. Written once at domain creation, never mutated.
4 // Current arena = domainRoot.fnStack[domainRoot.fnTop].
5 // Function entry pushes, function return pops. LIFO matches lifetime.
6 //
7 // Arena structs are self-contained in their own mmap region (offset 0).
8 // No linked lists. No heap allocation for metadata. Region ownership
9 // is a bounded table scan of the arena's flat region registry.
10
11 package runtime
12
13 import (
14 "unsafe"
15 )
16
17 const needsStaticHeap = true
18
19 var heapptr uintptr
20 var totalAlloc uint64
21 var mallocs uint64
22
23 // domainRoot: one TLS slot per domain. Written once, never mutated.
24 var domainRoot *DomainRoot
25
26 // fnStackMax is the maximum function arena nesting depth.
27 // 512 entries = 4096 bytes = one page. More call depth than any
28 // compiler pass will hit.
29 const fnStackMax = int32(512)
30
31 // domStackMax is the maximum child domain count per parent.
32 const domStackMax = int32(64)
33
34 // DomainRoot lives at offset 0 of the domain's root mmap region.
35 // Contains the function arena stack and child domain stack.
36 // The root arena is embedded inline - fnStack[0] points to it.
37 type DomainRoot struct {
38 fnStack [512]*Arena
39 fnTop int32
40 _pad1 int32
41 domStack [64]uintptr // child domain root addresses
42 domTop int32
43 _pad2 int32
44 root Arena // inline root arena, fnStack[0] = &root
45 }
46
47 // ArenaRegion is one mmap'd region owned by an Arena.
48 type ArenaRegion struct {
49 base uintptr
50 off uintptr // bytes allocated
51 cap uintptr
52 }
53
54 // arenaMaxRegions is the maximum number of regions per arena.
55 const arenaMaxRegions = int32(256)
56
57 // Arena is a bump-pointer allocator with a flat region registry.
58 // For function arenas, the Arena struct lives at offset 0 of its
59 // primary mmap region. No external allocation needed.
60 type Arena struct {
61 nregions int32
62 _pad int32
63 regions [256]ArenaRegion
64 }
65
66 // Arena pool: bounded freelist of reset arenas for reuse.
67 // Eliminates mmap/munmap syscalls for short-lived function arenas.
68 const arenaPoolMax = int32(8)
69
70 var arenaPool [8]*Arena
71 var arenaPoolN int32
72
73 //:noinline
74 func alloc(size uintptr, _ unsafe.Pointer) unsafe.Pointer {
75 if size == 0 {
76 size = 1
77 }
78 if domainRoot != nil {
79 a := domainRoot.fnStack[domainRoot.fnTop]
80 return ArenaAlloc(a, align(size))
81 }
82 // Startup bump allocator: used before domain root exists.
83 size = align(size)
84 addr := heapptr
85 heapptr += size
86 for heapptr >= heapEnd {
87 if growHeap() {
88 continue
89 }
90 runtimePanic("out of memory")
91 }
92 mallocs++
93 totalAlloc += uint64(size)
94 ptr := unsafe.Pointer(addr)
95 memzero(ptr, size)
96 return ptr
97 }
98
99 func ReadMemStats(m *MemStats) {
100 m.ArenaIdle = 0
101 m.ArenaInuse = totalAlloc
102 m.ArenaFreed = 0
103 m.ArenaSys = m.ArenaInuse + m.ArenaIdle
104 m.TotalAlloc = totalAlloc
105 m.Allocs = mallocs
106 m.Frees = 0
107 m.Sys = uint64(heapEnd - heapStart)
108 m.ArenaAlloc = totalAlloc
109 m.Alloc = m.ArenaAlloc
110 }
111
112 func initHeap() {
113 heapptr = heapStart
114 initRootArena()
115 }
116
117 // --- Domain lifecycle ---
118
119 // initDomain creates the domain root in a new mmap region.
120 // Sets the one TLS slot. Called once per domain lifetime.
121 func initDomain(capacity uintptr) {
122 capacity = (capacity + 4095) &^ 4095
123 headerSz := align(unsafe.Sizeof(DomainRoot{}))
124 if capacity < headerSz+4096 {
125 capacity = (headerSz + 4096 + 4095) &^ 4095
126 }
127 printstring("DOMAIN_INIT cap=")
128 printuint64(uint64(capacity))
129 printstring(" hdr=")
130 printuint64(uint64(headerSz))
131 printstring("\n")
132 addr := mmap(nil, capacity, flag_PROT_READ|flag_PROT_WRITE,
133 flag_MAP_PRIVATE|flag_MAP_ANONYMOUS, -1, 0)
134 if addr == unsafe.Pointer(^uintptr(0)) {
135 runtimePanic("domain: mmap failed")
136 }
137 memzero(addr, unsafe.Sizeof(DomainRoot{}))
138 dr := (*DomainRoot)(addr)
139 dr.root.nregions = 1
140 dr.root.regions[0].base = uintptr(addr)
141 dr.root.regions[0].off = headerSz
142 dr.root.regions[0].cap = capacity
143 dr.fnStack[0] = &dr.root
144 dr.fnTop = 0
145 dr.domTop = 0
146 domainRoot = dr // write TLS slot once
147 }
148
149 // --- Function arena stack ---
150
151 // FnArenaPush creates a new function arena and pushes it onto the stack.
152 // Returns the arena pointer (used by codegen for RelocInit).
153 func FnArenaPush(capacity uintptr) (a *Arena) {
154 domainRoot.fnTop++
155 if domainRoot.fnTop >= fnStackMax {
156 runtimePanic("function arena stack overflow")
157 }
158 a = ArenaAcquire(capacity)
159 domainRoot.fnStack[domainRoot.fnTop] = a
160 return a
161 }
162
163 // FnArenaPop removes the top function arena from the stack and returns it.
164 // The caller is responsible for calling ArenaFree on the returned arena.
165 func FnArenaPop() (a *Arena) {
166 a = domainRoot.fnStack[domainRoot.fnTop]
167 domainRoot.fnStack[domainRoot.fnTop] = nil
168 domainRoot.fnTop--
169 return a
170 }
171
172 // FnArenaPopFree pops the top function arena and frees it.
173 // Used by the no-copy return path.
174 func FnArenaPopFree() {
175 a := FnArenaPop()
176 ArenaRelease(a)
177 }
178
179 // --- Backward-compatible arena API ---
180
181 // RootArena returns the domain root arena (fnStack[0]).
182 func RootArena() (a *Arena) {
183 if domainRoot == nil {
184 return nil
185 }
186 return domainRoot.fnStack[0]
187 }
188
189 // CurrentArena returns the active arena (top of function stack).
190 func CurrentArena() (a *Arena) {
191 if domainRoot == nil {
192 return nil
193 }
194 return domainRoot.fnStack[domainRoot.fnTop]
195 }
196
197 // SetCurrentArena overwrites the top of the function arena stack.
198 // Used by hashmap owner-arena switching. Caller must save/restore.
199 func SetCurrentArena(a *Arena) {
200 if domainRoot == nil {
201 return
202 }
203 domainRoot.fnStack[domainRoot.fnTop] = a
204 }
205
206 // ArenaMark returns an opaque mark encoding the current allocation point.
207 // High 32 bits: active region index. Low 32 bits: offset in that region.
208 func ArenaMark() (mark uintptr) {
209 if domainRoot == nil {
210 return 0
211 }
212 a := domainRoot.fnStack[domainRoot.fnTop]
213 if a == nil {
214 return 0
215 }
216 idx := a.nregions - 1
217 return uintptr(idx)<<32 | a.regions[idx].off
218 }
219
220 // ArenaRestore resets the current arena to a previously saved mark.
221 func ArenaRestore(mark uintptr) {
222 if domainRoot == nil {
223 return
224 }
225 a := domainRoot.fnStack[domainRoot.fnTop]
226 if a == nil {
227 return
228 }
229 regionIdx := int32(mark >> 32)
230 off := mark & 0xFFFFFFFF
231 for i := a.nregions - 1; i > regionIdx; i-- {
232 munmap(unsafe.Pointer(a.regions[i].base), a.regions[i].cap)
233 }
234 a.nregions = regionIdx + 1
235 a.regions[regionIdx].off = off
236 }
237
238 // --- Arena lifecycle ---
239
240 const arenaDefaultRegion = uintptr(1024) * 1024 // 1MB chain regions
241
242 // ArenaNew creates an arena with the given initial capacity via mmap.
243 // The Arena struct is embedded at offset 0 of the mmap'd region.
244 // No external allocation needed.
245 func ArenaNew(capacity uintptr) (a *Arena) {
246 capacity = (capacity + 4095) &^ 4095
247 headerSz := align(unsafe.Sizeof(Arena{}))
248 if capacity < headerSz+4096 {
249 capacity = (headerSz + 4096 + 4095) &^ 4095
250 }
251 addr := mmap(nil, capacity, flag_PROT_READ|flag_PROT_WRITE,
252 flag_MAP_PRIVATE|flag_MAP_ANONYMOUS, -1, 0)
253 if addr == unsafe.Pointer(^uintptr(0)) {
254 runtimePanic("arena: mmap failed")
255 }
256 a = (*Arena)(addr)
257 a.nregions = 1
258 a.regions[0].base = uintptr(addr)
259 a.regions[0].off = headerSz // skip Arena header
260 a.regions[0].cap = capacity
261 return a
262 }
263
264 // ArenaAlloc allocates size bytes from the arena, 16-byte aligned, zeroed.
265 func ArenaAlloc(a *Arena, size uintptr) unsafe.Pointer {
266 if a == nil {
267 runtimePanic("ArenaAlloc: nil arena")
268 }
269 size = align(size)
270 active := a.nregions - 1
271 if a.regions[active].off+size > a.regions[active].cap {
272 if a.nregions >= arenaMaxRegions {
273 printstring("OVERFLOW nregions=")
274 printuint64(uint64(a.nregions))
275 printstring(" size=")
276 printuint64(uint64(size))
277 printstring(" r0.cap=")
278 printuint64(uint64(a.regions[0].cap))
279 printstring(" r0.off=")
280 printuint64(uint64(a.regions[0].off))
281 printstring(" active.cap=")
282 printuint64(uint64(a.regions[active].cap))
283 printstring(" active.off=")
284 printuint64(uint64(a.regions[active].off))
285 printstring(" arena=")
286 printuint64(uint64(uintptr(unsafe.Pointer(a))))
287 printstring("\n")
288 runtimePanic("arena: region table full")
289 }
290 newCap := arenaDefaultRegion
291 if size > newCap {
292 newCap = (size + 4095) &^ 4095
293 }
294 addr := mmap(nil, newCap, flag_PROT_READ|flag_PROT_WRITE,
295 flag_MAP_PRIVATE|flag_MAP_ANONYMOUS, -1, 0)
296 if addr == unsafe.Pointer(^uintptr(0)) {
297 runtimePanic("arena: region mmap failed")
298 }
299 a.regions[a.nregions].base = uintptr(addr)
300 a.regions[a.nregions].off = 0
301 a.regions[a.nregions].cap = newCap
302 a.nregions++
303 active = a.nregions - 1
304 }
305 ptr := unsafe.Pointer(a.regions[active].base + a.regions[active].off)
306 a.regions[active].off += size
307 memzero(ptr, size)
308 return ptr
309 }
310
311 // ArenaFree unmaps all regions in the arena.
312 // The primary region contains the Arena struct itself, so read
313 // base/cap into locals before the final munmap.
314 func ArenaFree(a *Arena) {
315 if a == nil {
316 return
317 }
318 for i := a.nregions - 1; i > 0; i-- {
319 munmap(unsafe.Pointer(a.regions[i].base), a.regions[i].cap)
320 }
321 b := a.regions[0].base
322 c := a.regions[0].cap
323 munmap(unsafe.Pointer(b), c)
324 }
325
326 // ArenaReset resets the arena for reuse. Frees all regions except
327 // the first, resets the first region's offset.
328 func ArenaReset(a *Arena) {
329 if a == nil {
330 return
331 }
332 for i := a.nregions - 1; i > 0; i-- {
333 munmap(unsafe.Pointer(a.regions[i].base), a.regions[i].cap)
334 }
335 a.nregions = 1
336 a.regions[0].off = align(unsafe.Sizeof(Arena{}))
337 }
338
339 // ArenaAcquire returns a pooled arena with cap >= capacity, or
340 // creates a new one via ArenaNew. First-fit scan of bounded pool.
341 func ArenaAcquire(capacity uintptr) (a *Arena) {
342 capacity = (capacity + 4095) &^ 4095
343 headerSz := align(unsafe.Sizeof(Arena{}))
344 if capacity < headerSz+4096 {
345 capacity = (headerSz + 4096 + 4095) &^ 4095
346 }
347 for i := int32(0); i < arenaPoolN; i++ {
348 if arenaPool[i].regions[0].cap >= capacity {
349 a = arenaPool[i]
350 arenaPoolN--
351 arenaPool[i] = arenaPool[arenaPoolN]
352 arenaPool[arenaPoolN] = nil
353 return a
354 }
355 }
356 return ArenaNew(capacity)
357 }
358
359 // ArenaRelease resets an arena and returns it to the pool.
360 // If the pool is full, frees the arena entirely via munmap.
361 func ArenaRelease(a *Arena) {
362 if a == nil {
363 return
364 }
365 if arenaPoolN >= arenaPoolMax {
366 ArenaFree(a)
367 return
368 }
369 ArenaReset(a)
370 arenaPool[arenaPoolN] = a
371 arenaPoolN++
372 }
373
374 // heapAlloc: direct startup bump allocation. Only used before
375 // domain root exists. Does not touch any TLS or arena state.
376 func heapAlloc(size uintptr) unsafe.Pointer {
377 size = align(size)
378 addr := heapptr
379 heapptr += size
380 for heapptr >= heapEnd {
381 if growHeap() {
382 continue
383 }
384 runtimePanic("out of memory")
385 }
386 ptr := unsafe.Pointer(addr)
387 memzero(ptr, size)
388 return ptr
389 }
390
391 // --- Region locality queries ---
392
393 // arenaContains checks if ptr is in any region of the arena above mark.
394 func arenaContains(a *Arena, ptr uintptr, mark uintptr) (ok bool) {
395 regionIdx := int32(mark >> 32)
396 regionOff := mark & 0xFFFFFFFF
397 for i := regionIdx; i < a.nregions; i++ {
398 r := a.regions[i]
399 start := r.base
400 if i == regionIdx {
401 start += regionOff
402 }
403 if ptr >= start && ptr < r.base+r.off {
404 return true
405 }
406 }
407 return false
408 }
409
410 // arenaCopySliceIfLocal copies a slice's backing array to the current arena
411 // position only if its data pointer is in the source arena region.
412 func arenaCopySliceIfLocal(s string, mark uintptr) (r string) {
413 if len(s) == 0 || domainRoot == nil {
414 return s
415 }
416 a := domainRoot.fnStack[domainRoot.fnTop]
417 p := uintptr(unsafe.Pointer(unsafe.SliceData([]byte(s))))
418 if !arenaContains(a, p, mark) {
419 return s
420 }
421 buf := []byte{:len(s)}
422 copy(buf, s)
423 return string(buf)
424 }
425
426 // arenaCopyPtrIfLocal copies size bytes at ptr only if in source region.
427 func arenaCopyPtrIfLocal(ptr unsafe.Pointer, mark uintptr, size uintptr) unsafe.Pointer {
428 if ptr == nil || domainRoot == nil {
429 return ptr
430 }
431 a := domainRoot.fnStack[domainRoot.fnTop]
432 p := uintptr(ptr)
433 if !arenaContains(a, p, mark) {
434 return ptr
435 }
436 dst := alloc(size, nil)
437 memcpy(dst, ptr, size)
438 return dst
439 }
440
441 func setHeapEnd(newHeapEnd uintptr) {
442 heapEnd = newHeapEnd
443 }
444
445 //:export moxie_scanstack
446 func scanstack(sp uintptr) {}
447