1 // Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2 // of this source code is governed by a BSD-style license that can be found in
3 // the LICENSE file.
4 5 //go:build jemalloc
6 // +build jemalloc
7 8 package z
9 10 /*
11 #cgo LDFLAGS: /usr/local/lib/libjemalloc.a -L/usr/local/lib -Wl,-rpath,/usr/local/lib -ljemalloc -lm -lstdc++ -pthread -ldl
12 #include <stdlib.h>
13 #include <jemalloc/jemalloc.h>
14 */
15 import "C"
16 import (
17 "bytes"
18 "fmt"
19 "sync"
20 "sync/atomic"
21 "unsafe"
22 23 "github.com/dustin/go-humanize"
24 )
25 26 // The go:linkname directives provides backdoor access to private functions in
27 // the runtime. Below we're accessing the throw function.
28 29 //go:linkname throw runtime.throw
30 func throw(s string)
31 32 // New allocates a slice of size n. The returned slice is from manually managed
33 // memory and MUST be released by calling Free. Failure to do so will result in
34 // a memory leak.
35 //
36 // Compile jemalloc with ./configure --with-jemalloc-prefix="je_"
37 // https://android.googlesource.com/platform/external/jemalloc_new/+/6840b22e8e11cb68b493297a5cd757d6eaa0b406/TUNING.md
38 // These two config options seems useful for frequent allocations and deallocations in
39 // multi-threaded programs (like we have).
40 // JE_MALLOC_CONF="background_thread:true,metadata_thp:auto"
41 //
42 // Compile Go program with `go build -tags=jemalloc` to enable this.
43 44 type dalloc struct {
45 t string
46 sz int
47 }
48 49 var dallocsMu sync.Mutex
50 var dallocs map[unsafe.Pointer]*dalloc
51 52 func init() {
53 // By initializing dallocs, we can start tracking allocations and deallocations via z.Calloc.
54 dallocs = make(map[unsafe.Pointer]*dalloc)
55 }
56 57 func Calloc(n int, tag string) []byte {
58 if n == 0 {
59 return make([]byte, 0)
60 }
61 // We need to be conscious of the Cgo pointer passing rules:
62 //
63 // https://golang.org/cmd/cgo/#hdr-Passing_pointers
64 //
65 // ...
66 // Note: the current implementation has a bug. While Go code is permitted
67 // to write nil or a C pointer (but not a Go pointer) to C memory, the
68 // current implementation may sometimes cause a runtime error if the
69 // contents of the C memory appear to be a Go pointer. Therefore, avoid
70 // passing uninitialized C memory to Go code if the Go code is going to
71 // store pointer values in it. Zero out the memory in C before passing it
72 // to Go.
73 74 ptr := C.je_calloc(C.size_t(n), 1)
75 if ptr == nil {
76 // NB: throw is like panic, except it guarantees the process will be
77 // terminated. The call below is exactly what the Go runtime invokes when
78 // it cannot allocate memory.
79 throw("out of memory")
80 }
81 82 uptr := unsafe.Pointer(ptr)
83 dallocsMu.Lock()
84 dallocs[uptr] = &dalloc{
85 t: tag,
86 sz: n,
87 }
88 dallocsMu.Unlock()
89 atomic.AddInt64(&numBytes, int64(n))
90 // Interpret the C pointer as a pointer to a Go array, then slice.
91 return (*[MaxArrayLen]byte)(uptr)[:n:n]
92 }
93 94 // CallocNoRef does the exact same thing as Calloc with jemalloc enabled.
95 func CallocNoRef(n int, tag string) []byte {
96 return Calloc(n, tag)
97 }
98 99 // Free frees the specified slice.
100 func Free(b []byte) {
101 if sz := cap(b); sz != 0 {
102 b = b[:cap(b)]
103 ptr := unsafe.Pointer(&b[0])
104 C.je_free(ptr)
105 atomic.AddInt64(&numBytes, -int64(sz))
106 dallocsMu.Lock()
107 delete(dallocs, ptr)
108 dallocsMu.Unlock()
109 }
110 }
111 112 func Leaks() string {
113 if dallocs == nil {
114 return "Leak detection disabled. Enable with 'leak' build flag."
115 }
116 dallocsMu.Lock()
117 defer dallocsMu.Unlock()
118 if len(dallocs) == 0 {
119 return "NO leaks found."
120 }
121 m := make(map[string]int)
122 for _, da := range dallocs {
123 m[da.t] += da.sz
124 }
125 var buf bytes.Buffer
126 fmt.Fprintf(&buf, "Allocations:\n")
127 for f, sz := range m {
128 fmt.Fprintf(&buf, "%s at file: %s\n", humanize.IBytes(uint64(sz)), f)
129 }
130 return buf.String()
131 }
132 133 // ReadMemStats populates stats with JE Malloc statistics.
134 func ReadMemStats(stats *MemStats) {
135 if stats == nil {
136 return
137 }
138 // Call an epoch mallclt to refresh the stats data as mentioned in the docs.
139 // http://jemalloc.net/jemalloc.3.html#epoch
140 // Note: This epoch mallctl is as expensive as a malloc call. It takes up the
141 // malloc_mutex_lock.
142 epoch := 1
143 sz := unsafe.Sizeof(&epoch)
144 C.je_mallctl(
145 (C.CString)("epoch"),
146 unsafe.Pointer(&epoch),
147 (*C.size_t)(unsafe.Pointer(&sz)),
148 unsafe.Pointer(&epoch),
149 (C.size_t)(unsafe.Sizeof(epoch)))
150 stats.Allocated = fetchStat("stats.allocated")
151 stats.Active = fetchStat("stats.active")
152 stats.Resident = fetchStat("stats.resident")
153 stats.Retained = fetchStat("stats.retained")
154 }
155 156 // fetchStat is used to read a specific attribute from je malloc stats using mallctl.
157 func fetchStat(s string) uint64 {
158 var out uint64
159 sz := unsafe.Sizeof(&out)
160 C.je_mallctl(
161 (C.CString)(s), // Query: eg: stats.allocated, stats.resident, etc.
162 unsafe.Pointer(&out), // Variable to store the output.
163 (*C.size_t)(unsafe.Pointer(&sz)), // Size of the output variable.
164 nil, // Input variable used to set a value.
165 0) // Size of the input variable.
166 return out
167 }
168 169 func StatsPrint() {
170 opts := C.CString("mdablxe")
171 C.je_malloc_stats_print(nil, nil, opts)
172 C.free(unsafe.Pointer(opts))
173 }
174