serial_cache.go raw
1 //go:build !(js && wasm)
2
3 package database
4
5 import (
6 "errors"
7
8 "github.com/dgraph-io/badger/v4"
9 "next.orly.dev/pkg/lol/chk"
10 "next.orly.dev/pkg/database/bufpool"
11 "next.orly.dev/pkg/database/indexes"
12 "next.orly.dev/pkg/database/indexes/types"
13 )
14
15 // SerialCache provides LRU caching for pubkey and event ID serial lookups.
16 // This is critical for compact event decoding performance since every event
17 // requires looking up the author pubkey and potentially multiple tag references.
18 //
19 // The cache uses LRU eviction and starts empty, growing on demand up to the
20 // configured limits. This provides better memory efficiency than pre-allocation
21 // and better hit rates than random eviction.
22 type SerialCache struct {
23 // Pubkey serial -> full pubkey (for decoding)
24 pubkeyBySerial *LRUCache[uint64, []byte]
25
26 // Pubkey bytes -> serial (for encoding)
27 // Uses [32]byte as key since []byte isn't comparable
28 serialByPubkey *LRUCache[[32]byte, uint64]
29
30 // Event serial -> full event ID (for decoding)
31 eventIdBySerial *LRUCache[uint64, []byte]
32
33 // Event ID bytes -> serial (for encoding)
34 serialByEventId *LRUCache[[32]byte, uint64]
35
36 // Limits (for stats reporting)
37 maxPubkeys int
38 maxEventIds int
39 }
40
41 // NewSerialCache creates a new serial cache with the specified maximum sizes.
42 // The cache starts empty and grows on demand up to these limits.
43 func NewSerialCache(maxPubkeys, maxEventIds int) *SerialCache {
44 if maxPubkeys <= 0 {
45 maxPubkeys = 100000 // Default 100k pubkeys
46 }
47 if maxEventIds <= 0 {
48 maxEventIds = 500000 // Default 500k event IDs
49 }
50 return &SerialCache{
51 pubkeyBySerial: NewLRUCache[uint64, []byte](maxPubkeys),
52 serialByPubkey: NewLRUCache[[32]byte, uint64](maxPubkeys),
53 eventIdBySerial: NewLRUCache[uint64, []byte](maxEventIds),
54 serialByEventId: NewLRUCache[[32]byte, uint64](maxEventIds),
55 maxPubkeys: maxPubkeys,
56 maxEventIds: maxEventIds,
57 }
58 }
59
60 // CachePubkey adds a pubkey to the cache in both directions.
61 func (c *SerialCache) CachePubkey(serial uint64, pubkey []byte) {
62 if len(pubkey) != 32 {
63 return
64 }
65
66 // Copy pubkey to avoid referencing external slice
67 pk := make([]byte, 32)
68 copy(pk, pubkey)
69
70 // Cache serial -> pubkey (for decoding)
71 c.pubkeyBySerial.Put(serial, pk)
72
73 // Cache pubkey -> serial (for encoding)
74 var key [32]byte
75 copy(key[:], pubkey)
76 c.serialByPubkey.Put(key, serial)
77 }
78
79 // GetPubkeyBySerial returns the pubkey for a serial from cache.
80 func (c *SerialCache) GetPubkeyBySerial(serial uint64) (pubkey []byte, found bool) {
81 return c.pubkeyBySerial.Get(serial)
82 }
83
84 // GetSerialByPubkey returns the serial for a pubkey from cache.
85 func (c *SerialCache) GetSerialByPubkey(pubkey []byte) (serial uint64, found bool) {
86 if len(pubkey) != 32 {
87 return 0, false
88 }
89 var key [32]byte
90 copy(key[:], pubkey)
91 return c.serialByPubkey.Get(key)
92 }
93
94 // CacheEventId adds an event ID to the cache in both directions.
95 func (c *SerialCache) CacheEventId(serial uint64, eventId []byte) {
96 if len(eventId) != 32 {
97 return
98 }
99
100 // Copy event ID to avoid referencing external slice
101 eid := make([]byte, 32)
102 copy(eid, eventId)
103
104 // Cache serial -> event ID (for decoding)
105 c.eventIdBySerial.Put(serial, eid)
106
107 // Cache event ID -> serial (for encoding)
108 var key [32]byte
109 copy(key[:], eventId)
110 c.serialByEventId.Put(key, serial)
111 }
112
113 // GetEventIdBySerial returns the event ID for a serial from cache.
114 func (c *SerialCache) GetEventIdBySerial(serial uint64) (eventId []byte, found bool) {
115 return c.eventIdBySerial.Get(serial)
116 }
117
118 // GetSerialByEventId returns the serial for an event ID from cache.
119 func (c *SerialCache) GetSerialByEventId(eventId []byte) (serial uint64, found bool) {
120 if len(eventId) != 32 {
121 return 0, false
122 }
123 var key [32]byte
124 copy(key[:], eventId)
125 return c.serialByEventId.Get(key)
126 }
127
128 // DatabaseSerialResolver implements SerialResolver using the database and cache.
129 type DatabaseSerialResolver struct {
130 db *D
131 cache *SerialCache
132 }
133
134 // NewDatabaseSerialResolver creates a new resolver.
135 func NewDatabaseSerialResolver(db *D, cache *SerialCache) *DatabaseSerialResolver {
136 return &DatabaseSerialResolver{db: db, cache: cache}
137 }
138
139 // GetOrCreatePubkeySerial implements SerialResolver.
140 func (r *DatabaseSerialResolver) GetOrCreatePubkeySerial(pubkey []byte) (serial uint64, err error) {
141 if len(pubkey) != 32 {
142 return 0, errors.New("pubkey must be 32 bytes")
143 }
144
145 // Check cache first
146 if s, found := r.cache.GetSerialByPubkey(pubkey); found {
147 return s, nil
148 }
149
150 // Use existing function which handles creation
151 ser, err := r.db.GetOrCreatePubkeySerial(pubkey)
152 if err != nil {
153 return 0, err
154 }
155
156 serial = ser.Get()
157
158 // Cache it
159 r.cache.CachePubkey(serial, pubkey)
160
161 return serial, nil
162 }
163
164 // GetPubkeyBySerial implements SerialResolver.
165 func (r *DatabaseSerialResolver) GetPubkeyBySerial(serial uint64) (pubkey []byte, err error) {
166 // Check cache first
167 if pk, found := r.cache.GetPubkeyBySerial(serial); found {
168 return pk, nil
169 }
170
171 // Look up in database
172 ser := new(types.Uint40)
173 if err = ser.Set(serial); err != nil {
174 return nil, err
175 }
176
177 pubkey, err = r.db.GetPubkeyBySerial(ser)
178 if err != nil {
179 return nil, err
180 }
181
182 // Cache it
183 r.cache.CachePubkey(serial, pubkey)
184
185 return pubkey, nil
186 }
187
188 // GetEventSerialById implements SerialResolver.
189 func (r *DatabaseSerialResolver) GetEventSerialById(eventId []byte) (serial uint64, found bool, err error) {
190 if len(eventId) != 32 {
191 return 0, false, errors.New("event ID must be 32 bytes")
192 }
193
194 // Check cache first
195 if s, ok := r.cache.GetSerialByEventId(eventId); ok {
196 return s, true, nil
197 }
198
199 // Look up in database using existing GetSerialById
200 ser, err := r.db.GetSerialById(eventId)
201 if err != nil {
202 // Not found is not an error - just return found=false
203 return 0, false, nil
204 }
205
206 serial = ser.Get()
207
208 // Cache it
209 r.cache.CacheEventId(serial, eventId)
210
211 return serial, true, nil
212 }
213
214 // GetEventIdBySerial implements SerialResolver.
215 func (r *DatabaseSerialResolver) GetEventIdBySerial(serial uint64) (eventId []byte, err error) {
216 // Check cache first
217 if eid, found := r.cache.GetEventIdBySerial(serial); found {
218 return eid, nil
219 }
220
221 // Look up in database - use SerialEventId index
222 ser := new(types.Uint40)
223 if err = ser.Set(serial); err != nil {
224 return nil, err
225 }
226
227 eventId, err = r.db.GetEventIdBySerial(ser)
228 if err != nil {
229 return nil, err
230 }
231
232 // Cache it
233 r.cache.CacheEventId(serial, eventId)
234
235 return eventId, nil
236 }
237
238 // GetEventIdBySerial looks up an event ID by its serial number.
239 // Uses the SerialEventId index (sei prefix).
240 func (d *D) GetEventIdBySerial(ser *types.Uint40) (eventId []byte, err error) {
241 keyBuf := bufpool.GetSmall()
242 defer bufpool.PutSmall(keyBuf)
243 if err = indexes.SerialEventIdEnc(ser).MarshalWrite(keyBuf); chk.E(err) {
244 return nil, err
245 }
246
247 err = d.View(func(txn *badger.Txn) error {
248 item, gerr := txn.Get(keyBuf.Bytes())
249 if gerr != nil {
250 // Don't log ErrKeyNotFound - it's expected for legacy events
251 // that don't have SerialEventId mappings
252 return gerr
253 }
254
255 return item.Value(func(val []byte) error {
256 // Validate that the stored value is exactly 32 bytes
257 if len(val) != 32 {
258 return errors.New("corrupted event ID: expected 32 bytes")
259 }
260 eventId = make([]byte, 32)
261 copy(eventId, val)
262 return nil
263 })
264 })
265
266 if err != nil {
267 return nil, errors.New("event ID not found for serial")
268 }
269
270 return eventId, nil
271 }
272
273 // StoreEventIdSerial stores the mapping from event serial to full event ID.
274 // This is called during event save to enable later reconstruction.
275 func (d *D) StoreEventIdSerial(txn *badger.Txn, serial uint64, eventId []byte) error {
276 if len(eventId) != 32 {
277 return errors.New("event ID must be 32 bytes")
278 }
279
280 ser := new(types.Uint40)
281 if err := ser.Set(serial); err != nil {
282 return err
283 }
284
285 keyBuf := bufpool.GetSmall()
286 defer bufpool.PutSmall(keyBuf)
287 if err := indexes.SerialEventIdEnc(ser).MarshalWrite(keyBuf); chk.E(err) {
288 return err
289 }
290
291 return txn.Set(bufpool.CopyBytes(keyBuf), eventId)
292 }
293
294 // SerialCacheStats holds statistics about the serial cache.
295 type SerialCacheStats struct {
296 PubkeysCached int // Number of pubkeys currently cached
297 PubkeysMaxSize int // Maximum pubkey cache size
298 EventIdsCached int // Number of event IDs currently cached
299 EventIdsMaxSize int // Maximum event ID cache size
300 PubkeyMemoryBytes int // Estimated memory usage for pubkey cache
301 EventIdMemoryBytes int // Estimated memory usage for event ID cache
302 TotalMemoryBytes int // Total estimated memory usage
303 }
304
305 // Stats returns statistics about the serial cache.
306 func (c *SerialCache) Stats() SerialCacheStats {
307 pubkeysCached := c.pubkeyBySerial.Len()
308 eventIdsCached := c.eventIdBySerial.Len()
309
310 // Memory estimation:
311 // Each entry has: key + value + list.Element overhead + map entry overhead
312 // - Pubkey by serial: 8 (key) + 32 (value) + ~80 (list) + ~16 (map) ≈ 136 bytes
313 // - Serial by pubkey: 32 (key) + 8 (value) + ~80 (list) + ~16 (map) ≈ 136 bytes
314 // Total per pubkey (both directions): ~272 bytes
315 // Similarly for event IDs: ~272 bytes per entry (both directions)
316 pubkeyMemory := pubkeysCached * 272
317 eventIdMemory := eventIdsCached * 272
318
319 return SerialCacheStats{
320 PubkeysCached: pubkeysCached,
321 PubkeysMaxSize: c.maxPubkeys,
322 EventIdsCached: eventIdsCached,
323 EventIdsMaxSize: c.maxEventIds,
324 PubkeyMemoryBytes: pubkeyMemory,
325 EventIdMemoryBytes: eventIdMemory,
326 TotalMemoryBytes: pubkeyMemory + eventIdMemory,
327 }
328 }
329
330 // SerialCacheStats returns statistics about the serial cache.
331 func (d *D) SerialCacheStats() SerialCacheStats {
332 if d.serialCache == nil {
333 return SerialCacheStats{}
334 }
335 return d.serialCache.Stats()
336 }
337