fetch-events-by-serials.go raw
1 //go:build !(js && wasm)
2
3 package database
4
5 import (
6 "bytes"
7
8 "github.com/dgraph-io/badger/v4"
9 "next.orly.dev/pkg/lol/chk"
10 "next.orly.dev/pkg/lol/log"
11 "next.orly.dev/pkg/database/indexes"
12 "next.orly.dev/pkg/database/indexes/types"
13 "next.orly.dev/pkg/nostr/encoders/event"
14 )
15
16 // FetchEventsBySerials fetches multiple events by their serials in a single database transaction.
17 // Returns a map of serial uint64 value to event, only including successfully fetched events.
18 //
19 // This function tries multiple storage formats in order:
20 // 1. cmp (compact format with serial references) - newest, most space-efficient
21 // 2. sev (small event inline) - legacy Reiser4 optimization
22 // 3. evt (traditional separate storage) - legacy fallback
23 func (d *D) FetchEventsBySerials(serials []*types.Uint40) (events map[uint64]*event.E, err error) {
24 // Pre-allocate map with estimated capacity to reduce reallocations
25 events = make(map[uint64]*event.E, len(serials))
26
27 if len(serials) == 0 {
28 return events, nil
29 }
30
31 // Create resolver for compact event decoding
32 resolver := NewDatabaseSerialResolver(d, d.serialCache)
33
34 if err = d.View(
35 func(txn *badger.Txn) (err error) {
36 // Create ONE iterator for sev prefix lookups - reused for all serials
37 // This dramatically reduces memory usage vs creating one per serial
38 sevOpts := badger.DefaultIteratorOptions
39 sevOpts.PrefetchValues = false // We read from key, not value
40 sevOpts.PrefetchSize = 1
41 sevIt := txn.NewIterator(sevOpts)
42 defer sevIt.Close()
43
44 for _, ser := range serials {
45 var ev *event.E
46 serialVal := ser.Get()
47
48 // Try cmp (compact format) first - most efficient
49 ev, err = d.fetchCompactEvent(txn, ser, resolver)
50 if err == nil && ev != nil {
51 events[serialVal] = ev
52 continue
53 }
54 err = nil // Reset error, try legacy formats
55
56 // Try sev (small event inline) using shared iterator
57 ev, err = d.fetchSmallEventWithIterator(txn, ser, sevIt)
58 if err == nil && ev != nil {
59 events[serialVal] = ev
60 continue
61 }
62 err = nil // Reset error, try evt
63
64 // Not found in sev table, try evt (traditional) prefix
65 ev, err = d.fetchLegacyEvent(txn, ser)
66 if err == nil && ev != nil {
67 events[serialVal] = ev
68 continue
69 }
70 err = nil // Reset error, event not found
71 }
72 return nil
73 },
74 ); err != nil {
75 return
76 }
77
78 return events, nil
79 }
80
81 // fetchSmallEventWithIterator uses a provided iterator for sev lookups (memory efficient)
82 func (d *D) fetchSmallEventWithIterator(txn *badger.Txn, ser *types.Uint40, it *badger.Iterator) (ev *event.E, err error) {
83 smallBuf := new(bytes.Buffer)
84 if err = indexes.SmallEventEnc(ser).MarshalWrite(smallBuf); chk.E(err) {
85 return nil, err
86 }
87 prefix := smallBuf.Bytes()
88
89 // Seek to the prefix for this serial
90 it.Seek(prefix)
91 if !it.ValidForPrefix(prefix) {
92 return nil, nil // Not found
93 }
94
95 // Found in sev table - extract inline data
96 key := it.Item().KeyCopy(nil) // Copy key as iterator may be reused
97 // Key format: sev|serial|size_uint16|event_data
98 if len(key) <= 8+2 { // prefix(3) + serial(5) + size(2) = 10 bytes minimum
99 return nil, nil
100 }
101
102 sizeIdx := 8 // After sev(3) + serial(5)
103 // Read uint16 big-endian size
104 size := int(key[sizeIdx])<<8 | int(key[sizeIdx+1])
105 dataStart := sizeIdx + 2
106
107 if len(key) < dataStart+size {
108 return nil, nil
109 }
110
111 eventData := key[dataStart : dataStart+size]
112
113 // Check if this is compact format (starts with version byte 1)
114 // Note: Legacy events whose ID starts with 0x01 will also match this check,
115 // so we fall back to legacy format if the SerialEventId mapping doesn't exist.
116 if len(eventData) > 0 && eventData[0] == CompactFormatVersion {
117 // Try to get event ID mapping - if it exists, this is truly compact format
118 eventId, idErr := d.GetEventIdBySerial(ser)
119 if idErr == nil {
120 // SerialEventId mapping exists - this is compact format
121 resolver := NewDatabaseSerialResolver(d, d.serialCache)
122 return UnmarshalCompactEvent(eventData, eventId, resolver)
123 }
124 // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01
125 // Fall through to legacy unmarshal
126 log.T.F("fetchSmallEventWithIterator: no sei mapping for serial %d, trying legacy format", ser.Get())
127 }
128
129 // Legacy binary format
130 ev = new(event.E)
131 if err = ev.UnmarshalBinary(bytes.NewBuffer(eventData)); err != nil {
132 return nil, err
133 }
134
135 return ev, nil
136 }
137
138 // fetchCompactEvent tries to fetch an event from the compact format (cmp prefix).
139 func (d *D) fetchCompactEvent(txn *badger.Txn, ser *types.Uint40, resolver SerialResolver) (ev *event.E, err error) {
140 // Build cmp key
141 keyBuf := new(bytes.Buffer)
142 if err = indexes.CompactEventEnc(ser).MarshalWrite(keyBuf); chk.E(err) {
143 return nil, err
144 }
145
146 item, err := txn.Get(keyBuf.Bytes())
147 if err != nil {
148 return nil, err
149 }
150
151 var compactData []byte
152 if compactData, err = item.ValueCopy(nil); chk.E(err) {
153 return nil, err
154 }
155
156 // Need to get the event ID from SerialEventId table
157 eventId, err := d.GetEventIdBySerial(ser)
158 if err != nil {
159 log.D.F("fetchCompactEvent: failed to get event ID for serial %d: %v", ser.Get(), err)
160 return nil, err
161 }
162
163 // Unmarshal compact event
164 ev, err = UnmarshalCompactEvent(compactData, eventId, resolver)
165 if err != nil {
166 log.D.F("fetchCompactEvent: failed to unmarshal compact event for serial %d: %v", ser.Get(), err)
167 return nil, err
168 }
169
170 return ev, nil
171 }
172
173 // fetchSmallEvent tries to fetch an event from the small event inline format (sev prefix).
174 func (d *D) fetchSmallEvent(txn *badger.Txn, ser *types.Uint40) (ev *event.E, err error) {
175 smallBuf := new(bytes.Buffer)
176 if err = indexes.SmallEventEnc(ser).MarshalWrite(smallBuf); chk.E(err) {
177 return nil, err
178 }
179
180 // Iterate with prefix to find the small event key
181 opts := badger.DefaultIteratorOptions
182 opts.Prefix = smallBuf.Bytes()
183 opts.PrefetchValues = true
184 opts.PrefetchSize = 1
185 it := txn.NewIterator(opts)
186 defer it.Close()
187
188 it.Rewind()
189 if !it.Valid() {
190 return nil, nil // Not found
191 }
192
193 // Found in sev table - extract inline data
194 key := it.Item().Key()
195 // Key format: sev|serial|size_uint16|event_data
196 if len(key) <= 8+2 { // prefix(3) + serial(5) + size(2) = 10 bytes minimum
197 return nil, nil
198 }
199
200 sizeIdx := 8 // After sev(3) + serial(5)
201 // Read uint16 big-endian size
202 size := int(key[sizeIdx])<<8 | int(key[sizeIdx+1])
203 dataStart := sizeIdx + 2
204
205 if len(key) < dataStart+size {
206 return nil, nil
207 }
208
209 eventData := key[dataStart : dataStart+size]
210
211 // Check if this is compact format (starts with version byte 1)
212 // Note: Legacy events whose ID starts with 0x01 will also match this check,
213 // so we fall back to legacy format if the SerialEventId mapping doesn't exist.
214 if len(eventData) > 0 && eventData[0] == CompactFormatVersion {
215 // Try to get event ID mapping - if it exists, this is truly compact format
216 eventId, idErr := d.GetEventIdBySerial(ser)
217 if idErr == nil {
218 // SerialEventId mapping exists - this is compact format
219 resolver := NewDatabaseSerialResolver(d, d.serialCache)
220 return UnmarshalCompactEvent(eventData, eventId, resolver)
221 }
222 // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01
223 // Fall through to legacy unmarshal
224 log.T.F("fetchSmallEvent: no sei mapping for serial %d, trying legacy format", ser.Get())
225 }
226
227 // Legacy binary format
228 ev = new(event.E)
229 if err = ev.UnmarshalBinary(bytes.NewBuffer(eventData)); err != nil {
230 return nil, err
231 }
232
233 return ev, nil
234 }
235
236 // fetchLegacyEvent tries to fetch an event from the legacy format (evt prefix).
237 func (d *D) fetchLegacyEvent(txn *badger.Txn, ser *types.Uint40) (ev *event.E, err error) {
238 buf := new(bytes.Buffer)
239 if err = indexes.EventEnc(ser).MarshalWrite(buf); chk.E(err) {
240 return nil, err
241 }
242
243 item, err := txn.Get(buf.Bytes())
244 if err != nil {
245 return nil, err
246 }
247
248 var v []byte
249 if v, err = item.ValueCopy(nil); chk.E(err) {
250 return nil, err
251 }
252
253 // Check if we have valid data before attempting to unmarshal
254 if len(v) < 32+32+1+2+1+1+64 { // ID + Pubkey + min varint fields + Sig
255 return nil, nil
256 }
257
258 // Check if this is compact format (starts with version byte 1)
259 // Note: Legacy events whose ID starts with 0x01 will also match this check,
260 // so we fall back to legacy format if the SerialEventId mapping doesn't exist.
261 if len(v) > 0 && v[0] == CompactFormatVersion {
262 // Try to get event ID mapping - if it exists, this is truly compact format
263 eventId, idErr := d.GetEventIdBySerial(ser)
264 if idErr == nil {
265 // SerialEventId mapping exists - this is compact format
266 resolver := NewDatabaseSerialResolver(d, d.serialCache)
267 return UnmarshalCompactEvent(v, eventId, resolver)
268 }
269 // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01
270 // Fall through to legacy unmarshal
271 log.T.F("fetchLegacyEvent: no sei mapping for serial %d, trying legacy format", ser.Get())
272 }
273
274 // Legacy binary format
275 ev = new(event.E)
276 if err = ev.UnmarshalBinary(bytes.NewBuffer(v)); err != nil {
277 return nil, err
278 }
279
280 return ev, nil
281 }
282
283