compact_stats.go raw
1 //go:build !(js && wasm)
2
3 package database
4
5 import (
6 "bytes"
7 "sync/atomic"
8
9 "github.com/dgraph-io/badger/v4"
10 "next.orly.dev/pkg/lol/chk"
11 "next.orly.dev/pkg/lol/log"
12 "next.orly.dev/pkg/database/indexes"
13 )
14
15 // CompactStorageStats holds statistics about compact vs legacy storage.
16 type CompactStorageStats struct {
17 // Event counts
18 CompactEvents int64 // Number of events in compact format (cmp prefix)
19 LegacyEvents int64 // Number of events in legacy format (evt/sev prefixes)
20 TotalEvents int64 // Total events
21
22 // Storage sizes
23 CompactBytes int64 // Total bytes used by compact format
24 LegacyBytes int64 // Total bytes used by legacy format (would be used without compact)
25
26 // Savings
27 BytesSaved int64 // Bytes saved by using compact format
28 PercentSaved float64 // Percentage of space saved
29 AverageCompact float64 // Average compact event size
30 AverageLegacy float64 // Average legacy event size (estimated)
31
32 // Serial mappings
33 SerialEventIdEntries int64 // Number of sei (serial -> event ID) mappings
34 SerialEventIdBytes int64 // Bytes used by sei mappings
35 }
36
37 // CompactStorageStats calculates storage statistics for compact event storage.
38 // This scans the database to provide accurate metrics on space savings.
39 func (d *D) CompactStorageStats() (stats CompactStorageStats, err error) {
40 if err = d.View(func(txn *badger.Txn) error {
41 // Count compact events (cmp prefix)
42 cmpPrf := new(bytes.Buffer)
43 if err = indexes.CompactEventEnc(nil).MarshalWrite(cmpPrf); chk.E(err) {
44 return err
45 }
46
47 it := txn.NewIterator(badger.IteratorOptions{Prefix: cmpPrf.Bytes()})
48 for it.Rewind(); it.Valid(); it.Next() {
49 item := it.Item()
50 stats.CompactEvents++
51 stats.CompactBytes += int64(len(item.Key())) + int64(item.ValueSize())
52 }
53 it.Close()
54
55 // Count legacy evt entries
56 evtPrf := new(bytes.Buffer)
57 if err = indexes.EventEnc(nil).MarshalWrite(evtPrf); chk.E(err) {
58 return err
59 }
60
61 it = txn.NewIterator(badger.IteratorOptions{Prefix: evtPrf.Bytes()})
62 for it.Rewind(); it.Valid(); it.Next() {
63 item := it.Item()
64 stats.LegacyEvents++
65 stats.LegacyBytes += int64(len(item.Key())) + int64(item.ValueSize())
66 }
67 it.Close()
68
69 // Count legacy sev entries
70 sevPrf := new(bytes.Buffer)
71 if err = indexes.SmallEventEnc(nil).MarshalWrite(sevPrf); chk.E(err) {
72 return err
73 }
74
75 it = txn.NewIterator(badger.IteratorOptions{Prefix: sevPrf.Bytes()})
76 for it.Rewind(); it.Valid(); it.Next() {
77 item := it.Item()
78 stats.LegacyEvents++
79 stats.LegacyBytes += int64(len(item.Key())) // sev stores data in key
80 }
81 it.Close()
82
83 // Count SerialEventId mappings (sei prefix)
84 seiPrf := new(bytes.Buffer)
85 if err = indexes.SerialEventIdEnc(nil).MarshalWrite(seiPrf); chk.E(err) {
86 return err
87 }
88
89 it = txn.NewIterator(badger.IteratorOptions{Prefix: seiPrf.Bytes()})
90 for it.Rewind(); it.Valid(); it.Next() {
91 item := it.Item()
92 stats.SerialEventIdEntries++
93 stats.SerialEventIdBytes += int64(len(item.Key())) + int64(item.ValueSize())
94 }
95 it.Close()
96
97 return nil
98 }); chk.E(err) {
99 return
100 }
101
102 stats.TotalEvents = stats.CompactEvents + stats.LegacyEvents
103
104 // Calculate averages
105 if stats.CompactEvents > 0 {
106 stats.AverageCompact = float64(stats.CompactBytes) / float64(stats.CompactEvents)
107 }
108 if stats.LegacyEvents > 0 {
109 stats.AverageLegacy = float64(stats.LegacyBytes) / float64(stats.LegacyEvents)
110 }
111
112 // Estimate savings: compare compact size to what legacy size would be
113 // For events that are in compact format, estimate legacy size based on typical ratios
114 // A typical event has:
115 // - 32 bytes event ID (saved in compact: stored separately in sei)
116 // - 32 bytes pubkey (saved: replaced by 5-byte serial)
117 // - For e-tags: 32 bytes each (saved: replaced by 5-byte serial when known)
118 // - For p-tags: 32 bytes each (saved: replaced by 5-byte serial)
119 // Conservative estimate: compact format is ~60% of legacy size for typical events
120 if stats.CompactEvents > 0 && stats.AverageCompact > 0 {
121 // Estimate what the legacy size would have been
122 estimatedLegacyForCompact := float64(stats.CompactBytes) / 0.60 // 60% compression ratio
123 stats.BytesSaved = int64(estimatedLegacyForCompact) - stats.CompactBytes - stats.SerialEventIdBytes
124 if stats.BytesSaved < 0 {
125 stats.BytesSaved = 0
126 }
127 totalWithoutCompact := estimatedLegacyForCompact + float64(stats.LegacyBytes)
128 totalWithCompact := float64(stats.CompactBytes + stats.LegacyBytes + stats.SerialEventIdBytes)
129 if totalWithoutCompact > 0 {
130 stats.PercentSaved = (1.0 - totalWithCompact/totalWithoutCompact) * 100.0
131 }
132 }
133
134 return stats, nil
135 }
136
137 // compactSaveCounter tracks cumulative bytes saved by compact format
138 var compactSaveCounter atomic.Int64
139
140 // LogCompactSavings logs the storage savings achieved by compact format.
141 // Call this periodically or after significant operations.
142 func (d *D) LogCompactSavings() {
143 stats, err := d.CompactStorageStats()
144 if err != nil {
145 log.W.F("failed to get compact storage stats: %v", err)
146 return
147 }
148
149 if stats.TotalEvents == 0 {
150 return
151 }
152
153 log.I.F("📊 Compact storage stats: %d compact events, %d legacy events",
154 stats.CompactEvents, stats.LegacyEvents)
155 log.I.F(" Compact size: %.2f MB, Legacy size: %.2f MB",
156 float64(stats.CompactBytes)/(1024.0*1024.0),
157 float64(stats.LegacyBytes)/(1024.0*1024.0))
158 log.I.F(" Serial mappings (sei): %d entries, %.2f KB",
159 stats.SerialEventIdEntries,
160 float64(stats.SerialEventIdBytes)/1024.0)
161
162 if stats.CompactEvents > 0 {
163 log.I.F(" Average compact event: %.0f bytes, estimated legacy: %.0f bytes",
164 stats.AverageCompact, stats.AverageCompact/0.60)
165 log.I.F(" Estimated savings: %.2f MB (%.1f%%)",
166 float64(stats.BytesSaved)/(1024.0*1024.0),
167 stats.PercentSaved)
168 }
169
170 // Also log serial cache stats
171 cacheStats := d.SerialCacheStats()
172 log.I.F(" Serial cache: %d/%d pubkeys, %d/%d event IDs, ~%.2f MB memory",
173 cacheStats.PubkeysCached, cacheStats.PubkeysMaxSize,
174 cacheStats.EventIdsCached, cacheStats.EventIdsMaxSize,
175 float64(cacheStats.TotalMemoryBytes)/(1024.0*1024.0))
176 }
177
178 // TrackCompactSaving records bytes saved for a single event.
179 // Call this during event save to track cumulative savings.
180 func TrackCompactSaving(legacySize, compactSize int) {
181 saved := legacySize - compactSize
182 if saved > 0 {
183 compactSaveCounter.Add(int64(saved))
184 }
185 }
186
187 // GetCumulativeCompactSavings returns total bytes saved across all compact saves.
188 func GetCumulativeCompactSavings() int64 {
189 return compactSaveCounter.Load()
190 }
191
192 // ResetCompactSavingsCounter resets the cumulative savings counter.
193 func ResetCompactSavingsCounter() {
194 compactSaveCounter.Store(0)
195 }
196