store.go raw
1 // Package neutrino is a compact-filter (BIP157/BIP158) light client: it syncs
2 // and verifies the block header chain, the compact filter header chain, and
3 // the filters themselves, then matches wallet scripts locally.
4 //
5 // The transport is a net.Conn, so the same client runs over TCP to a trusted
6 // node or over the Nostr relay tunnel to a bridge. The store is an interface:
7 // memory and append-only files are provided, and the wasm build can supply an
8 // IndexedDB implementation without touching the sync logic.
9 package neutrino
10
11 import (
12 "bufio"
13 "encoding/binary"
14 "errors"
15 "fmt"
16 "io"
17 "os"
18 "path/filepath"
19 "sort"
20 "sync"
21
22 "git.smesh.lol/nostrino/pkg/btcwire"
23 "git.smesh.lol/nostrino/pkg/crypto/chainhash"
24 )
25
26 // Anchor is the trusted starting point of a store. Headers are verified from
27 // Height+1 onward, and the filter headers from FilterHeight+1. A genesis
28 // anchor has FilterHeight -1 because the first filter header is not known
29 // until it has been downloaded; a checkpoint pins both values.
30 type Anchor struct {
31 Height uint32
32 BlockHash chainhash.Hash
33 FilterHeight int32 // -1 when no filter header is trusted yet
34 FilterHeader chainhash.Hash
35 }
36
37 // Store is the durable state a light client needs. Headers and filter headers
38 // are contiguous from the anchor; filters may start at a later height, because
39 // a swap wallet usually scans only recent history.
40 type Store interface {
41 // Anchor returns the trusted starting point. A store with no anchor has
42 // not been initialised.
43 Anchor() (Anchor, bool)
44 // SetAnchor installs the trusted starting point. It must be called before
45 // headers or filter headers are appended.
46 SetAnchor(anchor Anchor) error
47
48 // HeaderTip returns the highest header height and its hash. With no
49 // downloaded headers it is the anchor.
50 HeaderTip() (uint32, chainhash.Hash, bool)
51 // Header returns the header at height.
52 Header(height uint32) (btcwire.BlockHeader, bool)
53 // PutHeaders appends a contiguous run. start must be the anchor height
54 // plus one for an empty store, otherwise the previous tip plus one.
55 PutHeaders(start uint32, headers []btcwire.BlockHeader) error
56
57 // FilterHeaderTip returns the highest filter header height and its value.
58 // With none downloaded it is the anchor's filter header, when it has one.
59 FilterHeaderTip() (uint32, chainhash.Hash, bool)
60 // FilterHeader returns the filter header at height.
61 FilterHeader(height uint32) (chainhash.Hash, bool)
62 // FilterHash returns the hash of the compact filter at height, as
63 // committed to by the filter header chain.
64 FilterHash(height uint32) (chainhash.Hash, bool)
65 // PutFilterHeaders appends a contiguous run of filter hashes and their
66 // rolling filter headers. The two slices are parallel.
67 PutFilterHeaders(start uint32, filterHashes, filterHeaders []chainhash.Hash) error
68
69 // FilterTip returns the highest stored filter height.
70 FilterTip() (uint32, bool)
71 // Filter returns the compact filter at height.
72 Filter(height uint32) ([]byte, bool)
73 // PutFilter stores one verified filter.
74 PutFilter(height uint32, blockHash, filterHash chainhash.Hash, filter []byte) error
75
76 // Truncate removes every header, filter header and filter at or above
77 // from. It is the reorg primitive. from must be above the anchor.
78 Truncate(from uint32) error
79 }
80
81 // Memory is an in-memory Store. It is useful for tests and for a client that
82 // deliberately keeps no state.
83 type Memory struct {
84 mu sync.RWMutex
85
86 anchor Anchor
87 hasAnchor bool
88
89 headers map[uint32]btcwire.BlockHeader
90 headerTip uint32
91 hasHeader bool
92
93 fheaders map[uint32]chainhash.Hash
94 fhashes map[uint32]chainhash.Hash
95 fheaderTip uint32
96 hasFHeader bool
97
98 filters map[uint32][]byte
99 filterTip uint32
100 hasFilter bool
101 }
102
103 // NewMemory returns an empty memory store.
104 func NewMemory() *Memory {
105 return &Memory{
106 headers: make(map[uint32]btcwire.BlockHeader),
107 fheaders: make(map[uint32]chainhash.Hash),
108 fhashes: make(map[uint32]chainhash.Hash),
109 filters: make(map[uint32][]byte),
110 }
111 }
112
113 func (m *Memory) Anchor() (Anchor, bool) {
114 m.mu.RLock()
115 defer m.mu.RUnlock()
116 return m.anchor, m.hasAnchor
117 }
118
119 func (m *Memory) SetAnchor(anchor Anchor) error {
120 m.mu.Lock()
121 defer m.mu.Unlock()
122 return m.setAnchorLocked(anchor)
123 }
124
125 func (m *Memory) setAnchorLocked(anchor Anchor) error {
126 if anchor.BlockHash == (chainhash.Hash{}) {
127 return errors.New("neutrino: anchor has no block hash")
128 }
129 if anchor.FilterHeight < -1 {
130 return errors.New("neutrino: anchor filter height below -1")
131 }
132 if anchor.FilterHeight >= 0 && anchor.FilterHeader == (chainhash.Hash{}) {
133 return errors.New("neutrino: anchor has no filter header")
134 }
135 if m.hasAnchor {
136 return errors.New("neutrino: anchor already set")
137 }
138 if hdr, ok := m.headers[anchor.Height]; ok && hdr.BlockHash() != anchor.BlockHash {
139 return errors.New("neutrino: anchor block hash does not match the stored header")
140 }
141 if anchor.FilterHeight >= 0 {
142 if fh, ok := m.fheaders[uint32(anchor.FilterHeight)]; ok && fh != anchor.FilterHeader {
143 return errors.New("neutrino: anchor filter header does not match the stored filter header")
144 }
145 }
146 m.anchor = anchor
147 m.hasAnchor = true
148 if anchor.FilterHeight >= 0 {
149 m.fheaders[uint32(anchor.FilterHeight)] = anchor.FilterHeader
150 if !m.hasFHeader || uint32(anchor.FilterHeight) > m.fheaderTip {
151 m.fheaderTip = uint32(anchor.FilterHeight)
152 m.hasFHeader = true
153 }
154 }
155 return nil
156 }
157
158 func (m *Memory) HeaderTip() (uint32, chainhash.Hash, bool) {
159 m.mu.RLock()
160 defer m.mu.RUnlock()
161 if m.hasHeader {
162 for h := m.headerTip; ; h-- {
163 if hdr, ok := m.headers[h]; ok {
164 return h, hdr.BlockHash(), true
165 }
166 if h == 0 {
167 break
168 }
169 }
170 }
171 if m.hasAnchor {
172 return m.anchor.Height, m.anchor.BlockHash, true
173 }
174 return 0, chainhash.Hash{}, false
175 }
176
177 func (m *Memory) Header(height uint32) (btcwire.BlockHeader, bool) {
178 m.mu.RLock()
179 defer m.mu.RUnlock()
180 h, ok := m.headers[height]
181 return h, ok
182 }
183
184 func (m *Memory) PutHeaders(start uint32, headers []btcwire.BlockHeader) error {
185 m.mu.Lock()
186 defer m.mu.Unlock()
187 if len(headers) == 0 {
188 return nil
189 }
190 if m.hasHeader {
191 _, tipHash, _ := m.headerTipLocked()
192 if start != m.headerTip+1 {
193 return fmt.Errorf("neutrino: header batch starts at %d, want %d", start, m.headerTip+1)
194 }
195 if headers[0].PrevBlock != tipHash {
196 return fmt.Errorf("neutrino: header %d does not link to the tip", start)
197 }
198 } else if m.hasAnchor {
199 switch {
200 case start == m.anchor.Height+1:
201 if headers[0].PrevBlock != m.anchor.BlockHash {
202 return fmt.Errorf("neutrino: header %d does not link to the anchor", start)
203 }
204 case start == 0 && m.anchor.Height == 0:
205 // Legacy stores keep the genesis header itself.
206 if headers[0].BlockHash() != m.anchor.BlockHash {
207 return fmt.Errorf("neutrino: genesis header does not match the anchor")
208 }
209 if headers[0].PrevBlock != (chainhash.Hash{}) {
210 return fmt.Errorf("neutrino: genesis header has a previous block")
211 }
212 default:
213 return fmt.Errorf("neutrino: first header batch starts at %d, want %d", start, m.anchor.Height+1)
214 }
215 } else if start != 0 {
216 return fmt.Errorf("neutrino: first header batch starts at %d, want 0", start)
217 }
218 prev := headers[0].PrevBlock
219 for i, h := range headers {
220 if i > 0 && h.PrevBlock != prev {
221 return fmt.Errorf("neutrino: header %d does not link to header %d", start+uint32(i), start+uint32(i)-1)
222 }
223 m.headers[start+uint32(i)] = h
224 prev = h.BlockHash()
225 }
226 m.headerTip = start + uint32(len(headers)) - 1
227 m.hasHeader = true
228 return nil
229 }
230
231 func (m *Memory) headerTipLocked() (uint32, chainhash.Hash, bool) {
232 if !m.hasHeader {
233 if m.hasAnchor {
234 return m.anchor.Height, m.anchor.BlockHash, true
235 }
236 return 0, chainhash.Hash{}, false
237 }
238 h, ok := m.headers[m.headerTip]
239 return m.headerTip, h.BlockHash(), ok
240 }
241
242 func (m *Memory) FilterHeaderTip() (uint32, chainhash.Hash, bool) {
243 m.mu.RLock()
244 defer m.mu.RUnlock()
245 if m.hasFHeader {
246 return m.fheaderTip, m.fheaders[m.fheaderTip], true
247 }
248 if m.hasAnchor && m.anchor.FilterHeight >= 0 {
249 return uint32(m.anchor.FilterHeight), m.anchor.FilterHeader, true
250 }
251 return 0, chainhash.Hash{}, false
252 }
253
254 func (m *Memory) FilterHeader(height uint32) (chainhash.Hash, bool) {
255 m.mu.RLock()
256 defer m.mu.RUnlock()
257 h, ok := m.fheaders[height]
258 return h, ok
259 }
260
261 func (m *Memory) FilterHash(height uint32) (chainhash.Hash, bool) {
262 m.mu.RLock()
263 defer m.mu.RUnlock()
264 h, ok := m.fhashes[height]
265 return h, ok
266 }
267
268 func (m *Memory) PutFilterHeaders(start uint32, filterHashes, headers []chainhash.Hash) error {
269 m.mu.Lock()
270 defer m.mu.Unlock()
271 if len(headers) != len(filterHashes) {
272 return errors.New("neutrino: filter hash and filter header counts differ")
273 }
274 if len(headers) == 0 {
275 return nil
276 }
277 prev := chainhash.Hash{}
278 if m.hasFHeader {
279 if start != m.fheaderTip+1 {
280 return fmt.Errorf("neutrino: filter header batch starts at %d, want %d", start, m.fheaderTip+1)
281 }
282 prev = m.fheaders[m.fheaderTip]
283 } else if m.hasAnchor && m.anchor.FilterHeight >= 0 {
284 if start != uint32(m.anchor.FilterHeight)+1 {
285 return fmt.Errorf("neutrino: first filter header batch starts at %d, want %d", start, uint32(m.anchor.FilterHeight)+1)
286 }
287 prev = m.anchor.FilterHeader
288 } else if start != 0 {
289 return fmt.Errorf("neutrino: first filter header batch starts at %d, want 0", start)
290 }
291 for i, h := range headers {
292 want := btcwire.FilterHeader(filterHashes[i], prev)
293 if h != want {
294 return fmt.Errorf("neutrino: filter header %d does not commit to its filter hash", start+uint32(i))
295 }
296 m.fhashes[start+uint32(i)] = filterHashes[i]
297 m.fheaders[start+uint32(i)] = h
298 prev = h
299 }
300 m.fheaderTip = start + uint32(len(headers)) - 1
301 m.hasFHeader = true
302 return nil
303 }
304
305 func (m *Memory) FilterTip() (uint32, bool) {
306 m.mu.RLock()
307 defer m.mu.RUnlock()
308 return m.filterTip, m.hasFilter
309 }
310
311 func (m *Memory) Filter(height uint32) ([]byte, bool) {
312 m.mu.RLock()
313 defer m.mu.RUnlock()
314 f, ok := m.filters[height]
315 return f, ok
316 }
317
318 func (m *Memory) PutFilter(height uint32, blockHash, filterHash chainhash.Hash, filter []byte) error {
319 m.mu.Lock()
320 defer m.mu.Unlock()
321 hdr, ok := m.headers[height]
322 if !ok {
323 if m.hasAnchor && height == m.anchor.Height+1 {
324 // The first filter after a checkpoint still needs its header;
325 // callers store headers before filters.
326 return fmt.Errorf("neutrino: filter %d has no header", height)
327 }
328 return fmt.Errorf("neutrino: filter %d has no header", height)
329 }
330 if hdr.BlockHash() != blockHash {
331 return fmt.Errorf("neutrino: filter %d block hash mismatch", height)
332 }
333 if chainhash.DoubleHashH(filter) != filterHash {
334 return fmt.Errorf("neutrino: filter %d hash mismatch", height)
335 }
336 if _, exists := m.filters[height]; exists {
337 return fmt.Errorf("neutrino: filter %d already stored", height)
338 }
339 cp := make([]byte, len(filter))
340 copy(cp, filter)
341 m.filters[height] = cp
342 if !m.hasFilter || height > m.filterTip {
343 m.filterTip = height
344 m.hasFilter = true
345 }
346 return nil
347 }
348
349 // HeaderHeights lists the stored header heights in ascending order.
350 func (m *Memory) HeaderHeights() []uint32 {
351 m.mu.RLock()
352 defer m.mu.RUnlock()
353 out := make([]uint32, 0, len(m.headers))
354 for h := range m.headers {
355 out = append(out, h)
356 }
357 sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
358 return out
359 }
360
361 // FilterHeaderHeights lists the stored filter header heights.
362 func (m *Memory) FilterHeaderHeights() []uint32 {
363 m.mu.RLock()
364 defer m.mu.RUnlock()
365 out := make([]uint32, 0, len(m.fheaders))
366 for h := range m.fheaders {
367 out = append(out, h)
368 }
369 sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
370 return out
371 }
372
373 // FilterHeights lists the stored filter heights.
374 func (m *Memory) FilterHeights() []uint32 {
375 m.mu.RLock()
376 defer m.mu.RUnlock()
377 out := make([]uint32, 0, len(m.filters))
378 for h := range m.filters {
379 out = append(out, h)
380 }
381 sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
382 return out
383 }
384
385 // Truncate removes headers, filter headers and filters at or above from.
386 func (m *Memory) Truncate(from uint32) error {
387 m.mu.Lock()
388 defer m.mu.Unlock()
389 if m.hasAnchor && from <= m.anchor.Height {
390 return fmt.Errorf("neutrino: truncate at %d is at or below the anchor %d", from, m.anchor.Height)
391 }
392 for h := range m.headers {
393 if h >= from {
394 delete(m.headers, h)
395 }
396 }
397 m.recomputeHeaderTipLocked()
398 for h := range m.fheaders {
399 if h >= from {
400 delete(m.fheaders, h)
401 delete(m.fhashes, h)
402 }
403 }
404 m.recomputeFilterHeaderTipLocked()
405 for h := range m.filters {
406 if h >= from {
407 delete(m.filters, h)
408 }
409 }
410 m.recomputeFilterTipLocked()
411 return nil
412 }
413
414 func (m *Memory) recomputeHeaderTipLocked() {
415 m.hasHeader = false
416 for h := range m.headers {
417 if !m.hasHeader || h > m.headerTip {
418 m.headerTip = h
419 m.hasHeader = true
420 }
421 }
422 }
423
424 func (m *Memory) recomputeFilterHeaderTipLocked() {
425 m.hasFHeader = false
426 for h := range m.fheaders {
427 if !m.hasFHeader || h > m.fheaderTip {
428 m.fheaderTip = h
429 m.hasFHeader = true
430 }
431 }
432 }
433
434 func (m *Memory) recomputeFilterTipLocked() {
435 m.hasFilter = false
436 for h := range m.filters {
437 if !m.hasFilter || h > m.filterTip {
438 m.filterTip = h
439 m.hasFilter = true
440 }
441 }
442 }
443
444 // FileStore persists the same state as Memory in three append-only files.
445 // Each record carries its height, so a partial tail from a crash is detected
446 // and truncated on open rather than silently misread.
447 type FileStore struct {
448 *Memory
449 dir string
450
451 mu sync.Mutex
452 headers *os.File
453 fheaders *os.File
454 filters *os.File
455 }
456
457 const (
458 headerRecordSize = 4 + btcwire.BlockHeaderSize
459 fheaderRecordSize = 4 + 2*chainhash.HashSize
460 filterRecordPrefix = 4 + 2*chainhash.HashSize + 4
461 anchorRecordSize = 4 + 1 + 4 + chainhash.HashSize + 4 + chainhash.HashSize
462 anchorMagic = "NSTA"
463 anchorVersion = 1
464 )
465
466 // OpenFileStore opens or creates a store in dir.
467 func OpenFileStore(dir string) (*FileStore, error) {
468 if err := os.MkdirAll(dir, 0o700); err != nil {
469 return nil, fmt.Errorf("neutrino: store dir: %w", err)
470 }
471 s := &FileStore{Memory: NewMemory(), dir: dir}
472 var err error
473 if s.headers, err = os.OpenFile(filepath.Join(dir, "headers.bin"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o600); err != nil {
474 return nil, err
475 }
476 if s.fheaders, err = os.OpenFile(filepath.Join(dir, "cfheaders.bin"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o600); err != nil {
477 return nil, err
478 }
479 if s.filters, err = os.OpenFile(filepath.Join(dir, "filters.bin"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o600); err != nil {
480 return nil, err
481 }
482 if err := s.loadAnchor(); err != nil {
483 return nil, err
484 }
485 if err := s.loadHeaders(); err != nil {
486 return nil, err
487 }
488 if err := s.loadFilterHeaders(); err != nil {
489 return nil, err
490 }
491 if err := s.loadFilters(); err != nil {
492 return nil, err
493 }
494 // An older store has no anchor file. Its headers start at genesis, so the
495 // anchor is the genesis header and the filter chain starts from zero.
496 if _, ok := s.Memory.Anchor(); !ok {
497 if hdr, ok := s.Memory.Header(0); ok {
498 if err := s.SetAnchor(Anchor{Height: 0, BlockHash: hdr.BlockHash(), FilterHeight: -1}); err != nil {
499 return nil, err
500 }
501 }
502 }
503 return s, nil
504 }
505
506 func (s *FileStore) anchorPath() string { return filepath.Join(s.dir, "anchor.bin") }
507
508 func (s *FileStore) loadAnchor() error {
509 body, err := os.ReadFile(s.anchorPath())
510 if err != nil {
511 if os.IsNotExist(err) {
512 return nil
513 }
514 return fmt.Errorf("neutrino: read anchor: %w", err)
515 }
516 if len(body) != anchorRecordSize {
517 return fmt.Errorf("neutrino: anchor file has %d bytes, want %d", len(body), anchorRecordSize)
518 }
519 if string(body[:4]) != anchorMagic {
520 return errors.New("neutrino: anchor file has a bad magic")
521 }
522 if body[4] != anchorVersion {
523 return fmt.Errorf("neutrino: anchor file version %d", body[4])
524 }
525 var a Anchor
526 a.Height = binary.LittleEndian.Uint32(body[5:9])
527 copy(a.BlockHash[:], body[9:9+chainhash.HashSize])
528 off := 9 + chainhash.HashSize
529 a.FilterHeight = int32(binary.LittleEndian.Uint32(body[off : off+4]))
530 off += 4
531 copy(a.FilterHeader[:], body[off:off+chainhash.HashSize])
532 if err := s.Memory.SetAnchor(a); err != nil {
533 return fmt.Errorf("neutrino: load anchor: %w", err)
534 }
535 return nil
536 }
537
538 // SetAnchor installs and persists the trusted starting point.
539 func (s *FileStore) SetAnchor(a Anchor) error {
540 if err := s.Memory.SetAnchor(a); err != nil {
541 return err
542 }
543 var body [anchorRecordSize]byte
544 copy(body[:4], anchorMagic)
545 body[4] = anchorVersion
546 binary.LittleEndian.PutUint32(body[5:9], a.Height)
547 copy(body[9:9+chainhash.HashSize], a.BlockHash[:])
548 off := 9 + chainhash.HashSize
549 binary.LittleEndian.PutUint32(body[off:off+4], uint32(a.FilterHeight))
550 off += 4
551 copy(body[off:off+chainhash.HashSize], a.FilterHeader[:])
552 s.mu.Lock()
553 defer s.mu.Unlock()
554 tmp := s.anchorPath() + ".tmp"
555 if err := os.WriteFile(tmp, body[:], 0o600); err != nil {
556 return fmt.Errorf("neutrino: write anchor: %w", err)
557 }
558 if err := os.Rename(tmp, s.anchorPath()); err != nil {
559 return fmt.Errorf("neutrino: install anchor: %w", err)
560 }
561 return nil
562 }
563
564 func (s *FileStore) loadHeaders() error {
565 path := s.dir + "/headers.bin"
566 f, err := os.OpenFile(path, os.O_RDWR, 0)
567 if err != nil {
568 return err
569 }
570 defer f.Close()
571 br := bufio.NewReaderSize(f, 1<<20)
572 anchor, hasAnchor := s.Memory.Anchor()
573 expect := uint32(0)
574 if hasAnchor && anchor.Height > 0 {
575 expect = anchor.Height + 1
576 }
577 first := true
578 valid := int64(0)
579 for {
580 var rec [headerRecordSize]byte
581 if _, err := io.ReadFull(br, rec[:]); err != nil {
582 if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
583 return truncateTo(f, path, valid)
584 }
585 return err
586 }
587 height := binary.LittleEndian.Uint32(rec[:4])
588 if first {
589 first = false
590 switch {
591 case !hasAnchor:
592 if height != 0 {
593 return fmt.Errorf("neutrino: header file starts at %d, want 0", height)
594 }
595 expect = 0
596 case anchor.Height == 0 && height == 0:
597 expect = 0 // legacy store keeps the genesis header
598 case height == anchor.Height+1:
599 expect = height
600 default:
601 return fmt.Errorf("neutrino: header file starts at %d, want %d", height, anchor.Height+1)
602 }
603 }
604 if height != expect {
605 return fmt.Errorf("neutrino: header file has height %d, want %d", height, expect)
606 }
607 var h btcwire.BlockHeader
608 if err := h.Deserialize(rec[4:]); err != nil {
609 return err
610 }
611 if expect > 0 {
612 prev, ok := s.Memory.Header(expect - 1)
613 if !ok || prev.BlockHash() != h.PrevBlock {
614 return fmt.Errorf("neutrino: header file breaks linkage at %d", expect)
615 }
616 }
617 if err := s.Memory.PutHeaders(expect, []btcwire.BlockHeader{h}); err != nil {
618 return err
619 }
620 expect++
621 valid += headerRecordSize
622 }
623 }
624
625 func (s *FileStore) loadFilterHeaders() error {
626 path := s.dir + "/cfheaders.bin"
627 f, err := os.OpenFile(path, os.O_RDWR, 0)
628 if err != nil {
629 return err
630 }
631 defer f.Close()
632 br := bufio.NewReaderSize(f, 1<<20)
633 expect := uint32(0)
634 if a, ok := s.Memory.Anchor(); ok && a.FilterHeight >= 0 {
635 expect = uint32(a.FilterHeight) + 1
636 }
637 valid := int64(0)
638 for {
639 var rec [fheaderRecordSize]byte
640 if _, err := io.ReadFull(br, rec[:]); err != nil {
641 if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
642 return truncateTo(f, path, valid)
643 }
644 return err
645 }
646 height := binary.LittleEndian.Uint32(rec[:4])
647 if height != expect {
648 return fmt.Errorf("neutrino: filter header file has height %d, want %d", height, expect)
649 }
650 var fh, h chainhash.Hash
651 copy(fh[:], rec[4:36])
652 copy(h[:], rec[36:68])
653 if err := s.Memory.PutFilterHeaders(height, []chainhash.Hash{fh}, []chainhash.Hash{h}); err != nil {
654 return err
655 }
656 expect++
657 valid += fheaderRecordSize
658 }
659 }
660
661 func (s *FileStore) loadFilters() error {
662 path := s.dir + "/filters.bin"
663 f, err := os.OpenFile(path, os.O_RDWR, 0)
664 if err != nil {
665 return err
666 }
667 defer f.Close()
668 br := bufio.NewReaderSize(f, 1<<20)
669 valid := int64(0)
670 for {
671 var prefix [filterRecordPrefix]byte
672 if _, err := io.ReadFull(br, prefix[:]); err != nil {
673 if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
674 return truncateTo(f, path, valid)
675 }
676 return err
677 }
678 height := binary.LittleEndian.Uint32(prefix[:4])
679 var blockHash, filterHash chainhash.Hash
680 copy(blockHash[:], prefix[4:36])
681 copy(filterHash[:], prefix[36:68])
682 n := binary.LittleEndian.Uint32(prefix[68:72])
683 if n > 4<<20 {
684 return fmt.Errorf("neutrino: filter file record %d declares %d bytes", height, n)
685 }
686 filter := make([]byte, n)
687 if _, err := io.ReadFull(br, filter); err != nil {
688 if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
689 return truncateTo(f, path, valid)
690 }
691 return err
692 }
693 if err := s.Memory.PutFilter(height, blockHash, filterHash, filter); err != nil {
694 return err
695 }
696 valid += int64(filterRecordPrefix) + int64(n)
697 }
698 }
699
700 // truncateTo removes a partial record left by a crash, so the next append
701 // starts at a record boundary.
702 func truncateTo(f *os.File, path string, valid int64) error {
703 info, err := f.Stat()
704 if err != nil {
705 return err
706 }
707 if info.Size() == valid {
708 return nil
709 }
710 if err := f.Truncate(valid); err != nil {
711 return fmt.Errorf("neutrino: truncate %s: %w", path, err)
712 }
713 return nil
714 }
715
716 func (s *FileStore) PutHeaders(start uint32, headers []btcwire.BlockHeader) error {
717 if err := s.Memory.PutHeaders(start, headers); err != nil {
718 return err
719 }
720 s.mu.Lock()
721 defer s.mu.Unlock()
722 var rec [headerRecordSize]byte
723 for i, h := range headers {
724 binary.LittleEndian.PutUint32(rec[:4], start+uint32(i))
725 raw := h.Serialize()
726 copy(rec[4:], raw[:])
727 if _, err := s.headers.Write(rec[:]); err != nil {
728 return err
729 }
730 }
731 return s.headers.Sync()
732 }
733
734 func (s *FileStore) PutFilterHeaders(start uint32, filterHashes, headers []chainhash.Hash) error {
735 if err := s.Memory.PutFilterHeaders(start, filterHashes, headers); err != nil {
736 return err
737 }
738 s.mu.Lock()
739 defer s.mu.Unlock()
740 var rec [fheaderRecordSize]byte
741 for i, h := range headers {
742 binary.LittleEndian.PutUint32(rec[:4], start+uint32(i))
743 copy(rec[4:36], filterHashes[i][:])
744 copy(rec[36:68], h[:])
745 if _, err := s.fheaders.Write(rec[:]); err != nil {
746 return err
747 }
748 }
749 return s.fheaders.Sync()
750 }
751
752 func (s *FileStore) PutFilter(height uint32, blockHash, filterHash chainhash.Hash, filter []byte) error {
753 if err := s.Memory.PutFilter(height, blockHash, filterHash, filter); err != nil {
754 return err
755 }
756 s.mu.Lock()
757 defer s.mu.Unlock()
758 var prefix [filterRecordPrefix]byte
759 binary.LittleEndian.PutUint32(prefix[:4], height)
760 copy(prefix[4:36], blockHash[:])
761 copy(prefix[36:68], filterHash[:])
762 binary.LittleEndian.PutUint32(prefix[68:72], uint32(len(filter)))
763 if _, err := s.filters.Write(prefix[:]); err != nil {
764 return err
765 }
766 if _, err := s.filters.Write(filter); err != nil {
767 return err
768 }
769 return s.filters.Sync()
770 }
771
772 // Truncate removes the in-memory state and rewrites the three files. Reorgs
773 // are rare and a rewrite is much simpler than recording a truncation point in
774 // an append-only format.
775 func (s *FileStore) Truncate(from uint32) error {
776 if err := s.Memory.Truncate(from); err != nil {
777 return err
778 }
779 s.mu.Lock()
780 defer s.mu.Unlock()
781 if err := s.rewriteHeadersLocked(); err != nil {
782 return err
783 }
784 if err := s.rewriteFilterHeadersLocked(); err != nil {
785 return err
786 }
787 return s.rewriteFiltersLocked()
788 }
789
790 func (s *FileStore) rewriteHeadersLocked() error {
791 heights := s.Memory.HeaderHeights()
792 f, err := s.rewriteFile(s.headers, "headers.bin", func(f *os.File) error {
793 var rec [headerRecordSize]byte
794 for _, h := range heights {
795 hdr, ok := s.Memory.Header(h)
796 if !ok {
797 continue
798 }
799 binary.LittleEndian.PutUint32(rec[:4], h)
800 raw := hdr.Serialize()
801 copy(rec[4:], raw[:])
802 if _, err := f.Write(rec[:]); err != nil {
803 return err
804 }
805 }
806 return nil
807 })
808 if err != nil {
809 return err
810 }
811 s.headers = f
812 return nil
813 }
814
815 func (s *FileStore) rewriteFilterHeadersLocked() error {
816 anchor, hasAnchor := s.Memory.Anchor()
817 f, err := s.rewriteFile(s.fheaders, "cfheaders.bin", func(f *os.File) error {
818 var rec [fheaderRecordSize]byte
819 for _, h := range s.Memory.FilterHeaderHeights() {
820 if hasAnchor && h <= uint32(anchor.FilterHeight) {
821 continue
822 }
823 fh, ok := s.Memory.FilterHash(h)
824 if !ok {
825 continue
826 }
827 hd, ok := s.Memory.FilterHeader(h)
828 if !ok {
829 continue
830 }
831 binary.LittleEndian.PutUint32(rec[:4], h)
832 copy(rec[4:36], fh[:])
833 copy(rec[36:68], hd[:])
834 if _, err := f.Write(rec[:]); err != nil {
835 return err
836 }
837 }
838 return nil
839 })
840 if err != nil {
841 return err
842 }
843 s.fheaders = f
844 return nil
845 }
846
847 func (s *FileStore) rewriteFiltersLocked() error {
848 f, err := s.rewriteFile(s.filters, "filters.bin", func(f *os.File) error {
849 var prefix [filterRecordPrefix]byte
850 for _, h := range s.Memory.FilterHeights() {
851 data, ok := s.Memory.Filter(h)
852 if !ok {
853 continue
854 }
855 hdr, ok := s.Memory.Header(h)
856 if !ok {
857 return fmt.Errorf("neutrino: filter %d has no header", h)
858 }
859 fh, ok := s.Memory.FilterHash(h)
860 if !ok {
861 return fmt.Errorf("neutrino: filter %d has no filter hash", h)
862 }
863 bh := hdr.BlockHash()
864 binary.LittleEndian.PutUint32(prefix[:4], h)
865 copy(prefix[4:36], bh[:])
866 copy(prefix[36:68], fh[:])
867 binary.LittleEndian.PutUint32(prefix[68:72], uint32(len(data)))
868 if _, err := f.Write(prefix[:]); err != nil {
869 return err
870 }
871 if _, err := f.Write(data); err != nil {
872 return err
873 }
874 }
875 return nil
876 })
877 if err != nil {
878 return err
879 }
880 s.filters = f
881 return nil
882 }
883
884 func (s *FileStore) rewriteFile(old *os.File, name string, write func(*os.File) error) (*os.File, error) {
885 path := filepath.Join(s.dir, name)
886 tmp := path + ".tmp"
887 f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
888 if err != nil {
889 return nil, err
890 }
891 if err := write(f); err != nil {
892 f.Close()
893 os.Remove(tmp)
894 return nil, err
895 }
896 if err := f.Sync(); err != nil {
897 f.Close()
898 os.Remove(tmp)
899 return nil, err
900 }
901 if err := f.Close(); err != nil {
902 os.Remove(tmp)
903 return nil, err
904 }
905 if old != nil {
906 old.Close()
907 }
908 if err := os.Rename(tmp, path); err != nil {
909 return nil, err
910 }
911 return os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o600)
912 }
913
914 // Close closes the open files.
915 func (s *FileStore) Close() error {
916 s.mu.Lock()
917 defer s.mu.Unlock()
918 var first error
919 for _, f := range []*os.File{s.headers, s.fheaders, s.filters} {
920 if f != nil {
921 if err := f.Close(); err != nil && first == nil {
922 first = err
923 }
924 }
925 }
926 return first
927 }
928