//go:build js && wasm package idbstore import ( "context" "errors" "fmt" "sync" "syscall/js" "time" "git.smesh.lol/nostrino/pkg/btcwire" "git.smesh.lol/nostrino/pkg/crypto/chainhash" "git.smesh.lol/nostrino/pkg/neutrino" ) const ( storeHeaders = "headers" storeFHeaders = "fheaders" storeFilters = "filters" storeMeta = "meta" anchorKey = "anchor" queueDepth = 512 batchLimit = 200 opTimeout = 30 * time.Second ) // Config opens one chain's database. type Config struct { // DBName is the IndexedDB database. Empty means "nostrino-". DBName string // Chain is used for the default database name. Chain string // FiltersFrom skips loading filters below this height. Filters are the // large records; a swap wallet that scans recent history sets this to its // birth height. FiltersFrom uint32 } // Store is a neutrino.Store backed by IndexedDB, with an in-memory working // copy. Writes go to memory first and are persisted in batches. type Store struct { *neutrino.Memory db js.Value name string queue chan op stop chan struct{} wg sync.WaitGroup errMu sync.Mutex err error closeOnce sync.Once closed chan struct{} } type op struct { store string value js.Value barrier chan error isDelete bool deleteFrom uint32 } // Open opens or creates the database and loads its records into memory. func Open(ctx context.Context, cfg Config) (*Store, error) { name := cfg.DBName if name == "" { name = "nostrino-" + cfg.Chain } db, err := openDB(ctx, name) if err != nil { return nil, err } s := &Store{ Memory: neutrino.NewMemory(), db: db, name: name, queue: make(chan op, queueDepth), stop: make(chan struct{}), closed: make(chan struct{}), } s.wg.Add(1) go s.flusher() if err := s.load(ctx, cfg.FiltersFrom); err != nil { s.Close(context.Background()) return nil, err } return s, nil } // Name is the database name. func (s *Store) Name() string { return s.name } const ( headersSchema = "headers" fheadersSchema = "fheaders" filtersSchema = "filters" metaSchema = "meta" ) func openDB(ctx context.Context, name string) (js.Value, error) { idb := js.Global().Get("indexedDB") if idb.Type() == js.TypeUndefined { return js.Value{}, errors.New("idbstore: indexedDB is not available") } req := idb.Call("open", name, 1) var upgrade, success, failure js.Func done := make(chan error, 1) upgrade = js.FuncOf(func(this js.Value, args []js.Value) any { db := req.Get("result") names := db.Get("objectStoreNames") for _, spec := range []struct{ name, key string }{ {storeHeaders, "h"}, {storeFHeaders, "h"}, {storeFilters, "h"}, {storeMeta, "k"}, } { if names.Call("contains", spec.name).Bool() { continue } opts := js.Global().Get("Object").New() opts.Set("keyPath", spec.key) db.Call("createObjectStore", spec.name, opts) } return nil }) success = js.FuncOf(func(this js.Value, args []js.Value) any { select { case done <- nil: default: } return nil }) failure = js.FuncOf(func(this js.Value, args []js.Value) any { select { case done <- errors.New("idbstore: open failed"): default: } return nil }) req.Set("onupgradeneeded", upgrade) req.Set("onsuccess", success) req.Set("onerror", failure) defer func() { upgrade.Release() success.Release() failure.Release() }() select { case err := <-done: if err != nil { return js.Value{}, err } return req.Get("result"), nil case <-ctx.Done(): return js.Value{}, ctx.Err() } } func (s *Store) load(ctx context.Context, filtersFrom uint32) error { anchor, ok, err := s.loadAnchor(ctx) if err != nil { return err } if ok { if err := s.Memory.SetAnchor(anchor); err != nil { return fmt.Errorf("idbstore: anchor: %w", err) } } if err := s.loadHeaders(ctx); err != nil { return err } if err := s.loadFilterHeaders(ctx); err != nil { return err } return s.loadFilters(ctx, filtersFrom) } func (s *Store) loadAnchor(ctx context.Context) (neutrino.Anchor, bool, error) { arr, err := s.getAll(ctx, storeMeta, nil) if err != nil { return neutrino.Anchor{}, false, err } for i := 0; i < arr.Length(); i++ { rec := arr.Index(i) if rec.Get("k").String() != anchorKey { continue } var a neutrino.Anchor a.Height = uint32(rec.Get("h").Int()) if err := copyBytes(a.BlockHash[:], rec.Get("bh")); err != nil { return a, false, err } a.FilterHeight = int32(rec.Get("fh").Int()) if a.FilterHeight >= 0 { if err := copyBytes(a.FilterHeader[:], rec.Get("fhd")); err != nil { return a, false, err } } return a, true, nil } return neutrino.Anchor{}, false, nil } func (s *Store) loadHeaders(ctx context.Context) error { arr, err := s.getAll(ctx, storeHeaders, nil) if err != nil { return err } var raw [btcwire.BlockHeaderSize]byte for i := 0; i < arr.Length(); i++ { rec := arr.Index(i) height := uint32(rec.Get("h").Int()) if err := copyBytes(raw[:], rec.Get("d")); err != nil { return fmt.Errorf("idbstore: header %d: %w", height, err) } var h btcwire.BlockHeader if err := h.Deserialize(raw[:]); err != nil { return fmt.Errorf("idbstore: header %d: %w", height, err) } if err := s.Memory.PutHeaders(height, []btcwire.BlockHeader{h}); err != nil { return fmt.Errorf("idbstore: load headers: %w", err) } } return nil } func (s *Store) loadFilterHeaders(ctx context.Context) error { arr, err := s.getAll(ctx, storeFHeaders, nil) if err != nil { return err } var fh, hd chainhash.Hash for i := 0; i < arr.Length(); i++ { rec := arr.Index(i) height := uint32(rec.Get("h").Int()) if err := copyBytes(fh[:], rec.Get("fh")); err != nil { return fmt.Errorf("idbstore: filter header %d: %w", height, err) } if err := copyBytes(hd[:], rec.Get("hd")); err != nil { return fmt.Errorf("idbstore: filter header %d: %w", height, err) } if err := s.Memory.PutFilterHeaders(height, []chainhash.Hash{fh}, []chainhash.Hash{hd}); err != nil { return fmt.Errorf("idbstore: load filter headers: %w", err) } } return nil } func (s *Store) loadFilters(ctx context.Context, from uint32) error { var rangeVal js.Value if from > 0 { kr := js.Global().Get("IDBKeyRange") if kr.Type() != js.TypeUndefined { rangeVal = kr.Call("lowerBound", int(from)) } } arr, err := s.getAll(ctx, storeFilters, &rangeVal) if err != nil { return err } for i := 0; i < arr.Length(); i++ { rec := arr.Index(i) height := uint32(rec.Get("h").Int()) var bh, fh chainhash.Hash if err := copyBytes(bh[:], rec.Get("bh")); err != nil { return fmt.Errorf("idbstore: filter %d: %w", height, err) } if err := copyBytes(fh[:], rec.Get("fh")); err != nil { return fmt.Errorf("idbstore: filter %d: %w", height, err) } data := rec.Get("d") buf := make([]byte, data.Length()) if err := copyBytes(buf, data); err != nil { return fmt.Errorf("idbstore: filter %d: %w", height, err) } if err := s.Memory.PutFilter(height, bh, fh, buf); err != nil { return fmt.Errorf("idbstore: load filters: %w", err) } } return nil } func (s *Store) getAll(ctx context.Context, store string, rangeVal *js.Value) (js.Value, error) { tx := s.db.Call("transaction", store, "readonly") os := tx.Call("objectStore", store) var req js.Value if rangeVal != nil && rangeVal.Type() != js.TypeUndefined { req = os.Call("getAll", *rangeVal) } else { req = os.Call("getAll") } var success, failure js.Func done := make(chan error, 1) success = js.FuncOf(func(this js.Value, args []js.Value) any { select { case done <- nil: default: } return nil }) failure = js.FuncOf(func(this js.Value, args []js.Value) any { select { case done <- errors.New("idbstore: read " + store + " failed"): default: } return nil }) req.Set("onsuccess", success) req.Set("onerror", failure) select { case err := <-done: success.Release() failure.Release() if err != nil { return js.Value{}, err } return req.Get("result"), nil case <-ctx.Done(): return js.Value{}, ctx.Err() } } func (s *Store) flusher() { defer s.wg.Done() for { select { case <-s.stop: return case first := <-s.queue: batch := []op{first} for len(batch) < batchLimit { select { case next := <-s.queue: batch = append(batch, next) default: goto write } } write: err := s.writeBatch(batch) if err != nil { s.setErr(err) } for _, o := range batch { if o.barrier != nil { o.barrier <- err } } } } } func (s *Store) writeBatch(batch []op) error { var names []string seen := make(map[string]bool, 4) for _, o := range batch { if o.store == "" || seen[o.store] { continue } seen[o.store] = true names = append(names, o.store) } if len(names) == 0 { return nil } namesArray := js.Global().Get("Array").New() for _, name := range names { namesArray.Call("push", name) } tx := s.db.Call("transaction", namesArray, "readwrite") for _, o := range batch { if o.store == "" { continue } os := tx.Call("objectStore", o.store) if o.isDelete { kr := js.Global().Get("IDBKeyRange") if kr.Type() == js.TypeUndefined { return errors.New("idbstore: IDBKeyRange is not available") } os.Call("delete", kr.Call("lowerBound", int(o.deleteFrom))) continue } os.Call("put", o.value) } var success, failure js.Func done := make(chan error, 1) success = js.FuncOf(func(this js.Value, args []js.Value) any { select { case done <- nil: default: } return nil }) failure = js.FuncOf(func(this js.Value, args []js.Value) any { select { case done <- errors.New("idbstore: write failed"): default: } return nil }) tx.Set("oncomplete", success) tx.Set("onerror", failure) select { case err := <-done: success.Release() failure.Release() return err case <-time.After(opTimeout): success.Release() failure.Release() return errors.New("idbstore: write timed out") } } func (s *Store) setErr(err error) { s.errMu.Lock() if s.err == nil { s.err = err } s.errMu.Unlock() } func (s *Store) firstErr() error { s.errMu.Lock() defer s.errMu.Unlock() return s.err } func (s *Store) enqueue(o op) error { select { case <-s.closed: return errors.New("idbstore: closed") default: } select { case s.queue <- o: return nil case <-s.closed: return errors.New("idbstore: closed") } } // Flush waits until every queued write has committed. func (s *Store) Flush(ctx context.Context) error { barrier := make(chan error, 1) if err := s.enqueue(op{barrier: barrier}); err != nil { return err } select { case err := <-barrier: if err != nil { return err } return s.firstErr() case <-ctx.Done(): return ctx.Err() } } // Close flushes, stops the writer and closes the database. func (s *Store) Close(ctx context.Context) error { var err error s.closeOnce.Do(func() { err = s.Flush(ctx) close(s.stop) s.wg.Wait() close(s.closed) s.db.Call("close") }) return err } // SetAnchor stores the anchor through the same write-behind queue. func (s *Store) SetAnchor(a neutrino.Anchor) error { if err := s.Memory.SetAnchor(a); err != nil { return err } rec := js.Global().Get("Object").New() rec.Set("k", anchorKey) rec.Set("h", int(a.Height)) rec.Set("bh", bytesValue(a.BlockHash[:])) rec.Set("fh", int(a.FilterHeight)) rec.Set("fhd", bytesValue(a.FilterHeader[:])) barrier := make(chan error, 1) if err := s.enqueue(op{store: storeMeta, value: rec, barrier: barrier}); err != nil { return err } select { case err := <-barrier: return err case <-time.After(opTimeout): return errors.New("idbstore: anchor write timed out") } } // Truncate removes the in-memory state and deletes the persisted key ranges. func (s *Store) Truncate(from uint32) error { if err := s.Memory.Truncate(from); err != nil { return err } for _, name := range []string{storeHeaders, storeFHeaders, storeFilters} { if err := s.enqueue(op{store: name, isDelete: true, deleteFrom: from}); err != nil { return err } } barrier := make(chan error, 1) if err := s.enqueue(op{barrier: barrier}); err != nil { return err } select { case err := <-barrier: return err case <-time.After(opTimeout): return errors.New("idbstore: truncate timed out") } } // PutHeaders stores headers in memory and queues them for persistence. func (s *Store) PutHeaders(start uint32, headers []btcwire.BlockHeader) error { if err := s.Memory.PutHeaders(start, headers); err != nil { return err } for i, h := range headers { raw := h.Serialize() rec := js.Global().Get("Object").New() rec.Set("h", int(start)+i) rec.Set("d", bytesValue(raw[:])) if err := s.enqueue(op{store: storeHeaders, value: rec}); err != nil { return err } } return nil } // PutFilterHeaders stores filter hashes and headers, then queues them. func (s *Store) PutFilterHeaders(start uint32, filterHashes, headers []chainhash.Hash) error { if err := s.Memory.PutFilterHeaders(start, filterHashes, headers); err != nil { return err } for i, h := range headers { rec := js.Global().Get("Object").New() rec.Set("h", int(start)+i) rec.Set("fh", bytesValue(filterHashes[i][:])) rec.Set("hd", bytesValue(h[:])) if err := s.enqueue(op{store: storeFHeaders, value: rec}); err != nil { return err } } return nil } // PutFilter stores a verified filter, then queues it. func (s *Store) PutFilter(height uint32, blockHash, filterHash chainhash.Hash, filter []byte) error { if err := s.Memory.PutFilter(height, blockHash, filterHash, filter); err != nil { return err } rec := js.Global().Get("Object").New() rec.Set("h", int(height)) rec.Set("bh", bytesValue(blockHash[:])) rec.Set("fh", bytesValue(filterHash[:])) rec.Set("d", bytesValue(filter)) return s.enqueue(op{store: storeFilters, value: rec}) } func bytesValue(b []byte) js.Value { arr := js.Global().Get("Uint8Array").New(len(b)) js.CopyBytesToJS(arr, b) return arr } func copyBytes(dst []byte, v js.Value) error { if v.Type() != js.TypeObject { return errors.New("idbstore: field is not a typed array") } if v.Length() != len(dst) { return fmt.Errorf("idbstore: field has %d bytes, want %d", v.Length(), len(dst)) } js.CopyBytesToGo(dst, v) return nil } var _ neutrino.Store = (*Store)(nil)