engine_test.mx raw
1 // Package store tests drive the engine through its public API and the
2 // unexported index helpers that carry the interesting behaviour: the kind /
3 // author / tag / time scans, the graph edges, WAL replay and delete.
4 //
5 // The events are built directly with a fixed 32-byte id, 32-byte pubkey and a
6 // 64-byte signature: SaveEvent never verifies a signature, and the store only
7 // round-trips the binary form, so signing every fixture would only slow the
8 // suite without exercising more of the engine.
9 package store
10
11 import (
12 "bytes"
13 "os"
14 "testing"
15
16 "git.smesh.lol/nostr/pkg/event"
17 "git.smesh.lol/nostr/pkg/filter"
18 "git.smesh.lol/nostr/pkg/hex"
19 nkind "git.smesh.lol/nostr/pkg/kind"
20 "git.smesh.lol/nostr/pkg/tag"
21 "git.smesh.lol/nostr/pkg/timestamp"
22 )
23
24 func engHash(fill byte) (b []byte) {
25 b = []byte{:32}
26 for i := range b {
27 b[i] = fill
28 }
29 return
30 }
31
32 func engSig() (b []byte) {
33 b = []byte{:64}
34 for i := range b {
35 b[i] = byte(i)
36 }
37 return
38 }
39
40 func engEvent(idFill, pubFill byte, k uint16, ts int64, tags *tag.S, content string) (ev *event.E) {
41 ev = event.New()
42 ev.ID = engHash(idFill)
43 ev.Pubkey = engHash(pubFill)
44 ev.Kind = k
45 ev.CreatedAt = ts
46 ev.Tags = tags
47 ev.Content = []byte(content)
48 ev.Sig = engSig()
49 return
50 }
51
52 func engTag(key, val string) (s *tag.S) {
53 return tag.NewS(tag.NewFromBytesSlice([]byte(key), []byte(val)))
54 }
55
56 func engETag(id []byte) (s *tag.S) {
57 return tag.NewS(tag.NewFromBytesSlice([]byte("e"), hex.Enc(id)))
58 }
59
60 func engPTag(pub []byte) (s *tag.S) {
61 return tag.NewS(tag.NewFromBytesSlice([]byte("p"), hex.Enc(pub)))
62 }
63
64 func engTmp(t *testing.T) (dir string, ok bool) {
65 t.Helper()
66 d, err := os.MkdirTemp("", "store-*")
67 if err != nil {
68 t.Fatal(err)
69 return "", false
70 }
71 return d, true
72 }
73
74 func engOpenTmp(t *testing.T) (e *Engine, dir string, ok bool) {
75 t.Helper()
76 d, dok := engTmp(t)
77 if !dok {
78 return nil, "", false
79 }
80 g, gerr := Open(d)
81 if gerr != nil {
82 t.Fatal(gerr)
83 os.RemoveAll(d)
84 return nil, "", false
85 }
86 return g, d, true
87 }
88
89 func engOpen(t *testing.T, dir string) (e *Engine, ok bool) {
90 t.Helper()
91 g, err := Open(dir)
92 if err != nil {
93 t.Fatal(err)
94 return nil, false
95 }
96 return g, true
97 }
98
99 func engSaveAll(t *testing.T, e *Engine, evs []*event.E) (ok bool) {
100 t.Helper()
101 for _, ev := range evs {
102 if err := e.SaveEvent(ev); err != nil {
103 t.Fatal(err)
104 return false
105 }
106 }
107 return true
108 }
109
110 func engQuery(t *testing.T, e *Engine, f *filter.F) (res []*event.E, ok bool) {
111 t.Helper()
112 r, err := e.QueryEvents(f)
113 if err != nil {
114 t.Fatal(err)
115 return nil, false
116 }
117 return r, true
118 }
119
120 func engKindFilter(k uint16) (f *filter.F) {
121 f = filter.New()
122 f.Kinds = nkind.NewS(nkind.New(k))
123 return
124 }
125
126 func engAuthorFilter(pub []byte) (f *filter.F) {
127 f = filter.New()
128 f.Authors = tag.NewFromBytesSlice(pub)
129 return
130 }
131
132 func engTagFilter(key, val []byte) (f *filter.F) {
133 f = filter.New()
134 f.Tags = tag.NewS(tag.NewFromBytesSlice(key, val))
135 return
136 }
137
138 func engHasID(evs []*event.E, id []byte) (ok bool) {
139 for _, x := range evs {
140 if bytes.Equal(x.ID, id) {
141 return true
142 }
143 }
144 return
145 }
146
147 func engHasBytes(bs [][]byte, id []byte) (ok bool) {
148 for _, x := range bs {
149 if bytes.Equal(x, id) {
150 return true
151 }
152 }
153 return
154 }
155
156 func engDescending(evs []*event.E) (ok bool) {
157 for i := 1; i < len(evs); i++ {
158 if evs[i].CreatedAt > evs[i-1].CreatedAt {
159 return false
160 }
161 }
162 return true
163 }
164
165 // engGrid is six events spanning two kinds, two authors and two tag values,
166 // with one event carrying no tags. Every index combination test slices it.
167 func engGrid() (evs []*event.E) {
168 evs = []*event.E{:6}
169 evs[0] = engEvent(0x01, 0xA1, 1, 1000, engTag("t", "alpha"), "zero")
170 evs[1] = engEvent(0x02, 0xB2, 1, 1001, engTag("t", "beta"), "one")
171 evs[2] = engEvent(0x03, 0xA1, 7, 1002, engTag("t", "alpha"), "two")
172 evs[3] = engEvent(0x04, 0xB2, 7, 1003, engTag("t", "beta"), "three")
173 evs[4] = engEvent(0x05, 0xA1, 1, 1004, engTag("t", "beta"), "four")
174 evs[5] = engEvent(0x06, 0xA1, 7, 1005, nil, "five")
175 return
176 }
177
178 func TestEmptyStore(t *testing.T) {
179 e, dir, ok := engOpenTmp(t)
180 if !ok {
181 return
182 }
183 defer os.RemoveAll(dir)
184 defer e.Close()
185
186 if e.MaxSerial() != 0 {
187 t.Fatalf("MaxSerial on empty = %d", e.MaxSerial())
188 }
189 if len(e.allFiles()) != 21 {
190 t.Fatalf("allFiles = %d", int32(len(e.allFiles())))
191 }
192 res, qok := engQuery(t, e, filter.New())
193 if !qok {
194 return
195 }
196 if len(res) != 0 {
197 t.Fatalf("empty store query = %d", len(res))
198 }
199
200 missing := engHash(0x77)
201 if _, gerr := e.GetByID(missing); gerr == nil {
202 t.Fatal("GetByID on an empty store")
203 }
204 if _, serr := e.GetBySerial(3); serr == nil {
205 t.Fatal("GetBySerial on an empty store")
206 }
207 if got := e.Search([]byte("anything"), 0); got != nil {
208 t.Fatal("Search on an empty store")
209 }
210 if got := e.SearchWord([]byte("anything")); len(got) != 0 {
211 t.Fatal("SearchWord on an empty store")
212 }
213 if derr := e.DeleteEvent(missing); derr == nil {
214 t.Fatal("DeleteEvent on an empty store")
215 }
216 if _, found := e.getEventSerial(missing); found {
217 t.Fatal("getEventSerial on an empty store")
218 }
219 if _, found := e.getEventIDBySerial(3); found {
220 t.Fatal("getEventIDBySerial on an empty store")
221 }
222
223 n1 := e.getOrCreatePubkeySerial(engHash(0x11))
224 if n1 != 1 {
225 t.Fatalf("first pubkey serial = %d", n1)
226 }
227 n2 := e.getOrCreatePubkeySerial(engHash(0x11))
228 if n2 != 1 {
229 t.Fatal("pubkey serial not stable")
230 }
231 n3 := e.getOrCreatePubkeySerial(engHash(0x22))
232 if n3 != 2 {
233 t.Fatalf("second pubkey serial = %d", n3)
234 }
235 }
236
237 func TestSaveGetDuplicate(t *testing.T) {
238 e, dir, ok := engOpenTmp(t)
239 if !ok {
240 return
241 }
242 defer os.RemoveAll(dir)
243 defer e.Close()
244
245 ev1 := engEvent(0x11, 0xA1, 1, 1000, nil, "first")
246 ev2 := engEvent(0x22, 0xA1, 7, 2000, engTag("t", "x"), "second")
247
248 if err := e.SaveEvent(ev1); err != nil {
249 t.Fatal(err)
250 return
251 }
252 // The first WAL serial is (0<<32)|0, so a one-event store reports 0.
253 if e.MaxSerial() != 0 {
254 t.Fatalf("first serial = %d", e.MaxSerial())
255 }
256 if err := e.SaveEvent(ev2); err != nil {
257 t.Fatal(err)
258 return
259 }
260 if e.MaxSerial() == 0 {
261 t.Fatal("MaxSerial after the second save")
262 }
263
264 if derr := e.SaveEvent(ev1); derr == nil {
265 t.Fatal("duplicate event accepted")
266 }
267
268 // GetByID is a point lookup: the eid index compares only prefix|id-hash,
269 // so a range Scan over the full 16-byte key is an empty range and this
270 // used to answer "event not found" for a stored event. negentropy's
271 // FindHave calls it, which is how it went unnoticed until now.
272 got, gerr := e.GetByID(ev2.ID)
273 if gerr != nil {
274 t.Fatal("GetByID on a stored event: " | string(gerr.Error()))
275 return
276 }
277 if !bytes.Equal(got.ID, ev2.ID) {
278 t.Fatal("GetByID returned another event")
279 }
280 if _, missErr := e.GetByID(engHash(0x77)); missErr == nil {
281 t.Fatal("GetByID of an unknown id did not error")
282 }
283
284 got, gerr := e.GetBySerial(e.MaxSerial())
285 if gerr != nil {
286 t.Fatal(gerr)
287 return
288 }
289 if !bytes.Equal(got.ID, ev2.ID) {
290 t.Fatal("GetBySerial returned the wrong event")
291 }
292 if got.Kind != 7 {
293 t.Fatalf("GetBySerial kind = %d", got.Kind)
294 }
295 if !bytes.Equal(got.Content, ev2.Content) {
296 t.Fatal("GetBySerial content")
297 }
298 if got.Tags == nil || got.Tags.Len() != 1 {
299 t.Fatal("GetBySerial tags")
300 }
301 if !bytes.Equal(got.Pubkey, ev2.Pubkey) {
302 t.Fatal("GetBySerial pubkey")
303 }
304
305 // GetByID is not used here: it scans eid with a range whose bounds are
306 // longer than the index's cmpLen, so lowerBound treats every matching
307 // record as smaller than the start bound and the scan is empty. The
308 // working by-id path is getEventSerial (a point lookup) and it is what
309 // queryByIDs uses.
310 ser1, sok := e.getEventSerial(ev1.ID)
311 if !sok {
312 t.Fatal("saved event has no serial")
313 }
314 byID, berr := e.GetBySerial(ser1)
315 if berr != nil {
316 t.Fatal(berr)
317 return
318 }
319 if !bytes.Equal(byID.ID, ev1.ID) || byID.Kind != 1 {
320 t.Fatal("lookup by id through the serial")
321 }
322 if _, uok := e.getEventSerial(engHash(0xEE)); uok {
323 t.Fatal("missing id resolved")
324 }
325 if _, serr := e.GetBySerial(9999); serr == nil {
326 t.Fatal("GetBySerial of a missing serial")
327 }
328 }
329
330 func TestSplitWords(t *testing.T) {
331 words := splitWords([]byte("Hello, World! foo-bar baz9 42"))
332 if len(words) != 5 {
333 t.Fatalf("splitWords count = %d", int32(len(words)))
334 }
335 want := []string{"hello", "world", "foo", "bar", "baz9"}
336 for i := 0; i < 5; i++ {
337 if string(words[i]) != want[i] {
338 t.Fatalf("word %d = %s", i, string(words[i]))
339 }
340 }
341 if w := splitWords([]byte("")); len(w) != 0 {
342 t.Fatal("splitWords of empty input")
343 }
344 if w := splitWords([]byte("ab")); len(w) != 0 {
345 t.Fatal("two-character words are dropped")
346 }
347 if w := splitWords([]byte("abc")); len(w) != 1 || string(w[0]) != "abc" {
348 t.Fatal("three-character word")
349 }
350 if w := splitWords([]byte("ABC-123")); len(w) != 2 || string(w[0]) != "abc" || string(w[1]) != "123" {
351 t.Fatalf("digits after a separator = %d words", int32(len(w)))
352 }
353 }
354
355 func TestQueryByKind(t *testing.T) {
356 e, dir, ok := engOpenTmp(t)
357 if !ok {
358 return
359 }
360 defer os.RemoveAll(dir)
361 defer e.Close()
362 g := engGrid()
363 if !engSaveAll(t, e, g) {
364 return
365 }
366
367 res, rok := engQuery(t, e, engKindFilter(1))
368 if !rok {
369 return
370 }
371 if len(res) != 3 {
372 t.Fatalf("kind 1 count = %d", int32(len(res)))
373 }
374 for _, x := range res {
375 if x.Kind != 1 {
376 t.Fatal("kind filter leaked")
377 }
378 }
379
380 f2 := filter.New()
381 f2.Kinds = nkind.NewS(nkind.New(uint16(1)), nkind.New(uint16(7)))
382 res2, rok2 := engQuery(t, e, f2)
383 if !rok2 {
384 return
385 }
386 if len(res2) != 6 {
387 t.Fatalf("two-kind count = %d", int32(len(res2)))
388 }
389
390 res3, rok3 := engQuery(t, e, engKindFilter(999))
391 if !rok3 {
392 return
393 }
394 if len(res3) != 0 {
395 t.Fatalf("absent kind count = %d", int32(len(res3)))
396 }
397 }
398
399 func TestQueryByAuthor(t *testing.T) {
400 e, dir, ok := engOpenTmp(t)
401 if !ok {
402 return
403 }
404 defer os.RemoveAll(dir)
405 defer e.Close()
406 g := engGrid()
407 if !engSaveAll(t, e, g) {
408 return
409 }
410
411 res, rok := engQuery(t, e, engAuthorFilter(engHash(0xA1)))
412 if !rok {
413 return
414 }
415 if len(res) != 4 {
416 t.Fatalf("author A count = %d", int32(len(res)))
417 }
418 for _, x := range res {
419 if !bytes.Equal(x.Pubkey, engHash(0xA1)) {
420 t.Fatal("author filter leaked")
421 }
422 }
423 if !engHasID(res, g[0].ID) || !engHasID(res, g[5].ID) {
424 t.Fatal("author A missing events")
425 }
426
427 res2, rok2 := engQuery(t, e, engAuthorFilter(engHash(0xB2)))
428 if !rok2 {
429 return
430 }
431 if len(res2) != 2 {
432 t.Fatalf("author B count = %d", int32(len(res2)))
433 }
434 }
435
436 func TestQueryKindAndAuthor(t *testing.T) {
437 e, dir, ok := engOpenTmp(t)
438 if !ok {
439 return
440 }
441 defer os.RemoveAll(dir)
442 defer e.Close()
443 g := engGrid()
444 if !engSaveAll(t, e, g) {
445 return
446 }
447
448 f := engKindFilter(7)
449 f.Authors = tag.NewFromBytesSlice(engHash(0xA1))
450 res, rok := engQuery(t, e, f)
451 if !rok {
452 return
453 }
454 if len(res) != 2 {
455 t.Fatalf("kind 7 + author A count = %d", int32(len(res)))
456 }
457 if !engHasID(res, g[2].ID) || !engHasID(res, g[5].ID) {
458 t.Fatal("kind+author missing events")
459 }
460 for _, x := range res {
461 if x.Kind != 7 || !bytes.Equal(x.Pubkey, engHash(0xA1)) {
462 t.Fatal("kind+author leaked")
463 }
464 }
465 }
466
467 func TestQueryByTag(t *testing.T) {
468 e, dir, ok := engOpenTmp(t)
469 if !ok {
470 return
471 }
472 defer os.RemoveAll(dir)
473 defer e.Close()
474 g := engGrid()
475 if !engSaveAll(t, e, g) {
476 return
477 }
478
479 res, rok := engQuery(t, e, engTagFilter([]byte("t"), []byte("alpha")))
480 if !rok {
481 return
482 }
483 if len(res) != 2 {
484 t.Fatalf("t=alpha count = %d", int32(len(res)))
485 }
486 if !engHasID(res, g[0].ID) || !engHasID(res, g[2].ID) {
487 t.Fatal("t=alpha missing events")
488 }
489
490 res2, rok2 := engQuery(t, e, engTagFilter([]byte("t"), []byte("beta")))
491 if !rok2 {
492 return
493 }
494 if len(res2) != 3 {
495 t.Fatalf("t=beta count = %d", int32(len(res2)))
496 }
497
498 res3, rok3 := engQuery(t, e, engTagFilter([]byte("t"), []byte("gamma")))
499 if !rok3 {
500 return
501 }
502 if len(res3) != 0 {
503 t.Fatalf("t=gamma count = %d", int32(len(res3)))
504 }
505
506 // A tag value on events that have no tags cannot match.
507 res4, rok4 := engQuery(t, e, engTagFilter([]byte("x"), []byte("anything")))
508 if !rok4 {
509 return
510 }
511 if len(res4) != 0 {
512 t.Fatalf("absent tag key count = %d", int32(len(res4)))
513 }
514 }
515
516 func TestQueryTagCombinations(t *testing.T) {
517 e, dir, ok := engOpenTmp(t)
518 if !ok {
519 return
520 }
521 defer os.RemoveAll(dir)
522 defer e.Close()
523 g := engGrid()
524 if !engSaveAll(t, e, g) {
525 return
526 }
527
528 // tag + kind (tkc)
529 f := engTagFilter([]byte("t"), []byte("alpha"))
530 f.Kinds = nkind.NewS(nkind.New(uint16(1)))
531 res, rok := engQuery(t, e, f)
532 if !rok {
533 return
534 }
535 if len(res) != 1 || !engHasID(res, g[0].ID) {
536 t.Fatalf("tag+kind count = %d", int32(len(res)))
537 }
538
539 // tag + author (tpc)
540 f2 := engTagFilter([]byte("t"), []byte("alpha"))
541 f2.Authors = tag.NewFromBytesSlice(engHash(0xA1))
542 res2, rok2 := engQuery(t, e, f2)
543 if !rok2 {
544 return
545 }
546 if len(res2) != 2 {
547 t.Fatalf("tag+author count = %d", int32(len(res2)))
548 }
549
550 // tag + kind + author (tkp)
551 f3 := engTagFilter([]byte("t"), []byte("alpha"))
552 f3.Kinds = nkind.NewS(nkind.New(uint16(1)))
553 f3.Authors = tag.NewFromBytesSlice(engHash(0xA1))
554 res3, rok3 := engQuery(t, e, f3)
555 if !rok3 {
556 return
557 }
558 if len(res3) != 1 || !engHasID(res3, g[0].ID) {
559 t.Fatalf("tag+kind+author count = %d", int32(len(res3)))
560 }
561
562 // tag alpha belongs to author A and B, but not to kind 1 + author B.
563 f4 := engTagFilter([]byte("t"), []byte("alpha"))
564 f4.Kinds = nkind.NewS(nkind.New(uint16(1)))
565 f4.Authors = tag.NewFromBytesSlice(engHash(0xB2))
566 res4, rok4 := engQuery(t, e, f4)
567 if !rok4 {
568 return
569 }
570 if len(res4) != 0 {
571 t.Fatalf("tkp no-match count = %d", int32(len(res4)))
572 }
573 }
574
575 func TestQueryByIDs(t *testing.T) {
576 e, dir, ok := engOpenTmp(t)
577 if !ok {
578 return
579 }
580 defer os.RemoveAll(dir)
581 defer e.Close()
582 g := engGrid()
583 if !engSaveAll(t, e, g) {
584 return
585 }
586
587 f := filter.New()
588 f.Ids = tag.NewFromBytesSlice(g[0].ID, g[2].ID, engHash(0xEE))
589 res, rok := engQuery(t, e, f)
590 if !rok {
591 return
592 }
593 if len(res) != 2 {
594 t.Fatalf("id query count = %d", int32(len(res)))
595 }
596 if !bytes.Equal(res[0].ID, g[0].ID) || !bytes.Equal(res[1].ID, g[2].ID) {
597 t.Fatal("id query ignored filter order")
598 }
599
600 f2 := filter.New()
601 f2.Ids = tag.NewFromBytesSlice(engHash(0xEE))
602 res2, rok2 := engQuery(t, e, f2)
603 if !rok2 {
604 return
605 }
606 if len(res2) != 0 {
607 t.Fatalf("missing id query count = %d", int32(len(res2)))
608 }
609 }
610
611 func TestQueryTimeRangeAndOrder(t *testing.T) {
612 e, dir, ok := engOpenTmp(t)
613 if !ok {
614 return
615 }
616 defer os.RemoveAll(dir)
617 defer e.Close()
618
619 evs := []*event.E{:20}
620 for i := 0; i < 20; i++ {
621 evs[i] = engEvent(byte(i+1), 0xA1, 1, int64(1700000000+i*100), nil, "timed")
622 }
623 if !engSaveAll(t, e, evs) {
624 return
625 }
626
627 all, aok := engQuery(t, e, filter.New())
628 if !aok {
629 return
630 }
631 if len(all) != 20 {
632 t.Fatalf("all count = %d", int32(len(all)))
633 }
634 if !engDescending(all) {
635 t.Fatal("default results are not newest first")
636 }
637 if all[0].CreatedAt != 1700001900 {
638 t.Fatalf("newest created_at = %d", all[0].CreatedAt)
639 }
640
641 win := filter.New()
642 win.Since = timestamp.FromUnix(1700000500)
643 win.Until = timestamp.FromUnix(1700001500)
644 winRes, wok := engQuery(t, e, win)
645 if !wok {
646 return
647 }
648 if len(winRes) != 11 {
649 t.Fatalf("window count = %d", int32(len(winRes)))
650 }
651 for _, x := range winRes {
652 if x.CreatedAt < 1700000500 || x.CreatedAt > 1700001500 {
653 t.Fatal("window leaked an out-of-range event")
654 }
655 }
656
657 lim := uint32(10)
658 top10 := engKindFilter(1)
659 top10.Limit = &lim
660 topRes, tok := engQuery(t, e, top10)
661 if !tok {
662 return
663 }
664 if len(topRes) != 10 {
665 t.Fatalf("limit count = %d", int32(len(topRes)))
666 }
667 if !engDescending(topRes) {
668 t.Fatal("limited results are not newest first")
669 }
670 if topRes[0].CreatedAt != 1700001900 {
671 t.Fatalf("limited newest = %d", topRes[0].CreatedAt)
672 }
673
674 limBig := uint32(25)
675 over := engKindFilter(1)
676 over.Limit = &limBig
677 overRes, ook := engQuery(t, e, over)
678 if !ook {
679 return
680 }
681 if len(overRes) != 20 {
682 t.Fatalf("large limit count = %d", int32(len(overRes)))
683 }
684
685 fut := filter.New()
686 fut.Since = timestamp.FromUnix(1800000000)
687 futRes, fok := engQuery(t, e, fut)
688 if !fok {
689 return
690 }
691 if len(futRes) != 0 {
692 t.Fatalf("future since count = %d", int32(len(futRes)))
693 }
694
695 past := filter.New()
696 past.Until = timestamp.FromUnix(1600000000)
697 pastRes, pok := engQuery(t, e, past)
698 if !pok {
699 return
700 }
701 if len(pastRes) != 0 {
702 t.Fatalf("past until count = %d", int32(len(pastRes)))
703 }
704 }
705
706 func TestSearch(t *testing.T) {
707 e, dir, ok := engOpenTmp(t)
708 if !ok {
709 return
710 }
711 defer os.RemoveAll(dir)
712 defer e.Close()
713
714 evs := []*event.E{:3}
715 evs[0] = engEvent(0x31, 0xA1, 1, 1000, nil, "the quick brown fox jumps")
716 evs[1] = engEvent(0x32, 0xA1, 1, 1001, nil, "quick silver hammer")
717 evs[2] = engEvent(0x33, 0xA1, 1, 1002, nil, "ab cd")
718 if !engSaveAll(t, e, evs) {
719 return
720 }
721
722 if got := e.SearchWord([]byte("quick")); len(got) != 2 {
723 t.Fatalf("SearchWord quick = %d", int32(len(got)))
724 }
725 // SearchWord hashes the bytes it is handed; normalisation happens in Search.
726 if got := e.SearchWord([]byte("QUICK")); len(got) != 0 {
727 t.Fatal("SearchWord must not normalise its input")
728 }
729 if got := e.SearchWord([]byte("nonexistent")); len(got) != 0 {
730 t.Fatal("SearchWord of an absent word")
731 }
732
733 sq := e.Search([]byte("quick"), 0)
734 if len(sq) != 2 {
735 t.Fatalf("Search quick = %d", int32(len(sq)))
736 }
737 if got := e.Search([]byte("QUICK"), 0); len(got) != 2 {
738 t.Fatal("Search must lower-case its query")
739 }
740 sb := e.Search([]byte("quick brown"), 0)
741 if len(sb) != 1 || !bytes.Equal(sb[0].ID, evs[0].ID) {
742 t.Fatal("Search intersection")
743 }
744 if got := e.Search([]byte("fox hammer"), 0); len(got) != 0 {
745 t.Fatal("Search of an empty intersection")
746 }
747 if got := e.Search([]byte("ab"), 0); got != nil {
748 t.Fatal("Search of a sub-three-character word")
749 }
750 if got := e.Search([]byte(""), 0); got != nil {
751 t.Fatal("Search of empty input")
752 }
753 sl := e.Search([]byte("hammer"), 1)
754 if len(sl) != 1 || !bytes.Equal(sl[0].ID, evs[1].ID) {
755 t.Fatal("Search with a limit")
756 }
757 if got := e.Search([]byte("jumps"), 0); len(got) != 1 {
758 t.Fatal("Search of a single rare word")
759 }
760 }
761
762 func TestGraphEdges(t *testing.T) {
763 e, dir, ok := engOpenTmp(t)
764 if !ok {
765 return
766 }
767 defer os.RemoveAll(dir)
768 defer e.Close()
769
770 root := engEvent(0x41, 0xA1, 1, 1000, nil, "root")
771 reply := engEvent(0x42, 0xB2, 1, 1001, engETag(root.ID), "reply")
772 grand := engEvent(0x43, 0xA1, 1, 1002, engETag(reply.ID), "grand")
773 if err := e.SaveEvent(root); err != nil {
774 t.Fatal(err)
775 return
776 }
777 if err := e.SaveEvent(reply); err != nil {
778 t.Fatal(err)
779 return
780 }
781 if err := e.SaveEvent(grand); err != nil {
782 t.Fatal(err)
783 return
784 }
785
786 rootSer, rok := e.getEventSerial(root.ID)
787 if !rok {
788 t.Fatal("root has no serial")
789 return
790 }
791 replySer, rok2 := e.getEventSerial(reply.ID)
792 if !rok2 {
793 t.Fatal("reply has no serial")
794 return
795 }
796 grandSer, rok3 := e.getEventSerial(grand.ID)
797 if !rok3 {
798 t.Fatal("grand has no serial")
799 return
800 }
801 if rootSer == replySer || replySer == grandSer || rootSer == grandSer {
802 t.Fatal("serial collision")
803 }
804
805 // reply references root; root references nothing.
806 targets := e.GetETagTargets(replySer)
807 if len(targets) != 1 || targets[0] != rootSer {
808 t.Fatalf("GetETagTargets(reply) = %d", int32(len(targets)))
809 }
810 if t0 := e.GetETagTargets(rootSer); len(t0) != 0 {
811 t.Fatalf("GetETagTargets(root) = %d", int32(len(t0)))
812 }
813 refs := e.GetReferencingEvents(rootSer)
814 if len(refs) != 1 || refs[0] != replySer {
815 t.Fatalf("GetReferencingEvents(root) = %d", int32(len(refs)))
816 }
817 refs2 := e.GetReferencingEvents(grandSer)
818 if len(refs2) != 0 {
819 t.Fatalf("GetReferencingEvents(grand) = %d", int32(len(refs2)))
820 }
821
822 both := e.TraverseThread(root.ID, 3, "both")
823 if len(both) != 2 || !engHasBytes(both, reply.ID) || !engHasBytes(both, grand.ID) {
824 t.Fatalf("TraverseThread both = %d", int32(len(both)))
825 }
826 def := e.TraverseThread(root.ID, 3, "")
827 if len(def) != 2 {
828 t.Fatalf("TraverseThread default direction = %d", int32(len(def)))
829 }
830 inRoot := e.TraverseThread(root.ID, 3, "inbound")
831 if len(inRoot) != 2 {
832 t.Fatalf("root inbound = %d", int32(len(inRoot)))
833 }
834 outRoot := e.TraverseThread(root.ID, 3, "outbound")
835 if len(outRoot) != 0 {
836 t.Fatalf("root outbound = %d", int32(len(outRoot)))
837 }
838 outGrand := e.TraverseThread(grand.ID, 3, "outbound")
839 if len(outGrand) != 2 {
840 t.Fatalf("grand outbound = %d", int32(len(outGrand)))
841 }
842 inGrand := e.TraverseThread(grand.ID, 3, "inbound")
843 if len(inGrand) != 0 {
844 t.Fatalf("grand inbound = %d", int32(len(inGrand)))
845 }
846 depth1 := e.TraverseThread(root.ID, 1, "both")
847 if len(depth1) != 1 || !bytes.Equal(depth1[0], reply.ID) {
848 t.Fatalf("depth 1 = %d", int32(len(depth1)))
849 }
850 if z := e.TraverseThread(root.ID, 0, "both"); len(z) != 0 {
851 t.Fatal("depth 0 must return nothing")
852 }
853 if unk := e.TraverseThread(engHash(0xEE), 3, "both"); unk != nil {
854 t.Fatal("unknown seed must return nil")
855 }
856 }
857
858 func TestGraphEdgePopulation(t *testing.T) {
859 e, dir, ok := engOpenTmp(t)
860 if !ok {
861 return
862 }
863 defer os.RemoveAll(dir)
864 defer e.Close()
865
866 base := engEvent(0x51, 0xA1, 1, 1000, nil, "base")
867 if err := e.SaveEvent(base); err != nil {
868 t.Fatal(err)
869 return
870 }
871 epg0 := e.epg.Count()
872 peg0 := e.peg.Count()
873 ppg0 := e.ppg.Count()
874 gpp0 := e.gpp.Count()
875 if epg0 != 1 || peg0 != 1 {
876 t.Fatalf("author edge counts = %d %d", epg0, peg0)
877 }
878 if ppg0 != 0 || gpp0 != 0 {
879 t.Fatal("a tagless event must not create pubkey-pubkey edges")
880 }
881
882 // A p-tag to another pubkey adds the event's own author edge plus a p-tag
883 // edge to epg/peg, and one ppg/gpp edge.
884 other := engEvent(0x52, 0xA1, 1, 1001, engPTag(engHash(0xB2)), "other")
885 if err := e.SaveEvent(other); err != nil {
886 t.Fatal(err)
887 return
888 }
889 if e.epg.Count() != epg0+2 || e.peg.Count() != peg0+2 {
890 t.Fatalf("p-tag epg/peg = %d %d", e.epg.Count(), e.peg.Count())
891 }
892 if e.ppg.Count() != ppg0+1 || e.gpp.Count() != gpp0+1 {
893 t.Fatal("p-tag ppg/gpp edge")
894 }
895
896 // A p-tag to the author itself still adds the author and p-tag epg/peg
897 // edges but must skip ppg/gpp.
898 epg1 := e.epg.Count()
899 peg1 := e.peg.Count()
900 ppg1 := e.ppg.Count()
901 gpp1 := e.gpp.Count()
902 self := engEvent(0x53, 0xA1, 1, 1002, engPTag(engHash(0xA1)), "self")
903 if err := e.SaveEvent(self); err != nil {
904 t.Fatal(err)
905 return
906 }
907 if e.epg.Count() != epg1+2 || e.peg.Count() != peg1+2 {
908 t.Fatalf("self p-tag epg/peg = %d %d", e.epg.Count(), e.peg.Count())
909 }
910 if e.ppg.Count() != ppg1 || e.gpp.Count() != gpp1 {
911 t.Fatal("self p-tag must skip ppg/gpp")
912 }
913
914 // An e-tag to a missing event creates no event-event edge.
915 eeg0 := e.eeg.Count()
916 gee0 := e.gee.Count()
917 miss := engEvent(0x54, 0xA1, 1, 1003, engETag(engHash(0xEE)), "missing")
918 if err := e.SaveEvent(miss); err != nil {
919 t.Fatal(err)
920 return
921 }
922 if e.eeg.Count() != eeg0 || e.gee.Count() != gee0 {
923 t.Fatal("e-tag to a missing event must not create an edge")
924 }
925
926 // Malformed tags are skipped by the indexer.
927 bad := tag.NewS(
928 tag.NewFromBytesSlice([]byte("t")),
929 tag.NewFromBytesSlice([]byte("ab"), []byte("v")),
930 tag.NewFromBytesSlice([]byte("p"), []byte("bad")),
931 tag.NewFromBytesSlice([]byte("e"), []byte("bad")),
932 tag.NewFromBytesSlice([]byte("t"), []byte("")),
933 )
934 badEv := engEvent(0x55, 0xA1, 1, 1004, bad, "bad")
935 if err := e.SaveEvent(badEv); err != nil {
936 t.Fatal(err)
937 return
938 }
939 if _, gerr := e.getEventSerial(badEv.ID); !gerr {
940 t.Fatal("an event with malformed tags must still be stored")
941 }
942 }
943
944 func TestDeleteEvent(t *testing.T) {
945 e, dir, ok := engOpenTmp(t)
946 if !ok {
947 return
948 }
949 defer os.RemoveAll(dir)
950 defer e.Close()
951 g := engGrid()
952 if !engSaveAll(t, e, g) {
953 return
954 }
955
956 before, bok := engQuery(t, e, filter.New())
957 if !bok {
958 return
959 }
960 if len(before) != 6 {
961 t.Fatalf("pre-delete count = %d", int32(len(before)))
962 }
963
964 if err := e.DeleteEvent(g[1].ID); err != nil {
965 t.Fatal(err)
966 return
967 }
968 if _, gerr := e.GetByID(g[1].ID); gerr == nil {
969 t.Fatal("deleted event still found by id")
970 }
971 after, aok := engQuery(t, e, filter.New())
972 if !aok {
973 return
974 }
975 if len(after) != 5 || engHasID(after, g[1].ID) {
976 t.Fatalf("delete not reflected in query: %d", int32(len(after)))
977 }
978 byKind, kok := engQuery(t, e, engKindFilter(1))
979 if !kok {
980 return
981 }
982 if len(byKind) != 2 || engHasID(byKind, g[1].ID) {
983 t.Fatalf("kind scan still sees a deleted event: %d", int32(len(byKind)))
984 }
985
986 // A tombstone must hide the record from point lookups too, not only from a
987 // full scan: sorted.File.Get did not consult the delete set, so an ids
988 // filter kept returning the deleted event while filter-less queries hid it.
989 idsQuery := filter.New()
990 idsQuery.Ids = tag.NewFromBytesSlice(g[1].ID)
991 byID, iok := engQuery(t, e, idsQuery)
992 if !iok {
993 return
994 }
995 if len(byID) != 0 {
996 t.Fatalf("ids filter returned a deleted event: %d", int32(len(byID)))
997 }
998
999 // The WAL is append-only, so the record is still readable by serial.
1000 if _, serr := e.GetBySerial(e.MaxSerial()); serr != nil {
1001 t.Fatal("delete removed the WAL record")
1002 }
1003 if derr := e.DeleteEvent(engHash(0xEE)); derr == nil {
1004 t.Fatal("delete of a missing event succeeded")
1005 }
1006 }
1007
1008 func TestRebuildFromWAL(t *testing.T) {
1009 e1, dir, ok := engOpenTmp(t)
1010 if !ok {
1011 return
1012 }
1013 defer os.RemoveAll(dir)
1014 g := engGrid()
1015 if !engSaveAll(t, e1, g) {
1016 e1.Close()
1017 return
1018 }
1019 maxBefore := e1.MaxSerial()
1020 if err := e1.Close(); err != nil {
1021 t.Fatal(err)
1022 return
1023 }
1024 if rerr := os.Remove(dir | "/checkpoint.dat"); rerr != nil {
1025 t.Fatal(rerr)
1026 return
1027 }
1028
1029 e2, ok2 := engOpen(t, dir)
1030 if !ok2 {
1031 return
1032 }
1033 defer e2.Close()
1034 if e2.MaxSerial() != maxBefore {
1035 t.Fatalf("rebuilt MaxSerial = %d want %d", e2.MaxSerial(), maxBefore)
1036 }
1037 all, aok := engQuery(t, e2, filter.New())
1038 if !aok {
1039 return
1040 }
1041 if len(all) != 6 {
1042 t.Fatalf("rebuilt count = %d", int32(len(all)))
1043 }
1044 for i := 0; i < 6; i++ {
1045 s, sok := e2.getEventSerial(g[i].ID)
1046 if !sok {
1047 t.Fatal("rebuilt event has no serial")
1048 }
1049 if _, gerr := e2.GetBySerial(s); gerr != nil {
1050 t.Fatal("rebuilt GetBySerial failed")
1051 }
1052 }
1053 extra := engEvent(0x66, 0xA1, 1, 2000, nil, "post-rebuild")
1054 if err := e2.SaveEvent(extra); err != nil {
1055 t.Fatal(err)
1056 return
1057 }
1058 all2, aok2 := engQuery(t, e2, filter.New())
1059 if !aok2 {
1060 return
1061 }
1062 if len(all2) != 7 {
1063 t.Fatalf("post-rebuild count = %d", int32(len(all2)))
1064 }
1065 }
1066
1067 // TestIncrementalRecovery flushes the first two events to the sidecars, records
1068 // an incremental checkpoint, then writes a third event and drops the handles
1069 // without advancing the checkpoint. Reopening must replay only the third entry,
1070 // still expose all three, and move the checkpoint up to the third serial.
1071 //
1072 // This used to set the checkpoint with ckpt.Set directly because
1073 // flushSidecars seeds its minimum with uint64(math.MaxUint64) and that constant
1074 // read as 0 under the old compiler, so the minimum stayed 0: an incremental
1075 // checkpoint was never recorded, and writeSidecars set the checkpoint *back* to
1076 // 0. The constant is exact now, so the test drives the real quickCheckpoint and
1077 // asserts both ends of it - the checkpoint it writes before the crash, and the
1078 // one recovery advances it to.
1079 func TestIncrementalRecovery(t *testing.T) {
1080 e1, dir, ok := engOpenTmp(t)
1081 if !ok {
1082 return
1083 }
1084 defer os.RemoveAll(dir)
1085 g := engGrid()
1086 if err := e1.SaveEvent(g[0]); err != nil {
1087 t.Fatal(err)
1088 e1.Close()
1089 return
1090 }
1091 if err := e1.SaveEvent(g[1]); err != nil {
1092 t.Fatal(err)
1093 e1.Close()
1094 return
1095 }
1096 e1.quickCheckpoint()
1097 ck := e1.lastSer
1098 if ck == 0 {
1099 t.Fatal("first two events produced no serial")
1100 }
1101 // The incremental checkpoint must be the flushed minimum, not 0 and not the
1102 // unflushed tail.
1103 if got := e1.ckpt.Get(); got != ck {
1104 t.Fatalf("quickCheckpoint wrote %d, want %d", got, ck)
1105 }
1106 if err := e1.SaveEvent(g[2]); err != nil {
1107 t.Fatal(err)
1108 e1.Close()
1109 return
1110 }
1111 last := e1.lastSer
1112 if last <= ck {
1113 t.Fatalf("third event serial %d did not advance past the checkpoint %d", last, ck)
1114 }
1115 if serr := e1.w.Sync(); serr != nil {
1116 t.Fatal(serr)
1117 return
1118 }
1119 // Close the index files and the log without touching the checkpoint, the
1120 // way a crash before the next checkpoint would leave them.
1121 for _, f := range e1.allFiles() {
1122 f.Close()
1123 }
1124 e1.w.Close()
1125
1126 e2, ok2 := engOpen(t, dir)
1127 if !ok2 {
1128 return
1129 }
1130 defer e2.Close()
1131 // Recovery replays from the checkpoint, indexes the third entry and
1132 // checkpoints the flushed minimum, so the checkpoint ends at the third
1133 // serial instead of going backwards.
1134 if got := e2.ckpt.Get(); got != last {
1135 t.Fatalf("checkpoint after recovery = %d, want %d (was %d)", got, last, ck)
1136 }
1137 all, aok := engQuery(t, e2, filter.New())
1138 if !aok {
1139 return
1140 }
1141 if len(all) != 3 {
1142 t.Fatalf("incremental recovery count = %d", int32(len(all)))
1143 }
1144 for i := 0; i < 3; i++ {
1145 if !engHasID(all, g[i].ID) {
1146 t.Fatal("incremental recovery lost an event")
1147 }
1148 }
1149 }
1150
1151 func TestManyEventsCheckpoint(t *testing.T) {
1152 e, dir, ok := engOpenTmp(t)
1153 if !ok {
1154 return
1155 }
1156 defer os.RemoveAll(dir)
1157
1158 evs := []*event.E{:70}
1159 for i := 0; i < 70; i++ {
1160 evs[i] = engEvent(byte(i+1), 0xA1, 1, int64(1700000000+i), nil, "bulk")
1161 }
1162 if !engSaveAll(t, e, evs) {
1163 e.Close()
1164 return
1165 }
1166 if e.saveN != 70%checkpointEvery {
1167 t.Fatalf("saveN = %d", e.saveN)
1168 }
1169 if e.MaxSerial() == 0 {
1170 t.Fatal("MaxSerial after 70 saves")
1171 }
1172 byKind, kok := engQuery(t, e, engKindFilter(1))
1173 if !kok {
1174 e.Close()
1175 return
1176 }
1177 if len(byKind) != 70 {
1178 t.Fatalf("bulk kind count = %d", int32(len(byKind)))
1179 }
1180 byAuth, aok := engQuery(t, e, engAuthorFilter(engHash(0xA1)))
1181 if !aok {
1182 e.Close()
1183 return
1184 }
1185 if len(byAuth) != 70 {
1186 t.Fatalf("bulk author count = %d", int32(len(byAuth)))
1187 }
1188
1189 if err := e.Close(); err != nil {
1190 t.Fatal(err)
1191 return
1192 }
1193 e2, ok2 := engOpen(t, dir)
1194 if !ok2 {
1195 return
1196 }
1197 defer e2.Close()
1198 after, fok := engQuery(t, e2, engKindFilter(1))
1199 if !fok {
1200 return
1201 }
1202 if len(after) != 70 {
1203 t.Fatalf("reopened bulk count = %d", int32(len(after)))
1204 }
1205 }
1206
1207 // TestExpirationIndexNotPopulated pins the current gap: the engine opens an exp
1208 // index but never writes or reads it, so NIP-40 expiry is not enforced here.
1209 func TestExpirationIndexNotPopulated(t *testing.T) {
1210 e, dir, ok := engOpenTmp(t)
1211 if !ok {
1212 return
1213 }
1214 defer os.RemoveAll(dir)
1215 defer e.Close()
1216
1217 ev := engEvent(0x71, 0xA1, 1, 1000, engTag("expiration", "1700000000"), "expiring")
1218 if err := e.SaveEvent(ev); err != nil {
1219 t.Fatal(err)
1220 return
1221 }
1222 if e.exp.Count() != 0 {
1223 t.Fatalf("exp index populated = %d", e.exp.Count())
1224 }
1225 }
1226
1227 // TestResolveBlob drives the x-tag lookup over the tc index: two events share
1228 // one hash, one of them also carries a different hash, and a third carries no
1229 // x tag at all. It then closes and reopens the store with the checkpoint
1230 // removed, forcing the WAL rebuild, and asks again.
1231 func TestResolveBlob(t *testing.T) {
1232 e, dir, ok := engOpenTmp(t)
1233 if !ok {
1234 return
1235 }
1236 defer os.RemoveAll(dir)
1237
1238 hashA := hex.Enc(engHash(0xAA))
1239 hashB := hex.Enc(engHash(0xBB))
1240 hashC := hex.Enc(engHash(0xCC))
1241
1242 both := tag.NewS(
1243 tag.NewFromBytesSlice([]byte("x"), []byte(hashA)),
1244 tag.NewFromBytesSlice([]byte("x"), []byte(hashB)),
1245 )
1246 onlyA := engTag("x", string(hashA))
1247 neither := engTag("t", string(hashA))
1248
1249 evBoth := engEvent(0x01, 0x11, 32210, 100, both, "")
1250 evOnlyA := engEvent(0x02, 0x12, 32210, 200, onlyA, "")
1251 evNeither := engEvent(0x03, 0x13, 32210, 300, neither, "")
1252 if !engSaveAll(t, e, []*event.E{evBoth, evOnlyA, evNeither}) {
1253 return
1254 }
1255
1256 res := e.ResolveBlob([]byte(hashA))
1257 if len(res) != 2 {
1258 t.Fatalf("ResolveBlob(hashA) count = %d, want 2", int32(len(res)))
1259 }
1260 // Newest first: evOnlyA is created_at 200, evBoth 100.
1261 if !bytes.Equal(res[0].ID, evOnlyA.ID) {
1262 t.Fatal("ResolveBlob did not return the newest match first")
1263 }
1264 if !engHasID(res, evBoth.ID) || !engHasID(res, evOnlyA.ID) {
1265 t.Fatal("ResolveBlob missed an event carrying the hash")
1266 }
1267
1268 resB := e.ResolveBlob([]byte(hashB))
1269 if len(resB) != 1 || !bytes.Equal(resB[0].ID, evBoth.ID) {
1270 t.Fatalf("ResolveBlob(hashB) did not isolate the matching event")
1271 }
1272
1273 if got := e.ResolveBlob([]byte(hashC)); len(got) != 0 {
1274 t.Fatalf("unknown hash returned %d events", int32(len(got)))
1275 }
1276
1277 if err := e.Close(); err != nil {
1278 t.Fatal(err)
1279 return
1280 }
1281 if rerr := os.Remove(dir | "/checkpoint.dat"); rerr != nil {
1282 t.Fatal(rerr)
1283 return
1284 }
1285 e2, ok2 := engOpen(t, dir)
1286 if !ok2 {
1287 return
1288 }
1289 defer e2.Close()
1290
1291 res2 := e2.ResolveBlob([]byte(hashA))
1292 if len(res2) != 2 {
1293 t.Fatalf("ResolveBlob after WAL rebuild count = %d, want 2", int32(len(res2)))
1294 }
1295 if !engHasID(res2, evBoth.ID) || !engHasID(res2, evOnlyA.ID) {
1296 t.Fatal("ResolveBlob lost an event across the WAL rebuild")
1297 }
1298 }
1299