filter.go raw
1 package filter
2
3 import (
4 "bytes"
5 "sort"
6
7 "next.orly.dev/pkg/nostr/crypto/ec/schnorr"
8 "next.orly.dev/pkg/nostr/encoders/event"
9 "next.orly.dev/pkg/nostr/encoders/ints"
10 "next.orly.dev/pkg/nostr/encoders/kind"
11 "next.orly.dev/pkg/nostr/encoders/tag"
12 "next.orly.dev/pkg/nostr/encoders/text"
13 "next.orly.dev/pkg/nostr/encoders/timestamp"
14 "next.orly.dev/pkg/nostr/utils"
15 "next.orly.dev/pkg/nostr/utils/pointers"
16 "github.com/minio/sha256-simd"
17 "next.orly.dev/pkg/lol/chk"
18 "next.orly.dev/pkg/lol/errorf"
19 )
20
21 // F is the primary query form for requesting events from a nostr relay.
22 //
23 // The ordering of fields of filters is not specified as in the protocol there
24 // is no requirement to generate a hash for fast recognition of identical
25 // filters. However, for internal use in a relay, by applying a consistent sort
26 // order, this library will produce an identical JSON from the same *set* of
27 // fields no matter what order they were provided.
28 //
29 // This is to facilitate the deduplication of filters so an effective identical
30 // match is not performed on an identical filter.
31 type F struct {
32 Ids *tag.T `json:"ids,omitempty"`
33 Kinds *kind.S `json:"kinds,omitempty"`
34 Authors *tag.T `json:"authors,omitempty"`
35 Tags *tag.S `json:"-,omitempty"`
36 Since *timestamp.T `json:"since,omitempty"`
37 Until *timestamp.T `json:"until,omitempty"`
38 Search []byte `json:"search,omitempty"`
39 Limit *uint `json:"limit,omitempty"`
40
41 // Extra holds unknown JSON fields for relay extensions (e.g., _graph).
42 // Per NIP-01, unknown fields should be ignored by relays that don't support them,
43 // but relays implementing extensions can access them here.
44 // Keys are stored without quotes, values are raw JSON bytes.
45 Extra map[string][]byte `json:"-"`
46 }
47
48 // New creates a new, reasonably initialized filter that will be ready for most uses without
49 // further allocations.
50 func New() (f *F) {
51 return &F{
52 Ids: tag.NewWithCap(10),
53 Kinds: kind.NewWithCap(10),
54 Authors: tag.NewWithCap(10),
55 Tags: tag.NewSWithCap(10),
56 Since: timestamp.New(),
57 Until: timestamp.New(),
58 }
59 }
60
61 var (
62 // IDs is the JSON object key for IDs.
63 IDs = []byte("ids")
64 // Kinds is the JSON object key for Kinds.
65 Kinds = []byte("kinds")
66 // Authors is the JSON object key for Authors.
67 Authors = []byte("authors")
68 // Since is the JSON object key for Since.
69 Since = []byte("since")
70 // Until is the JSON object key for Until.
71 Until = []byte("until")
72 // Limit is the JSON object key for Limit.
73 Limit = []byte("limit")
74 // Search is the JSON object key for Search.
75 Search = []byte("search")
76 )
77
78 // Sort the fields of a filter so a fingerprint on a filter that has the same set of content
79 // produces the same fingerprint.
80 func (f *F) Sort() {
81 if f.Ids != nil {
82 sort.Sort(f.Ids)
83 }
84 if f.Kinds != nil {
85 sort.Sort(f.Kinds)
86 }
87 if f.Authors != nil {
88 sort.Sort(f.Authors)
89 }
90 if f.Tags != nil {
91 for i, v := range *f.Tags {
92 if len(v.T) > 2 {
93 vv := (v.T)[1:]
94 sort.Slice(
95 vv, func(i, j int) bool {
96 return bytes.Compare((v.T)[i+1], (v.T)[j+1]) < 0
97 },
98 )
99 // keep the first as is, this is the #x prefix
100 first := (v.T)[:1]
101 // append the sorted values to the prefix
102 v.T = append(first, vv...)
103 // replace the old value with the sorted one
104 (*f.Tags)[i] = v
105 }
106 }
107 sort.Sort(f.Tags)
108 }
109 }
110
111 // MatchesIgnoringTimestampConstraints checks a filter against an event and
112 // determines if the event matches the filter, ignoring timestamp constraints..
113 func (f *F) MatchesIgnoringTimestampConstraints(ev *event.E) bool {
114 if ev == nil {
115 return false
116 }
117 if f.Ids.Len() > 0 && !f.Ids.Contains(ev.ID) {
118 return false
119 }
120 if f.Kinds.Len() > 0 && !f.Kinds.Contains(ev.Kind) {
121 return false
122 }
123 if f.Authors.Len() > 0 {
124 found := false
125 for _, author := range f.Authors.T {
126 if utils.FastEqual(author, ev.Pubkey) {
127 found = true
128 break
129 }
130 }
131 if !found {
132 return false
133 }
134 }
135 if f.Tags.Len() > 0 {
136 for _, v := range *f.Tags {
137 if v.Len() < 2 {
138 continue
139 }
140 key := v.Key()
141 values := v.T[1:]
142 if !ev.Tags.ContainsAny(key, values) {
143 return false
144 }
145 }
146 }
147 return true
148 }
149
150 // Matches checks a filter against an event and determines if the event matches the filter.
151 func (f *F) Matches(ev *event.E) (match bool) {
152 if !f.MatchesIgnoringTimestampConstraints(ev) {
153 return
154 }
155 if f.Since.Int() != 0 && ev.CreatedAt < f.Since.I64() {
156 return
157 }
158 if f.Until.Int() != 0 && ev.CreatedAt > f.Until.I64() {
159 return
160 }
161 return true
162 }
163
164 // EstimateSize returns an estimated size for marshaling the filter to JSON.
165 // This accounts for worst-case expansion of escaped content and hex encoding.
166 func (f *F) EstimateSize() (size int) {
167 // JSON structure overhead: {, }, commas, quotes, keys
168 size = 50
169
170 // IDs: "ids":["hex1","hex2",...]
171 if f.Ids != nil && f.Ids.Len() > 0 {
172 size += 7 // "ids":[
173 for _, id := range f.Ids.T {
174 size += 2*len(id) + 4 // hex encoding + quotes + comma
175 }
176 size += 1 // closing ]
177 }
178
179 // Kinds: "kinds":[1,2,3,...]
180 if f.Kinds.Len() > 0 {
181 size += 9 // "kinds":[
182 size += f.Kinds.Len() * 5 // assume average 5 bytes per kind number
183 size += 1 // closing ]
184 }
185
186 // Authors: "authors":["hex1","hex2",...]
187 if f.Authors.Len() > 0 {
188 size += 11 // "authors":[
189 for _, auth := range f.Authors.T {
190 size += 2*len(auth) + 4 // hex encoding + quotes + comma
191 }
192 size += 1 // closing ]
193 }
194
195 // Tags: "#x":["val1","val2",...]
196 if f.Tags != nil && f.Tags.Len() > 0 {
197 for _, tg := range *f.Tags {
198 if tg == nil || tg.Len() < 2 {
199 continue
200 }
201 size += 6 // "#x":[
202 for _, val := range tg.T[1:] {
203 size += len(val)*2 + 4 // escaped value + quotes + comma
204 }
205 size += 1 // closing ]
206 }
207 }
208
209 // Since: "since":1234567890
210 if f.Since != nil && f.Since.U64() > 0 {
211 size += 10 // "since": + timestamp
212 }
213
214 // Until: "until":1234567890
215 if f.Until != nil && f.Until.U64() > 0 {
216 size += 10 // "until": + timestamp
217 }
218
219 // Search: "search":"escaped text"
220 if len(f.Search) > 0 {
221 size += 11 // "search":"
222 size += len(f.Search) * 2 // worst case escaping
223 size += 1 // closing quote
224 }
225
226 // Limit: "limit":100
227 if pointers.Present(f.Limit) {
228 size += 11 // "limit": + number
229 }
230
231 return
232 }
233
234 // Marshal a filter into raw JSON bytes, minified. The field ordering and sort
235 // of fields is canonicalized so that a hash can identify the same filter.
236 func (f *F) Marshal(dst []byte) (b []byte) {
237 var err error
238 _ = err
239 var first bool
240 // Pre-allocate buffer if nil to reduce reallocations
241 if dst == nil {
242 estimatedSize := f.EstimateSize()
243 dst = make([]byte, 0, estimatedSize)
244 }
245 // sort the fields so they come out the same
246 f.Sort()
247 // open parentheses
248 b = dst
249 b = append(b, '{')
250 if f.Ids != nil && f.Ids.Len() > 0 {
251 first = true
252 b = text.JSONKey(b, IDs)
253 b = text.MarshalHexArray(b, f.Ids.T)
254 }
255 if f.Kinds.Len() > 0 {
256 if first {
257 b = append(b, ',')
258 } else {
259 first = true
260 }
261 b = text.JSONKey(b, Kinds)
262 b = f.Kinds.Marshal(b)
263 }
264 if f.Authors.Len() > 0 {
265 if first {
266 b = append(b, ',')
267 } else {
268 first = true
269 }
270 b = text.JSONKey(b, Authors)
271 b = text.MarshalHexArray(b, f.Authors.T)
272 }
273 if f.Tags != nil && f.Tags.Len() > 0 {
274 // tags are stored as tags with the initial element the "#a" and the rest the list in
275 // each element of the tags list. eg:
276 //
277 // [["#p","<pubkey1>","<pubkey3"],["#t","hashtag","stuff"]]
278 //
279 for _, tg := range *f.Tags {
280 if tg == nil {
281 // nothing here
282 continue
283 }
284 if tg.Len() < 2 {
285 // must have at least key and one value
286 continue
287 }
288 tKey := tg.T[0]
289 if len(tKey) != 1 ||
290 ((tKey[0] < 'a' || tKey[0] > 'z') && (tKey[0] < 'A' || tKey[0] > 'Z')) {
291 // key must be single alpha character
292 continue
293 }
294 values := tg.T[1:]
295 if len(values) == 0 {
296 continue
297 }
298 if first {
299 b = append(b, ',')
300 } else {
301 first = true
302 }
303 // append the key with # prefix
304 b = append(b, '"', '#', tKey[0], '"', ':')
305 b = append(b, '[')
306 for i, value := range values {
307 b = text.AppendQuote(b, value, text.NostrEscape)
308 if i < len(values)-1 {
309 b = append(b, ',')
310 }
311 }
312 b = append(b, ']')
313 }
314 }
315 if f.Since != nil && f.Since.U64() > 0 {
316 if first {
317 b = append(b, ',')
318 } else {
319 first = true
320 }
321 b = text.JSONKey(b, Since)
322 b = f.Since.Marshal(b)
323 }
324 if f.Until != nil && f.Until.U64() > 0 {
325 if first {
326 b = append(b, ',')
327 } else {
328 first = true
329 }
330 b = text.JSONKey(b, Until)
331 b = f.Until.Marshal(b)
332 }
333 if len(f.Search) > 0 {
334 if first {
335 b = append(b, ',')
336 } else {
337 first = true
338 }
339 b = text.JSONKey(b, Search)
340 b = text.AppendQuote(b, f.Search, text.NostrEscape)
341 }
342 if pointers.Present(f.Limit) {
343 if first {
344 b = append(b, ',')
345 } else {
346 first = true
347 }
348 b = text.JSONKey(b, Limit)
349 b = ints.New(*f.Limit).Marshal(b)
350 }
351 // close parentheses
352 b = append(b, '}')
353 return
354 }
355
356 // Serialize a filter.F into raw minified JSON bytes.
357 func (f *F) Serialize() (b []byte) { return f.Marshal(nil) }
358
359 // states of the unmarshaler
360 const (
361 beforeOpen = iota
362 openParen
363 inKey
364 inKV
365 inVal
366 betweenKV
367 afterClose
368 )
369
370 // Unmarshal a filter from raw (minified) JSON bytes into the runtime format.
371 //
372 // todo: this may tolerate whitespace, not certain currently.
373 func (f *F) Unmarshal(b []byte) (r []byte, err error) {
374 r = b
375 var key []byte
376 var state int
377 for ; len(r) > 0; r = r[1:] {
378 // log.I.ToSliceOfBytes("%c", rem[0])
379 switch state {
380 case beforeOpen:
381 if r[0] == '{' {
382 state = openParen
383 // log.I.Ln("openParen")
384 }
385 case openParen:
386 if r[0] == '"' {
387 state = inKey
388 // log.I.Ln("inKey")
389 }
390 case inKey:
391 if r[0] == '"' {
392 state = inKV
393 // log.I.Ln("inKV")
394 } else {
395 // Pre-allocate key buffer if needed
396 if key == nil {
397 key = make([]byte, 0, 16)
398 }
399 key = append(key, r[0])
400 }
401 case inKV:
402 if r[0] == ':' {
403 state = inVal
404 }
405 case inVal:
406 if len(key) < 1 {
407 err = errorf.E("filter key zero length: '%s'\n'%s", b, r)
408 return
409 }
410 switch key[0] {
411 case '#':
412 // tags start with # and have 1 letter
413 l := len(key)
414 if l != 2 {
415 err = errorf.E(
416 "filter tag keys can only be # and one alpha character: '%s'\n%s",
417 key, b,
418 )
419 return
420 }
421 // Store just the single-character tag key (e.g., 'p'), not the '#' prefix
422 // This matches how event tags store their keys (e.g., ["p", "pubkey"])
423 k := make([]byte, 1)
424 k[0] = key[1]
425 var ff [][]byte
426 if ff, r, err = text.UnmarshalStringArray(r); chk.E(err) {
427 return
428 }
429 ff = append([][]byte{k}, ff...)
430 if f.Tags == nil {
431 f.Tags = tag.NewSWithCap(1)
432 }
433 s := append(*f.Tags, tag.NewFromBytesSlice(ff...))
434 f.Tags = &s
435 state = betweenKV
436 case IDs[0]:
437 if len(key) < len(IDs) {
438 goto invalid
439 }
440 var ff [][]byte
441 if ff, r, err = text.UnmarshalHexArray(
442 r, sha256.Size,
443 ); chk.E(err) {
444 return
445 }
446 f.Ids = tag.NewFromBytesSlice(ff...)
447 state = betweenKV
448 case Kinds[0]:
449 if len(key) < len(Kinds) {
450 goto invalid
451 }
452 f.Kinds = kind.NewWithCap(0)
453 if r, err = f.Kinds.Unmarshal(r); chk.E(err) {
454 return
455 }
456 state = betweenKV
457 case Authors[0]:
458 if len(key) < len(Authors) {
459 goto invalid
460 }
461 var ff [][]byte
462 if ff, r, err = text.UnmarshalHexArray(
463 r, schnorr.PubKeyBytesLen,
464 ); chk.E(err) {
465 return
466 }
467 f.Authors = tag.NewFromBytesSlice(ff...)
468 state = betweenKV
469 case Until[0]:
470 if len(key) < len(Until) {
471 goto invalid
472 }
473 u := ints.New(0)
474 if r, err = u.Unmarshal(r); chk.E(err) {
475 return
476 }
477 f.Until = timestamp.FromUnix(int64(u.N))
478 state = betweenKV
479 case Limit[0]:
480 if len(key) < len(Limit) {
481 goto invalid
482 }
483 l := ints.New(0)
484 if r, err = l.Unmarshal(r); chk.E(err) {
485 return
486 }
487 u := uint(l.N)
488 f.Limit = &u
489 state = betweenKV
490 case Search[0]:
491 if len(key) < len(Since) {
492 goto invalid
493 }
494 switch key[1] {
495 case Search[1]:
496 if len(key) < len(Search) {
497 goto invalid
498 }
499 var txt []byte
500 if txt, r, err = text.UnmarshalQuoted(r); chk.E(err) {
501 return
502 }
503 f.Search = txt
504 // log.I.ToSliceOfBytes("\n%s\n%s", txt, rem)
505 state = betweenKV
506 // log.I.Ln("betweenKV")
507 case Since[1]:
508 if len(key) < len(Since) {
509 goto invalid
510 }
511 s := ints.New(0)
512 if r, err = s.Unmarshal(r); chk.E(err) {
513 return
514 }
515 f.Since = timestamp.FromUnix(int64(s.N))
516 state = betweenKV
517 // log.I.Ln("betweenKV")
518 }
519 default:
520 // Per NIP-01, unknown filter fields should be ignored by relays that
521 // don't support them. Store them in Extra for extensions to use.
522 var val []byte
523 if val, r, err = skipJSONValue(r); err != nil {
524 goto invalid
525 }
526 if f.Extra == nil {
527 f.Extra = make(map[string][]byte)
528 }
529 // Store the raw JSON value keyed by the field name
530 f.Extra[string(key)] = val
531 state = betweenKV
532 }
533 key = key[:0]
534 case betweenKV:
535 if len(r) == 0 {
536 return
537 }
538 if r[0] == '}' {
539 state = afterClose
540 // log.I.Ln("afterClose")
541 // rem = rem[1:]
542 } else if r[0] == ',' {
543 state = openParen
544 // log.I.Ln("openParen")
545 } else if r[0] == '"' {
546 state = inKey
547 // log.I.Ln("inKey")
548 }
549 }
550 if len(r) == 0 {
551 return
552 }
553 if r[0] == '}' {
554 r = r[1:]
555 return
556 }
557 }
558 invalid:
559 err = errorf.E("invalid key,\n'%s'\n'%s'", string(b), string(r))
560 return
561 }
562