lexer.go raw
1 // Package jlexer contains a JSON lexer implementation.
2 //
3 // It is expected that it is mostly used with generated parser code, so the interface is tuned
4 // for a parser that knows what kind of data is expected.
5 package jlexer
6
7 import (
8 "bytes"
9 "encoding/base64"
10 "encoding/json"
11 "errors"
12 "fmt"
13 "io"
14 "strconv"
15 "unicode"
16 "unicode/utf16"
17 "unicode/utf8"
18
19 "github.com/josharian/intern"
20 )
21
22 // TokenKind determines type of a token.
23 type TokenKind byte
24
25 const (
26 TokenUndef TokenKind = iota // No token.
27 TokenDelim // Delimiter: one of '{', '}', '[' or ']'.
28 TokenString // A string literal, e.g. "abc\u1234"
29 TokenNumber // Number literal, e.g. 1.5e5
30 TokenBool // Boolean literal: true or false.
31 TokenNull // null keyword.
32 )
33
34 // token describes a single token: type, position in the input and value.
35 type token struct {
36 kind TokenKind // Type of a token.
37
38 boolValue bool // Value if a boolean literal token.
39 byteValueCloned bool // true if byteValue was allocated and does not refer to original json body
40 byteValue []byte // Raw value of a token.
41 delimValue byte
42 }
43
44 // Lexer is a JSON lexer: it iterates over JSON tokens in a byte slice.
45 type Lexer struct {
46 Data []byte // Input data given to the lexer.
47
48 start int // Start of the current token.
49 pos int // Current unscanned position in the input stream.
50 token token // Last scanned token, if token.kind != TokenUndef.
51
52 firstElement bool // Whether current element is the first in array or an object.
53 wantSep byte // A comma or a colon character, which need to occur before a token.
54
55 UseMultipleErrors bool // If we want to use multiple errors.
56 fatalError error // Fatal error occurred during lexing. It is usually a syntax error.
57 multipleErrors []*LexerError // Semantic errors occurred during lexing. Marshalling will be continued after finding this errors.
58 }
59
60 // FetchToken scans the input for the next token.
61 func (r *Lexer) FetchToken() {
62 r.token.kind = TokenUndef
63 r.start = r.pos
64
65 // Check if r.Data has r.pos element
66 // If it doesn't, it mean corrupted input data
67 if len(r.Data) < r.pos {
68 r.errParse("Unexpected end of data")
69 return
70 }
71 // Determine the type of a token by skipping whitespace and reading the
72 // first character.
73 for _, c := range r.Data[r.pos:] {
74 switch c {
75 case ':', ',':
76 if r.wantSep == c {
77 r.pos++
78 r.start++
79 r.wantSep = 0
80 } else {
81 r.errSyntax()
82 }
83
84 case ' ', '\t', '\r', '\n':
85 r.pos++
86 r.start++
87
88 case '"':
89 if r.wantSep != 0 {
90 r.errSyntax()
91 }
92
93 r.token.kind = TokenString
94 r.fetchString()
95 return
96
97 case '{', '[':
98 if r.wantSep != 0 {
99 r.errSyntax()
100 }
101 r.firstElement = true
102 r.token.kind = TokenDelim
103 r.token.delimValue = r.Data[r.pos]
104 r.pos++
105 return
106
107 case '}', ']':
108 if !r.firstElement && (r.wantSep != ',') {
109 r.errSyntax()
110 }
111 r.wantSep = 0
112 r.token.kind = TokenDelim
113 r.token.delimValue = r.Data[r.pos]
114 r.pos++
115 return
116
117 case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-':
118 if r.wantSep != 0 {
119 r.errSyntax()
120 }
121 r.token.kind = TokenNumber
122 r.fetchNumber()
123 return
124
125 case 'n':
126 if r.wantSep != 0 {
127 r.errSyntax()
128 }
129
130 r.token.kind = TokenNull
131 r.fetchNull()
132 return
133
134 case 't':
135 if r.wantSep != 0 {
136 r.errSyntax()
137 }
138
139 r.token.kind = TokenBool
140 r.token.boolValue = true
141 r.fetchTrue()
142 return
143
144 case 'f':
145 if r.wantSep != 0 {
146 r.errSyntax()
147 }
148
149 r.token.kind = TokenBool
150 r.token.boolValue = false
151 r.fetchFalse()
152 return
153
154 default:
155 r.errSyntax()
156 return
157 }
158 }
159 r.fatalError = io.EOF
160 return
161 }
162
163 // isTokenEnd returns true if the char can follow a non-delimiter token
164 func isTokenEnd(c byte) bool {
165 return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '[' || c == ']' || c == '{' || c == '}' || c == ',' || c == ':'
166 }
167
168 // fetchNull fetches and checks remaining bytes of null keyword.
169 func (r *Lexer) fetchNull() {
170 r.pos += 4
171 if r.pos > len(r.Data) ||
172 r.Data[r.pos-3] != 'u' ||
173 r.Data[r.pos-2] != 'l' ||
174 r.Data[r.pos-1] != 'l' ||
175 (r.pos != len(r.Data) && !isTokenEnd(r.Data[r.pos])) {
176
177 r.pos -= 4
178 r.errSyntax()
179 }
180 }
181
182 // fetchTrue fetches and checks remaining bytes of true keyword.
183 func (r *Lexer) fetchTrue() {
184 r.pos += 4
185 if r.pos > len(r.Data) ||
186 r.Data[r.pos-3] != 'r' ||
187 r.Data[r.pos-2] != 'u' ||
188 r.Data[r.pos-1] != 'e' ||
189 (r.pos != len(r.Data) && !isTokenEnd(r.Data[r.pos])) {
190
191 r.pos -= 4
192 r.errSyntax()
193 }
194 }
195
196 // fetchFalse fetches and checks remaining bytes of false keyword.
197 func (r *Lexer) fetchFalse() {
198 r.pos += 5
199 if r.pos > len(r.Data) ||
200 r.Data[r.pos-4] != 'a' ||
201 r.Data[r.pos-3] != 'l' ||
202 r.Data[r.pos-2] != 's' ||
203 r.Data[r.pos-1] != 'e' ||
204 (r.pos != len(r.Data) && !isTokenEnd(r.Data[r.pos])) {
205
206 r.pos -= 5
207 r.errSyntax()
208 }
209 }
210
211 // fetchNumber scans a number literal token.
212 func (r *Lexer) fetchNumber() {
213 hasE := false
214 afterE := false
215 hasDot := false
216
217 r.pos++
218 for i, c := range r.Data[r.pos:] {
219 switch {
220 case c >= '0' && c <= '9':
221 afterE = false
222 case c == '.' && !hasDot:
223 hasDot = true
224 case (c == 'e' || c == 'E') && !hasE:
225 hasE = true
226 hasDot = true
227 afterE = true
228 case (c == '+' || c == '-') && afterE:
229 afterE = false
230 default:
231 r.pos += i
232 if !isTokenEnd(c) {
233 r.errSyntax()
234 } else {
235 r.token.byteValue = r.Data[r.start:r.pos]
236 }
237 return
238 }
239 }
240
241 r.pos = len(r.Data)
242 r.token.byteValue = r.Data[r.start:]
243 }
244
245 // findStringLen tries to scan into the string literal for ending quote char to determine required size.
246 // The size will be exact if no escapes are present and may be inexact if there are escaped chars.
247 func findStringLen(data []byte) (isValid bool, length int) {
248 for {
249 idx := bytes.IndexByte(data, '"')
250 if idx == -1 {
251 return false, len(data)
252 }
253 if idx == 0 || (idx > 0 && data[idx-1] != '\\') {
254 return true, length + idx
255 }
256
257 // count \\\\\\\ sequences. even number of slashes means quote is not really escaped
258 cnt := 1
259 for idx-cnt-1 >= 0 && data[idx-cnt-1] == '\\' {
260 cnt++
261 }
262 if cnt%2 == 0 {
263 return true, length + idx
264 }
265
266 length += idx + 1
267 data = data[idx+1:]
268 }
269 }
270
271 // unescapeStringToken performs unescaping of string token.
272 // if no escaping is needed, original string is returned, otherwise - a new one allocated
273 func (r *Lexer) unescapeStringToken() (err error) {
274 data := r.token.byteValue
275 var unescapedData []byte
276
277 for {
278 i := bytes.IndexByte(data, '\\')
279 if i == -1 {
280 break
281 }
282
283 escapedRune, escapedBytes, err := decodeEscape(data[i:])
284 if err != nil {
285 r.errParse(err.Error())
286 return err
287 }
288
289 if unescapedData == nil {
290 unescapedData = make([]byte, 0, len(r.token.byteValue))
291 }
292
293 var d [4]byte
294 s := utf8.EncodeRune(d[:], escapedRune)
295 unescapedData = append(unescapedData, data[:i]...)
296 unescapedData = append(unescapedData, d[:s]...)
297
298 data = data[i+escapedBytes:]
299 }
300
301 if unescapedData != nil {
302 r.token.byteValue = append(unescapedData, data...)
303 r.token.byteValueCloned = true
304 }
305 return
306 }
307
308 // getu4 decodes \uXXXX from the beginning of s, returning the hex value,
309 // or it returns -1.
310 func getu4(s []byte) rune {
311 if len(s) < 6 || s[0] != '\\' || s[1] != 'u' {
312 return -1
313 }
314 var val rune
315 for i := 2; i < len(s) && i < 6; i++ {
316 var v byte
317 c := s[i]
318 switch c {
319 case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
320 v = c - '0'
321 case 'a', 'b', 'c', 'd', 'e', 'f':
322 v = c - 'a' + 10
323 case 'A', 'B', 'C', 'D', 'E', 'F':
324 v = c - 'A' + 10
325 default:
326 return -1
327 }
328
329 val <<= 4
330 val |= rune(v)
331 }
332 return val
333 }
334
335 // decodeEscape processes a single escape sequence and returns number of bytes processed.
336 func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) {
337 if len(data) < 2 {
338 return 0, 0, errors.New("incorrect escape symbol \\ at the end of token")
339 }
340
341 c := data[1]
342 switch c {
343 case '"', '/', '\\':
344 return rune(c), 2, nil
345 case 'b':
346 return '\b', 2, nil
347 case 'f':
348 return '\f', 2, nil
349 case 'n':
350 return '\n', 2, nil
351 case 'r':
352 return '\r', 2, nil
353 case 't':
354 return '\t', 2, nil
355 case 'u':
356 rr := getu4(data)
357 if rr < 0 {
358 return 0, 0, errors.New("incorrectly escaped \\uXXXX sequence")
359 }
360
361 read := 6
362 if utf16.IsSurrogate(rr) {
363 rr1 := getu4(data[read:])
364 if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar {
365 read += 6
366 rr = dec
367 } else {
368 rr = unicode.ReplacementChar
369 }
370 }
371 return rr, read, nil
372 }
373
374 return 0, 0, errors.New("incorrectly escaped bytes")
375 }
376
377 // fetchString scans a string literal token.
378 func (r *Lexer) fetchString() {
379 r.pos++
380 data := r.Data[r.pos:]
381
382 isValid, length := findStringLen(data)
383 if !isValid {
384 r.pos += length
385 r.errParse("unterminated string literal")
386 return
387 }
388 r.token.byteValue = data[:length]
389 r.pos += length + 1 // skip closing '"' as well
390 }
391
392 // scanToken scans the next token if no token is currently available in the lexer.
393 func (r *Lexer) scanToken() {
394 if r.token.kind != TokenUndef || r.fatalError != nil {
395 return
396 }
397
398 r.FetchToken()
399 }
400
401 // consume resets the current token to allow scanning the next one.
402 func (r *Lexer) consume() {
403 r.token.kind = TokenUndef
404 r.token.byteValueCloned = false
405 r.token.delimValue = 0
406 }
407
408 // Ok returns true if no error (including io.EOF) was encountered during scanning.
409 func (r *Lexer) Ok() bool {
410 return r.fatalError == nil
411 }
412
413 const maxErrorContextLen = 13
414
415 func (r *Lexer) errParse(what string) {
416 if r.fatalError == nil {
417 var str string
418 if len(r.Data)-r.pos <= maxErrorContextLen {
419 str = string(r.Data)
420 } else {
421 str = string(r.Data[r.pos:r.pos+maxErrorContextLen-3]) + "..."
422 }
423 r.fatalError = &LexerError{
424 Reason: what,
425 Offset: r.pos,
426 Data: str,
427 }
428 }
429 }
430
431 func (r *Lexer) errSyntax() {
432 r.errParse("syntax error")
433 }
434
435 func (r *Lexer) errInvalidToken(expected string) {
436 if r.fatalError != nil {
437 return
438 }
439 if r.UseMultipleErrors {
440 r.pos = r.start
441 r.consume()
442 r.SkipRecursive()
443 switch expected {
444 case "[":
445 r.token.delimValue = ']'
446 r.token.kind = TokenDelim
447 case "{":
448 r.token.delimValue = '}'
449 r.token.kind = TokenDelim
450 }
451 r.addNonfatalError(&LexerError{
452 Reason: fmt.Sprintf("expected %s", expected),
453 Offset: r.start,
454 Data: string(r.Data[r.start:r.pos]),
455 })
456 return
457 }
458
459 var str string
460 if len(r.token.byteValue) <= maxErrorContextLen {
461 str = string(r.token.byteValue)
462 } else {
463 str = string(r.token.byteValue[:maxErrorContextLen-3]) + "..."
464 }
465 r.fatalError = &LexerError{
466 Reason: fmt.Sprintf("expected %s", expected),
467 Offset: r.pos,
468 Data: str,
469 }
470 }
471
472 func (r *Lexer) GetPos() int {
473 return r.pos
474 }
475
476 // Delim consumes a token and verifies that it is the given delimiter.
477 func (r *Lexer) Delim(c byte) {
478 if r.token.kind == TokenUndef && r.Ok() {
479 r.FetchToken()
480 }
481
482 if !r.Ok() || r.token.delimValue != c {
483 r.consume() // errInvalidToken can change token if UseMultipleErrors is enabled.
484 r.errInvalidToken(string([]byte{c}))
485 } else {
486 r.consume()
487 }
488 }
489
490 // IsDelim returns true if there was no scanning error and next token is the given delimiter.
491 func (r *Lexer) IsDelim(c byte) bool {
492 if r.token.kind == TokenUndef && r.Ok() {
493 r.FetchToken()
494 }
495 return !r.Ok() || r.token.delimValue == c
496 }
497
498 // Null verifies that the next token is null and consumes it.
499 func (r *Lexer) Null() {
500 if r.token.kind == TokenUndef && r.Ok() {
501 r.FetchToken()
502 }
503 if !r.Ok() || r.token.kind != TokenNull {
504 r.errInvalidToken("null")
505 }
506 r.consume()
507 }
508
509 // IsNull returns true if the next token is a null keyword.
510 func (r *Lexer) IsNull() bool {
511 if r.token.kind == TokenUndef && r.Ok() {
512 r.FetchToken()
513 }
514 return r.Ok() && r.token.kind == TokenNull
515 }
516
517 // Skip skips a single token.
518 func (r *Lexer) Skip() {
519 if r.token.kind == TokenUndef && r.Ok() {
520 r.FetchToken()
521 }
522 r.consume()
523 }
524
525 // SkipRecursive skips next array or object completely, or just skips a single token if not
526 // an array/object.
527 //
528 // Note: no syntax validation is performed on the skipped data.
529 func (r *Lexer) SkipRecursive() {
530 r.scanToken()
531 var start, end byte
532 startPos := r.start
533
534 switch r.token.delimValue {
535 case '{':
536 start, end = '{', '}'
537 case '[':
538 start, end = '[', ']'
539 default:
540 r.consume()
541 return
542 }
543
544 r.consume()
545
546 level := 1
547 inQuotes := false
548 wasEscape := false
549
550 for i, c := range r.Data[r.pos:] {
551 switch {
552 case c == start && !inQuotes:
553 level++
554 case c == end && !inQuotes:
555 level--
556 if level == 0 {
557 r.pos += i + 1
558 if !json.Valid(r.Data[startPos:r.pos]) {
559 r.pos = len(r.Data)
560 r.fatalError = &LexerError{
561 Reason: "skipped array/object json value is invalid",
562 Offset: r.pos,
563 Data: string(r.Data[r.pos:]),
564 }
565 }
566 return
567 }
568 case c == '\\' && inQuotes:
569 wasEscape = !wasEscape
570 continue
571 case c == '"' && inQuotes:
572 inQuotes = wasEscape
573 case c == '"':
574 inQuotes = true
575 }
576 wasEscape = false
577 }
578 r.pos = len(r.Data)
579 r.fatalError = &LexerError{
580 Reason: "EOF reached while skipping array/object or token",
581 Offset: r.pos,
582 Data: string(r.Data[r.pos:]),
583 }
584 }
585
586 // Raw fetches the next item recursively as a data slice
587 func (r *Lexer) Raw() []byte {
588 r.SkipRecursive()
589 if !r.Ok() {
590 return nil
591 }
592 return r.Data[r.start:r.pos]
593 }
594
595 // IsStart returns whether the lexer is positioned at the start
596 // of an input string.
597 func (r *Lexer) IsStart() bool {
598 return r.pos == 0
599 }
600
601 // Consumed reads all remaining bytes from the input, publishing an error if
602 // there is anything but whitespace remaining.
603 func (r *Lexer) Consumed() {
604 if r.pos > len(r.Data) || !r.Ok() {
605 return
606 }
607
608 for _, c := range r.Data[r.pos:] {
609 if c != ' ' && c != '\t' && c != '\r' && c != '\n' {
610 r.AddError(&LexerError{
611 Reason: "invalid character '" + string(c) + "' after top-level value",
612 Offset: r.pos,
613 Data: string(r.Data[r.pos:]),
614 })
615 return
616 }
617
618 r.pos++
619 r.start++
620 }
621 }
622
623 func (r *Lexer) unsafeString(skipUnescape bool) (string, []byte) {
624 if r.token.kind == TokenUndef && r.Ok() {
625 r.FetchToken()
626 }
627 if !r.Ok() || r.token.kind != TokenString {
628 r.errInvalidToken("string")
629 return "", nil
630 }
631 if !skipUnescape {
632 if err := r.unescapeStringToken(); err != nil {
633 r.errInvalidToken("string")
634 return "", nil
635 }
636 }
637
638 bytes := r.token.byteValue
639 ret := bytesToStr(r.token.byteValue)
640 r.consume()
641 return ret, bytes
642 }
643
644 // UnsafeString returns the string value if the token is a string literal.
645 //
646 // Warning: returned string may point to the input buffer, so the string should not outlive
647 // the input buffer. Intended pattern of usage is as an argument to a switch statement.
648 func (r *Lexer) UnsafeString() string {
649 ret, _ := r.unsafeString(false)
650 return ret
651 }
652
653 // UnsafeBytes returns the byte slice if the token is a string literal.
654 func (r *Lexer) UnsafeBytes() []byte {
655 _, ret := r.unsafeString(false)
656 return ret
657 }
658
659 // UnsafeFieldName returns current member name string token
660 func (r *Lexer) UnsafeFieldName(skipUnescape bool) string {
661 ret, _ := r.unsafeString(skipUnescape)
662 return ret
663 }
664
665 // String reads a string literal.
666 func (r *Lexer) String() string {
667 if r.token.kind == TokenUndef && r.Ok() {
668 r.FetchToken()
669 }
670 if !r.Ok() || r.token.kind != TokenString {
671 r.errInvalidToken("string")
672 return ""
673 }
674 if err := r.unescapeStringToken(); err != nil {
675 r.errInvalidToken("string")
676 return ""
677 }
678 var ret string
679 if r.token.byteValueCloned {
680 ret = bytesToStr(r.token.byteValue)
681 } else {
682 ret = string(r.token.byteValue)
683 }
684 r.consume()
685 return ret
686 }
687
688 // StringIntern reads a string literal, and performs string interning on it.
689 func (r *Lexer) StringIntern() string {
690 if r.token.kind == TokenUndef && r.Ok() {
691 r.FetchToken()
692 }
693 if !r.Ok() || r.token.kind != TokenString {
694 r.errInvalidToken("string")
695 return ""
696 }
697 if err := r.unescapeStringToken(); err != nil {
698 r.errInvalidToken("string")
699 return ""
700 }
701 ret := intern.Bytes(r.token.byteValue)
702 r.consume()
703 return ret
704 }
705
706 // Bytes reads a string literal and base64 decodes it into a byte slice.
707 func (r *Lexer) Bytes() []byte {
708 if r.token.kind == TokenUndef && r.Ok() {
709 r.FetchToken()
710 }
711 if !r.Ok() || r.token.kind != TokenString {
712 r.errInvalidToken("string")
713 return nil
714 }
715 if err := r.unescapeStringToken(); err != nil {
716 r.errInvalidToken("string")
717 return nil
718 }
719 ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue)))
720 n, err := base64.StdEncoding.Decode(ret, r.token.byteValue)
721 if err != nil {
722 r.fatalError = &LexerError{
723 Reason: err.Error(),
724 }
725 return nil
726 }
727
728 r.consume()
729 return ret[:n]
730 }
731
732 // Bool reads a true or false boolean keyword.
733 func (r *Lexer) Bool() bool {
734 if r.token.kind == TokenUndef && r.Ok() {
735 r.FetchToken()
736 }
737 if !r.Ok() || r.token.kind != TokenBool {
738 r.errInvalidToken("bool")
739 return false
740 }
741 ret := r.token.boolValue
742 r.consume()
743 return ret
744 }
745
746 func (r *Lexer) number() string {
747 if r.token.kind == TokenUndef && r.Ok() {
748 r.FetchToken()
749 }
750 if !r.Ok() || r.token.kind != TokenNumber {
751 r.errInvalidToken("number")
752 return ""
753 }
754 ret := bytesToStr(r.token.byteValue)
755 r.consume()
756 return ret
757 }
758
759 func (r *Lexer) Uint8() uint8 {
760 s := r.number()
761 if !r.Ok() {
762 return 0
763 }
764
765 n, err := strconv.ParseUint(s, 10, 8)
766 if err != nil {
767 r.addNonfatalError(&LexerError{
768 Offset: r.start,
769 Reason: err.Error(),
770 Data: s,
771 })
772 }
773 return uint8(n)
774 }
775
776 func (r *Lexer) Uint16() uint16 {
777 s := r.number()
778 if !r.Ok() {
779 return 0
780 }
781
782 n, err := strconv.ParseUint(s, 10, 16)
783 if err != nil {
784 r.addNonfatalError(&LexerError{
785 Offset: r.start,
786 Reason: err.Error(),
787 Data: s,
788 })
789 }
790 return uint16(n)
791 }
792
793 func (r *Lexer) Uint32() uint32 {
794 s := r.number()
795 if !r.Ok() {
796 return 0
797 }
798
799 n, err := strconv.ParseUint(s, 10, 32)
800 if err != nil {
801 r.addNonfatalError(&LexerError{
802 Offset: r.start,
803 Reason: err.Error(),
804 Data: s,
805 })
806 }
807 return uint32(n)
808 }
809
810 func (r *Lexer) Uint64() uint64 {
811 s := r.number()
812 if !r.Ok() {
813 return 0
814 }
815
816 n, err := strconv.ParseUint(s, 10, 64)
817 if err != nil {
818 r.addNonfatalError(&LexerError{
819 Offset: r.start,
820 Reason: err.Error(),
821 Data: s,
822 })
823 }
824 return n
825 }
826
827 func (r *Lexer) Uint() uint {
828 return uint(r.Uint64())
829 }
830
831 func (r *Lexer) Int8() int8 {
832 s := r.number()
833 if !r.Ok() {
834 return 0
835 }
836
837 n, err := strconv.ParseInt(s, 10, 8)
838 if err != nil {
839 r.addNonfatalError(&LexerError{
840 Offset: r.start,
841 Reason: err.Error(),
842 Data: s,
843 })
844 }
845 return int8(n)
846 }
847
848 func (r *Lexer) Int16() int16 {
849 s := r.number()
850 if !r.Ok() {
851 return 0
852 }
853
854 n, err := strconv.ParseInt(s, 10, 16)
855 if err != nil {
856 r.addNonfatalError(&LexerError{
857 Offset: r.start,
858 Reason: err.Error(),
859 Data: s,
860 })
861 }
862 return int16(n)
863 }
864
865 func (r *Lexer) Int32() int32 {
866 s := r.number()
867 if !r.Ok() {
868 return 0
869 }
870
871 n, err := strconv.ParseInt(s, 10, 32)
872 if err != nil {
873 r.addNonfatalError(&LexerError{
874 Offset: r.start,
875 Reason: err.Error(),
876 Data: s,
877 })
878 }
879 return int32(n)
880 }
881
882 func (r *Lexer) Int64() int64 {
883 s := r.number()
884 if !r.Ok() {
885 return 0
886 }
887
888 n, err := strconv.ParseInt(s, 10, 64)
889 if err != nil {
890 r.addNonfatalError(&LexerError{
891 Offset: r.start,
892 Reason: err.Error(),
893 Data: s,
894 })
895 }
896 return n
897 }
898
899 func (r *Lexer) Int() int {
900 return int(r.Int64())
901 }
902
903 func (r *Lexer) Uint8Str() uint8 {
904 s, b := r.unsafeString(false)
905 if !r.Ok() {
906 return 0
907 }
908
909 n, err := strconv.ParseUint(s, 10, 8)
910 if err != nil {
911 r.addNonfatalError(&LexerError{
912 Offset: r.start,
913 Reason: err.Error(),
914 Data: string(b),
915 })
916 }
917 return uint8(n)
918 }
919
920 func (r *Lexer) Uint16Str() uint16 {
921 s, b := r.unsafeString(false)
922 if !r.Ok() {
923 return 0
924 }
925
926 n, err := strconv.ParseUint(s, 10, 16)
927 if err != nil {
928 r.addNonfatalError(&LexerError{
929 Offset: r.start,
930 Reason: err.Error(),
931 Data: string(b),
932 })
933 }
934 return uint16(n)
935 }
936
937 func (r *Lexer) Uint32Str() uint32 {
938 s, b := r.unsafeString(false)
939 if !r.Ok() {
940 return 0
941 }
942
943 n, err := strconv.ParseUint(s, 10, 32)
944 if err != nil {
945 r.addNonfatalError(&LexerError{
946 Offset: r.start,
947 Reason: err.Error(),
948 Data: string(b),
949 })
950 }
951 return uint32(n)
952 }
953
954 func (r *Lexer) Uint64Str() uint64 {
955 s, b := r.unsafeString(false)
956 if !r.Ok() {
957 return 0
958 }
959
960 n, err := strconv.ParseUint(s, 10, 64)
961 if err != nil {
962 r.addNonfatalError(&LexerError{
963 Offset: r.start,
964 Reason: err.Error(),
965 Data: string(b),
966 })
967 }
968 return n
969 }
970
971 func (r *Lexer) UintStr() uint {
972 return uint(r.Uint64Str())
973 }
974
975 func (r *Lexer) UintptrStr() uintptr {
976 return uintptr(r.Uint64Str())
977 }
978
979 func (r *Lexer) Int8Str() int8 {
980 s, b := r.unsafeString(false)
981 if !r.Ok() {
982 return 0
983 }
984
985 n, err := strconv.ParseInt(s, 10, 8)
986 if err != nil {
987 r.addNonfatalError(&LexerError{
988 Offset: r.start,
989 Reason: err.Error(),
990 Data: string(b),
991 })
992 }
993 return int8(n)
994 }
995
996 func (r *Lexer) Int16Str() int16 {
997 s, b := r.unsafeString(false)
998 if !r.Ok() {
999 return 0
1000 }
1001
1002 n, err := strconv.ParseInt(s, 10, 16)
1003 if err != nil {
1004 r.addNonfatalError(&LexerError{
1005 Offset: r.start,
1006 Reason: err.Error(),
1007 Data: string(b),
1008 })
1009 }
1010 return int16(n)
1011 }
1012
1013 func (r *Lexer) Int32Str() int32 {
1014 s, b := r.unsafeString(false)
1015 if !r.Ok() {
1016 return 0
1017 }
1018
1019 n, err := strconv.ParseInt(s, 10, 32)
1020 if err != nil {
1021 r.addNonfatalError(&LexerError{
1022 Offset: r.start,
1023 Reason: err.Error(),
1024 Data: string(b),
1025 })
1026 }
1027 return int32(n)
1028 }
1029
1030 func (r *Lexer) Int64Str() int64 {
1031 s, b := r.unsafeString(false)
1032 if !r.Ok() {
1033 return 0
1034 }
1035
1036 n, err := strconv.ParseInt(s, 10, 64)
1037 if err != nil {
1038 r.addNonfatalError(&LexerError{
1039 Offset: r.start,
1040 Reason: err.Error(),
1041 Data: string(b),
1042 })
1043 }
1044 return n
1045 }
1046
1047 func (r *Lexer) IntStr() int {
1048 return int(r.Int64Str())
1049 }
1050
1051 func (r *Lexer) Float32() float32 {
1052 s := r.number()
1053 if !r.Ok() {
1054 return 0
1055 }
1056
1057 n, err := strconv.ParseFloat(s, 32)
1058 if err != nil {
1059 r.addNonfatalError(&LexerError{
1060 Offset: r.start,
1061 Reason: err.Error(),
1062 Data: s,
1063 })
1064 }
1065 return float32(n)
1066 }
1067
1068 func (r *Lexer) Float32Str() float32 {
1069 s, b := r.unsafeString(false)
1070 if !r.Ok() {
1071 return 0
1072 }
1073 n, err := strconv.ParseFloat(s, 32)
1074 if err != nil {
1075 r.addNonfatalError(&LexerError{
1076 Offset: r.start,
1077 Reason: err.Error(),
1078 Data: string(b),
1079 })
1080 }
1081 return float32(n)
1082 }
1083
1084 func (r *Lexer) Float64() float64 {
1085 s := r.number()
1086 if !r.Ok() {
1087 return 0
1088 }
1089
1090 n, err := strconv.ParseFloat(s, 64)
1091 if err != nil {
1092 r.addNonfatalError(&LexerError{
1093 Offset: r.start,
1094 Reason: err.Error(),
1095 Data: s,
1096 })
1097 }
1098 return n
1099 }
1100
1101 func (r *Lexer) Float64Str() float64 {
1102 s, b := r.unsafeString(false)
1103 if !r.Ok() {
1104 return 0
1105 }
1106 n, err := strconv.ParseFloat(s, 64)
1107 if err != nil {
1108 r.addNonfatalError(&LexerError{
1109 Offset: r.start,
1110 Reason: err.Error(),
1111 Data: string(b),
1112 })
1113 }
1114 return n
1115 }
1116
1117 func (r *Lexer) Error() error {
1118 return r.fatalError
1119 }
1120
1121 func (r *Lexer) AddError(e error) {
1122 if r.fatalError == nil {
1123 r.fatalError = e
1124 }
1125 }
1126
1127 func (r *Lexer) AddNonFatalError(e error) {
1128 r.addNonfatalError(&LexerError{
1129 Offset: r.start,
1130 Data: string(r.Data[r.start:r.pos]),
1131 Reason: e.Error(),
1132 })
1133 }
1134
1135 func (r *Lexer) addNonfatalError(err *LexerError) {
1136 if r.UseMultipleErrors {
1137 // We don't want to add errors with the same offset.
1138 if len(r.multipleErrors) != 0 && r.multipleErrors[len(r.multipleErrors)-1].Offset == err.Offset {
1139 return
1140 }
1141 r.multipleErrors = append(r.multipleErrors, err)
1142 return
1143 }
1144 r.fatalError = err
1145 }
1146
1147 func (r *Lexer) GetNonFatalErrors() []*LexerError {
1148 return r.multipleErrors
1149 }
1150
1151 // JsonNumber fetches and json.Number from 'encoding/json' package.
1152 // Both int, float or string, contains them are valid values
1153 func (r *Lexer) JsonNumber() json.Number {
1154 if r.token.kind == TokenUndef && r.Ok() {
1155 r.FetchToken()
1156 }
1157 if !r.Ok() {
1158 r.errInvalidToken("json.Number")
1159 return json.Number("")
1160 }
1161
1162 switch r.token.kind {
1163 case TokenString:
1164 return json.Number(r.String())
1165 case TokenNumber:
1166 return json.Number(r.Raw())
1167 case TokenNull:
1168 r.Null()
1169 return json.Number("")
1170 default:
1171 r.errSyntax()
1172 return json.Number("")
1173 }
1174 }
1175
1176 // Interface fetches an interface{} analogous to the 'encoding/json' package.
1177 func (r *Lexer) Interface() interface{} {
1178 if r.token.kind == TokenUndef && r.Ok() {
1179 r.FetchToken()
1180 }
1181
1182 if !r.Ok() {
1183 return nil
1184 }
1185 switch r.token.kind {
1186 case TokenString:
1187 return r.String()
1188 case TokenNumber:
1189 return r.Float64()
1190 case TokenBool:
1191 return r.Bool()
1192 case TokenNull:
1193 r.Null()
1194 return nil
1195 }
1196
1197 if r.token.delimValue == '{' {
1198 r.consume()
1199
1200 ret := map[string]interface{}{}
1201 for !r.IsDelim('}') {
1202 key := r.String()
1203 r.WantColon()
1204 ret[key] = r.Interface()
1205 r.WantComma()
1206 }
1207 r.Delim('}')
1208
1209 if r.Ok() {
1210 return ret
1211 } else {
1212 return nil
1213 }
1214 } else if r.token.delimValue == '[' {
1215 r.consume()
1216
1217 ret := []interface{}{}
1218 for !r.IsDelim(']') {
1219 ret = append(ret, r.Interface())
1220 r.WantComma()
1221 }
1222 r.Delim(']')
1223
1224 if r.Ok() {
1225 return ret
1226 } else {
1227 return nil
1228 }
1229 }
1230 r.errSyntax()
1231 return nil
1232 }
1233
1234 // WantComma requires a comma to be present before fetching next token.
1235 func (r *Lexer) WantComma() {
1236 r.wantSep = ','
1237 r.firstElement = false
1238 }
1239
1240 // WantColon requires a colon to be present before fetching next token.
1241 func (r *Lexer) WantColon() {
1242 r.wantSep = ':'
1243 r.firstElement = false
1244 }
1245
1246 // CurrentToken returns current token kind if there were no errors and TokenUndef otherwise
1247 func (r *Lexer) CurrentToken() TokenKind {
1248 if r.token.kind == TokenUndef && r.Ok() {
1249 r.FetchToken()
1250 }
1251
1252 if !r.Ok() {
1253 return TokenUndef
1254 }
1255
1256 return r.token.kind
1257 }
1258