scanner.mx raw
1 package main
2
3 import (
4 "fmt"
5 "io"
6 "unicode"
7 "unicode/utf8"
8 )
9
10 const (
11 comments uint = 1 << iota
12 directives
13 )
14
15 var sharedUtab *unicode.Tables
16
17 func getSharedUtab() *unicode.Tables {
18 if sharedUtab == nil {
19 sharedUtab = unicode.NewTables()
20 }
21 return sharedUtab
22 }
23
24 type Scanner struct {
25 Source
26 mode uint
27 nlsemi bool
28
29 Line, Col uint32
30 Blank bool
31 Tok Token
32 Lit string
33 Bad bool
34 Kind LitKind
35 Op Operator
36 Prec int32
37
38 keywordMap [1 << 6]Token
39 keywordsReady bool
40 utab *unicode.Tables
41 }
42
43 func (s *Scanner) Init(src io.Reader, errh func(line, col uint32, msg string), mode uint) {
44 s.Source.init(src, errh)
45 s.mode = mode
46 s.nlsemi = false
47 s.utab = getSharedUtab()
48 s.initKeywords()
49 }
50
51 func (s *Scanner) InitBytes(src []byte, errh func(line, col uint32, msg string), mode uint) {
52 s.Source.initBytes(src, errh)
53 s.mode = 0
54 s.nlsemi = false
55 s.utab = getSharedUtab()
56 s.initKeywords()
57 }
58
59 func (s *Scanner) Errorf(format string, args ...interface{}) {
60 s.error(fmt.Sprintf(format, args...))
61 }
62
63 func (s *Scanner) ErrorAtf(offset int32, format string, args ...interface{}) {
64 s.errh(s.line, s.col+uint32(offset), fmt.Sprintf(format, args...))
65 }
66
67 func (s *Scanner) SetLit(kind LitKind, ok bool) {
68 s.nlsemi = true
69 s.Tok = Literal
70 s.Lit = string(s.segmentCopy())
71 s.Bad = !ok
72 s.Kind = kind
73 }
74
75 func (s *Scanner) Next() {
76 nlsemi := s.nlsemi
77 s.nlsemi = false
78
79 redo:
80 s.stop()
81 startLine, startCol := s.pos()
82 for s.ch == ' ' || s.ch == '\t' || s.ch == '\n' && !nlsemi || s.ch == '\r' {
83 s.nextch()
84 }
85
86 s.Line, s.Col = s.pos()
87 s.Blank = s.line > startLine || startCol == Colbase
88 s.start()
89 if IsLetter(s.ch) || s.ch >= utf8.RuneSelf && s.AtIdentChar(true) {
90 s.nextch()
91 s.Ident()
92 return
93 }
94
95 switch s.ch {
96 case -1:
97 if nlsemi {
98 s.Lit = "EOF"
99 s.Tok = Semi
100 break
101 }
102 s.Tok = EOF
103
104 case '\n':
105 s.nextch()
106 s.Lit = "newline"
107 s.Tok = Semi
108
109 case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
110 s.Number(false)
111
112 case '"':
113 s.stdString()
114
115 case '`':
116 s.rawString()
117
118 case '\'':
119 s.rune()
120
121 case '(':
122 s.nextch()
123 s.Tok = Lparen
124
125 case '[':
126 s.nextch()
127 s.Tok = Lbrack
128
129 case '{':
130 s.nextch()
131 s.Tok = Lbrace
132
133 case ',':
134 s.nextch()
135 s.Tok = Comma
136
137 case ';':
138 s.nextch()
139 s.Lit = "semicolon"
140 s.Tok = Semi
141
142 case ')':
143 s.nextch()
144 s.nlsemi = true
145 s.Tok = Rparen
146
147 case ']':
148 s.nextch()
149 s.nlsemi = true
150 s.Tok = Rbrack
151
152 case '}':
153 s.nextch()
154 s.nlsemi = true
155 s.Tok = Rbrace
156
157 case ':':
158 s.nextch()
159 if s.ch == '=' {
160 s.nextch()
161 s.Tok = Define
162 break
163 }
164 s.Tok = Colon
165
166 case '.':
167 s.nextch()
168 if IsDecimal(s.ch) {
169 s.Number(true)
170 break
171 }
172 if s.ch == '.' {
173 s.nextch()
174 if s.ch == '.' {
175 s.nextch()
176 s.Tok = DotDotDot
177 break
178 }
179 s.rewind()
180 s.nextch()
181 }
182 s.Tok = Dot
183
184 case '+':
185 s.nextch()
186 s.Op, s.Prec = Add, PrecAdd
187 if s.ch != '+' {
188 s.assignOp()
189 return
190 }
191 s.nextch()
192 s.nlsemi = true
193 s.Tok = IncOp
194
195 case '-':
196 s.nextch()
197 s.Op, s.Prec = Sub, PrecAdd
198 if s.ch != '-' {
199 s.assignOp()
200 return
201 }
202 s.nextch()
203 s.nlsemi = true
204 s.Tok = IncOp
205
206 case '*':
207 s.nextch()
208 s.Op, s.Prec = Mul, PrecMul
209 if s.ch == '=' {
210 s.nextch()
211 s.Tok = AssignOp
212 break
213 }
214 s.Tok = Star
215
216 case '/':
217 s.nextch()
218 if s.ch == '/' {
219 s.nextch()
220 s.lineComment()
221 goto redo
222 }
223 if s.ch == '*' {
224 s.nextch()
225 s.fullComment()
226 if line, _ := s.pos(); line > s.Line && nlsemi {
227 s.Lit = "newline"
228 s.Tok = Semi
229 break
230 }
231 goto redo
232 }
233 s.Op, s.Prec = Div, PrecMul
234 if s.ch == '=' {
235 s.nextch()
236 s.Tok = AssignOp
237 } else {
238 s.Tok = OperatorType
239 }
240
241 case '%':
242 s.nextch()
243 s.Op, s.Prec = Rem, PrecMul
244 if s.ch == '=' {
245 s.nextch()
246 s.Tok = AssignOp
247 } else {
248 s.Tok = OperatorType
249 }
250
251 case '&':
252 s.nextch()
253 if s.ch == '&' {
254 s.nextch()
255 s.Op, s.Prec = AndAnd, PrecAndAnd
256 s.Tok = OperatorType
257 break
258 }
259 s.Op, s.Prec = And, PrecMul
260 if s.ch == '^' {
261 s.nextch()
262 s.Op = AndNot
263 }
264 if s.ch == '=' {
265 s.nextch()
266 s.Tok = AssignOp
267 } else {
268 s.Tok = OperatorType
269 }
270
271 case '|':
272 s.nextch()
273 if s.ch == '|' {
274 s.nextch()
275 s.Op, s.Prec = OrOr, PrecOrOr
276 s.Tok = OperatorType
277 break
278 }
279 s.Op, s.Prec = Or, PrecAdd
280 if s.ch == '=' {
281 s.nextch()
282 s.Tok = AssignOp
283 } else {
284 s.Tok = OperatorType
285 }
286
287 case '^':
288 s.nextch()
289 s.Op, s.Prec = Xor, PrecAdd
290 if s.ch == '=' {
291 s.nextch()
292 s.Tok = AssignOp
293 } else {
294 s.Tok = OperatorType
295 }
296
297 case '<':
298 s.nextch()
299 if s.ch == '=' {
300 s.nextch()
301 s.Op, s.Prec = Leq, PrecCmp
302 s.Tok = OperatorType
303 break
304 }
305 if s.ch == '<' {
306 s.nextch()
307 s.Op, s.Prec = Shl, PrecMul
308 s.assignOp()
309 return
310 }
311 if s.ch == '-' {
312 s.nextch()
313 s.Tok = Arrow
314 break
315 }
316 s.Op, s.Prec = Lss, PrecCmp
317 s.Tok = OperatorType
318
319 case '>':
320 s.nextch()
321 if s.ch == '=' {
322 s.nextch()
323 s.Op, s.Prec = Geq, PrecCmp
324 s.Tok = OperatorType
325 break
326 }
327 if s.ch == '>' {
328 s.nextch()
329 s.Op, s.Prec = Shr, PrecMul
330 s.assignOp()
331 return
332 }
333 s.Op, s.Prec = Gtr, PrecCmp
334 s.Tok = OperatorType
335
336 case '=':
337 s.nextch()
338 if s.ch == '=' {
339 s.nextch()
340 s.Op, s.Prec = Eql, PrecCmp
341 s.Tok = OperatorType
342 break
343 }
344 s.Tok = Assign
345
346 case '!':
347 s.nextch()
348 if s.ch == '=' {
349 s.nextch()
350 s.Op, s.Prec = Neq, PrecCmp
351 s.Tok = OperatorType
352 break
353 }
354 s.Op, s.Prec = Not, 0
355 s.Tok = OperatorType
356
357 case '~':
358 s.nextch()
359 s.Op, s.Prec = Tilde, 0
360 s.Tok = OperatorType
361
362 default:
363 s.Errorf("invalid character %#U", s.ch)
364 s.nextch()
365 goto redo
366 }
367
368 return
369 }
370
371 func (s *Scanner) assignOp() {
372 if s.ch == '=' {
373 s.nextch()
374 s.Tok = AssignOp
375 return
376 }
377 s.Tok = OperatorType
378 }
379
380 func (s *Scanner) Ident() {
381 for IsLetter(s.ch) || IsDecimal(s.ch) {
382 s.nextch()
383 }
384
385 if s.ch >= utf8.RuneSelf {
386 for s.AtIdentChar(false) {
387 s.nextch()
388 }
389 }
390
391 lit := s.segment()
392 if len(lit) >= 2 {
393 h := (uint(lit[0])<<4 ^ uint(lit[1]) + uint(len(lit))) & 63
394 if tok := s.keywordMap[h]; tok != 0 && tokStrFast(tok) == string(lit) {
395 s.nlsemi = contains(1<<Break|1<<Continue|1<<Fallthrough|1<<Return, tok)
396 s.Tok = tok
397 return
398 }
399 }
400
401 s.nlsemi = true
402 c := []byte{:len(lit)}
403 copy(c, lit)
404 s.Lit = string(c)
405 s.Tok = NameType
406 }
407
408 func tokStrFast(tok Token) string {
409 idx := [48]uint8{0, 3, 7, 14, 16, 19, 23, 24, 26, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 42, 47, 51, 55, 60, 68, 75, 80, 84, 95, 98, 102, 104, 108, 110, 116, 125, 128, 135, 140, 146, 152, 158, 164, 168, 171, 171}
410 return token_name[idx[tok-1]:idx[tok]]
411 }
412
413 func (s *Scanner) AtIdentChar(first bool) bool {
414 switch {
415 case s.utab.IsLetter(s.ch) || s.ch == '_':
416 case s.utab.IsDigit(s.ch):
417 if first {
418 s.Errorf("identifier cannot begin with digit %#U", s.ch)
419 }
420 case s.ch >= utf8.RuneSelf:
421 s.Errorf("invalid character %#U in identifier", s.ch)
422 default:
423 return false
424 }
425 return true
426 }
427
428 func (s *Scanner) initKeywords() {
429 if s.keywordsReady {
430 return
431 }
432 s.keywordsReady = true
433 for tok := Break; tok <= Var; tok++ {
434 b := []byte(tok.String())
435 h := (uint(b[0])<<4 ^ uint(b[1]) + uint(len(b))) & 63
436 if s.keywordMap[h] != 0 {
437 panic("imperfect hash")
438 }
439 s.keywordMap[h] = tok
440 }
441 }
442
443 func Lower(ch rune) rune { return ('a' - 'A') | ch }
444 func IsLetter(ch rune) bool { return 'a' <= Lower(ch) && Lower(ch) <= 'z' || ch == '_' }
445 func IsDecimal(ch rune) bool { return '0' <= ch && ch <= '9' }
446 func IsHex(ch rune) bool { return '0' <= ch && ch <= '9' || 'a' <= Lower(ch) && Lower(ch) <= 'f' }
447
448 func (s *Scanner) Digits(base int32, invalid *int32) (digsep int32) {
449 if base <= 10 {
450 max := rune('0' + base)
451 for IsDecimal(s.ch) || s.ch == '_' {
452 ds := int32(1)
453 if s.ch == '_' {
454 ds = 2
455 } else if s.ch >= max && *invalid < 0 {
456 _, col := s.pos()
457 *invalid = int32(col - s.col)
458 }
459 digsep |= ds
460 s.nextch()
461 }
462 } else {
463 for IsHex(s.ch) || s.ch == '_' {
464 ds := int32(1)
465 if s.ch == '_' {
466 ds = 2
467 }
468 digsep |= ds
469 s.nextch()
470 }
471 }
472 return
473 }
474
475 func (s *Scanner) Number(seenPoint bool) {
476 ok := true
477 kind := IntLit
478 base := int32(10)
479 prefix := rune(0)
480 digsep := int32(0)
481 invalid := int32(-1)
482
483 if !seenPoint {
484 if s.ch == '0' {
485 s.nextch()
486 switch Lower(s.ch) {
487 case 'x':
488 s.nextch()
489 base, prefix = 16, 'x'
490 case 'o':
491 s.nextch()
492 base, prefix = 8, 'o'
493 case 'b':
494 s.nextch()
495 base, prefix = 2, 'b'
496 default:
497 base, prefix = 8, '0'
498 digsep = 1
499 }
500 }
501 digsep |= s.Digits(base, &invalid)
502 if s.ch == '.' {
503 if prefix == 'o' || prefix == 'b' {
504 s.Errorf("invalid radix point in %s literal", baseName(base))
505 ok = false
506 }
507 s.nextch()
508 seenPoint = true
509 }
510 }
511
512 if seenPoint {
513 kind = FloatLit
514 digsep |= s.Digits(base, &invalid)
515 }
516
517 if digsep&1 == 0 && ok {
518 s.Errorf("%s literal has no digits", baseName(base))
519 ok = false
520 }
521
522 if e := Lower(s.ch); e == 'e' || e == 'p' {
523 if ok {
524 switch {
525 case e == 'e' && prefix != 0 && prefix != '0':
526 s.Errorf("%q exponent requires decimal mantissa", s.ch)
527 ok = false
528 case e == 'p' && prefix != 'x':
529 s.Errorf("%q exponent requires hexadecimal mantissa", s.ch)
530 ok = false
531 }
532 }
533 s.nextch()
534 kind = FloatLit
535 if s.ch == '+' || s.ch == '-' {
536 s.nextch()
537 }
538 digsep = s.Digits(10, nil) | digsep&2
539 if digsep&1 == 0 && ok {
540 s.Errorf("exponent has no digits")
541 ok = false
542 }
543 } else if prefix == 'x' && kind == FloatLit && ok {
544 s.Errorf("hexadecimal mantissa requires a 'p' exponent")
545 ok = false
546 }
547
548 if s.ch == 'i' {
549 kind = ImagLit
550 s.nextch()
551 }
552
553 s.SetLit(kind, ok)
554
555 if kind == IntLit && invalid >= 0 && ok {
556 s.ErrorAtf(invalid, "invalid digit %q in %s literal", s.Lit[invalid], baseName(base))
557 ok = false
558 }
559
560 if digsep&2 != 0 && ok {
561 if i := invalidSep(s.Lit); i >= 0 {
562 s.ErrorAtf(i, "'_' must separate successive digits")
563 ok = false
564 }
565 }
566
567 s.Bad = !ok
568 }
569
570 func baseName(base int32) string {
571 switch base {
572 case 2:
573 return "binary"
574 case 8:
575 return "octal"
576 case 10:
577 return "decimal"
578 case 16:
579 return "hexadecimal"
580 }
581 panic("invalid base")
582 }
583
584 func invalidSep(x string) int32 {
585 x1 := ' '
586 d := '.'
587 i := int32(0)
588
589 if len(x) >= 2 && x[0] == '0' {
590 x1 = Lower(rune(x[1]))
591 if x1 == 'x' || x1 == 'o' || x1 == 'b' {
592 d = '0'
593 i = 2
594 }
595 }
596
597 for ; i < int32(len(x)); i++ {
598 p := d
599 d = rune(x[i])
600 switch {
601 case d == '_':
602 if p != '0' {
603 return i
604 }
605 case IsDecimal(d) || x1 == 'x' && IsHex(d):
606 d = '0'
607 default:
608 if p == '_' {
609 return i - 1
610 }
611 d = '.'
612 }
613 }
614 if d == '_' {
615 return int32(len(x)) - 1
616 }
617
618 return -1
619 }
620
621 func (s *Scanner) rune() {
622 ok := true
623 s.nextch()
624
625 n := 0
626 for ; ; n++ {
627 if s.ch == '\'' {
628 if ok {
629 if n == 0 {
630 s.Errorf("empty rune literal or unescaped '")
631 ok = false
632 } else if n != 1 {
633 s.ErrorAtf(0, "more than one character in rune literal")
634 ok = false
635 }
636 }
637 s.nextch()
638 break
639 }
640 if s.ch == '\\' {
641 s.nextch()
642 if !s.escape('\'') {
643 ok = false
644 }
645 continue
646 }
647 if s.ch == '\n' {
648 if ok {
649 s.Errorf("newline in rune literal")
650 ok = false
651 }
652 break
653 }
654 if s.ch < 0 {
655 if ok {
656 s.ErrorAtf(0, "rune literal not terminated")
657 ok = false
658 }
659 break
660 }
661 s.nextch()
662 }
663
664 s.SetLit(RuneLit, ok)
665 }
666
667 func (s *Scanner) stdString() {
668 ok := true
669 s.nextch()
670
671 for {
672 if s.ch == '"' {
673 s.nextch()
674 break
675 }
676 if s.ch == '\\' {
677 s.nextch()
678 if !s.escape('"') {
679 ok = false
680 }
681 continue
682 }
683 if s.ch == '\n' {
684 s.Errorf("newline in string")
685 ok = false
686 break
687 }
688 if s.ch < 0 {
689 s.ErrorAtf(0, "string not terminated")
690 ok = false
691 break
692 }
693 s.nextch()
694 }
695
696 s.SetLit(StringLit, ok)
697 }
698
699 func (s *Scanner) rawString() {
700 ok := true
701 s.nextch()
702
703 for {
704 if s.ch == '`' {
705 s.nextch()
706 break
707 }
708 if s.ch < 0 {
709 s.ErrorAtf(0, "string not terminated")
710 ok = false
711 break
712 }
713 s.nextch()
714 }
715
716 s.SetLit(StringLit, ok)
717 }
718
719 func (s *Scanner) comment(text string) {
720 s.ErrorAtf(0, "%s", text)
721 }
722
723 func (s *Scanner) skipLine() {
724 for s.ch >= 0 && s.ch != '\n' {
725 s.nextch()
726 }
727 }
728
729 func (s *Scanner) lineComment() {
730 if s.mode&comments != 0 {
731 s.skipLine()
732 s.comment(string(s.segment()))
733 return
734 }
735
736 if s.mode&directives == 0 || (s.ch != 'g' && s.ch != 'l') {
737 s.stop()
738 s.skipLine()
739 return
740 }
741
742 prefix := "go:"
743 if s.ch == 'l' {
744 prefix = "line "
745 }
746
747 for _, r := range prefix {
748 if s.ch != rune(r) {
749 s.stop()
750 s.skipLine()
751 return
752 }
753 s.nextch()
754 }
755 s.skipLine()
756 s.comment(string(s.segment()))
757 }
758
759 func (s *Scanner) skipComment() bool {
760 for s.ch >= 0 {
761 for s.ch == '*' {
762 s.nextch()
763 if s.ch == '/' {
764 s.nextch()
765 return true
766 }
767 }
768 s.nextch()
769 }
770 s.ErrorAtf(0, "comment not terminated")
771 return false
772 }
773
774 func (s *Scanner) fullComment() {
775 if s.mode&comments != 0 {
776 if s.skipComment() {
777 s.comment(string(s.segment()))
778 }
779 return
780 }
781
782 if s.mode&directives == 0 || s.ch != 'l' {
783 s.stop()
784 s.skipComment()
785 return
786 }
787
788 const prefix = "line "
789
790 for _, r := range prefix {
791 if s.ch != rune(r) {
792 s.stop()
793 s.skipComment()
794 return
795 }
796 s.nextch()
797 }
798 if s.skipComment() {
799 s.comment(string(s.segment()))
800 }
801 }
802
803 func (s *Scanner) escape(quote rune) bool {
804 var n int32
805 var base, max uint32
806
807 switch s.ch {
808 case quote, 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\':
809 s.nextch()
810 return true
811 case '0', '1', '2', '3', '4', '5', '6', '7':
812 n, base, max = 3, 8, 255
813 case 'x':
814 s.nextch()
815 n, base, max = 2, 16, 255
816 case 'u':
817 s.nextch()
818 n, base, max = 4, 16, unicode.MaxRune
819 case 'U':
820 s.nextch()
821 n, base, max = 8, 16, unicode.MaxRune
822 default:
823 if s.ch < 0 {
824 return true
825 }
826 s.Errorf("unknown escape")
827 return false
828 }
829
830 var x uint32
831 for i := n; i > 0; i-- {
832 if s.ch < 0 {
833 return true
834 }
835 d := base
836 if IsDecimal(s.ch) {
837 d = uint32(s.ch) - '0'
838 } else if 'a' <= Lower(s.ch) && Lower(s.ch) <= 'f' {
839 d = uint32(Lower(s.ch)) - 'a' + 10
840 }
841 if d >= base {
842 s.Errorf("invalid character %q in %s escape", s.ch, baseName(int32(base)))
843 return false
844 }
845 x = x*base + d
846 s.nextch()
847 }
848
849 if x > max && base == 8 {
850 s.Errorf("octal escape value %d > 255", x)
851 return false
852 }
853
854 if x > max || 0xD800 <= x && x < 0xE000 {
855 s.Errorf("escape is invalid Unicode code point %#U", x)
856 return false
857 }
858
859 return true
860 }
861
862 func String(n Node) string {
863 return fmt.Sprintf("%T", n)
864 }
865
866 func StartPos(n Node) Pos {
867 return n.Pos()
868 }
869