scanner.mx raw
1 package main
2
3 import (
4 "fmt"
5 "io"
6 "unicode"
7 "unicode/utf8"
8 )
9
10 const (
11 comments uint32 = 1 << iota
12 directives
13 )
14
15 var sharedUtab *unicode.Tables
16
17 func getSharedUtab() *unicode.Tables {
18 if sharedUtab == nil {
19 sharedUtab = unicode.NewTables()
20 }
21 return sharedUtab
22 }
23
24 type Scanner struct {
25 Source
26 mode uint32
27 nlsemi bool
28
29 Line, Col uint32
30 Blank bool
31 Tok Token
32 Lit string
33 Bad bool
34 Kind LitKind
35 Op Operator
36 Prec int32
37 utab *unicode.Tables
38 }
39
40 func (s *Scanner) Init(src io.Reader, errh func(line, col uint32, msg string), mode uint32) {
41 s.Source.init(src, errh)
42 s.mode = mode
43 s.nlsemi = false
44 s.utab = getSharedUtab()
45 }
46
47 func (s *Scanner) Errorf(format string, args ...interface{}) {
48 s.error(fmt.Sprintf(format, args...))
49 }
50
51 func (s *Scanner) ErrorAtf(offset int32, format string, args ...interface{}) {
52 s.errh(s.line, s.col+uint32(offset), fmt.Sprintf(format, args...))
53 }
54
55 func (s *Scanner) SetLit(kind LitKind, ok bool) {
56 s.nlsemi = true
57 s.Tok = Literal
58 s.Lit = string(s.segment())
59 s.Bad = !ok
60 s.Kind = kind
61 }
62
63 func (s *Scanner) Next() {
64 nlsemi := s.nlsemi
65 s.nlsemi = false
66
67 redo:
68 s.stop()
69 startLine, startCol := s.pos()
70 for s.ch == ' ' || s.ch == '\t' || s.ch == '\n' && !nlsemi || s.ch == '\r' {
71 s.nextch()
72 }
73
74 s.Line, s.Col = s.pos()
75 s.Blank = s.line > startLine || startCol == Colbase
76 s.start()
77 if IsLetter(s.ch) || s.ch >= utf8.RuneSelf && s.AtIdentChar(true) {
78 s.nextch()
79 s.Ident()
80 return
81 }
82
83 switch s.ch {
84 case -1:
85 if nlsemi {
86 s.Lit = "EOF"
87 s.Tok = Semi
88 break
89 }
90 s.Tok = EOF
91
92 case '\n':
93 s.nextch()
94 s.Lit = "newline"
95 s.Tok = Semi
96
97 case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
98 s.Number(false)
99
100 case '"':
101 s.stdString()
102
103 case '`':
104 s.rawString()
105
106 case '\'':
107 s.rune()
108
109 case '(':
110 s.nextch()
111 s.Tok = Lparen
112
113 case '[':
114 s.nextch()
115 s.Tok = Lbrack
116
117 case '{':
118 s.nextch()
119 s.Tok = Lbrace
120
121 case ',':
122 s.nextch()
123 s.Tok = Comma
124
125 case ';':
126 s.nextch()
127 s.Lit = "semicolon"
128 s.Tok = Semi
129
130 case ')':
131 s.nextch()
132 s.nlsemi = true
133 s.Tok = Rparen
134
135 case ']':
136 s.nextch()
137 s.nlsemi = true
138 s.Tok = Rbrack
139
140 case '}':
141 s.nextch()
142 s.nlsemi = true
143 s.Tok = Rbrace
144
145 case ':':
146 s.nextch()
147 if s.ch == '=' {
148 s.nextch()
149 s.Tok = Define
150 break
151 }
152 s.Tok = Colon
153
154 case '.':
155 s.nextch()
156 if IsDecimal(s.ch) {
157 s.Number(true)
158 break
159 }
160 if s.ch == '.' {
161 s.nextch()
162 if s.ch == '.' {
163 s.nextch()
164 s.Tok = DotDotDot
165 break
166 }
167 s.rewind()
168 s.nextch()
169 }
170 s.Tok = Dot
171
172 case '+':
173 s.nextch()
174 s.Op, s.Prec = Add, PrecAdd
175 if s.ch != '+' {
176 s.assignOp()
177 return
178 }
179 s.nextch()
180 s.nlsemi = true
181 s.Tok = IncOp
182
183 case '-':
184 s.nextch()
185 s.Op, s.Prec = Sub, PrecAdd
186 if s.ch != '-' {
187 s.assignOp()
188 return
189 }
190 s.nextch()
191 s.nlsemi = true
192 s.Tok = IncOp
193
194 case '*':
195 s.nextch()
196 s.Op, s.Prec = Mul, PrecMul
197 if s.ch == '=' {
198 s.nextch()
199 s.Tok = AssignOp
200 break
201 }
202 s.Tok = Star
203
204 case '/':
205 s.nextch()
206 if s.ch == '/' {
207 s.nextch()
208 s.lineComment()
209 goto redo
210 }
211 if s.ch == '*' {
212 s.nextch()
213 s.fullComment()
214 if line, _ := s.pos(); line > s.Line && nlsemi {
215 s.Lit = "newline"
216 s.Tok = Semi
217 break
218 }
219 goto redo
220 }
221 s.Op, s.Prec = Div, PrecMul
222 if s.ch == '=' {
223 s.nextch()
224 s.Tok = AssignOp
225 } else {
226 s.Tok = OperatorType
227 }
228
229 case '%':
230 s.nextch()
231 s.Op, s.Prec = Rem, PrecMul
232 if s.ch == '=' {
233 s.nextch()
234 s.Tok = AssignOp
235 } else {
236 s.Tok = OperatorType
237 }
238
239 case '&':
240 s.nextch()
241 if s.ch == '&' {
242 s.nextch()
243 s.Op, s.Prec = AndAnd, PrecAndAnd
244 s.Tok = OperatorType
245 break
246 }
247 s.Op, s.Prec = And, PrecMul
248 if s.ch == '^' {
249 s.nextch()
250 s.Op = AndNot
251 }
252 if s.ch == '=' {
253 s.nextch()
254 s.Tok = AssignOp
255 } else {
256 s.Tok = OperatorType
257 }
258
259 case '|':
260 s.nextch()
261 if s.ch == '|' {
262 s.nextch()
263 s.Op, s.Prec = OrOr, PrecOrOr
264 s.Tok = OperatorType
265 break
266 }
267 s.Op, s.Prec = Or, PrecAdd
268 if s.ch == '=' {
269 s.nextch()
270 s.Tok = AssignOp
271 } else {
272 s.Tok = OperatorType
273 }
274
275 case '^':
276 s.nextch()
277 s.Op, s.Prec = Xor, PrecAdd
278 if s.ch == '=' {
279 s.nextch()
280 s.Tok = AssignOp
281 } else {
282 s.Tok = OperatorType
283 }
284
285 case '<':
286 s.nextch()
287 if s.ch == '=' {
288 s.nextch()
289 s.Op, s.Prec = Leq, PrecCmp
290 s.Tok = OperatorType
291 break
292 }
293 if s.ch == '<' {
294 s.nextch()
295 s.Op, s.Prec = Shl, PrecMul
296 s.assignOp()
297 return
298 }
299 if s.ch == '-' {
300 s.nextch()
301 s.Tok = Arrow
302 break
303 }
304 s.Op, s.Prec = Lss, PrecCmp
305 s.Tok = OperatorType
306
307 case '>':
308 s.nextch()
309 if s.ch == '=' {
310 s.nextch()
311 s.Op, s.Prec = Geq, PrecCmp
312 s.Tok = OperatorType
313 break
314 }
315 if s.ch == '>' {
316 s.nextch()
317 s.Op, s.Prec = Shr, PrecMul
318 s.assignOp()
319 return
320 }
321 s.Op, s.Prec = Gtr, PrecCmp
322 s.Tok = OperatorType
323
324 case '=':
325 s.nextch()
326 if s.ch == '=' {
327 s.nextch()
328 s.Op, s.Prec = Eql, PrecCmp
329 s.Tok = OperatorType
330 break
331 }
332 s.Tok = Assign
333
334 case '!':
335 s.nextch()
336 if s.ch == '=' {
337 s.nextch()
338 s.Op, s.Prec = Neq, PrecCmp
339 s.Tok = OperatorType
340 break
341 }
342 s.Op, s.Prec = Not, 0
343 s.Tok = OperatorType
344
345 case '~':
346 s.nextch()
347 s.Op, s.Prec = Tilde, 0
348 s.Tok = OperatorType
349
350 default:
351 s.Errorf("invalid character %#U", s.ch)
352 s.nextch()
353 goto redo
354 }
355
356 return
357 }
358
359 func (s *Scanner) assignOp() {
360 if s.ch == '=' {
361 s.nextch()
362 s.Tok = AssignOp
363 return
364 }
365 s.Tok = OperatorType
366 }
367
368 func (s *Scanner) Ident() {
369 for IsLetter(s.ch) || IsDecimal(s.ch) {
370 s.nextch()
371 }
372
373 if s.ch >= utf8.RuneSelf {
374 for s.AtIdentChar(false) {
375 s.nextch()
376 }
377 }
378
379 lit := s.segment()
380 if len(lit) >= 2 {
381 if tok := keywordMap[Hash(lit)]; tok != 0 && TokStrFast(tok) == string(lit) {
382 s.nlsemi = contains(1<<Break|1<<Continue|1<<Fallthrough|1<<Return, tok)
383 s.Tok = tok
384 return
385 }
386 }
387
388 s.nlsemi = true
389 s.Lit = string(lit)
390 s.Tok = NameType
391 }
392
393 func TokStrFast(tok Token) string {
394 return token_name[token_index[tok-1]:token_index[tok]]
395 }
396
397 func (s *Scanner) AtIdentChar(first bool) bool {
398 switch {
399 case s.utab.IsLetter(s.ch) || s.ch == '_':
400 case s.utab.IsDigit(s.ch):
401 if first {
402 s.Errorf("identifier cannot begin with digit %#U", s.ch)
403 }
404 case s.ch >= utf8.RuneSelf:
405 s.Errorf("invalid character %#U in identifier", s.ch)
406 default:
407 return false
408 }
409 return true
410 }
411
412 func Hash(s []byte) uint32 {
413 return (uint32(s[0])<<4 ^ uint32(s[1]) + uint32(len(s))) & uint32(len(keywordMap)-1)
414 }
415
416 var keywordMap [1 << 6]Token
417
418 var keywordsInitialized bool
419
420 func InitKeywords() {
421 if keywordsInitialized {
422 return
423 }
424 keywordsInitialized = true
425 for tok := Break; tok <= Var; tok++ {
426 h := Hash([]byte(tok.String()))
427 if keywordMap[h] != 0 {
428 panic("imperfect hash")
429 }
430 keywordMap[h] = tok
431 }
432 }
433
434 func Lower(ch rune) rune { return ('a' - 'A') | ch }
435 func IsLetter(ch rune) bool { return 'a' <= Lower(ch) && Lower(ch) <= 'z' || ch == '_' }
436 func IsDecimal(ch rune) bool { return '0' <= ch && ch <= '9' }
437 func IsHex(ch rune) bool { return '0' <= ch && ch <= '9' || 'a' <= Lower(ch) && Lower(ch) <= 'f' }
438
439 func (s *Scanner) Digits(base int32, invalid *int32) (digsep int32) {
440 if base <= 10 {
441 max := rune('0' + base)
442 for IsDecimal(s.ch) || s.ch == '_' {
443 ds := int32(1)
444 if s.ch == '_' {
445 ds = 2
446 } else if s.ch >= max && *invalid < 0 {
447 _, col := s.pos()
448 *invalid = int32(col - s.col)
449 }
450 digsep |= ds
451 s.nextch()
452 }
453 } else {
454 for IsHex(s.ch) || s.ch == '_' {
455 ds := int32(1)
456 if s.ch == '_' {
457 ds = 2
458 }
459 digsep |= ds
460 s.nextch()
461 }
462 }
463 return
464 }
465
466 func (s *Scanner) Number(seenPoint bool) {
467 ok := true
468 kind := IntLit
469 base := int32(10)
470 prefix := rune(0)
471 digsep := int32(0)
472 invalid := int32(-1)
473
474 if !seenPoint {
475 if s.ch == '0' {
476 s.nextch()
477 switch Lower(s.ch) {
478 case 'x':
479 s.nextch()
480 base, prefix = 16, 'x'
481 case 'o':
482 s.nextch()
483 base, prefix = 8, 'o'
484 case 'b':
485 s.nextch()
486 base, prefix = 2, 'b'
487 default:
488 base, prefix = 8, '0'
489 digsep = 1
490 }
491 }
492 digsep |= s.Digits(base, &invalid)
493 if s.ch == '.' {
494 if prefix == 'o' || prefix == 'b' {
495 s.Errorf("invalid radix point in %s literal", baseName(base))
496 ok = false
497 }
498 s.nextch()
499 seenPoint = true
500 }
501 }
502
503 if seenPoint {
504 kind = FloatLit
505 digsep |= s.Digits(base, &invalid)
506 }
507
508 if digsep&1 == 0 && ok {
509 s.Errorf("%s literal has no digits", baseName(base))
510 ok = false
511 }
512
513 if e := Lower(s.ch); e == 'e' || e == 'p' {
514 if ok {
515 switch {
516 case e == 'e' && prefix != 0 && prefix != '0':
517 s.Errorf("%q exponent requires decimal mantissa", s.ch)
518 ok = false
519 case e == 'p' && prefix != 'x':
520 s.Errorf("%q exponent requires hexadecimal mantissa", s.ch)
521 ok = false
522 }
523 }
524 s.nextch()
525 kind = FloatLit
526 if s.ch == '+' || s.ch == '-' {
527 s.nextch()
528 }
529 digsep = s.Digits(10, nil) | digsep&2
530 if digsep&1 == 0 && ok {
531 s.Errorf("exponent has no digits")
532 ok = false
533 }
534 } else if prefix == 'x' && kind == FloatLit && ok {
535 s.Errorf("hexadecimal mantissa requires a 'p' exponent")
536 ok = false
537 }
538
539 if s.ch == 'i' {
540 kind = ImagLit
541 s.nextch()
542 }
543
544 s.SetLit(kind, ok)
545
546 if kind == IntLit && invalid >= 0 && ok {
547 s.ErrorAtf(invalid, "invalid digit %q in %s literal", s.Lit[invalid], baseName(base))
548 ok = false
549 }
550
551 if digsep&2 != 0 && ok {
552 if i := invalidSep(s.Lit); i >= 0 {
553 s.ErrorAtf(i, "'_' must separate successive digits")
554 ok = false
555 }
556 }
557
558 s.Bad = !ok
559 }
560
561 func baseName(base int32) string {
562 switch base {
563 case 2:
564 return "binary"
565 case 8:
566 return "octal"
567 case 10:
568 return "decimal"
569 case 16:
570 return "hexadecimal"
571 }
572 panic("invalid base")
573 }
574
575 func invalidSep(x string) int32 {
576 x1 := ' '
577 d := '.'
578 i := int32(0)
579
580 if len(x) >= 2 && x[0] == '0' {
581 x1 = Lower(rune(x[1]))
582 if x1 == 'x' || x1 == 'o' || x1 == 'b' {
583 d = '0'
584 i = 2
585 }
586 }
587
588 for ; i < int32(len(x)); i++ {
589 p := d
590 d = rune(x[i])
591 switch {
592 case d == '_':
593 if p != '0' {
594 return i
595 }
596 case IsDecimal(d) || x1 == 'x' && IsHex(d):
597 d = '0'
598 default:
599 if p == '_' {
600 return i - 1
601 }
602 d = '.'
603 }
604 }
605 if d == '_' {
606 return int32(len(x)) - 1
607 }
608
609 return -1
610 }
611
612 func (s *Scanner) rune() {
613 ok := true
614 s.nextch()
615
616 n := 0
617 for ; ; n++ {
618 if s.ch == '\'' {
619 if ok {
620 if n == 0 {
621 s.Errorf("empty rune literal or unescaped '")
622 ok = false
623 } else if n != 1 {
624 s.ErrorAtf(0, "more than one character in rune literal")
625 ok = false
626 }
627 }
628 s.nextch()
629 break
630 }
631 if s.ch == '\\' {
632 s.nextch()
633 if !s.escape('\'') {
634 ok = false
635 }
636 continue
637 }
638 if s.ch == '\n' {
639 if ok {
640 s.Errorf("newline in rune literal")
641 ok = false
642 }
643 break
644 }
645 if s.ch < 0 {
646 if ok {
647 s.ErrorAtf(0, "rune literal not terminated")
648 ok = false
649 }
650 break
651 }
652 s.nextch()
653 }
654
655 s.SetLit(RuneLit, ok)
656 }
657
658 func (s *Scanner) stdString() {
659 ok := true
660 s.nextch()
661
662 for {
663 if s.ch == '"' {
664 s.nextch()
665 break
666 }
667 if s.ch == '\\' {
668 s.nextch()
669 if !s.escape('"') {
670 ok = false
671 }
672 continue
673 }
674 if s.ch == '\n' {
675 s.Errorf("newline in string")
676 ok = false
677 break
678 }
679 if s.ch < 0 {
680 s.ErrorAtf(0, "string not terminated")
681 ok = false
682 break
683 }
684 s.nextch()
685 }
686
687 s.SetLit(StringLit, ok)
688 }
689
690 func (s *Scanner) rawString() {
691 ok := true
692 s.nextch()
693
694 for {
695 if s.ch == '`' {
696 s.nextch()
697 break
698 }
699 if s.ch < 0 {
700 s.ErrorAtf(0, "string not terminated")
701 ok = false
702 break
703 }
704 s.nextch()
705 }
706
707 s.SetLit(StringLit, ok)
708 }
709
710 func (s *Scanner) comment(text string) {
711 s.ErrorAtf(0, "%s", text)
712 }
713
714 func (s *Scanner) skipLine() {
715 for s.ch >= 0 && s.ch != '\n' {
716 s.nextch()
717 }
718 }
719
720 func (s *Scanner) lineComment() {
721 if s.mode&comments != 0 {
722 s.skipLine()
723 s.comment(string(s.segment()))
724 return
725 }
726
727 if s.mode&directives == 0 || (s.ch != 'g' && s.ch != 'l') {
728 s.stop()
729 s.skipLine()
730 return
731 }
732
733 prefix := "go:"
734 if s.ch == 'l' {
735 prefix = "line "
736 }
737
738 for _, r := range prefix {
739 if s.ch != rune(r) {
740 s.stop()
741 s.skipLine()
742 return
743 }
744 s.nextch()
745 }
746 s.skipLine()
747 s.comment(string(s.segment()))
748 }
749
750 func (s *Scanner) skipComment() bool {
751 for s.ch >= 0 {
752 for s.ch == '*' {
753 s.nextch()
754 if s.ch == '/' {
755 s.nextch()
756 return true
757 }
758 }
759 s.nextch()
760 }
761 s.ErrorAtf(0, "comment not terminated")
762 return false
763 }
764
765 func (s *Scanner) fullComment() {
766 if s.mode&comments != 0 {
767 if s.skipComment() {
768 s.comment(string(s.segment()))
769 }
770 return
771 }
772
773 if s.mode&directives == 0 || s.ch != 'l' {
774 s.stop()
775 s.skipComment()
776 return
777 }
778
779 const prefix = "line "
780
781 for _, r := range prefix {
782 if s.ch != rune(r) {
783 s.stop()
784 s.skipComment()
785 return
786 }
787 s.nextch()
788 }
789 if s.skipComment() {
790 s.comment(string(s.segment()))
791 }
792 }
793
794 func (s *Scanner) escape(quote rune) bool {
795 var n int32
796 var base, max uint32
797
798 switch s.ch {
799 case quote, 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\':
800 s.nextch()
801 return true
802 case '0', '1', '2', '3', '4', '5', '6', '7':
803 n, base, max = 3, 8, 255
804 case 'x':
805 s.nextch()
806 n, base, max = 2, 16, 255
807 case 'u':
808 s.nextch()
809 n, base, max = 4, 16, unicode.MaxRune
810 case 'U':
811 s.nextch()
812 n, base, max = 8, 16, unicode.MaxRune
813 default:
814 if s.ch < 0 {
815 return true
816 }
817 s.Errorf("unknown escape")
818 return false
819 }
820
821 var x uint32
822 for i := n; i > 0; i-- {
823 if s.ch < 0 {
824 return true
825 }
826 d := base
827 if IsDecimal(s.ch) {
828 d = uint32(s.ch) - '0'
829 } else if 'a' <= Lower(s.ch) && Lower(s.ch) <= 'f' {
830 d = uint32(Lower(s.ch)) - 'a' + 10
831 }
832 if d >= base {
833 s.Errorf("invalid character %q in %s escape", s.ch, baseName(int32(base)))
834 return false
835 }
836 x = x*base + d
837 s.nextch()
838 }
839
840 if x > max && base == 8 {
841 s.Errorf("octal escape value %d > 255", x)
842 return false
843 }
844
845 if x > max || 0xD800 <= x && x < 0xE000 {
846 s.Errorf("escape is invalid Unicode code point %#U", x)
847 return false
848 }
849
850 return true
851 }
852
853 func String(n Node) string {
854 return fmt.Sprintf("%T", n)
855 }
856
857 func StartPos(n Node) Pos {
858 return n.Pos()
859 }
860