scanner.mx raw
1 package main
2
3 import (
4 "fmt"
5 "io"
6 "unicode"
7 "unicode/utf8"
8 )
9
10 const (
11 comments uint32 = 1 << iota
12 directives
13 )
14
15 var sharedUtab *unicode.Tables
16
17 func getSharedUtab() *unicode.Tables {
18 if sharedUtab == nil {
19 sharedUtab = unicode.NewTables()
20 }
21 return sharedUtab
22 }
23
24 type Scanner struct {
25 Source
26 mode uint32
27 nlsemi bool
28
29 Line, Col uint32
30 Blank bool
31 Tok Token
32 Lit string
33 Bad bool
34 Kind LitKind
35 Op Operator
36 Prec int32
37 utab *unicode.Tables
38 }
39
40 func (s *Scanner) Init(src io.Reader, errh func(line, col uint32, msg string), mode uint32) {
41 s.Source.init(src, errh)
42 s.mode = mode
43 s.nlsemi = false
44 s.utab = getSharedUtab()
45 }
46
47 func (s *Scanner) Errorf(format string, args ...interface{}) {
48 s.error(fmt.Sprintf(format, args...))
49 }
50
51 func (s *Scanner) ErrorAtf(offset int32, format string, args ...interface{}) {
52 s.errh(s.line, s.col+uint32(offset), fmt.Sprintf(format, args...))
53 }
54
55 func (s *Scanner) SetLit(kind LitKind, ok bool) {
56 s.nlsemi = true
57 s.Tok = Literal
58 s.Lit = string(s.segmentCopy())
59 s.Bad = !ok
60 s.Kind = kind
61 }
62
63 func (s *Scanner) Next() {
64 nlsemi := s.nlsemi
65 s.nlsemi = false
66
67 redo:
68 s.stop()
69 startLine, startCol := s.pos()
70 for s.ch == ' ' || s.ch == '\t' || s.ch == '\n' && !nlsemi || s.ch == '\r' {
71 s.nextch()
72 }
73
74 s.Line, s.Col = s.pos()
75 s.Blank = s.line > startLine || startCol == Colbase
76 s.start()
77 if IsLetter(s.ch) || s.ch >= utf8.RuneSelf && s.AtIdentChar(true) {
78 s.nextch()
79 s.Ident()
80 return
81 }
82
83 switch s.ch {
84 case -1:
85 if nlsemi {
86 s.Lit = "EOF"
87 s.Tok = Semi
88 break
89 }
90 s.Tok = EOF
91
92 case '\n':
93 s.nextch()
94 s.Lit = "newline"
95 s.Tok = Semi
96
97 case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
98 s.Number(false)
99
100 case '"':
101 s.stdString()
102
103 case '`':
104 s.rawString()
105
106 case '\'':
107 s.rune()
108
109 case '(':
110 s.nextch()
111 s.Tok = Lparen
112
113 case '[':
114 s.nextch()
115 s.Tok = Lbrack
116
117 case '{':
118 s.nextch()
119 s.Tok = Lbrace
120
121 case ',':
122 s.nextch()
123 s.Tok = Comma
124
125 case ';':
126 s.nextch()
127 s.Lit = "semicolon"
128 s.Tok = Semi
129
130 case ')':
131 s.nextch()
132 s.nlsemi = true
133 s.Tok = Rparen
134
135 case ']':
136 s.nextch()
137 s.nlsemi = true
138 s.Tok = Rbrack
139
140 case '}':
141 s.nextch()
142 s.nlsemi = true
143 s.Tok = Rbrace
144
145 case ':':
146 s.nextch()
147 if s.ch == '=' {
148 s.nextch()
149 s.Tok = Define
150 break
151 }
152 s.Tok = Colon
153
154 case '.':
155 s.nextch()
156 if IsDecimal(s.ch) {
157 s.Number(true)
158 break
159 }
160 if s.ch == '.' {
161 s.nextch()
162 if s.ch == '.' {
163 s.nextch()
164 s.Tok = DotDotDot
165 break
166 }
167 s.rewind()
168 s.nextch()
169 }
170 s.Tok = Dot
171
172 case '+':
173 s.nextch()
174 s.Op, s.Prec = Add, PrecAdd
175 if s.ch != '+' {
176 s.assignOp()
177 return
178 }
179 s.nextch()
180 s.nlsemi = true
181 s.Tok = IncOp
182
183 case '-':
184 s.nextch()
185 s.Op, s.Prec = Sub, PrecAdd
186 if s.ch != '-' {
187 s.assignOp()
188 return
189 }
190 s.nextch()
191 s.nlsemi = true
192 s.Tok = IncOp
193
194 case '*':
195 s.nextch()
196 s.Op, s.Prec = Mul, PrecMul
197 if s.ch == '=' {
198 s.nextch()
199 s.Tok = AssignOp
200 break
201 }
202 s.Tok = Star
203
204 case '/':
205 s.nextch()
206 if s.ch == '/' {
207 s.nextch()
208 s.lineComment()
209 goto redo
210 }
211 if s.ch == '*' {
212 s.nextch()
213 s.fullComment()
214 if line, _ := s.pos(); line > s.Line && nlsemi {
215 s.Lit = "newline"
216 s.Tok = Semi
217 break
218 }
219 goto redo
220 }
221 s.Op, s.Prec = Div, PrecMul
222 if s.ch == '=' {
223 s.nextch()
224 s.Tok = AssignOp
225 } else {
226 s.Tok = OperatorType
227 }
228
229 case '%':
230 s.nextch()
231 s.Op, s.Prec = Rem, PrecMul
232 if s.ch == '=' {
233 s.nextch()
234 s.Tok = AssignOp
235 } else {
236 s.Tok = OperatorType
237 }
238
239 case '&':
240 s.nextch()
241 if s.ch == '&' {
242 s.nextch()
243 s.Op, s.Prec = AndAnd, PrecAndAnd
244 s.Tok = OperatorType
245 break
246 }
247 s.Op, s.Prec = And, PrecMul
248 if s.ch == '^' {
249 s.nextch()
250 s.Op = AndNot
251 }
252 if s.ch == '=' {
253 s.nextch()
254 s.Tok = AssignOp
255 } else {
256 s.Tok = OperatorType
257 }
258
259 case '|':
260 s.nextch()
261 if s.ch == '|' {
262 s.nextch()
263 s.Op, s.Prec = OrOr, PrecOrOr
264 s.Tok = OperatorType
265 break
266 }
267 s.Op, s.Prec = Or, PrecAdd
268 if s.ch == '=' {
269 s.nextch()
270 s.Tok = AssignOp
271 } else {
272 s.Tok = OperatorType
273 }
274
275 case '^':
276 s.nextch()
277 s.Op, s.Prec = Xor, PrecAdd
278 if s.ch == '=' {
279 s.nextch()
280 s.Tok = AssignOp
281 } else {
282 s.Tok = OperatorType
283 }
284
285 case '<':
286 s.nextch()
287 if s.ch == '=' {
288 s.nextch()
289 s.Op, s.Prec = Leq, PrecCmp
290 s.Tok = OperatorType
291 break
292 }
293 if s.ch == '<' {
294 s.nextch()
295 s.Op, s.Prec = Shl, PrecMul
296 s.assignOp()
297 return
298 }
299 if s.ch == '-' {
300 s.nextch()
301 s.Tok = Arrow
302 break
303 }
304 s.Op, s.Prec = Lss, PrecCmp
305 s.Tok = OperatorType
306
307 case '>':
308 s.nextch()
309 if s.ch == '=' {
310 s.nextch()
311 s.Op, s.Prec = Geq, PrecCmp
312 s.Tok = OperatorType
313 break
314 }
315 if s.ch == '>' {
316 s.nextch()
317 s.Op, s.Prec = Shr, PrecMul
318 s.assignOp()
319 return
320 }
321 s.Op, s.Prec = Gtr, PrecCmp
322 s.Tok = OperatorType
323
324 case '=':
325 s.nextch()
326 if s.ch == '=' {
327 s.nextch()
328 s.Op, s.Prec = Eql, PrecCmp
329 s.Tok = OperatorType
330 break
331 }
332 s.Tok = Assign
333
334 case '!':
335 s.nextch()
336 if s.ch == '=' {
337 s.nextch()
338 s.Op, s.Prec = Neq, PrecCmp
339 s.Tok = OperatorType
340 break
341 }
342 s.Op, s.Prec = Not, 0
343 s.Tok = OperatorType
344
345 case '~':
346 s.nextch()
347 s.Op, s.Prec = Tilde, 0
348 s.Tok = OperatorType
349
350 default:
351 s.Errorf("invalid character %#U", s.ch)
352 s.nextch()
353 goto redo
354 }
355
356 return
357 }
358
359 func (s *Scanner) assignOp() {
360 if s.ch == '=' {
361 s.nextch()
362 s.Tok = AssignOp
363 return
364 }
365 s.Tok = OperatorType
366 }
367
368 func (s *Scanner) Ident() {
369 for IsLetter(s.ch) || IsDecimal(s.ch) {
370 s.nextch()
371 }
372
373 if s.ch >= utf8.RuneSelf {
374 for s.AtIdentChar(false) {
375 s.nextch()
376 }
377 }
378
379 lit := s.segment()
380 if len(lit) >= 2 {
381 if tok := keywordMap[Hash(lit)]; tok != 0 && TokStrFast(tok) == string(lit) {
382 s.nlsemi = contains(1<<Break|1<<Continue|1<<Fallthrough|1<<Return, tok)
383 s.Tok = tok
384 return
385 }
386 }
387
388 s.nlsemi = true
389 c := []byte{:len(lit)}
390 copy(c, lit)
391 s.Lit = string(c)
392 s.Tok = NameType
393 }
394
395 func TokStrFast(tok Token) string {
396 return token_name[token_index[tok-1]:token_index[tok]]
397 }
398
399 func (s *Scanner) AtIdentChar(first bool) bool {
400 switch {
401 case s.utab.IsLetter(s.ch) || s.ch == '_':
402 case s.utab.IsDigit(s.ch):
403 if first {
404 s.Errorf("identifier cannot begin with digit %#U", s.ch)
405 }
406 case s.ch >= utf8.RuneSelf:
407 s.Errorf("invalid character %#U in identifier", s.ch)
408 default:
409 return false
410 }
411 return true
412 }
413
414 func Hash(s []byte) uint32 {
415 return (uint32(s[0])<<4 ^ uint32(s[1]) + uint32(len(s))) & uint32(len(keywordMap)-1)
416 }
417
418 var keywordMap [1 << 6]Token
419
420 var keywordsInitialized bool
421
422 func InitKeywords() {
423 if keywordsInitialized {
424 return
425 }
426 keywordsInitialized = true
427 for tok := Break; tok <= Var; tok++ {
428 h := Hash([]byte(tok.String()))
429 if keywordMap[h] != 0 {
430 panic("imperfect hash")
431 }
432 keywordMap[h] = tok
433 }
434 }
435
436 func Lower(ch rune) rune { return ('a' - 'A') | ch }
437 func IsLetter(ch rune) bool { return 'a' <= Lower(ch) && Lower(ch) <= 'z' || ch == '_' }
438 func IsDecimal(ch rune) bool { return '0' <= ch && ch <= '9' }
439 func IsHex(ch rune) bool { return '0' <= ch && ch <= '9' || 'a' <= Lower(ch) && Lower(ch) <= 'f' }
440
441 func (s *Scanner) Digits(base int32, invalid *int32) (digsep int32) {
442 if base <= 10 {
443 max := rune('0' + base)
444 for IsDecimal(s.ch) || s.ch == '_' {
445 ds := int32(1)
446 if s.ch == '_' {
447 ds = 2
448 } else if s.ch >= max && *invalid < 0 {
449 _, col := s.pos()
450 *invalid = int32(col - s.col)
451 }
452 digsep |= ds
453 s.nextch()
454 }
455 } else {
456 for IsHex(s.ch) || s.ch == '_' {
457 ds := int32(1)
458 if s.ch == '_' {
459 ds = 2
460 }
461 digsep |= ds
462 s.nextch()
463 }
464 }
465 return
466 }
467
468 func (s *Scanner) Number(seenPoint bool) {
469 ok := true
470 kind := IntLit
471 base := int32(10)
472 prefix := rune(0)
473 digsep := int32(0)
474 invalid := int32(-1)
475
476 if !seenPoint {
477 if s.ch == '0' {
478 s.nextch()
479 switch Lower(s.ch) {
480 case 'x':
481 s.nextch()
482 base, prefix = 16, 'x'
483 case 'o':
484 s.nextch()
485 base, prefix = 8, 'o'
486 case 'b':
487 s.nextch()
488 base, prefix = 2, 'b'
489 default:
490 base, prefix = 8, '0'
491 digsep = 1
492 }
493 }
494 digsep |= s.Digits(base, &invalid)
495 if s.ch == '.' {
496 if prefix == 'o' || prefix == 'b' {
497 s.Errorf("invalid radix point in %s literal", baseName(base))
498 ok = false
499 }
500 s.nextch()
501 seenPoint = true
502 }
503 }
504
505 if seenPoint {
506 kind = FloatLit
507 digsep |= s.Digits(base, &invalid)
508 }
509
510 if digsep&1 == 0 && ok {
511 s.Errorf("%s literal has no digits", baseName(base))
512 ok = false
513 }
514
515 if e := Lower(s.ch); e == 'e' || e == 'p' {
516 if ok {
517 switch {
518 case e == 'e' && prefix != 0 && prefix != '0':
519 s.Errorf("%q exponent requires decimal mantissa", s.ch)
520 ok = false
521 case e == 'p' && prefix != 'x':
522 s.Errorf("%q exponent requires hexadecimal mantissa", s.ch)
523 ok = false
524 }
525 }
526 s.nextch()
527 kind = FloatLit
528 if s.ch == '+' || s.ch == '-' {
529 s.nextch()
530 }
531 digsep = s.Digits(10, nil) | digsep&2
532 if digsep&1 == 0 && ok {
533 s.Errorf("exponent has no digits")
534 ok = false
535 }
536 } else if prefix == 'x' && kind == FloatLit && ok {
537 s.Errorf("hexadecimal mantissa requires a 'p' exponent")
538 ok = false
539 }
540
541 if s.ch == 'i' {
542 kind = ImagLit
543 s.nextch()
544 }
545
546 s.SetLit(kind, ok)
547
548 if kind == IntLit && invalid >= 0 && ok {
549 s.ErrorAtf(invalid, "invalid digit %q in %s literal", s.Lit[invalid], baseName(base))
550 ok = false
551 }
552
553 if digsep&2 != 0 && ok {
554 if i := invalidSep(s.Lit); i >= 0 {
555 s.ErrorAtf(i, "'_' must separate successive digits")
556 ok = false
557 }
558 }
559
560 s.Bad = !ok
561 }
562
563 func baseName(base int32) string {
564 switch base {
565 case 2:
566 return "binary"
567 case 8:
568 return "octal"
569 case 10:
570 return "decimal"
571 case 16:
572 return "hexadecimal"
573 }
574 panic("invalid base")
575 }
576
577 func invalidSep(x string) int32 {
578 x1 := ' '
579 d := '.'
580 i := int32(0)
581
582 if len(x) >= 2 && x[0] == '0' {
583 x1 = Lower(rune(x[1]))
584 if x1 == 'x' || x1 == 'o' || x1 == 'b' {
585 d = '0'
586 i = 2
587 }
588 }
589
590 for ; i < int32(len(x)); i++ {
591 p := d
592 d = rune(x[i])
593 switch {
594 case d == '_':
595 if p != '0' {
596 return i
597 }
598 case IsDecimal(d) || x1 == 'x' && IsHex(d):
599 d = '0'
600 default:
601 if p == '_' {
602 return i - 1
603 }
604 d = '.'
605 }
606 }
607 if d == '_' {
608 return int32(len(x)) - 1
609 }
610
611 return -1
612 }
613
614 func (s *Scanner) rune() {
615 ok := true
616 s.nextch()
617
618 n := 0
619 for ; ; n++ {
620 if s.ch == '\'' {
621 if ok {
622 if n == 0 {
623 s.Errorf("empty rune literal or unescaped '")
624 ok = false
625 } else if n != 1 {
626 s.ErrorAtf(0, "more than one character in rune literal")
627 ok = false
628 }
629 }
630 s.nextch()
631 break
632 }
633 if s.ch == '\\' {
634 s.nextch()
635 if !s.escape('\'') {
636 ok = false
637 }
638 continue
639 }
640 if s.ch == '\n' {
641 if ok {
642 s.Errorf("newline in rune literal")
643 ok = false
644 }
645 break
646 }
647 if s.ch < 0 {
648 if ok {
649 s.ErrorAtf(0, "rune literal not terminated")
650 ok = false
651 }
652 break
653 }
654 s.nextch()
655 }
656
657 s.SetLit(RuneLit, ok)
658 }
659
660 func (s *Scanner) stdString() {
661 ok := true
662 s.nextch()
663
664 for {
665 if s.ch == '"' {
666 s.nextch()
667 break
668 }
669 if s.ch == '\\' {
670 s.nextch()
671 if !s.escape('"') {
672 ok = false
673 }
674 continue
675 }
676 if s.ch == '\n' {
677 s.Errorf("newline in string")
678 ok = false
679 break
680 }
681 if s.ch < 0 {
682 s.ErrorAtf(0, "string not terminated")
683 ok = false
684 break
685 }
686 s.nextch()
687 }
688
689 s.SetLit(StringLit, ok)
690 }
691
692 func (s *Scanner) rawString() {
693 ok := true
694 s.nextch()
695
696 for {
697 if s.ch == '`' {
698 s.nextch()
699 break
700 }
701 if s.ch < 0 {
702 s.ErrorAtf(0, "string not terminated")
703 ok = false
704 break
705 }
706 s.nextch()
707 }
708
709 s.SetLit(StringLit, ok)
710 }
711
712 func (s *Scanner) comment(text string) {
713 s.ErrorAtf(0, "%s", text)
714 }
715
716 func (s *Scanner) skipLine() {
717 for s.ch >= 0 && s.ch != '\n' {
718 s.nextch()
719 }
720 }
721
722 func (s *Scanner) lineComment() {
723 if s.mode&comments != 0 {
724 s.skipLine()
725 s.comment(string(s.segment()))
726 return
727 }
728
729 if s.mode&directives == 0 || (s.ch != 'g' && s.ch != 'l') {
730 s.stop()
731 s.skipLine()
732 return
733 }
734
735 prefix := "go:"
736 if s.ch == 'l' {
737 prefix = "line "
738 }
739
740 for _, r := range prefix {
741 if s.ch != rune(r) {
742 s.stop()
743 s.skipLine()
744 return
745 }
746 s.nextch()
747 }
748 s.skipLine()
749 s.comment(string(s.segment()))
750 }
751
752 func (s *Scanner) skipComment() bool {
753 for s.ch >= 0 {
754 for s.ch == '*' {
755 s.nextch()
756 if s.ch == '/' {
757 s.nextch()
758 return true
759 }
760 }
761 s.nextch()
762 }
763 s.ErrorAtf(0, "comment not terminated")
764 return false
765 }
766
767 func (s *Scanner) fullComment() {
768 if s.mode&comments != 0 {
769 if s.skipComment() {
770 s.comment(string(s.segment()))
771 }
772 return
773 }
774
775 if s.mode&directives == 0 || s.ch != 'l' {
776 s.stop()
777 s.skipComment()
778 return
779 }
780
781 const prefix = "line "
782
783 for _, r := range prefix {
784 if s.ch != rune(r) {
785 s.stop()
786 s.skipComment()
787 return
788 }
789 s.nextch()
790 }
791 if s.skipComment() {
792 s.comment(string(s.segment()))
793 }
794 }
795
796 func (s *Scanner) escape(quote rune) bool {
797 var n int32
798 var base, max uint32
799
800 switch s.ch {
801 case quote, 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\':
802 s.nextch()
803 return true
804 case '0', '1', '2', '3', '4', '5', '6', '7':
805 n, base, max = 3, 8, 255
806 case 'x':
807 s.nextch()
808 n, base, max = 2, 16, 255
809 case 'u':
810 s.nextch()
811 n, base, max = 4, 16, unicode.MaxRune
812 case 'U':
813 s.nextch()
814 n, base, max = 8, 16, unicode.MaxRune
815 default:
816 if s.ch < 0 {
817 return true
818 }
819 s.Errorf("unknown escape")
820 return false
821 }
822
823 var x uint32
824 for i := n; i > 0; i-- {
825 if s.ch < 0 {
826 return true
827 }
828 d := base
829 if IsDecimal(s.ch) {
830 d = uint32(s.ch) - '0'
831 } else if 'a' <= Lower(s.ch) && Lower(s.ch) <= 'f' {
832 d = uint32(Lower(s.ch)) - 'a' + 10
833 }
834 if d >= base {
835 s.Errorf("invalid character %q in %s escape", s.ch, baseName(int32(base)))
836 return false
837 }
838 x = x*base + d
839 s.nextch()
840 }
841
842 if x > max && base == 8 {
843 s.Errorf("octal escape value %d > 255", x)
844 return false
845 }
846
847 if x > max || 0xD800 <= x && x < 0xE000 {
848 s.Errorf("escape is invalid Unicode code point %#U", x)
849 return false
850 }
851
852 return true
853 }
854
855 func String(n Node) string {
856 return fmt.Sprintf("%T", n)
857 }
858
859 func StartPos(n Node) Pos {
860 return n.Pos()
861 }
862