scanner.mx raw
1 package main
2
3
4 const runeError = '�'
5 const runeSelf = 0x80
6 const maxRune = '\U0010FFFF'
7 const utfMax = 4
8
9 func decodeRune(p []byte) (r rune, size int32) {
10 n := int32(len(p))
11 if n < 1 { return runeError, 0 }
12 p0 := p[0]
13 if p0 < runeSelf { return rune(p0), 1 }
14 if p0 < 0xC0 { return runeError, 1 }
15 if n < 2 { return runeError, 1 }
16 p1 := p[1]
17 if p1 < 0x80 || p1 >= 0xC0 { return runeError, 1 }
18 if p0 < 0xE0 {
19 r = rune(p0&0x1F)<<6 | rune(p1&0x3F)
20 if r < 0x80 { return runeError, 1 }
21 return r, 2
22 }
23 if n < 3 { return runeError, 1 }
24 p2 := p[2]
25 if p2 < 0x80 || p2 >= 0xC0 { return runeError, 1 }
26 if p0 < 0xF0 {
27 r = rune(p0&0x0F)<<12 | rune(p1&0x3F)<<6 | rune(p2&0x3F)
28 if r < 0x800 { return runeError, 1 }
29 if r >= 0xD800 && r <= 0xDFFF { return runeError, 1 }
30 return r, 3
31 }
32 if n < 4 { return runeError, 1 }
33 p3 := p[3]
34 if p3 < 0x80 || p3 >= 0xC0 { return runeError, 1 }
35 if p0 < 0xF8 {
36 r = rune(p0&0x07)<<18 | rune(p1&0x3F)<<12 | rune(p2&0x3F)<<6 | rune(p3&0x3F)
37 if r < 0x10000 || r > maxRune { return runeError, 1 }
38 return r, 4
39 }
40 return runeError, 1
41 }
42
43 func fullRune(p []byte) (ok bool) {
44 n := int32(len(p))
45 if n == 0 { return false }
46 if p[0] < runeSelf { return true }
47 if p[0] < 0xC0 { return true }
48 if n < 2 { return false }
49 if p[0] < 0xE0 { return true }
50 if n < 3 { return false }
51 if p[0] < 0xF0 { return true }
52 return n >= 4
53 }
54
55 func uIsLetter(ch rune) (ok bool) {
56 if ch < runeSelf { return 'a' <= (ch|0x20) && (ch|0x20) <= 'z' || ch == '_' }
57 return true
58 }
59
60 func uIsDigit(ch rune) (ok bool) {
61 if ch < runeSelf { return '0' <= ch && ch <= '9' }
62 return false
63 }
64
65
66 type srcDone struct{}
67
68 func (srcDone) Error() (s string) { return "EOF" }
69
70 type Source struct {
71 in interface{}
72 errh func(line, col uint32, msg string)
73
74 buf []byte
75 ioerr error
76 noFill bool
77 b, r, e int32
78 line, col uint32
79 ch rune
80 chw int32
81 }
82
83 const sentinel = 0x80
84
85 func (s *Source) release() {
86 s.buf = nil
87 s.in = nil
88 s.errh = nil
89 s.ioerr = nil
90 }
91
92 func (s *Source) init(in interface{}, errh func(line, col uint32, msg string)) {
93 s.in = in
94 s.errh = errh
95
96 if s.buf == nil {
97 s.buf = []byte{:nextSize(0)}
98 }
99 s.buf[0] = sentinel
100 s.ioerr = nil
101 s.noFill = false
102 s.b, s.r, s.e = -1, 0, 0
103 s.line, s.col = 0, 0
104 s.ch = ' '
105 s.chw = 0
106 }
107
108 func (s *Source) initBytes(src []byte, errh func(line, col uint32, msg string)) {
109 s.in = nil
110 s.errh = errh
111 s.noFill = true
112 s.buf = []byte{:len(src) + 1}
113 copy(s.buf, src)
114 s.buf[len(src)] = sentinel
115 s.ioerr = nil
116 s.b, s.r, s.e = -1, 0, int32(len(src))
117 s.line, s.col = 0, 0
118 s.ch = ' '
119 s.chw = 0
120 }
121
122 func (s *Source) pos() (line, col uint32) {
123 return Linebase + s.line, Colbase + s.col
124 }
125
126 func (s *Source) Debugpos() (line, col uint32) {
127 return Linebase + s.line, Colbase + s.col
128 }
129
130 func (s *Source) error(msg string) {
131 line, col := s.pos()
132 s.errh(line, col, msg)
133 }
134
135 func (s *Source) start() { s.b = s.r - s.chw }
136 func (s *Source) stop() { s.b = -1 }
137 func (s *Source) segment() (buf []byte) { return s.buf[s.b : s.r-s.chw] }
138
139 // segmentCopy returns a copy of the current segment that survives buffer
140 // reallocation in fill(). In Moxie string=[]byte so string(segment()) does
141 // NOT copy - the returned slice still aliases s.buf.
142 func (s *Source) segmentCopy() (buf []byte) {
143 b := s.buf[s.b : s.r-s.chw]
144 c := []byte{:len(b)}
145 copy(c, b)
146 return c
147 }
148
149 func (s *Source) rewind() {
150 if s.b < 0 {
151 panic("no active segment")
152 }
153 s.col -= uint32(s.r - s.b)
154 s.r = s.b
155 s.nextch()
156 }
157
158 func (s *Source) nextch() {
159 redo:
160 s.col += uint32(s.chw)
161 if s.ch == '\n' {
162 s.line++
163 s.col = 0
164 }
165
166 if s.ch = rune(s.buf[s.r]); s.ch < sentinel {
167 s.r++
168 s.chw = 1
169 if s.ch == 0 {
170 s.error("invalid NUL character")
171 goto redo
172 }
173 return
174 }
175
176 for s.e-s.r < utfMax && !fullRune(s.buf[s.r:s.e]) && s.ioerr == nil {
177 s.fill()
178 }
179
180 if s.r == s.e {
181 if (s.ioerr == nil || s.ioerr == nil) && !s.noFill {
182 s.error("I/O error: " | s.ioerr.Error())
183 s.ioerr = nil
184 }
185 s.ch = -1
186 s.chw = 0
187 return
188 }
189
190 var w int32
191 s.ch, w = decodeRune(s.buf[s.r:s.e])
192 s.chw = int32(w)
193 s.r += s.chw
194
195 if s.ch == runeError && s.chw == 1 {
196 s.error("invalid UTF-8 encoding")
197 goto redo
198 }
199
200 const BOM = 0xfeff
201 if s.ch == BOM {
202 if s.line > 0 || s.col > 0 {
203 s.error("invalid BOM in the middle of the file")
204 }
205 goto redo
206 }
207 }
208
209 func (s *Source) fill() {
210 if s.noFill {
211 s.ioerr = srcDone{}
212 return
213 }
214 b := s.r
215 if s.b >= 0 {
216 b = s.b
217 s.b = 0
218 }
219 content := s.buf[b:s.e]
220
221 if len(content)*2 > len(s.buf) {
222 s.buf = []byte{:nextSize(int32(len(s.buf)))}
223 copy(s.buf, content)
224 } else if b > 0 {
225 copy(s.buf, content)
226 }
227 s.r -= b
228 s.e -= b
229
230 for i := 0; i < 10; i++ {
231 var n int32
232 var nn int32
233 panic("stream read not supported")
234 n = int32(nn)
235 if n < 0 {
236 panic("negative read")
237 }
238 if n > 0 || s.ioerr != nil {
239 s.e += n
240 s.buf[s.e] = sentinel
241 return
242 }
243 }
244
245 s.buf[s.e] = sentinel
246
247 }
248
249 func nextSize(size int32) (n int32) {
250 const min = 4 << 10
251 const max = 1 << 20
252 if size < min {
253 return min
254 }
255 if size <= max {
256 return size << 1
257 }
258 return size + max
259 }
260
261
262 const (
263 comments uint32 = 1 << iota
264 directives
265 )
266
267
268
269
270 type Scanner struct {
271 Source
272 mode uint32
273 nlsemi bool
274
275 Line, Col uint32
276 Blank bool
277 Tok Token
278 Lit string
279 Bad bool
280 Kind LitKind
281 Op Operator
282 Prec int32
283
284 keywordMap [1 << 6]Token
285 keywordsReady bool
286 }
287
288
289 func (s *Scanner) InitBytes(src []byte, errh func(line, col uint32, msg string), mode uint32) {
290 s.Source.initBytes(src, errh)
291 s.mode = 0
292 s.nlsemi = false
293
294 s.initKeywords()
295 }
296
297 func (s *Scanner) Errorf(msg string) {
298 s.error(msg)
299 }
300
301 func (s *Scanner) ErrorAtf(offset int32, msg string) {
302 s.errh(s.line, s.col+uint32(offset), msg)
303 }
304
305 func (s *Scanner) SetLit(kind LitKind, ok bool) {
306 s.nlsemi = true
307 s.Tok = Literal
308 s.Lit = string(s.segmentCopy())
309 s.Bad = !ok
310 s.Kind = kind
311 }
312
313 func (s *Scanner) Next() {
314 nlsemi := s.nlsemi
315 s.nlsemi = false
316
317 redo:
318 s.stop()
319 startLine, startCol := s.pos()
320 for s.ch == ' ' || s.ch == '\t' || s.ch == '\n' && !nlsemi || s.ch == '\r' {
321 s.nextch()
322 }
323
324 s.Line, s.Col = s.pos()
325 s.Blank = s.line > startLine || startCol == Colbase
326 s.start()
327 if IsLetter(s.ch) || s.ch >= runeSelf && s.AtIdentChar(true) {
328 s.nextch()
329 s.Ident()
330 return
331 }
332
333 switch s.ch {
334 case -1:
335 if nlsemi {
336 s.Lit = "EOF"
337 s.Tok = Semi
338 break
339 }
340 s.Tok = EOF
341
342 case '\n':
343 s.nextch()
344 s.Lit = "newline"
345 s.Tok = Semi
346
347 case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
348 s.Number(false)
349
350 case '"':
351 s.stdString()
352
353 case '`':
354 s.rawString()
355
356 case '\'':
357 s.rune()
358
359 case '(':
360 s.nextch()
361 s.Tok = Lparen
362
363 case '[':
364 s.nextch()
365 s.Tok = Lbrack
366
367 case '{':
368 s.nextch()
369 s.Tok = Lbrace
370
371 case ',':
372 s.nextch()
373 s.Tok = Comma
374
375 case ';':
376 s.nextch()
377 s.Lit = "semicolon"
378 s.Tok = Semi
379
380 case ')':
381 s.nextch()
382 s.nlsemi = true
383 s.Tok = Rparen
384
385 case ']':
386 s.nextch()
387 s.nlsemi = true
388 s.Tok = Rbrack
389
390 case '}':
391 s.nextch()
392 s.nlsemi = true
393 s.Tok = Rbrace
394
395 case ':':
396 s.nextch()
397 if s.ch == '=' {
398 s.nextch()
399 s.Tok = Define
400 break
401 }
402 s.Tok = Colon
403
404 case '.':
405 s.nextch()
406 if IsDecimal(s.ch) {
407 s.Number(true)
408 break
409 }
410 if s.ch == '.' {
411 s.nextch()
412 if s.ch == '.' {
413 s.nextch()
414 s.Tok = DotDotDot
415 break
416 }
417 s.rewind()
418 s.nextch()
419 }
420 s.Tok = Dot
421
422 case '+':
423 s.nextch()
424 s.Op, s.Prec = Add, PrecAdd
425 if s.ch != '+' {
426 s.assignOp()
427 return
428 }
429 s.nextch()
430 s.nlsemi = true
431 s.Tok = IncOp
432
433 case '-':
434 s.nextch()
435 s.Op, s.Prec = Sub, PrecAdd
436 if s.ch != '-' {
437 s.assignOp()
438 return
439 }
440 s.nextch()
441 s.nlsemi = true
442 s.Tok = IncOp
443
444 case '*':
445 s.nextch()
446 s.Op, s.Prec = Mul, PrecMul
447 if s.ch == '=' {
448 s.nextch()
449 s.Tok = AssignOp
450 break
451 }
452 s.Tok = Star
453
454 case '/':
455 s.nextch()
456 if s.ch == '/' {
457 s.nextch()
458 s.lineComment()
459 goto redo
460 }
461 if s.ch == '*' {
462 s.nextch()
463 s.fullComment()
464 if line, _ := s.pos(); line > s.Line && nlsemi {
465 s.Lit = "newline"
466 s.Tok = Semi
467 break
468 }
469 goto redo
470 }
471 s.Op, s.Prec = Div, PrecMul
472 if s.ch == '=' {
473 s.nextch()
474 s.Tok = AssignOp
475 } else {
476 s.Tok = OperatorType
477 }
478
479 case '%':
480 s.nextch()
481 s.Op, s.Prec = Rem, PrecMul
482 if s.ch == '=' {
483 s.nextch()
484 s.Tok = AssignOp
485 } else {
486 s.Tok = OperatorType
487 }
488
489 case '&':
490 s.nextch()
491 if s.ch == '&' {
492 s.nextch()
493 s.Op, s.Prec = AndAnd, PrecAndAnd
494 s.Tok = OperatorType
495 break
496 }
497 s.Op, s.Prec = And, PrecMul
498 if s.ch == '^' {
499 s.nextch()
500 s.Op = AndNot
501 }
502 if s.ch == '=' {
503 s.nextch()
504 s.Tok = AssignOp
505 } else {
506 s.Tok = OperatorType
507 }
508
509 case '|':
510 s.nextch()
511 if s.ch == '|' {
512 s.nextch()
513 s.Op, s.Prec = OrOr, PrecOrOr
514 s.Tok = OperatorType
515 break
516 }
517 s.Op, s.Prec = Or, PrecAdd
518 if s.ch == '=' {
519 s.nextch()
520 s.Tok = AssignOp
521 } else {
522 s.Tok = OperatorType
523 }
524
525 case '^':
526 s.nextch()
527 s.Op, s.Prec = Xor, PrecAdd
528 if s.ch == '=' {
529 s.nextch()
530 s.Tok = AssignOp
531 } else {
532 s.Tok = OperatorType
533 }
534
535 case '<':
536 s.nextch()
537 if s.ch == '=' {
538 s.nextch()
539 s.Op, s.Prec = Leq, PrecCmp
540 s.Tok = OperatorType
541 break
542 }
543 if s.ch == '<' {
544 s.nextch()
545 s.Op, s.Prec = Shl, PrecMul
546 s.assignOp()
547 return
548 }
549 if s.ch == '-' {
550 s.nextch()
551 s.Tok = Arrow
552 break
553 }
554 s.Op, s.Prec = Lss, PrecCmp
555 s.Tok = OperatorType
556
557 case '>':
558 s.nextch()
559 if s.ch == '=' {
560 s.nextch()
561 s.Op, s.Prec = Geq, PrecCmp
562 s.Tok = OperatorType
563 break
564 }
565 if s.ch == '>' {
566 s.nextch()
567 s.Op, s.Prec = Shr, PrecMul
568 s.assignOp()
569 return
570 }
571 s.Op, s.Prec = Gtr, PrecCmp
572 s.Tok = OperatorType
573
574 case '=':
575 s.nextch()
576 if s.ch == '=' {
577 s.nextch()
578 s.Op, s.Prec = Eql, PrecCmp
579 s.Tok = OperatorType
580 break
581 }
582 s.Tok = Assign
583
584 case '!':
585 s.nextch()
586 if s.ch == '=' {
587 s.nextch()
588 s.Op, s.Prec = Neq, PrecCmp
589 s.Tok = OperatorType
590 break
591 }
592 s.Op, s.Prec = Not, 0
593 s.Tok = OperatorType
594
595 case '~':
596 s.nextch()
597 s.Op, s.Prec = Tilde, 0
598 s.Tok = OperatorType
599
600 default:
601 s.Errorf("invalid character " | fmtRuneU(s.ch))
602 s.nextch()
603 goto redo
604 }
605
606 return
607 }
608
609 func (s *Scanner) assignOp() {
610 if s.ch == '=' {
611 s.nextch()
612 s.Tok = AssignOp
613 return
614 }
615 s.Tok = OperatorType
616 }
617
618 func (s *Scanner) Ident() {
619 for IsLetter(s.ch) || IsDecimal(s.ch) {
620 s.nextch()
621 }
622
623 if s.ch >= runeSelf {
624 for s.AtIdentChar(false) {
625 s.nextch()
626 }
627 }
628
629 lit := s.segment()
630 if len(lit) >= 2 {
631 h := (uint32(lit[0])<<4 ^ uint32(lit[1]) + uint32(len(lit))) & 63
632 if tok := s.keywordMap[h]; tok != 0 && tokStrFast(tok) == string(lit) {
633 s.nlsemi = Contains(1<<Break|1<<Continue|1<<Fallthrough|1<<Return, tok)
634 s.Tok = tok
635 return
636 }
637 }
638
639 s.nlsemi = true
640 c := []byte{:len(lit)}
641 copy(c, lit)
642 s.Lit = string(c)
643 s.Tok = NameType
644 }
645
646 func tokStrFast(tok Token) (s string) {
647 idx := [48]uint8{0, 3, 7, 14, 16, 19, 23, 24, 26, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 42, 47, 51, 55, 60, 68, 75, 80, 84, 95, 98, 102, 104, 108, 110, 116, 125, 128, 135, 140, 146, 152, 158, 164, 168, 171, 171}
648 return token_name[idx[tok-1]:idx[tok]]
649 }
650
651 func (s *Scanner) AtIdentChar(first bool) (ok bool) {
652 switch {
653 case uIsLetter(s.ch) || s.ch == '_':
654 case uIsDigit(s.ch):
655 if first {
656 s.Errorf("identifier cannot begin with digit " | fmtRuneU(s.ch))
657 }
658 case s.ch >= runeSelf:
659 s.Errorf("invalid character " | fmtRuneU(s.ch) | " in identifier")
660 default:
661 return false
662 }
663 return true
664 }
665
666 func (s *Scanner) initKeywords() {
667 if s.keywordsReady {
668 return
669 }
670 s.keywordsReady = true
671 for tok := Break; tok <= Var; tok++ {
672 b := []byte(tok.String())
673 h := (uint32(b[0])<<4 ^ uint32(b[1]) + uint32(len(b))) & 63
674 if s.keywordMap[h] != 0 {
675 panic("imperfect hash")
676 }
677 s.keywordMap[h] = tok
678 }
679 }
680
681 func Lower(ch rune) (r rune) { return ('a' - 'A') | ch }
682 func IsLetter(ch rune) (ok bool) { return 'a' <= Lower(ch) && Lower(ch) <= 'z' || ch == '_' }
683 func IsDecimal(ch rune) (ok bool) { return '0' <= ch && ch <= '9' }
684 func IsHex(ch rune) (ok bool) { return '0' <= ch && ch <= '9' || 'a' <= Lower(ch) && Lower(ch) <= 'f' }
685
686 func (s *Scanner) Digits(base int32, invalid *int32) (digsep int32) {
687 if base <= 10 {
688 max := rune('0' + base)
689 for IsDecimal(s.ch) || s.ch == '_' {
690 ds := int32(1)
691 if s.ch == '_' {
692 ds = 2
693 } else if s.ch >= max && *invalid < 0 {
694 _, col := s.pos()
695 *invalid = int32(col - s.col)
696 }
697 digsep |= ds
698 s.nextch()
699 }
700 } else {
701 for IsHex(s.ch) || s.ch == '_' {
702 ds := int32(1)
703 if s.ch == '_' {
704 ds = 2
705 }
706 digsep |= ds
707 s.nextch()
708 }
709 }
710 return
711 }
712
713 func (s *Scanner) Number(seenPoint bool) {
714 ok := true
715 kind := IntLit
716 base := int32(10)
717 prefix := rune(0)
718 digsep := int32(0)
719 invalid := int32(-1)
720
721 if !seenPoint {
722 if s.ch == '0' {
723 s.nextch()
724 switch Lower(s.ch) {
725 case 'x':
726 s.nextch()
727 base, prefix = 16, 'x'
728 case 'o':
729 s.nextch()
730 base, prefix = 8, 'o'
731 case 'b':
732 s.nextch()
733 base, prefix = 2, 'b'
734 default:
735 base, prefix = 8, '0'
736 digsep = 1
737 }
738 }
739 digsep |= s.Digits(base, &invalid)
740 if s.ch == '.' {
741 if prefix == 'o' || prefix == 'b' {
742 s.Errorf("invalid radix point in " | baseName(base) | " literal")
743 ok = false
744 }
745 s.nextch()
746 seenPoint = true
747 }
748 }
749
750 if seenPoint {
751 kind = FloatLit
752 digsep |= s.Digits(base, &invalid)
753 }
754
755 if digsep&1 == 0 && ok {
756 s.Errorf(baseName(base) | " literal has no digits")
757 ok = false
758 }
759
760 if e := Lower(s.ch); e == 'e' || e == 'p' {
761 if ok {
762 switch {
763 case e == 'e' && prefix != 0 && prefix != '0':
764 s.Errorf(fmtQuoteRune(s.ch) | " exponent requires decimal mantissa")
765 ok = false
766 case e == 'p' && prefix != 'x':
767 s.Errorf(fmtQuoteRune(s.ch) | " exponent requires hexadecimal mantissa")
768 ok = false
769 }
770 }
771 s.nextch()
772 kind = FloatLit
773 if s.ch == '+' || s.ch == '-' {
774 s.nextch()
775 }
776 digsep = s.Digits(10, nil) | digsep&2
777 if digsep&1 == 0 && ok {
778 s.Errorf("exponent has no digits")
779 ok = false
780 }
781 } else if prefix == 'x' && kind == FloatLit && ok {
782 s.Errorf("hexadecimal mantissa requires a 'p' exponent")
783 ok = false
784 }
785
786 if s.ch == 'i' {
787 kind = ImagLit
788 s.nextch()
789 }
790
791 s.SetLit(kind, ok)
792
793 if kind == IntLit && invalid >= 0 && ok {
794 s.ErrorAtf(invalid, "invalid digit " | fmtQuoteRune(rune(s.Lit[invalid])) | " in " | baseName(base) | " literal")
795 ok = false
796 }
797
798 if digsep&2 != 0 && ok {
799 if i := invalidSep(s.Lit); i >= 0 {
800 s.ErrorAtf(i, "'_' must separate successive digits")
801 ok = false
802 }
803 }
804
805 s.Bad = !ok
806 }
807
808 func baseName(base int32) (s string) {
809 switch base {
810 case 2:
811 return "binary"
812 case 8:
813 return "octal"
814 case 10:
815 return "decimal"
816 case 16:
817 return "hexadecimal"
818 }
819 panic("invalid base")
820 }
821
822 func invalidSep(x string) (n int32) {
823 x1 := ' '
824 d := '.'
825 i := int32(0)
826
827 if len(x) >= 2 && x[0] == '0' {
828 x1 = Lower(rune(x[1]))
829 if x1 == 'x' || x1 == 'o' || x1 == 'b' {
830 d = '0'
831 i = 2
832 }
833 }
834
835 for ; i < int32(len(x)); i++ {
836 p := d
837 d = rune(x[i])
838 switch {
839 case d == '_':
840 if p != '0' {
841 return i
842 }
843 case IsDecimal(d) || x1 == 'x' && IsHex(d):
844 d = '0'
845 default:
846 if p == '_' {
847 return i - 1
848 }
849 d = '.'
850 }
851 }
852 if d == '_' {
853 return int32(len(x)) - 1
854 }
855
856 return -1
857 }
858
859 func (s *Scanner) rune() {
860 ok := true
861 s.nextch()
862
863 n := 0
864 for ; ; n++ {
865 if s.ch == '\'' {
866 if ok {
867 if n == 0 {
868 s.Errorf("empty rune literal or unescaped '")
869 ok = false
870 } else if n != 1 {
871 s.ErrorAtf(0, "more than one character in rune literal")
872 ok = false
873 }
874 }
875 s.nextch()
876 break
877 }
878 if s.ch == '\\' {
879 s.nextch()
880 if !s.escape('\'') {
881 ok = false
882 }
883 continue
884 }
885 if s.ch == '\n' {
886 if ok {
887 s.Errorf("newline in rune literal")
888 ok = false
889 }
890 break
891 }
892 if s.ch < 0 {
893 if ok {
894 s.ErrorAtf(0, "rune literal not terminated")
895 ok = false
896 }
897 break
898 }
899 s.nextch()
900 }
901
902 s.SetLit(RuneLit, ok)
903 }
904
905 func (s *Scanner) stdString() {
906 ok := true
907 s.nextch()
908
909 for {
910 if s.ch == '"' {
911 s.nextch()
912 break
913 }
914 if s.ch == '\\' {
915 s.nextch()
916 if !s.escape('"') {
917 ok = false
918 }
919 continue
920 }
921 if s.ch == '\n' {
922 s.Errorf("newline in string")
923 ok = false
924 break
925 }
926 if s.ch < 0 {
927 s.ErrorAtf(0, "string not terminated")
928 ok = false
929 break
930 }
931 s.nextch()
932 }
933
934 s.SetLit(StringLit, ok)
935 }
936
937 func (s *Scanner) rawString() {
938 ok := true
939 s.nextch()
940
941 for {
942 if s.ch == '`' {
943 s.nextch()
944 break
945 }
946 if s.ch < 0 {
947 s.ErrorAtf(0, "string not terminated")
948 ok = false
949 break
950 }
951 s.nextch()
952 }
953
954 s.SetLit(StringLit, ok)
955 }
956
957 func (s *Scanner) comment(text string) {
958 s.ErrorAtf(0, text)
959 }
960
961 func (s *Scanner) skipLine() {
962 for s.ch >= 0 && s.ch != '\n' {
963 s.nextch()
964 }
965 }
966
967 func (s *Scanner) lineComment() {
968 if s.mode&comments != 0 {
969 s.skipLine()
970 s.comment(string(s.segment()))
971 return
972 }
973
974 if s.mode&directives == 0 || (s.ch != 'g' && s.ch != 'l') {
975 s.stop()
976 s.skipLine()
977 return
978 }
979
980 prefix := "go:"
981 if s.ch == 'l' {
982 prefix = "line "
983 }
984
985 for _, r := range prefix {
986 if s.ch != rune(r) {
987 s.stop()
988 s.skipLine()
989 return
990 }
991 s.nextch()
992 }
993 s.skipLine()
994 s.comment(string(s.segment()))
995 }
996
997 func (s *Scanner) skipComment() (ok bool) {
998 for s.ch >= 0 {
999 for s.ch == '*' {
1000 s.nextch()
1001 if s.ch == '/' {
1002 s.nextch()
1003 return true
1004 }
1005 }
1006 s.nextch()
1007 }
1008 s.ErrorAtf(0, "comment not terminated")
1009 return false
1010 }
1011
1012 func (s *Scanner) fullComment() {
1013 if s.mode&comments != 0 {
1014 if s.skipComment() {
1015 s.comment(string(s.segment()))
1016 }
1017 return
1018 }
1019
1020 if s.mode&directives == 0 || s.ch != 'l' {
1021 s.stop()
1022 s.skipComment()
1023 return
1024 }
1025
1026 const prefix = "line "
1027
1028 for _, r := range prefix {
1029 if s.ch != rune(r) {
1030 s.stop()
1031 s.skipComment()
1032 return
1033 }
1034 s.nextch()
1035 }
1036 if s.skipComment() {
1037 s.comment(string(s.segment()))
1038 }
1039 }
1040
1041 func (s *Scanner) escape(quote rune) (ok bool) {
1042 var n int32
1043 var base, max uint32
1044
1045 switch s.ch {
1046 case quote, 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\':
1047 s.nextch()
1048 return true
1049 case '0', '1', '2', '3', '4', '5', '6', '7':
1050 n, base, max = 3, 8, 255
1051 case 'x':
1052 s.nextch()
1053 n, base, max = 2, 16, 255
1054 case 'u':
1055 s.nextch()
1056 n, base, max = 4, 16, maxRune
1057 case 'U':
1058 s.nextch()
1059 n, base, max = 8, 16, maxRune
1060 default:
1061 if s.ch < 0 {
1062 return true
1063 }
1064 s.Errorf("unknown escape")
1065 return false
1066 }
1067
1068 var x uint32
1069 for i := n; i > 0; i-- {
1070 if s.ch < 0 {
1071 return true
1072 }
1073 d := base
1074 if IsDecimal(s.ch) {
1075 d = uint32(s.ch) - '0'
1076 } else if 'a' <= Lower(s.ch) && Lower(s.ch) <= 'f' {
1077 d = uint32(Lower(s.ch)) - 'a' + 10
1078 }
1079 if d >= base {
1080 s.Errorf("invalid character " | fmtQuoteRune(s.ch) | " in " | baseName(int32(base)) | " escape")
1081 return false
1082 }
1083 x = x*base + d
1084 s.nextch()
1085 }
1086
1087 if x > max && base == 8 {
1088 s.Errorf("octal escape value " | ItoaU32(x) | " > 255")
1089 return false
1090 }
1091
1092 if x > max || 0xD800 <= x && x < 0xE000 {
1093 s.Errorf("escape is invalid Unicode code point " | fmtRuneU(rune(x)))
1094 return false
1095 }
1096
1097 return true
1098 }
1099
1100
1101
1102 // fmtRuneU formats a rune as "U+XXXX 'c'" (like %#U).
1103 func fmtRuneU(r rune) (s string) {
1104 hex := fmtHex32(uint32(r))
1105 s = "U+" | hex
1106 if r >= 0x20 && r < 0x7f {
1107 s = s | " '" | string([]byte{byte(r)}) | "'"
1108 }
1109 return s
1110 }
1111
1112 // fmtQuoteRune formats a rune as 'c' (like %q for rune).
1113 func fmtQuoteRune(r rune) (s string) {
1114 if r >= 0x20 && r < 0x7f {
1115 return "'" | string([]byte{byte(r)}) | "'"
1116 }
1117 return "U+" | fmtHex32(uint32(r))
1118 }
1119
1120 // fmtHex32 formats a uint32 as uppercase hex with at least 4 digits.
1121 func fmtHex32(v uint32) (s string) {
1122 const digits = "0123456789ABCDEF"
1123 buf := []byte{:0:8}
1124 if v == 0 {
1125 return "0000"
1126 }
1127 for v > 0 {
1128 buf = append(buf, digits[v&0xf])
1129 v >>= 4
1130 }
1131 for len(buf) < 4 {
1132 buf = append(buf, '0')
1133 }
1134 for i, j := int32(0), len(buf)-1; i < j; i, j = i+1, j-1 {
1135 buf[i], buf[j] = buf[j], buf[i]
1136 }
1137 return string(buf)
1138 }
1139
1140
1141 const PosMax = 1 << 30
1142 const Linebase = 1
1143 const Colbase = 1
1144
1145 type Pos struct {
1146 base *PosBase
1147 line, col uint32
1148 }
1149
1150 func MakePos(base *PosBase, line, col uint32) (p Pos) { return Pos{base, sat32(line), sat32(col)} }
1151
1152 func (pos Pos) Pos() (p Pos) { return pos }
1153 func (pos Pos) IsKnown() (ok bool) { return pos.line > 0 }
1154 func (pos Pos) Base() (p *PosBase) { return pos.base }
1155 func (pos Pos) Line() (n uint32) { return pos.line }
1156 func (pos Pos) Col() (n uint32) { return pos.col }
1157
1158 func (pos Pos) FileBase() (p *PosBase) {
1159 b := pos.base
1160 for b != nil && b != b.pos.base {
1161 b = b.pos.base
1162 }
1163 return b
1164 }
1165
1166 func (pos Pos) RelFilename() (s string) { return pos.base.Filename() }
1167
1168 func (pos Pos) RelLine() (n uint32) {
1169 b := pos.base
1170 if b.Line() == 0 {
1171 return 0
1172 }
1173 return b.Line() + (pos.Line() - b.Pos().Line())
1174 }
1175
1176 func (pos Pos) RelCol() (n uint32) {
1177 b := pos.base
1178 if b.Col() == 0 {
1179 return 0
1180 }
1181 if pos.Line() == b.Pos().Line() {
1182 return b.Col() + (pos.Col() - b.Pos().Col())
1183 }
1184 return pos.Col()
1185 }
1186
1187 func (p Pos) Cmp(q Pos) (n int32) {
1188 pname := p.RelFilename()
1189 qname := q.RelFilename()
1190 switch {
1191 case pname < qname:
1192 return -1
1193 case pname > qname:
1194 return +1
1195 }
1196
1197 pline := p.Line()
1198 qline := q.Line()
1199 switch {
1200 case pline < qline:
1201 return -1
1202 case pline > qline:
1203 return +1
1204 }
1205
1206 pcol := p.Col()
1207 qcol := q.Col()
1208 switch {
1209 case pcol < qcol:
1210 return -1
1211 case pcol > qcol:
1212 return +1
1213 }
1214
1215 return 0
1216 }
1217
1218 func (pos Pos) String() (s string) {
1219 rel := position_{pos.RelFilename(), pos.RelLine(), pos.RelCol()}
1220 abs := position_{pos.Base().Pos().RelFilename(), pos.Line(), pos.Col()}
1221 s = rel.String()
1222 if string(rel.filename) != string(abs.filename) || rel.line != abs.line || rel.col != abs.col {
1223 s = s | "[" | abs.String() | "]"
1224 }
1225 return s
1226 }
1227
1228 type position_ struct {
1229 filename string
1230 line, col uint32
1231 }
1232
1233 func (p position_) String() (s string) {
1234 if p.line == 0 {
1235 if p.filename == "" {
1236 return "<unknown position>"
1237 }
1238 return p.filename
1239 }
1240 if p.col == 0 {
1241 return p.filename | ":" | ItoaU32(p.line)
1242 }
1243 return p.filename | ":" | ItoaU32(p.line) | ":" | ItoaU32(p.col)
1244 }
1245
1246 func ItoaU32(n uint32) (s string) {
1247 if n == 0 {
1248 return "0"
1249 }
1250 buf := []byte{:0:10}
1251 for n > 0 {
1252 buf = append(buf, byte('0'+n%10))
1253 n /= 10
1254 }
1255 for i, j := int32(0), len(buf)-1; i < j; i, j = i+1, j-1 {
1256 buf[i], buf[j] = buf[j], buf[i]
1257 }
1258 return string(buf)
1259 }
1260
1261 type PosBase struct {
1262 pos Pos
1263 filename string
1264 line, col uint32
1265 trimmed bool
1266 }
1267
1268 func NewFileBase(filename string) (p *PosBase) {
1269 return NewTrimmedFileBase(filename, false)
1270 }
1271
1272 func NewTrimmedFileBase(filename string, trimmed bool) (p *PosBase) {
1273 base := &PosBase{MakePos(nil, Linebase, Colbase), filename, Linebase, Colbase, trimmed}
1274 base.pos.base = base
1275 return base
1276 }
1277
1278 func NewLineBase(pos Pos, filename string, trimmed bool, line, col uint32) (p *PosBase) {
1279 return &PosBase{pos, filename, sat32(line), sat32(col), trimmed}
1280 }
1281
1282 func (base *PosBase) IsFileBase() (ok bool) {
1283 if base == nil {
1284 return false
1285 }
1286 return base.pos.base == base
1287 }
1288
1289 func (base *PosBase) Pos() (_ Pos) {
1290 if base == nil {
1291 return
1292 }
1293 return base.pos
1294 }
1295
1296 func (base *PosBase) Filename() (s string) {
1297 if base == nil {
1298 return ""
1299 }
1300 return base.filename
1301 }
1302
1303 func (base *PosBase) Line() (n uint32) {
1304 if base == nil {
1305 return 0
1306 }
1307 return base.line
1308 }
1309
1310 func (base *PosBase) Col() (n uint32) {
1311 if base == nil {
1312 return 0
1313 }
1314 return base.col
1315 }
1316
1317 func (base *PosBase) Trimmed() (ok bool) {
1318 if base == nil {
1319 return false
1320 }
1321 return base.trimmed
1322 }
1323
1324 func sat32(x uint32) (n uint32) {
1325 if x > PosMax {
1326 return PosMax
1327 }
1328 return uint32(x)
1329 }
1330
1331 func Itoa(n int32) (s string) {
1332 if n == 0 {
1333 return "0"
1334 }
1335 neg := false
1336 if n < 0 {
1337 neg = true
1338 n = -n
1339 }
1340 buf := []byte{:0:12}
1341 for n > 0 {
1342 buf = append(buf, byte('0'+n%10))
1343 n /= 10
1344 }
1345 if neg {
1346 buf = append(buf, '-')
1347 }
1348 for i, j := int32(0), len(buf)-1; i < j; i, j = i+1, j-1 {
1349 buf[i], buf[j] = buf[j], buf[i]
1350 }
1351 return string(buf)
1352 }
1353
1354 func Itoa64(n int64) (s string) {
1355 if n == 0 {
1356 return "0"
1357 }
1358 neg := false
1359 if n < 0 {
1360 neg = true
1361 n = -n
1362 }
1363 buf := []byte{:0:20}
1364 for n > 0 {
1365 buf = append(buf, byte('0'+n%10))
1366 n /= 10
1367 }
1368 if neg {
1369 buf = append(buf, '-')
1370 }
1371 for i, j := int32(0), len(buf)-1; i < j; i, j = i+1, j-1 {
1372 buf[i], buf[j] = buf[j], buf[i]
1373 }
1374 return string(buf)
1375 }
1376
1377 func Contains(tokset uint64, tok Token) (ok bool) {
1378 var bit uint64 = 1
1379 bit = bit << tok
1380 return tokset&bit != 0
1381 }
1382
1383
1384 type Token uint32
1385
1386 const (
1387 _ Token = iota
1388 EOF
1389
1390 NameType
1391 Literal
1392
1393 OperatorType
1394 AssignOp
1395 IncOp
1396 Assign
1397 Define
1398 Arrow
1399 Star
1400
1401 Lparen
1402 Lbrack
1403 Lbrace
1404 Rparen
1405 Rbrack
1406 Rbrace
1407 Comma
1408 Semi
1409 Colon
1410 Dot
1411 DotDotDot
1412
1413 Break
1414 Case
1415 Chan
1416 Const
1417 Continue
1418 Default
1419 Defer
1420 Else
1421 Fallthrough
1422 For
1423 Func
1424 Go
1425 Goto
1426 If
1427 Import
1428 Interface
1429 Map
1430 Package
1431 Range
1432 Return
1433 Select
1434 Struct
1435 Switch
1436 TypeType
1437 Var
1438
1439 tokenCount
1440 )
1441
1442 const _ uint64 = 1 << (tokenCount - 1)
1443
1444 type LitKind uint8
1445
1446 const (
1447 IntLit LitKind = iota
1448 FloatLit
1449 ImagLit
1450 RuneLit
1451 StringLit
1452 )
1453
1454 type Operator uint32
1455
1456 const (
1457 _ Operator = iota
1458
1459 Def
1460 Not
1461 Recv
1462 Tilde
1463
1464 OrOr
1465
1466 AndAnd
1467
1468 Eql
1469 Neq
1470 Lss
1471 Leq
1472 Gtr
1473 Geq
1474
1475 Add
1476 Sub
1477 Or
1478 Xor
1479
1480 Mul
1481 Div
1482 Rem
1483 And
1484 AndNot
1485 Shl
1486 Shr
1487 )
1488
1489 const (
1490 _ = iota
1491 PrecOrOr
1492 PrecAndAnd
1493 PrecCmp
1494 PrecAdd
1495 PrecMul
1496 )
1497
1498 const token_name = "EOFnameliteralopop=opop=:=<-*([{)]},;:....breakcasechanconstcontinuedefaultdeferelsefallthroughforfuncgogotoifimportinterfacemappackagerangereturnselectstructswitchtypevar"
1499
1500 func (i Token) String() (s string) {
1501 idx := [48]uint8{0, 3, 7, 14, 16, 19, 23, 24, 26, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 42, 47, 51, 55, 60, 68, 75, 80, 84, 95, 98, 102, 104, 108, 110, 116, 125, 128, 135, 140, 146, 152, 158, 164, 168, 171, 171}
1502 i -= 1
1503 if i >= Token(len(idx)-1) {
1504 return "token(" | Itoa64(int64(i+1)) | ")"
1505 }
1506 return token_name[idx[i]:idx[i+1]]
1507 }
1508
1509 const _Operator_name = ":!<-~||&&==!=<<=>>=+-|^*/%&&^<<>>"
1510
1511 func (i Operator) String() (s string) {
1512 idx := [24]uint8{0, 1, 2, 4, 5, 7, 9, 11, 13, 14, 16, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 29, 31, 33}
1513 i -= 1
1514 if i >= Operator(len(idx)-1) {
1515 return "Operator(" | Itoa64(int64(i+1)) | ")"
1516 }
1517 return _Operator_name[idx[i]:idx[i+1]]
1518 }
1519