scan.mx raw
1 package main
2
3 import (
4 "unicode/utf8"
5 )
6
7 type Token uint8
8
9 const (
10 TokEOF Token = iota
11 TokName
12 TokLit
13 TokOp
14 TokAssignOp
15 TokIncOp
16 TokAssign
17 TokDefine
18 TokArrow
19 TokStar
20 TokLparen
21 TokLbrack
22 TokLbrace
23 TokRparen
24 TokRbrack
25 TokRbrace
26 TokComma
27 TokSemi
28 TokColon
29 TokDot
30 TokDotDotDot
31 TokBreak
32 TokCase
33 TokChan
34 TokConst
35 TokContinue
36 TokDefault
37 TokDefer
38 TokElse
39 TokFor
40 TokFunc
41 TokGo
42 TokGoto
43 TokIf
44 TokImport
45 TokInterface
46 TokMap
47 TokPackage
48 TokRange
49 TokReturn
50 TokSelect
51 TokStruct
52 TokSwitch
53 TokType
54 TokVar
55 TokComment
56 TokFallthrough
57 )
58
59 type FTok struct {
60 tok Token
61 lit string
62 line uint32
63 col uint32
64 nlBefore bool
65 blankBef bool
66 }
67
68 var tokNames [48]string
69
70 var tokNamesReady bool
71
72 func initTokNames() {
73 if tokNamesReady { return }
74 tokNamesReady = true
75 tokNames[TokEOF] = ""
76 tokNames[TokAssign] = "="
77 tokNames[TokDefine] = ":="
78 tokNames[TokArrow] = "<-"
79 tokNames[TokStar] = "*"
80 tokNames[TokLparen] = "("
81 tokNames[TokLbrack] = "["
82 tokNames[TokLbrace] = "{"
83 tokNames[TokRparen] = ")"
84 tokNames[TokRbrack] = "]"
85 tokNames[TokRbrace] = "}"
86 tokNames[TokComma] = ","
87 tokNames[TokSemi] = ";"
88 tokNames[TokColon] = ":"
89 tokNames[TokDot] = "."
90 tokNames[TokDotDotDot] = "..."
91 tokNames[TokBreak] = "break"
92 tokNames[TokCase] = "case"
93 tokNames[TokChan] = "chan"
94 tokNames[TokConst] = "const"
95 tokNames[TokContinue] = "continue"
96 tokNames[TokDefault] = "default"
97 tokNames[TokDefer] = "defer"
98 tokNames[TokElse] = "else"
99 tokNames[TokFor] = "for"
100 tokNames[TokFunc] = "func"
101 tokNames[TokFallthrough] = "fallthrough"
102 tokNames[TokGo] = "go"
103 tokNames[TokGoto] = "goto"
104 tokNames[TokIf] = "if"
105 tokNames[TokImport] = "import"
106 tokNames[TokInterface] = "interface"
107 tokNames[TokMap] = "map"
108 tokNames[TokPackage] = "package"
109 tokNames[TokRange] = "range"
110 tokNames[TokReturn] = "return"
111 tokNames[TokSelect] = "select"
112 tokNames[TokStruct] = "struct"
113 tokNames[TokSwitch] = "switch"
114 tokNames[TokType] = "type"
115 tokNames[TokVar] = "var"
116 }
117
118 func tokText(t *FTok) string {
119 initTokNames()
120 tt := t.tok
121 if tt == TokName || tt == TokLit || tt == TokComment ||
122 tt == TokOp || tt == TokAssignOp || tt == TokIncOp {
123 return t.lit
124 }
125 if int32(tt) < int32(len(tokNames)) && tokNames[tt] != "" {
126 return tokNames[tt]
127 }
128 return "?"
129 }
130
131 func tokDbg(t *FTok) string {
132 tt := t.tok
133 if tt == TokEOF { return "EOF" }
134 if tt == TokSemi { return "SEMI" }
135 if tt == TokComment { return "COMMENT:" | t.lit }
136 if tt == TokName { return "NAME:" | t.lit }
137 if tt == TokLit { return "LIT:" | t.lit }
138 if tt == TokOp { return "OP:" | t.lit }
139 txt := tokText(t)
140 if txt != "?" { return txt }
141 return "KW:" | txt
142 }
143
144 func isNlSemiTok(tok Token) (yes bool) {
145 if tok == TokName || tok == TokLit || tok == TokBreak || tok == TokContinue ||
146 tok == TokFallthrough || tok == TokReturn || tok == TokIncOp ||
147 tok == TokRparen || tok == TokRbrack || tok == TokRbrace {
148 yes = true
149 }
150 return
151 }
152
153 type Scan struct {
154 src string
155 pos int32
156 line uint32
157 col uint32
158 ch rune
159 chw int32
160 nlsemi bool
161 }
162
163 func (s *Scan) init(src string) {
164 s.src = src
165 s.pos = 0
166 s.line = 1
167 s.col = 1
168 s.ch = ' '
169 s.chw = 0
170 s.nlsemi = false
171 s.advance()
172 }
173
174 func (s *Scan) advance() {
175 if s.pos >= int32(len(s.src)) {
176 s.ch = -1
177 s.chw = 0
178 return
179 }
180 b := s.src[s.pos]
181 if b < 0x80 {
182 s.ch = rune(b)
183 s.chw = 1
184 s.pos++
185 return
186 }
187 r, w := utf8.DecodeRune([]byte(s.src[s.pos:]))
188 s.ch = r
189 s.chw = int32(w)
190 s.pos += s.chw
191 }
192
193 func (s *Scan) nextch() {
194 if s.ch == '\n' {
195 s.line++
196 s.col = 0
197 }
198 s.col += uint32(s.chw)
199 s.advance()
200 }
201
202 func isLetter(ch rune) (yes bool) {
203 if 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' {
204 yes = true
205 }
206 return
207 }
208
209 func isDecimal(ch rune) (yes bool) {
210 if '0' <= ch && ch <= '9' {
211 yes = true
212 }
213 return
214 }
215
216 func isHex(ch rune) (yes bool) {
217 if '0' <= ch && ch <= '9' || 'a' <= ch && ch <= 'f' || 'A' <= ch && ch <= 'F' {
218 yes = true
219 }
220 return
221 }
222
223 func lower(ch rune) (r rune) {
224 r = ch | ('a' - 'A')
225 return
226 }
227
228 func (s *Scan) curPos() int32 { return s.pos - s.chw }
229
230 func (s *Scan) scanIdent(start int32) string {
231 for isLetter(s.ch) || isDecimal(s.ch) || s.ch >= utf8.RuneSelf {
232 s.nextch()
233 }
234 return s.src[start:s.curPos()]
235 }
236
237 func (s *Scan) scanNumber() string {
238 start := s.curPos()
239 if s.ch == '0' {
240 s.nextch()
241 switch lower(s.ch) {
242 case 'x':
243 s.nextch()
244 for isHex(s.ch) || s.ch == '_' { s.nextch() }
245 case 'o':
246 s.nextch()
247 for '0' <= s.ch && s.ch <= '7' || s.ch == '_' { s.nextch() }
248 case 'b':
249 s.nextch()
250 for s.ch == '0' || s.ch == '1' || s.ch == '_' { s.nextch() }
251 default:
252 for isDecimal(s.ch) || s.ch == '_' { s.nextch() }
253 }
254 } else {
255 for isDecimal(s.ch) || s.ch == '_' { s.nextch() }
256 }
257 if s.ch == '.' {
258 s.nextch()
259 for isDecimal(s.ch) || s.ch == '_' { s.nextch() }
260 }
261 e := lower(s.ch)
262 if e == 'e' || e == 'p' {
263 s.nextch()
264 if s.ch == '+' || s.ch == '-' { s.nextch() }
265 for isDecimal(s.ch) || s.ch == '_' { s.nextch() }
266 }
267 if s.ch == 'i' { s.nextch() }
268 return s.src[start:s.curPos()]
269 }
270
271 func (s *Scan) scanString(quote rune) string {
272 start := s.curPos()
273 s.nextch()
274 for s.ch >= 0 && s.ch != quote {
275 if s.ch == '\\' && quote != '`' {
276 s.nextch()
277 }
278 if s.ch == '\n' && quote == '"' { break }
279 s.nextch()
280 }
281 if s.ch == quote { s.nextch() }
282 return s.src[start:s.curPos()]
283 }
284
285 func (s *Scan) scanLineComment(start int32) string {
286 for s.ch >= 0 && s.ch != '\n' { s.nextch() }
287 return s.src[start:s.curPos()]
288 }
289
290 func (s *Scan) scanBlockComment(start int32) string {
291 for s.ch >= 0 {
292 if s.ch == '*' {
293 s.nextch()
294 if s.ch == '/' {
295 s.nextch()
296 return s.src[start:s.curPos()]
297 }
298 continue
299 }
300 s.nextch()
301 }
302 return s.src[start:s.curPos()]
303 }
304
305 var kwMap [64]kwEntry
306
307 type kwEntry struct {
308 word string
309 tok Token
310 }
311
312 var kwReady bool
313
314 func initKW() {
315 if kwReady { return }
316 kwReady = true
317 kws := [26]kwEntry{
318 {"break", TokBreak},
319 {"case", TokCase},
320 {"chan", TokChan},
321 {"const", TokConst},
322 {"continue", TokContinue},
323 {"default", TokDefault},
324 {"defer", TokDefer},
325 {"else", TokElse},
326 {"fallthrough", TokFallthrough},
327 {"for", TokFor},
328 {"func", TokFunc},
329 {"go", TokGo},
330 {"goto", TokGoto},
331 {"if", TokIf},
332 {"import", TokImport},
333 {"interface", TokInterface},
334 {"map", TokMap},
335 {"package", TokPackage},
336 {"range", TokRange},
337 {"return", TokReturn},
338 {"select", TokSelect},
339 {"struct", TokStruct},
340 {"switch", TokSwitch},
341 {"type", TokType},
342 {"var", TokVar},
343 {"spawn", TokName},
344 }
345 for i := int32(0); i < int32(len(kws)); i++ {
346 e := &kws[i]
347 if len(e.word) < 2 { continue }
348 h := kwHash(e.word)
349 kwMap[h] = *e
350 }
351 }
352
353 func kwHash(w string) uint32 {
354 return (uint32(w[0])<<4 ^ uint32(w[1]) + uint32(len(w))) & 63
355 }
356
357 func lookupKW(word string) (tok Token) {
358 tok = TokName
359 if len(word) < 2 { return }
360 h := kwHash(word)
361 e := &kwMap[h]
362 if e.word == word { tok = e.tok }
363 return
364 }
365
366 func tokenize(src string) []FTok {
367 initKW()
368 var s Scan
369 s.init(src)
370
371 toks := []FTok{:0:256}
372 prevLine := uint32(1)
373 prevWasNl := false
374 hadBlank := false
375
376 for {
377 for s.ch == ' ' || s.ch == '\t' || s.ch == '\r' || s.ch == '\n' && !s.nlsemi {
378 if s.ch == '\n' {
379 if prevWasNl { hadBlank = true }
380 prevWasNl = true
381 } else {
382 prevWasNl = false
383 }
384 s.nextch()
385 }
386
387 tline := s.line
388 tcol := s.col
389 nlBefore := tline > prevLine
390 if s.ch == '\n' && s.nlsemi {
391 nlBefore = true
392 }
393
394 var tok Token
395 var lit string
396 s.nlsemi = false
397
398 switch {
399 case s.ch < 0:
400 if nlBefore {
401 tok = TokSemi
402 lit = "\n"
403 toks = append(toks, FTok{tok, lit, tline, tcol, nlBefore, hadBlank})
404 hadBlank = false
405 }
406 toks = append(toks, FTok{TokEOF, "", tline, tcol, false, false})
407 return toks
408
409 case s.ch == '\n':
410 s.nextch()
411 tok = TokSemi
412 lit = "\n"
413 s.nlsemi = false
414 toks = append(toks, FTok{tok, lit, tline, tcol, true, hadBlank})
415 hadBlank = false
416 prevLine = tline
417 prevWasNl = true
418 continue
419
420 case isLetter(s.ch) || s.ch >= utf8.RuneSelf:
421 start := s.curPos()
422 s.nextch()
423 lit = s.scanIdent(start)
424 tok = lookupKW(lit)
425 s.nlsemi = isNlSemiTok(tok)
426
427 case isDecimal(s.ch):
428 lit = s.scanNumber()
429 tok = TokLit
430 s.nlsemi = true
431
432 case s.ch == '"' || s.ch == '\'' || s.ch == '`':
433 lit = s.scanString(s.ch)
434 tok = TokLit
435 s.nlsemi = true
436
437 case s.ch == '(':
438 s.nextch()
439 tok = TokLparen
440 case s.ch == ')':
441 s.nextch()
442 tok = TokRparen
443 s.nlsemi = true
444 case s.ch == '[':
445 s.nextch()
446 tok = TokLbrack
447 case s.ch == ']':
448 s.nextch()
449 tok = TokRbrack
450 s.nlsemi = true
451 case s.ch == '{':
452 s.nextch()
453 tok = TokLbrace
454 case s.ch == '}':
455 s.nextch()
456 tok = TokRbrace
457 s.nlsemi = true
458 case s.ch == ',':
459 s.nextch()
460 tok = TokComma
461 case s.ch == ';':
462 s.nextch()
463 tok = TokSemi
464 lit = ";"
465 case s.ch == ':':
466 s.nextch()
467 if s.ch == '=' {
468 s.nextch()
469 tok = TokDefine
470 } else {
471 tok = TokColon
472 }
473 case s.ch == '.':
474 dotStart := s.curPos()
475 s.nextch()
476 if isDecimal(s.ch) {
477 for isDecimal(s.ch) || s.ch == '_' { s.nextch() }
478 e := lower(s.ch)
479 if e == 'e' {
480 s.nextch()
481 if s.ch == '+' || s.ch == '-' { s.nextch() }
482 for isDecimal(s.ch) { s.nextch() }
483 }
484 if s.ch == 'i' { s.nextch() }
485 lit = s.src[dotStart:s.curPos()]
486 tok = TokLit
487 s.nlsemi = true
488 } else if s.ch == '.' {
489 s.nextch()
490 if s.ch == '.' {
491 s.nextch()
492 tok = TokDotDotDot
493 } else {
494 tok = TokDot
495 }
496 } else {
497 tok = TokDot
498 }
499
500 case s.ch == '/':
501 cstart := s.curPos()
502 s.nextch()
503 if s.ch == '/' {
504 s.nextch()
505 lit = s.scanLineComment(cstart)
506 tok = TokComment
507 toks = append(toks, FTok{tok, lit, tline, tcol, nlBefore, hadBlank})
508 hadBlank = false
509 prevLine = tline
510 prevWasNl = false
511 continue
512 }
513 if s.ch == '*' {
514 s.nextch()
515 lit = s.scanBlockComment(cstart)
516 tok = TokComment
517 toks = append(toks, FTok{tok, lit, tline, tcol, nlBefore, hadBlank})
518 hadBlank = false
519 prevLine = s.line
520 prevWasNl = false
521 continue
522 }
523 if s.ch == '=' {
524 s.nextch()
525 tok = TokAssignOp
526 lit = "/="
527 } else {
528 tok = TokOp
529 lit = "/"
530 }
531
532 case s.ch == '+':
533 s.nextch()
534 if s.ch == '+' {
535 s.nextch()
536 tok = TokIncOp
537 lit = "++"
538 s.nlsemi = true
539 } else if s.ch == '=' {
540 s.nextch()
541 tok = TokAssignOp
542 lit = "+="
543 } else {
544 tok = TokOp
545 lit = "+"
546 }
547 case s.ch == '-':
548 s.nextch()
549 if s.ch == '-' {
550 s.nextch()
551 tok = TokIncOp
552 lit = "--"
553 s.nlsemi = true
554 } else if s.ch == '=' {
555 s.nextch()
556 tok = TokAssignOp
557 lit = "-="
558 } else {
559 tok = TokOp
560 lit = "-"
561 }
562 case s.ch == '*':
563 s.nextch()
564 if s.ch == '=' {
565 s.nextch()
566 tok = TokAssignOp
567 lit = "*="
568 } else {
569 tok = TokStar
570 }
571 case s.ch == '%':
572 s.nextch()
573 if s.ch == '=' {
574 s.nextch()
575 tok = TokAssignOp
576 lit = "%="
577 } else {
578 tok = TokOp
579 lit = "%"
580 }
581 case s.ch == '&':
582 s.nextch()
583 if s.ch == '&' {
584 s.nextch()
585 tok = TokOp
586 lit = "&&"
587 } else if s.ch == '^' {
588 s.nextch()
589 if s.ch == '=' {
590 s.nextch()
591 tok = TokAssignOp
592 lit = "&^="
593 } else {
594 tok = TokOp
595 lit = "&^"
596 }
597 } else if s.ch == '=' {
598 s.nextch()
599 tok = TokAssignOp
600 lit = "&="
601 } else {
602 tok = TokOp
603 lit = "&"
604 }
605 case s.ch == '|':
606 s.nextch()
607 if s.ch == '|' {
608 s.nextch()
609 tok = TokOp
610 lit = "||"
611 } else if s.ch == '=' {
612 s.nextch()
613 tok = TokAssignOp
614 lit = "|="
615 } else {
616 tok = TokOp
617 lit = "|"
618 }
619 case s.ch == '^':
620 s.nextch()
621 if s.ch == '=' {
622 s.nextch()
623 tok = TokAssignOp
624 lit = "^="
625 } else {
626 tok = TokOp
627 lit = "^"
628 }
629 case s.ch == '<':
630 s.nextch()
631 if s.ch == '=' {
632 s.nextch()
633 tok = TokOp
634 lit = "<="
635 } else if s.ch == '<' {
636 s.nextch()
637 if s.ch == '=' {
638 s.nextch()
639 tok = TokAssignOp
640 lit = "<<="
641 } else {
642 tok = TokOp
643 lit = "<<"
644 }
645 } else if s.ch == '-' {
646 s.nextch()
647 tok = TokArrow
648 } else {
649 tok = TokOp
650 lit = "<"
651 }
652 case s.ch == '>':
653 s.nextch()
654 if s.ch == '=' {
655 s.nextch()
656 tok = TokOp
657 lit = ">="
658 } else if s.ch == '>' {
659 s.nextch()
660 if s.ch == '=' {
661 s.nextch()
662 tok = TokAssignOp
663 lit = ">>="
664 } else {
665 tok = TokOp
666 lit = ">>"
667 }
668 } else {
669 tok = TokOp
670 lit = ">"
671 }
672 case s.ch == '=':
673 s.nextch()
674 if s.ch == '=' {
675 s.nextch()
676 tok = TokOp
677 lit = "=="
678 } else {
679 tok = TokAssign
680 }
681 case s.ch == '!':
682 s.nextch()
683 if s.ch == '=' {
684 s.nextch()
685 tok = TokOp
686 lit = "!="
687 } else {
688 tok = TokOp
689 lit = "!"
690 }
691 case s.ch == '~':
692 s.nextch()
693 tok = TokOp
694 lit = "~"
695
696 default:
697 s.nextch()
698 continue
699 }
700
701 toks = append(toks, FTok{tok, lit, tline, tcol, nlBefore, hadBlank})
702 hadBlank = false
703 prevLine = s.line
704 prevWasNl = false
705 }
706 }
707