package main

import (
	"unicode/utf8"
)

type Token uint8

const (
	TokEOF Token = iota
	TokName
	TokLit
	TokOp
	TokAssignOp
	TokIncOp
	TokAssign
	TokDefine
	TokArrow
	TokStar
	TokLparen
	TokLbrack
	TokLbrace
	TokRparen
	TokRbrack
	TokRbrace
	TokComma
	TokSemi
	TokColon
	TokDot
	TokDotDotDot
	TokBreak
	TokCase
	TokChan
	TokConst
	TokContinue
	TokDefault
	TokDefer
	TokElse
	TokFor
	TokFunc
	TokGo
	TokGoto
	TokIf
	TokImport
	TokInterface
	TokMap
	TokPackage
	TokRange
	TokReturn
	TokSelect
	TokStruct
	TokSwitch
	TokType
	TokVar
	TokComment
	TokFallthrough
)

type FTok struct {
	tok      Token
	lit      string
	line     uint32
	col      uint32
	nlBefore bool
	blankBef bool
}

var tokNames [48]string

var tokNamesReady bool

func initTokNames() {
	if tokNamesReady { return }
	tokNamesReady = true
	tokNames[TokEOF] = ""
	tokNames[TokAssign] = "="
	tokNames[TokDefine] = ":="
	tokNames[TokArrow] = "<-"
	tokNames[TokStar] = "*"
	tokNames[TokLparen] = "("
	tokNames[TokLbrack] = "["
	tokNames[TokLbrace] = "{"
	tokNames[TokRparen] = ")"
	tokNames[TokRbrack] = "]"
	tokNames[TokRbrace] = "}"
	tokNames[TokComma] = ","
	tokNames[TokSemi] = ";"
	tokNames[TokColon] = ":"
	tokNames[TokDot] = "."
	tokNames[TokDotDotDot] = "..."
	tokNames[TokBreak] = "break"
	tokNames[TokCase] = "case"
	tokNames[TokChan] = "chan"
	tokNames[TokConst] = "const"
	tokNames[TokContinue] = "continue"
	tokNames[TokDefault] = "default"
	tokNames[TokDefer] = "defer"
	tokNames[TokElse] = "else"
	tokNames[TokFor] = "for"
	tokNames[TokFunc] = "func"
	tokNames[TokFallthrough] = "fallthrough"
	tokNames[TokGo] = "go"
	tokNames[TokGoto] = "goto"
	tokNames[TokIf] = "if"
	tokNames[TokImport] = "import"
	tokNames[TokInterface] = "interface"
	tokNames[TokMap] = "map"
	tokNames[TokPackage] = "package"
	tokNames[TokRange] = "range"
	tokNames[TokReturn] = "return"
	tokNames[TokSelect] = "select"
	tokNames[TokStruct] = "struct"
	tokNames[TokSwitch] = "switch"
	tokNames[TokType] = "type"
	tokNames[TokVar] = "var"
}

func tokText(t *FTok) string {
	initTokNames()
	tt := t.tok
	if tt == TokName || tt == TokLit || tt == TokComment ||
		tt == TokOp || tt == TokAssignOp || tt == TokIncOp {
		return t.lit
	}
	if int32(tt) < int32(len(tokNames)) && tokNames[tt] != "" {
		return tokNames[tt]
	}
	return "?"
}

func tokDbg(t *FTok) string {
	tt := t.tok
	if tt == TokEOF { return "EOF" }
	if tt == TokSemi { return "SEMI" }
	if tt == TokComment { return "COMMENT:" | t.lit }
	if tt == TokName { return "NAME:" | t.lit }
	if tt == TokLit { return "LIT:" | t.lit }
	if tt == TokOp { return "OP:" | t.lit }
	txt := tokText(t)
	if txt != "?" { return txt }
	return "KW:" | txt
}

func isNlSemiTok(tok Token) (yes bool) {
	if tok == TokName || tok == TokLit || tok == TokBreak || tok == TokContinue ||
		tok == TokFallthrough || tok == TokReturn || tok == TokIncOp ||
		tok == TokRparen || tok == TokRbrack || tok == TokRbrace {
		yes = true
	}
	return
}

type Scan struct {
	src    string
	pos    int32
	line   uint32
	col    uint32
	ch     rune
	chw    int32
	nlsemi bool
}

func (s *Scan) init(src string) {
	s.src = src
	s.pos = 0
	s.line = 1
	s.col = 1
	s.ch = ' '
	s.chw = 0
	s.nlsemi = false
	s.advance()
}

func (s *Scan) advance() {
	if s.pos >= int32(len(s.src)) {
		s.ch = -1
		s.chw = 0
		return
	}
	b := s.src[s.pos]
	if b < 0x80 {
		s.ch = rune(b)
		s.chw = 1
		s.pos++
		return
	}
	r, w := utf8.DecodeRune([]byte(s.src[s.pos:]))
	s.ch = r
	s.chw = int32(w)
	s.pos += s.chw
}

func (s *Scan) nextch() {
	if s.ch == '\n' {
		s.line++
		s.col = 0
	}
	s.col += uint32(s.chw)
	s.advance()
}

func isLetter(ch rune) (yes bool) {
	if 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' {
		yes = true
	}
	return
}

func isDecimal(ch rune) (yes bool) {
	if '0' <= ch && ch <= '9' {
		yes = true
	}
	return
}

func isHex(ch rune) (yes bool) {
	if '0' <= ch && ch <= '9' || 'a' <= ch && ch <= 'f' || 'A' <= ch && ch <= 'F' {
		yes = true
	}
	return
}

func lower(ch rune) (r rune) {
	r = ch | ('a' - 'A')
	return
}

func (s *Scan) curPos() int32 { return s.pos - s.chw }

func (s *Scan) scanIdent(start int32) string {
	for isLetter(s.ch) || isDecimal(s.ch) || s.ch >= utf8.RuneSelf {
		s.nextch()
	}
	return s.src[start:s.curPos()]
}

func (s *Scan) scanNumber() string {
	start := s.curPos()
	if s.ch == '0' {
		s.nextch()
		switch lower(s.ch) {
		case 'x':
			s.nextch()
			for isHex(s.ch) || s.ch == '_' { s.nextch() }
		case 'o':
			s.nextch()
			for '0' <= s.ch && s.ch <= '7' || s.ch == '_' { s.nextch() }
		case 'b':
			s.nextch()
			for s.ch == '0' || s.ch == '1' || s.ch == '_' { s.nextch() }
		default:
			for isDecimal(s.ch) || s.ch == '_' { s.nextch() }
		}
	} else {
		for isDecimal(s.ch) || s.ch == '_' { s.nextch() }
	}
	if s.ch == '.' {
		s.nextch()
		for isDecimal(s.ch) || s.ch == '_' { s.nextch() }
	}
	e := lower(s.ch)
	if e == 'e' || e == 'p' {
		s.nextch()
		if s.ch == '+' || s.ch == '-' { s.nextch() }
		for isDecimal(s.ch) || s.ch == '_' { s.nextch() }
	}
	if s.ch == 'i' { s.nextch() }
	return s.src[start:s.curPos()]
}

func (s *Scan) scanString(quote rune) string {
	start := s.curPos()
	s.nextch()
	for s.ch >= 0 && s.ch != quote {
		if s.ch == '\\' && quote != '`' {
			s.nextch()
		}
		if s.ch == '\n' && quote == '"' { break }
		s.nextch()
	}
	if s.ch == quote { s.nextch() }
	return s.src[start:s.curPos()]
}

func (s *Scan) scanLineComment(start int32) string {
	for s.ch >= 0 && s.ch != '\n' { s.nextch() }
	return s.src[start:s.curPos()]
}

func (s *Scan) scanBlockComment(start int32) string {
	for s.ch >= 0 {
		if s.ch == '*' {
			s.nextch()
			if s.ch == '/' {
				s.nextch()
				return s.src[start:s.curPos()]
			}
			continue
		}
		s.nextch()
	}
	return s.src[start:s.curPos()]
}

var kwMap [64]kwEntry

type kwEntry struct {
	word string
	tok  Token
}

var kwReady bool

func initKW() {
	if kwReady { return }
	kwReady = true
	kws := [26]kwEntry{
		{"break", TokBreak},
		{"case", TokCase},
		{"chan", TokChan},
		{"const", TokConst},
		{"continue", TokContinue},
		{"default", TokDefault},
		{"defer", TokDefer},
		{"else", TokElse},
		{"fallthrough", TokFallthrough},
		{"for", TokFor},
		{"func", TokFunc},
		{"go", TokGo},
		{"goto", TokGoto},
		{"if", TokIf},
		{"import", TokImport},
		{"interface", TokInterface},
		{"map", TokMap},
		{"package", TokPackage},
		{"range", TokRange},
		{"return", TokReturn},
		{"select", TokSelect},
		{"struct", TokStruct},
		{"switch", TokSwitch},
		{"type", TokType},
		{"var", TokVar},
		{"spawn", TokName},
	}
	for i := int32(0); i < int32(len(kws)); i++ {
		e := &kws[i]
		if len(e.word) < 2 { continue }
		h := kwHash(e.word)
		kwMap[h] = *e
	}
}

func kwHash(w string) uint32 {
	return (uint32(w[0])<<4 ^ uint32(w[1]) + uint32(len(w))) & 63
}

func lookupKW(word string) (tok Token) {
	tok = TokName
	if len(word) < 2 { return }
	h := kwHash(word)
	e := &kwMap[h]
	if e.word == word { tok = e.tok }
	return
}

func tokenize(src string) []FTok {
	initKW()
	var s Scan
	s.init(src)

	toks := []FTok{:0:256}
	prevLine := uint32(1)
	prevWasNl := false
	hadBlank := false

	for {
		for s.ch == ' ' || s.ch == '\t' || s.ch == '\r' || s.ch == '\n' && !s.nlsemi {
			if s.ch == '\n' {
				if prevWasNl { hadBlank = true }
				prevWasNl = true
			} else {
				prevWasNl = false
			}
			s.nextch()
		}

		tline := s.line
		tcol := s.col
		nlBefore := tline > prevLine
		if s.ch == '\n' && s.nlsemi {
			nlBefore = true
		}

		var tok Token
		var lit string
		s.nlsemi = false

		switch {
		case s.ch < 0:
			if nlBefore {
				tok = TokSemi
				lit = "\n"
				toks = append(toks, FTok{tok, lit, tline, tcol, nlBefore, hadBlank})
				hadBlank = false
			}
			toks = append(toks, FTok{TokEOF, "", tline, tcol, false, false})
			return toks

		case s.ch == '\n':
			s.nextch()
			tok = TokSemi
			lit = "\n"
			s.nlsemi = false
			toks = append(toks, FTok{tok, lit, tline, tcol, true, hadBlank})
			hadBlank = false
			prevLine = tline
			prevWasNl = true
			continue

		case isLetter(s.ch) || s.ch >= utf8.RuneSelf:
			start := s.curPos()
			s.nextch()
			lit = s.scanIdent(start)
			tok = lookupKW(lit)
			s.nlsemi = isNlSemiTok(tok)

		case isDecimal(s.ch):
			lit = s.scanNumber()
			tok = TokLit
			s.nlsemi = true

		case s.ch == '"' || s.ch == '\'' || s.ch == '`':
			lit = s.scanString(s.ch)
			tok = TokLit
			s.nlsemi = true

		case s.ch == '(':
			s.nextch()
			tok = TokLparen
		case s.ch == ')':
			s.nextch()
			tok = TokRparen
			s.nlsemi = true
		case s.ch == '[':
			s.nextch()
			tok = TokLbrack
		case s.ch == ']':
			s.nextch()
			tok = TokRbrack
			s.nlsemi = true
		case s.ch == '{':
			s.nextch()
			tok = TokLbrace
		case s.ch == '}':
			s.nextch()
			tok = TokRbrace
			s.nlsemi = true
		case s.ch == ',':
			s.nextch()
			tok = TokComma
		case s.ch == ';':
			s.nextch()
			tok = TokSemi
			lit = ";"
		case s.ch == ':':
			s.nextch()
			if s.ch == '=' {
				s.nextch()
				tok = TokDefine
			} else {
				tok = TokColon
			}
		case s.ch == '.':
			dotStart := s.curPos()
			s.nextch()
			if isDecimal(s.ch) {
				for isDecimal(s.ch) || s.ch == '_' { s.nextch() }
				e := lower(s.ch)
				if e == 'e' {
					s.nextch()
					if s.ch == '+' || s.ch == '-' { s.nextch() }
					for isDecimal(s.ch) { s.nextch() }
				}
				if s.ch == 'i' { s.nextch() }
				lit = s.src[dotStart:s.curPos()]
				tok = TokLit
				s.nlsemi = true
			} else if s.ch == '.' {
				s.nextch()
				if s.ch == '.' {
					s.nextch()
					tok = TokDotDotDot
				} else {
					tok = TokDot
				}
			} else {
				tok = TokDot
			}

		case s.ch == '/':
			cstart := s.curPos()
			s.nextch()
			if s.ch == '/' {
				s.nextch()
				lit = s.scanLineComment(cstart)
				tok = TokComment
				toks = append(toks, FTok{tok, lit, tline, tcol, nlBefore, hadBlank})
				hadBlank = false
				prevLine = tline
				prevWasNl = false
				continue
			}
			if s.ch == '*' {
				s.nextch()
				lit = s.scanBlockComment(cstart)
				tok = TokComment
				toks = append(toks, FTok{tok, lit, tline, tcol, nlBefore, hadBlank})
				hadBlank = false
				prevLine = s.line
				prevWasNl = false
				continue
			}
			if s.ch == '=' {
				s.nextch()
				tok = TokAssignOp
				lit = "/="
			} else {
				tok = TokOp
				lit = "/"
			}

		case s.ch == '+':
			s.nextch()
			if s.ch == '+' {
				s.nextch()
				tok = TokIncOp
				lit = "++"
				s.nlsemi = true
			} else if s.ch == '=' {
				s.nextch()
				tok = TokAssignOp
				lit = "+="
			} else {
				tok = TokOp
				lit = "+"
			}
		case s.ch == '-':
			s.nextch()
			if s.ch == '-' {
				s.nextch()
				tok = TokIncOp
				lit = "--"
				s.nlsemi = true
			} else if s.ch == '=' {
				s.nextch()
				tok = TokAssignOp
				lit = "-="
			} else {
				tok = TokOp
				lit = "-"
			}
		case s.ch == '*':
			s.nextch()
			if s.ch == '=' {
				s.nextch()
				tok = TokAssignOp
				lit = "*="
			} else {
				tok = TokStar
			}
		case s.ch == '%':
			s.nextch()
			if s.ch == '=' {
				s.nextch()
				tok = TokAssignOp
				lit = "%="
			} else {
				tok = TokOp
				lit = "%"
			}
		case s.ch == '&':
			s.nextch()
			if s.ch == '&' {
				s.nextch()
				tok = TokOp
				lit = "&&"
			} else if s.ch == '^' {
				s.nextch()
				if s.ch == '=' {
					s.nextch()
					tok = TokAssignOp
					lit = "&^="
				} else {
					tok = TokOp
					lit = "&^"
				}
			} else if s.ch == '=' {
				s.nextch()
				tok = TokAssignOp
				lit = "&="
			} else {
				tok = TokOp
				lit = "&"
			}
		case s.ch == '|':
			s.nextch()
			if s.ch == '|' {
				s.nextch()
				tok = TokOp
				lit = "||"
			} else if s.ch == '=' {
				s.nextch()
				tok = TokAssignOp
				lit = "|="
			} else {
				tok = TokOp
				lit = "|"
			}
		case s.ch == '^':
			s.nextch()
			if s.ch == '=' {
				s.nextch()
				tok = TokAssignOp
				lit = "^="
			} else {
				tok = TokOp
				lit = "^"
			}
		case s.ch == '<':
			s.nextch()
			if s.ch == '=' {
				s.nextch()
				tok = TokOp
				lit = "<="
			} else if s.ch == '<' {
				s.nextch()
				if s.ch == '=' {
					s.nextch()
					tok = TokAssignOp
					lit = "<<="
				} else {
					tok = TokOp
					lit = "<<"
				}
			} else if s.ch == '-' {
				s.nextch()
				tok = TokArrow
			} else {
				tok = TokOp
				lit = "<"
			}
		case s.ch == '>':
			s.nextch()
			if s.ch == '=' {
				s.nextch()
				tok = TokOp
				lit = ">="
			} else if s.ch == '>' {
				s.nextch()
				if s.ch == '=' {
					s.nextch()
					tok = TokAssignOp
					lit = ">>="
				} else {
					tok = TokOp
					lit = ">>"
				}
			} else {
				tok = TokOp
				lit = ">"
			}
		case s.ch == '=':
			s.nextch()
			if s.ch == '=' {
				s.nextch()
				tok = TokOp
				lit = "=="
			} else {
				tok = TokAssign
			}
		case s.ch == '!':
			s.nextch()
			if s.ch == '=' {
				s.nextch()
				tok = TokOp
				lit = "!="
			} else {
				tok = TokOp
				lit = "!"
			}
		case s.ch == '~':
			s.nextch()
			tok = TokOp
			lit = "~"

		default:
			s.nextch()
			continue
		}

		toks = append(toks, FTok{tok, lit, tline, tcol, nlBefore, hadBlank})
		hadBlank = false
		prevLine = s.line
		prevWasNl = false
	}
}