lexer.go raw

   1  package lexer
   2  
   3  import (
   4  	"io"
   5  
   6  	"github.com/goccy/go-yaml/scanner"
   7  	"github.com/goccy/go-yaml/token"
   8  )
   9  
  10  // Tokenize split to token instances from string
  11  func Tokenize(src string) token.Tokens {
  12  	var s scanner.Scanner
  13  	s.Init(src)
  14  	var tokens token.Tokens
  15  	for {
  16  		subTokens, err := s.Scan()
  17  		if err == io.EOF {
  18  			break
  19  		}
  20  		tokens.Add(subTokens...)
  21  	}
  22  	return tokens
  23  }
  24