tokenize.go raw

   1  package ini
   2  
   3  import (
   4  	"strings"
   5  )
   6  
   7  func tokenize(lines []string) ([]lineToken, error) {
   8  	tokens := make([]lineToken, 0, len(lines))
   9  	for _, line := range lines {
  10  		if len(strings.TrimSpace(line)) == 0 || isLineComment(line) {
  11  			continue
  12  		}
  13  
  14  		if tok := asProfile(line); tok != nil {
  15  			tokens = append(tokens, tok)
  16  		} else if tok := asProperty(line); tok != nil {
  17  			tokens = append(tokens, tok)
  18  		} else if tok := asSubProperty(line); tok != nil {
  19  			tokens = append(tokens, tok)
  20  		} else if tok := asContinuation(line); tok != nil {
  21  			tokens = append(tokens, tok)
  22  		} // unrecognized tokens are effectively ignored
  23  	}
  24  	return tokens, nil
  25  }
  26  
  27  func isLineComment(line string) bool {
  28  	trimmed := strings.TrimLeft(line, " \t")
  29  	return strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, ";")
  30  }
  31  
  32  func asProfile(line string) *lineTokenProfile { // " [ type name ] ; comment"
  33  	trimmed := strings.TrimSpace(trimProfileComment(line)) // "[ type name ]"
  34  	if !isBracketed(trimmed) {
  35  		return nil
  36  	}
  37  	trimmed = trimmed[1 : len(trimmed)-1] // " type name " (or just " name ")
  38  	trimmed = strings.TrimSpace(trimmed)  // "type name" / "name"
  39  	typ, name := splitProfile(trimmed)
  40  	return &lineTokenProfile{
  41  		Type: typ,
  42  		Name: name,
  43  	}
  44  }
  45  
  46  func asProperty(line string) *lineTokenProperty {
  47  	if isLineSpace(rune(line[0])) {
  48  		return nil
  49  	}
  50  
  51  	trimmed := trimPropertyComment(line)
  52  	trimmed = strings.TrimRight(trimmed, " \t")
  53  	k, v, ok := splitProperty(trimmed)
  54  	if !ok {
  55  		return nil
  56  	}
  57  
  58  	return &lineTokenProperty{
  59  		Key:   strings.ToLower(k), // LEGACY: normalize key case
  60  		Value: legacyStrconv(v),   // LEGACY: see func docs
  61  	}
  62  }
  63  
  64  func asSubProperty(line string) *lineTokenSubProperty {
  65  	if !isLineSpace(rune(line[0])) {
  66  		return nil
  67  	}
  68  
  69  	// comments on sub-properties are included in the value
  70  	trimmed := strings.TrimLeft(line, " \t")
  71  	k, v, ok := splitProperty(trimmed)
  72  	if !ok {
  73  		return nil
  74  	}
  75  
  76  	return &lineTokenSubProperty{ // same LEGACY constraints as in normal property
  77  		Key:   strings.ToLower(k),
  78  		Value: legacyStrconv(v),
  79  	}
  80  }
  81  
  82  func asContinuation(line string) *lineTokenContinuation {
  83  	if !isLineSpace(rune(line[0])) {
  84  		return nil
  85  	}
  86  
  87  	// includes comments like sub-properties
  88  	trimmed := strings.TrimLeft(line, " \t")
  89  	return &lineTokenContinuation{
  90  		Value: trimmed,
  91  	}
  92  }
  93