token.go raw
1 package token
2
3 import (
4 "fmt"
5 "strings"
6 )
7
8 // Character type for character
9 type Character byte
10
11 const (
12 // SequenceEntryCharacter character for sequence entry
13 SequenceEntryCharacter Character = '-'
14 // MappingKeyCharacter character for mapping key
15 MappingKeyCharacter Character = '?'
16 // MappingValueCharacter character for mapping value
17 MappingValueCharacter Character = ':'
18 // CollectEntryCharacter character for collect entry
19 CollectEntryCharacter Character = ','
20 // SequenceStartCharacter character for sequence start
21 SequenceStartCharacter Character = '['
22 // SequenceEndCharacter character for sequence end
23 SequenceEndCharacter Character = ']'
24 // MappingStartCharacter character for mapping start
25 MappingStartCharacter Character = '{'
26 // MappingEndCharacter character for mapping end
27 MappingEndCharacter Character = '}'
28 // CommentCharacter character for comment
29 CommentCharacter Character = '#'
30 // AnchorCharacter character for anchor
31 AnchorCharacter Character = '&'
32 // AliasCharacter character for alias
33 AliasCharacter Character = '*'
34 // TagCharacter character for tag
35 TagCharacter Character = '!'
36 // LiteralCharacter character for literal
37 LiteralCharacter Character = '|'
38 // FoldedCharacter character for folded
39 FoldedCharacter Character = '>'
40 // SingleQuoteCharacter character for single quote
41 SingleQuoteCharacter Character = '\''
42 // DoubleQuoteCharacter character for double quote
43 DoubleQuoteCharacter Character = '"'
44 // DirectiveCharacter character for directive
45 DirectiveCharacter Character = '%'
46 // SpaceCharacter character for space
47 SpaceCharacter Character = ' '
48 // LineBreakCharacter character for line break
49 LineBreakCharacter Character = '\n'
50 )
51
52 // Type type identifier for token
53 type Type int
54
55 const (
56 // UnknownType reserve for invalid type
57 UnknownType Type = iota
58 // DocumentHeaderType type for DocumentHeader token
59 DocumentHeaderType
60 // DocumentEndType type for DocumentEnd token
61 DocumentEndType
62 // SequenceEntryType type for SequenceEntry token
63 SequenceEntryType
64 // MappingKeyType type for MappingKey token
65 MappingKeyType
66 // MappingValueType type for MappingValue token
67 MappingValueType
68 // MergeKeyType type for MergeKey token
69 MergeKeyType
70 // CollectEntryType type for CollectEntry token
71 CollectEntryType
72 // SequenceStartType type for SequenceStart token
73 SequenceStartType
74 // SequenceEndType type for SequenceEnd token
75 SequenceEndType
76 // MappingStartType type for MappingStart token
77 MappingStartType
78 // MappingEndType type for MappingEnd token
79 MappingEndType
80 // CommentType type for Comment token
81 CommentType
82 // AnchorType type for Anchor token
83 AnchorType
84 // AliasType type for Alias token
85 AliasType
86 // TagType type for Tag token
87 TagType
88 // LiteralType type for Literal token
89 LiteralType
90 // FoldedType type for Folded token
91 FoldedType
92 // SingleQuoteType type for SingleQuote token
93 SingleQuoteType
94 // DoubleQuoteType type for DoubleQuote token
95 DoubleQuoteType
96 // DirectiveType type for Directive token
97 DirectiveType
98 // SpaceType type for Space token
99 SpaceType
100 // NullType type for Null token
101 NullType
102 // InfinityType type for Infinity token
103 InfinityType
104 // NanType type for Nan token
105 NanType
106 // IntegerType type for Integer token
107 IntegerType
108 // BinaryIntegerType type for BinaryInteger token
109 BinaryIntegerType
110 // OctetIntegerType type for OctetInteger token
111 OctetIntegerType
112 // HexIntegerType type for HexInteger token
113 HexIntegerType
114 // FloatType type for Float token
115 FloatType
116 // StringType type for String token
117 StringType
118 // BoolType type for Bool token
119 BoolType
120 )
121
122 // String type identifier to text
123 func (t Type) String() string {
124 switch t {
125 case UnknownType:
126 return "Unknown"
127 case DocumentHeaderType:
128 return "DocumentHeader"
129 case DocumentEndType:
130 return "DocumentEnd"
131 case SequenceEntryType:
132 return "SequenceEntry"
133 case MappingKeyType:
134 return "MappingKey"
135 case MappingValueType:
136 return "MappingValue"
137 case MergeKeyType:
138 return "MergeKey"
139 case CollectEntryType:
140 return "CollectEntry"
141 case SequenceStartType:
142 return "SequenceStart"
143 case SequenceEndType:
144 return "SequenceEnd"
145 case MappingStartType:
146 return "MappingStart"
147 case MappingEndType:
148 return "MappingEnd"
149 case CommentType:
150 return "Comment"
151 case AnchorType:
152 return "Anchor"
153 case AliasType:
154 return "Alias"
155 case TagType:
156 return "Tag"
157 case LiteralType:
158 return "Literal"
159 case FoldedType:
160 return "Folded"
161 case SingleQuoteType:
162 return "SingleQuote"
163 case DoubleQuoteType:
164 return "DoubleQuote"
165 case DirectiveType:
166 return "Directive"
167 case SpaceType:
168 return "Space"
169 case StringType:
170 return "String"
171 case BoolType:
172 return "Bool"
173 case IntegerType:
174 return "Integer"
175 case BinaryIntegerType:
176 return "BinaryInteger"
177 case OctetIntegerType:
178 return "OctetInteger"
179 case HexIntegerType:
180 return "HexInteger"
181 case FloatType:
182 return "Float"
183 case NullType:
184 return "Null"
185 case InfinityType:
186 return "Infinity"
187 case NanType:
188 return "Nan"
189 }
190 return ""
191 }
192
193 // CharacterType type for character category
194 type CharacterType int
195
196 const (
197 // CharacterTypeIndicator type of indicator character
198 CharacterTypeIndicator CharacterType = iota
199 // CharacterTypeWhiteSpace type of white space character
200 CharacterTypeWhiteSpace
201 // CharacterTypeMiscellaneous type of miscellaneous character
202 CharacterTypeMiscellaneous
203 // CharacterTypeEscaped type of escaped character
204 CharacterTypeEscaped
205 )
206
207 // String character type identifier to text
208 func (c CharacterType) String() string {
209 switch c {
210 case CharacterTypeIndicator:
211 return "Indicator"
212 case CharacterTypeWhiteSpace:
213 return "WhiteSpcae"
214 case CharacterTypeMiscellaneous:
215 return "Miscellaneous"
216 case CharacterTypeEscaped:
217 return "Escaped"
218 }
219 return ""
220 }
221
222 // Indicator type for indicator
223 type Indicator int
224
225 const (
226 // NotIndicator not indicator
227 NotIndicator Indicator = iota
228 // BlockStructureIndicator indicator for block structure ( '-', '?', ':' )
229 BlockStructureIndicator
230 // FlowCollectionIndicator indicator for flow collection ( '[', ']', '{', '}', ',' )
231 FlowCollectionIndicator
232 // CommentIndicator indicator for comment ( '#' )
233 CommentIndicator
234 // NodePropertyIndicator indicator for node property ( '!', '&', '*' )
235 NodePropertyIndicator
236 // BlockScalarIndicator indicator for block scalar ( '|', '>' )
237 BlockScalarIndicator
238 // QuotedScalarIndicator indicator for quoted scalar ( ''', '"' )
239 QuotedScalarIndicator
240 // DirectiveIndicator indicator for directive ( '%' )
241 DirectiveIndicator
242 // InvalidUseOfReservedIndicator indicator for invalid use of reserved keyword ( '@', '`' )
243 InvalidUseOfReservedIndicator
244 )
245
246 // String indicator to text
247 func (i Indicator) String() string {
248 switch i {
249 case NotIndicator:
250 return "NotIndicator"
251 case BlockStructureIndicator:
252 return "BlockStructure"
253 case FlowCollectionIndicator:
254 return "FlowCollection"
255 case CommentIndicator:
256 return "Comment"
257 case NodePropertyIndicator:
258 return "NodeProperty"
259 case BlockScalarIndicator:
260 return "BlockScalar"
261 case QuotedScalarIndicator:
262 return "QuotedScalar"
263 case DirectiveIndicator:
264 return "Directive"
265 case InvalidUseOfReservedIndicator:
266 return "InvalidUseOfReserved"
267 }
268 return ""
269 }
270
271 var (
272 reservedNullKeywords = []string{
273 "null",
274 "Null",
275 "NULL",
276 "~",
277 }
278 reservedBoolKeywords = []string{
279 "true",
280 "True",
281 "TRUE",
282 "false",
283 "False",
284 "FALSE",
285 }
286 reservedInfKeywords = []string{
287 ".inf",
288 ".Inf",
289 ".INF",
290 "-.inf",
291 "-.Inf",
292 "-.INF",
293 }
294 reservedNanKeywords = []string{
295 ".nan",
296 ".NaN",
297 ".NAN",
298 }
299 reservedKeywordMap = map[string]func(string, string, *Position) *Token{}
300 )
301
302 func reservedKeywordToken(typ Type, value, org string, pos *Position) *Token {
303 return &Token{
304 Type: typ,
305 CharacterType: CharacterTypeMiscellaneous,
306 Indicator: NotIndicator,
307 Value: value,
308 Origin: org,
309 Position: pos,
310 }
311 }
312
313 func init() {
314 for _, keyword := range reservedNullKeywords {
315 reservedKeywordMap[keyword] = func(value, org string, pos *Position) *Token {
316 return reservedKeywordToken(NullType, value, org, pos)
317 }
318 }
319 for _, keyword := range reservedBoolKeywords {
320 reservedKeywordMap[keyword] = func(value, org string, pos *Position) *Token {
321 return reservedKeywordToken(BoolType, value, org, pos)
322 }
323 }
324 for _, keyword := range reservedInfKeywords {
325 reservedKeywordMap[keyword] = func(value, org string, pos *Position) *Token {
326 return reservedKeywordToken(InfinityType, value, org, pos)
327 }
328 }
329 for _, keyword := range reservedNanKeywords {
330 reservedKeywordMap[keyword] = func(value, org string, pos *Position) *Token {
331 return reservedKeywordToken(NanType, value, org, pos)
332 }
333 }
334 }
335
336 // ReservedTagKeyword type of reserved tag keyword
337 type ReservedTagKeyword string
338
339 const (
340 // IntegerTag `!!int` tag
341 IntegerTag ReservedTagKeyword = "!!int"
342 // FloatTag `!!float` tag
343 FloatTag ReservedTagKeyword = "!!float"
344 // NullTag `!!null` tag
345 NullTag ReservedTagKeyword = "!!null"
346 // SequenceTag `!!seq` tag
347 SequenceTag ReservedTagKeyword = "!!seq"
348 // MappingTag `!!map` tag
349 MappingTag ReservedTagKeyword = "!!map"
350 // StringTag `!!str` tag
351 StringTag ReservedTagKeyword = "!!str"
352 // BinaryTag `!!binary` tag
353 BinaryTag ReservedTagKeyword = "!!binary"
354 // OrderedMapTag `!!omap` tag
355 OrderedMapTag ReservedTagKeyword = "!!omap"
356 // SetTag `!!set` tag
357 SetTag ReservedTagKeyword = "!!set"
358 // TimestampTag `!!timestamp` tag
359 TimestampTag ReservedTagKeyword = "!!timestamp"
360 )
361
362 var (
363 // ReservedTagKeywordMap map for reserved tag keywords
364 ReservedTagKeywordMap = map[ReservedTagKeyword]func(string, string, *Position) *Token{
365 IntegerTag: func(value, org string, pos *Position) *Token {
366 return &Token{
367 Type: TagType,
368 CharacterType: CharacterTypeIndicator,
369 Indicator: NodePropertyIndicator,
370 Value: value,
371 Origin: org,
372 Position: pos,
373 }
374 },
375 FloatTag: func(value, org string, pos *Position) *Token {
376 return &Token{
377 Type: TagType,
378 CharacterType: CharacterTypeIndicator,
379 Indicator: NodePropertyIndicator,
380 Value: value,
381 Origin: org,
382 Position: pos,
383 }
384 },
385 NullTag: func(value, org string, pos *Position) *Token {
386 return &Token{
387 Type: TagType,
388 CharacterType: CharacterTypeIndicator,
389 Indicator: NodePropertyIndicator,
390 Value: value,
391 Origin: org,
392 Position: pos,
393 }
394 },
395 SequenceTag: func(value, org string, pos *Position) *Token {
396 return &Token{
397 Type: TagType,
398 CharacterType: CharacterTypeIndicator,
399 Indicator: NodePropertyIndicator,
400 Value: value,
401 Origin: org,
402 Position: pos,
403 }
404 },
405 MappingTag: func(value, org string, pos *Position) *Token {
406 return &Token{
407 Type: TagType,
408 CharacterType: CharacterTypeIndicator,
409 Indicator: NodePropertyIndicator,
410 Value: value,
411 Origin: org,
412 Position: pos,
413 }
414 },
415 StringTag: func(value, org string, pos *Position) *Token {
416 return &Token{
417 Type: TagType,
418 CharacterType: CharacterTypeIndicator,
419 Indicator: NodePropertyIndicator,
420 Value: value,
421 Origin: org,
422 Position: pos,
423 }
424 },
425 BinaryTag: func(value, org string, pos *Position) *Token {
426 return &Token{
427 Type: TagType,
428 CharacterType: CharacterTypeIndicator,
429 Indicator: NodePropertyIndicator,
430 Value: value,
431 Origin: org,
432 Position: pos,
433 }
434 },
435 OrderedMapTag: func(value, org string, pos *Position) *Token {
436 return &Token{
437 Type: TagType,
438 CharacterType: CharacterTypeIndicator,
439 Indicator: NodePropertyIndicator,
440 Value: value,
441 Origin: org,
442 Position: pos,
443 }
444 },
445 SetTag: func(value, org string, pos *Position) *Token {
446 return &Token{
447 Type: TagType,
448 CharacterType: CharacterTypeIndicator,
449 Indicator: NodePropertyIndicator,
450 Value: value,
451 Origin: org,
452 Position: pos,
453 }
454 },
455 TimestampTag: func(value, org string, pos *Position) *Token {
456 return &Token{
457 Type: TagType,
458 CharacterType: CharacterTypeIndicator,
459 Indicator: NodePropertyIndicator,
460 Value: value,
461 Origin: org,
462 Position: pos,
463 }
464 },
465 }
466 )
467
468 type numType int
469
470 const (
471 numTypeNone numType = iota
472 numTypeBinary
473 numTypeOctet
474 numTypeHex
475 numTypeFloat
476 )
477
478 type numStat struct {
479 isNum bool
480 typ numType
481 }
482
483 func getNumberStat(str string) *numStat {
484 stat := &numStat{}
485 if str == "" {
486 return stat
487 }
488 if str == "-" || str == "." || str == "+" || str == "_" {
489 return stat
490 }
491 if str[0] == '_' {
492 return stat
493 }
494 dotFound := false
495 isNegative := false
496 isExponent := false
497 if str[0] == '-' {
498 isNegative = true
499 }
500 for idx, c := range str {
501 switch c {
502 case 'x':
503 if (isNegative && idx == 2) || (!isNegative && idx == 1) {
504 continue
505 }
506 case 'o':
507 if (isNegative && idx == 2) || (!isNegative && idx == 1) {
508 continue
509 }
510 case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
511 continue
512 case 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F':
513 if (len(str) > 2 && str[0] == '0' && str[1] == 'x') ||
514 (len(str) > 3 && isNegative && str[1] == '0' && str[2] == 'x') {
515 // hex number
516 continue
517 }
518 if c == 'b' && ((isNegative && idx == 2) || (!isNegative && idx == 1)) {
519 // binary number
520 continue
521 }
522 if (c == 'e' || c == 'E') && dotFound {
523 // exponent
524 isExponent = true
525 continue
526 }
527 case '.':
528 if dotFound {
529 // multiple dot
530 return stat
531 }
532 dotFound = true
533 continue
534 case '-':
535 if idx == 0 || isExponent {
536 continue
537 }
538 case '+':
539 if idx == 0 || isExponent {
540 continue
541 }
542 case '_':
543 continue
544 }
545 return stat
546 }
547 stat.isNum = true
548 switch {
549 case dotFound:
550 stat.typ = numTypeFloat
551 case strings.HasPrefix(str, "0b") || strings.HasPrefix(str, "-0b"):
552 stat.typ = numTypeBinary
553 case strings.HasPrefix(str, "0x") || strings.HasPrefix(str, "-0x"):
554 stat.typ = numTypeHex
555 case strings.HasPrefix(str, "0o") || strings.HasPrefix(str, "-0o"):
556 stat.typ = numTypeOctet
557 case (len(str) > 1 && str[0] == '0') || (len(str) > 1 && str[0] == '-' && str[1] == '0'):
558 stat.typ = numTypeOctet
559 }
560 return stat
561 }
562
563 func looksLikeTimeValue(value string) bool {
564 for i, c := range value {
565 switch c {
566 case ':', '1', '2', '3', '4', '5', '6', '7', '8', '9':
567 continue
568 case '0':
569 if i == 0 {
570 return false
571 }
572 continue
573 }
574 return false
575 }
576 return true
577 }
578
579 // IsNeedQuoted whether need quote for passed string or not
580 func IsNeedQuoted(value string) bool {
581 if value == "" {
582 return true
583 }
584 if _, exists := reservedKeywordMap[value]; exists {
585 return true
586 }
587 if stat := getNumberStat(value); stat.isNum {
588 return true
589 }
590 first := value[0]
591 switch first {
592 case '*', '&', '[', '{', '}', ']', ',', '!', '|', '>', '%', '\'', '"':
593 return true
594 }
595 last := value[len(value)-1]
596 switch last {
597 case ':':
598 return true
599 }
600 if looksLikeTimeValue(value) {
601 return true
602 }
603 for i, c := range value {
604 switch c {
605 case '#', '\\':
606 return true
607 case ':':
608 if i+1 < len(value) && value[i+1] == ' ' {
609 return true
610 }
611 }
612 }
613 return false
614 }
615
616 // LiteralBlockHeader detect literal block scalar header
617 func LiteralBlockHeader(value string) string {
618 lbc := DetectLineBreakCharacter(value)
619
620 switch {
621 case !strings.Contains(value, lbc):
622 return ""
623 case strings.HasSuffix(value, fmt.Sprintf("%s%s", lbc, lbc)):
624 return "|+"
625 case strings.HasSuffix(value, lbc):
626 return "|"
627 default:
628 return "|-"
629 }
630 }
631
632 // New create reserved keyword token or number token and other string token
633 func New(value string, org string, pos *Position) *Token {
634 fn := reservedKeywordMap[value]
635 if fn != nil {
636 return fn(value, org, pos)
637 }
638 if stat := getNumberStat(value); stat.isNum {
639 tk := &Token{
640 Type: IntegerType,
641 CharacterType: CharacterTypeMiscellaneous,
642 Indicator: NotIndicator,
643 Value: value,
644 Origin: org,
645 Position: pos,
646 }
647 switch stat.typ {
648 case numTypeFloat:
649 tk.Type = FloatType
650 case numTypeBinary:
651 tk.Type = BinaryIntegerType
652 case numTypeOctet:
653 tk.Type = OctetIntegerType
654 case numTypeHex:
655 tk.Type = HexIntegerType
656 }
657 return tk
658 }
659 return String(value, org, pos)
660 }
661
662 // Position type for position in YAML document
663 type Position struct {
664 Line int
665 Column int
666 Offset int
667 IndentNum int
668 IndentLevel int
669 }
670
671 // String position to text
672 func (p *Position) String() string {
673 return fmt.Sprintf("[level:%d,line:%d,column:%d,offset:%d]", p.IndentLevel, p.Line, p.Column, p.Offset)
674 }
675
676 // Token type for token
677 type Token struct {
678 Type Type
679 CharacterType CharacterType
680 Indicator Indicator
681 Value string
682 Origin string
683 Position *Position
684 Next *Token
685 Prev *Token
686 }
687
688 // PreviousType previous token type
689 func (t *Token) PreviousType() Type {
690 if t.Prev != nil {
691 return t.Prev.Type
692 }
693 return UnknownType
694 }
695
696 // NextType next token type
697 func (t *Token) NextType() Type {
698 if t.Next != nil {
699 return t.Next.Type
700 }
701 return UnknownType
702 }
703
704 // AddColumn append column number to current position of column
705 func (t *Token) AddColumn(col int) {
706 if t == nil {
707 return
708 }
709 t.Position.Column += col
710 }
711
712 // Clone copy token ( preserve Prev/Next reference )
713 func (t *Token) Clone() *Token {
714 if t == nil {
715 return nil
716 }
717 copied := *t
718 if t.Position != nil {
719 pos := *(t.Position)
720 copied.Position = &pos
721 }
722 return &copied
723 }
724
725 // Tokens type of token collection
726 type Tokens []*Token
727
728 func (t *Tokens) add(tk *Token) {
729 tokens := *t
730 if len(tokens) == 0 {
731 tokens = append(tokens, tk)
732 } else {
733 last := tokens[len(tokens)-1]
734 last.Next = tk
735 tk.Prev = last
736 tokens = append(tokens, tk)
737 }
738 *t = tokens
739 }
740
741 // Add append new some tokens
742 func (t *Tokens) Add(tks ...*Token) {
743 for _, tk := range tks {
744 t.add(tk)
745 }
746 }
747
748 // Dump dump all token structures for debugging
749 func (t Tokens) Dump() {
750 for _, tk := range t {
751 fmt.Printf("- %+v\n", tk)
752 }
753 }
754
755 // String create token for String
756 func String(value string, org string, pos *Position) *Token {
757 return &Token{
758 Type: StringType,
759 CharacterType: CharacterTypeMiscellaneous,
760 Indicator: NotIndicator,
761 Value: value,
762 Origin: org,
763 Position: pos,
764 }
765 }
766
767 // SequenceEntry create token for SequenceEntry
768 func SequenceEntry(org string, pos *Position) *Token {
769 return &Token{
770 Type: SequenceEntryType,
771 CharacterType: CharacterTypeIndicator,
772 Indicator: BlockStructureIndicator,
773 Value: string(SequenceEntryCharacter),
774 Origin: org,
775 Position: pos,
776 }
777 }
778
779 // MappingKey create token for MappingKey
780 func MappingKey(pos *Position) *Token {
781 return &Token{
782 Type: MappingKeyType,
783 CharacterType: CharacterTypeIndicator,
784 Indicator: BlockStructureIndicator,
785 Value: string(MappingKeyCharacter),
786 Origin: string(MappingKeyCharacter),
787 Position: pos,
788 }
789 }
790
791 // MappingValue create token for MappingValue
792 func MappingValue(pos *Position) *Token {
793 return &Token{
794 Type: MappingValueType,
795 CharacterType: CharacterTypeIndicator,
796 Indicator: BlockStructureIndicator,
797 Value: string(MappingValueCharacter),
798 Origin: string(MappingValueCharacter),
799 Position: pos,
800 }
801 }
802
803 // CollectEntry create token for CollectEntry
804 func CollectEntry(org string, pos *Position) *Token {
805 return &Token{
806 Type: CollectEntryType,
807 CharacterType: CharacterTypeIndicator,
808 Indicator: FlowCollectionIndicator,
809 Value: string(CollectEntryCharacter),
810 Origin: org,
811 Position: pos,
812 }
813 }
814
815 // SequenceStart create token for SequenceStart
816 func SequenceStart(org string, pos *Position) *Token {
817 return &Token{
818 Type: SequenceStartType,
819 CharacterType: CharacterTypeIndicator,
820 Indicator: FlowCollectionIndicator,
821 Value: string(SequenceStartCharacter),
822 Origin: org,
823 Position: pos,
824 }
825 }
826
827 // SequenceEnd create token for SequenceEnd
828 func SequenceEnd(org string, pos *Position) *Token {
829 return &Token{
830 Type: SequenceEndType,
831 CharacterType: CharacterTypeIndicator,
832 Indicator: FlowCollectionIndicator,
833 Value: string(SequenceEndCharacter),
834 Origin: org,
835 Position: pos,
836 }
837 }
838
839 // MappingStart create token for MappingStart
840 func MappingStart(org string, pos *Position) *Token {
841 return &Token{
842 Type: MappingStartType,
843 CharacterType: CharacterTypeIndicator,
844 Indicator: FlowCollectionIndicator,
845 Value: string(MappingStartCharacter),
846 Origin: org,
847 Position: pos,
848 }
849 }
850
851 // MappingEnd create token for MappingEnd
852 func MappingEnd(org string, pos *Position) *Token {
853 return &Token{
854 Type: MappingEndType,
855 CharacterType: CharacterTypeIndicator,
856 Indicator: FlowCollectionIndicator,
857 Value: string(MappingEndCharacter),
858 Origin: org,
859 Position: pos,
860 }
861 }
862
863 // Comment create token for Comment
864 func Comment(value string, org string, pos *Position) *Token {
865 return &Token{
866 Type: CommentType,
867 CharacterType: CharacterTypeIndicator,
868 Indicator: CommentIndicator,
869 Value: value,
870 Origin: org,
871 Position: pos,
872 }
873 }
874
875 // Anchor create token for Anchor
876 func Anchor(org string, pos *Position) *Token {
877 return &Token{
878 Type: AnchorType,
879 CharacterType: CharacterTypeIndicator,
880 Indicator: NodePropertyIndicator,
881 Value: string(AnchorCharacter),
882 Origin: org,
883 Position: pos,
884 }
885 }
886
887 // Alias create token for Alias
888 func Alias(org string, pos *Position) *Token {
889 return &Token{
890 Type: AliasType,
891 CharacterType: CharacterTypeIndicator,
892 Indicator: NodePropertyIndicator,
893 Value: string(AliasCharacter),
894 Origin: org,
895 Position: pos,
896 }
897 }
898
899 // Tag create token for Tag
900 func Tag(value string, org string, pos *Position) *Token {
901 fn := ReservedTagKeywordMap[ReservedTagKeyword(value)]
902 if fn != nil {
903 return fn(value, org, pos)
904 }
905 return &Token{
906 Type: TagType,
907 CharacterType: CharacterTypeIndicator,
908 Indicator: NodePropertyIndicator,
909 Value: value,
910 Origin: org,
911 Position: pos,
912 }
913 }
914
915 // Literal create token for Literal
916 func Literal(value string, org string, pos *Position) *Token {
917 return &Token{
918 Type: LiteralType,
919 CharacterType: CharacterTypeIndicator,
920 Indicator: BlockScalarIndicator,
921 Value: value,
922 Origin: org,
923 Position: pos,
924 }
925 }
926
927 // Folded create token for Folded
928 func Folded(value string, org string, pos *Position) *Token {
929 return &Token{
930 Type: FoldedType,
931 CharacterType: CharacterTypeIndicator,
932 Indicator: BlockScalarIndicator,
933 Value: value,
934 Origin: org,
935 Position: pos,
936 }
937 }
938
939 // SingleQuote create token for SingleQuote
940 func SingleQuote(value string, org string, pos *Position) *Token {
941 return &Token{
942 Type: SingleQuoteType,
943 CharacterType: CharacterTypeIndicator,
944 Indicator: QuotedScalarIndicator,
945 Value: value,
946 Origin: org,
947 Position: pos,
948 }
949 }
950
951 // DoubleQuote create token for DoubleQuote
952 func DoubleQuote(value string, org string, pos *Position) *Token {
953 return &Token{
954 Type: DoubleQuoteType,
955 CharacterType: CharacterTypeIndicator,
956 Indicator: QuotedScalarIndicator,
957 Value: value,
958 Origin: org,
959 Position: pos,
960 }
961 }
962
963 // Directive create token for Directive
964 func Directive(org string, pos *Position) *Token {
965 return &Token{
966 Type: DirectiveType,
967 CharacterType: CharacterTypeIndicator,
968 Indicator: DirectiveIndicator,
969 Value: string(DirectiveCharacter),
970 Origin: org,
971 Position: pos,
972 }
973 }
974
975 // Space create token for Space
976 func Space(pos *Position) *Token {
977 return &Token{
978 Type: SpaceType,
979 CharacterType: CharacterTypeWhiteSpace,
980 Indicator: NotIndicator,
981 Value: string(SpaceCharacter),
982 Origin: string(SpaceCharacter),
983 Position: pos,
984 }
985 }
986
987 // MergeKey create token for MergeKey
988 func MergeKey(org string, pos *Position) *Token {
989 return &Token{
990 Type: MergeKeyType,
991 CharacterType: CharacterTypeMiscellaneous,
992 Indicator: NotIndicator,
993 Value: "<<",
994 Origin: org,
995 Position: pos,
996 }
997 }
998
999 // DocumentHeader create token for DocumentHeader
1000 func DocumentHeader(org string, pos *Position) *Token {
1001 return &Token{
1002 Type: DocumentHeaderType,
1003 CharacterType: CharacterTypeMiscellaneous,
1004 Indicator: NotIndicator,
1005 Value: "---",
1006 Origin: org,
1007 Position: pos,
1008 }
1009 }
1010
1011 // DocumentEnd create token for DocumentEnd
1012 func DocumentEnd(org string, pos *Position) *Token {
1013 return &Token{
1014 Type: DocumentEndType,
1015 CharacterType: CharacterTypeMiscellaneous,
1016 Indicator: NotIndicator,
1017 Value: "...",
1018 Origin: org,
1019 Position: pos,
1020 }
1021 }
1022
1023 // DetectLineBreakCharacter detect line break character in only one inside scalar content scope.
1024 func DetectLineBreakCharacter(src string) string {
1025 nc := strings.Count(src, "\n")
1026 rc := strings.Count(src, "\r")
1027 rnc := strings.Count(src, "\r\n")
1028 switch {
1029 case nc == rnc && rc == rnc:
1030 return "\r\n"
1031 case rc > nc:
1032 return "\r"
1033 default:
1034 return "\n"
1035 }
1036 }
1037