|
66
|
1 package chroma
|
|
|
2
|
|
|
3 import "strings"
|
|
|
4
|
|
|
5 // An Iterator across tokens.
|
|
|
6 //
|
|
|
7 // EOF will be returned at the end of the Token stream.
|
|
|
8 //
|
|
|
9 // If an error occurs within an Iterator, it may propagate this in a panic. Formatters should recover.
|
|
|
10 type Iterator func() Token
|
|
|
11
|
|
|
12 // Tokens consumes all tokens from the iterator and returns them as a slice.
|
|
|
13 func (i Iterator) Tokens() []Token {
|
|
|
14 var out []Token
|
|
|
15 for t := i(); t != EOF; t = i() {
|
|
|
16 out = append(out, t)
|
|
|
17 }
|
|
|
18 return out
|
|
|
19 }
|
|
|
20
|
|
|
21 // Concaterator concatenates tokens from a series of iterators.
|
|
|
22 func Concaterator(iterators ...Iterator) Iterator {
|
|
|
23 return func() Token {
|
|
|
24 for len(iterators) > 0 {
|
|
|
25 t := iterators[0]()
|
|
|
26 if t != EOF {
|
|
|
27 return t
|
|
|
28 }
|
|
|
29 iterators = iterators[1:]
|
|
|
30 }
|
|
|
31 return EOF
|
|
|
32 }
|
|
|
33 }
|
|
|
34
|
|
|
35 // Literator converts a sequence of literal Tokens into an Iterator.
|
|
|
36 func Literator(tokens ...Token) Iterator {
|
|
|
37 return func() Token {
|
|
|
38 if len(tokens) == 0 {
|
|
|
39 return EOF
|
|
|
40 }
|
|
|
41 token := tokens[0]
|
|
|
42 tokens = tokens[1:]
|
|
|
43 return token
|
|
|
44 }
|
|
|
45 }
|
|
|
46
|
|
|
47 // SplitTokensIntoLines splits tokens containing newlines in two.
|
|
|
48 func SplitTokensIntoLines(tokens []Token) (out [][]Token) {
|
|
|
49 var line []Token // nolint: prealloc
|
|
|
50 for _, token := range tokens {
|
|
|
51 for strings.Contains(token.Value, "\n") {
|
|
|
52 parts := strings.SplitAfterN(token.Value, "\n", 2)
|
|
|
53 // Token becomes the tail.
|
|
|
54 token.Value = parts[1]
|
|
|
55
|
|
|
56 // Append the head to the line and flush the line.
|
|
|
57 clone := token.Clone()
|
|
|
58 clone.Value = parts[0]
|
|
|
59 line = append(line, clone)
|
|
|
60 out = append(out, line)
|
|
|
61 line = nil
|
|
|
62 }
|
|
|
63 line = append(line, token)
|
|
|
64 }
|
|
|
65 if len(line) > 0 {
|
|
|
66 out = append(out, line)
|
|
|
67 }
|
|
|
68 // Strip empty trailing token line.
|
|
|
69 if len(out) > 0 {
|
|
|
70 last := out[len(out)-1]
|
|
|
71 if len(last) == 1 && last[0].Value == "" {
|
|
|
72 out = out[:len(out)-1]
|
|
|
73 }
|
|
|
74 }
|
|
|
75 return
|
|
|
76 }
|
