|
66
|
1 package chroma
|
|
|
2
|
|
|
3 // Coalesce is a Lexer interceptor that collapses runs of common types into a single token.
|
|
|
4 func Coalesce(lexer Lexer) Lexer { return &coalescer{lexer} }
|
|
|
5
|
|
|
6 type coalescer struct{ Lexer }
|
|
|
7
|
|
|
8 func (d *coalescer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) {
|
|
|
9 var prev Token
|
|
|
10 it, err := d.Lexer.Tokenise(options, text)
|
|
|
11 if err != nil {
|
|
|
12 return nil, err
|
|
|
13 }
|
|
|
14 return func() Token {
|
|
|
15 for token := it(); token != (EOF); token = it() {
|
|
|
16 if len(token.Value) == 0 {
|
|
|
17 continue
|
|
|
18 }
|
|
|
19 if prev == EOF {
|
|
|
20 prev = token
|
|
|
21 } else {
|
|
|
22 if prev.Type == token.Type && len(prev.Value) < 8192 {
|
|
|
23 prev.Value += token.Value
|
|
|
24 } else {
|
|
|
25 out := prev
|
|
|
26 prev = token
|
|
|
27 return out
|
|
|
28 }
|
|
|
29 }
|
|
|
30 }
|
|
|
31 out := prev
|
|
|
32 prev = EOF
|
|
|
33 return out
|
|
|
34 }, nil
|
|
|
35 }
|
