Mercurial > yakumo_izuru > aya
diff vendor/github.com/alecthomas/chroma/v2/coalesce.go @ 66:787b5ee0289d draft
Use vendored modules
Signed-off-by: Izuru Yakumo <yakumo.izuru@chaotic.ninja>
author | yakumo.izuru |
---|---|
date | Sun, 23 Jul 2023 13:18:53 +0000 |
parents | |
children |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/vendor/github.com/alecthomas/chroma/v2/coalesce.go Sun Jul 23 13:18:53 2023 +0000 @@ -0,0 +1,35 @@ +package chroma + +// Coalesce is a Lexer interceptor that collapses runs of common types into a single token. +func Coalesce(lexer Lexer) Lexer { return &coalescer{lexer} } + +type coalescer struct{ Lexer } + +func (d *coalescer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) { + var prev Token + it, err := d.Lexer.Tokenise(options, text) + if err != nil { + return nil, err + } + return func() Token { + for token := it(); token != (EOF); token = it() { + if len(token.Value) == 0 { + continue + } + if prev == EOF { + prev = token + } else { + if prev.Type == token.Type && len(prev.Value) < 8192 { + prev.Value += token.Value + } else { + out := prev + prev = token + return out + } + } + } + out := prev + prev = EOF + return out + }, nil +}