Added token tests

This commit is contained in:
Eduard Urbach 2024-06-06 21:35:14 +02:00
parent c0f399df7f
commit 18993e0855
Signed by: eduard
GPG key ID: 49226B848C78F6C8
7 changed files with 241 additions and 75 deletions

View file

@ -1,16 +1,24 @@
package token
import "strings"
import (
"bytes"
)
// List is a slice of tokens.
type List []Token
// String implements string serialization.
func (list List) String() string {
builder := strings.Builder{}
builder := bytes.Buffer{}
var last Token
for _, t := range list {
builder.WriteString(t.String())
if t.Kind == Identifier && last.Kind == Separator {
builder.WriteByte(' ')
}
builder.Write(t.Bytes)
last = t
}
return builder.String()