Added token tests

This commit is contained in:
Eduard Urbach 2024-06-06 21:35:14 +02:00
parent c0f399df7f
commit 18993e0855
Signed by: eduard
GPG key ID: 49226B848C78F6C8
7 changed files with 241 additions and 75 deletions

View file

@ -24,17 +24,23 @@ func Tokenize(buffer []byte) List {
switch buffer[i] {
// Texts
case '"':
start := i
end := len(buffer)
i++
position := i
for i < len(buffer) && buffer[i] != '"' {
for i < len(buffer) {
if buffer[i] == '"' {
end = i + 1
break
}
i++
}
tokens = append(tokens, Token{
Text,
position,
buffer[position:i],
String,
start,
buffer[start:end],
})
// Parentheses start