cli/q
1
0
Fork 0
Code Issues Pull requests Releases Activity

Improved Windows support in the tokenizer

Browse source
This commit is contained in:
Eduard Urbach 2025-04-18 14:48:59 +02:00
parent 487c7fb3a6
commit 42bbe5d4ed
Signed by: eduard
GPG key ID: 49226B848C78F6C8
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/token/Tokenize.go
View file

@ -9,7 +9,7 @@ func Tokenize(buffer []byte) List {
for i < Position(len(buffer)) { for i < Position(len(buffer)) {
switch buffer[i] { switch buffer[i] {
case ' ', '\t': case ' ', '\t', '\r':
case ',': case ',':
tokens = append(tokens, Token{Kind: Separator, Position: i, Length: 1}) tokens = append(tokens, Token{Kind: Separator, Position: i, Length: 1})
case '(': case '(':