diff --git a/src/build/token/Tokenize.go b/src/build/token/Tokenize.go index bd2d4c6..b2fc798 100644 --- a/src/build/token/Tokenize.go +++ b/src/build/token/Tokenize.go @@ -24,24 +24,9 @@ func Tokenize(buffer []byte) List { switch buffer[i] { // Whitespace case ' ', '\t': - // Texts - case '"': - start := i - end := len(buffer) - i++ - - for i < len(buffer) { - if buffer[i] == '"' { - end = i + 1 - i++ - break - } - - i++ - } - - tokens = append(tokens, Token{String, start, buffer[start:end]}) - continue + // Separator + case ',': + tokens = append(tokens, Token{Separator, i, separatorBytes}) // Parentheses start case '(': @@ -67,10 +52,6 @@ func Tokenize(buffer []byte) List { case ']': tokens = append(tokens, Token{ArrayEnd, i, arrayEndBytes}) - // Separator - case ',': - tokens = append(tokens, Token{Separator, i, separatorBytes}) - // New line case '\n': tokens = append(tokens, Token{NewLine, i, newLineBytes}) @@ -89,8 +70,27 @@ func Tokenize(buffer []byte) List { continue + // String + case '"': + start := i + end := len(buffer) + i++ + + for i < len(buffer) { + if buffer[i] == '"' { + end = i + 1 + i++ + break + } + + i++ + } + + tokens = append(tokens, Token{String, start, buffer[start:end]}) + continue + default: - // Identifiers + // Identifier if isIdentifierStart(buffer[i]) { position := i i++ @@ -109,7 +109,7 @@ func Tokenize(buffer []byte) List { continue } - // Numbers + // Number if isNumber(buffer[i]) { position := i i++ @@ -122,7 +122,7 @@ func Tokenize(buffer []byte) List { continue } - // Operators + // Operator if isOperator(buffer[i]) { position := i i++ @@ -135,7 +135,7 @@ func Tokenize(buffer []byte) List { continue } - // Invalid characters + // Invalid character tokens = append(tokens, Token{Invalid, i, buffer[i : i+1]}) }