Added token tests
This commit is contained in:
parent
c0f399df7f
commit
18993e0855
7 changed files with 241 additions and 75 deletions
|
@ -24,17 +24,23 @@ func Tokenize(buffer []byte) List {
|
|||
switch buffer[i] {
|
||||
// Texts
|
||||
case '"':
|
||||
start := i
|
||||
end := len(buffer)
|
||||
i++
|
||||
position := i
|
||||
|
||||
for i < len(buffer) && buffer[i] != '"' {
|
||||
for i < len(buffer) {
|
||||
if buffer[i] == '"' {
|
||||
end = i + 1
|
||||
break
|
||||
}
|
||||
|
||||
i++
|
||||
}
|
||||
|
||||
tokens = append(tokens, Token{
|
||||
Text,
|
||||
position,
|
||||
buffer[position:i],
|
||||
String,
|
||||
start,
|
||||
buffer[start:end],
|
||||
})
|
||||
|
||||
// Parentheses start
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue