Implemented runes
This commit is contained in:
parent
eaeeba495e
commit
f7645104fb
11 changed files with 88 additions and 27 deletions
|
@ -9,6 +9,7 @@ const (
|
|||
NewLine // NewLine is the newline character.
|
||||
Identifier // Identifier is a series of characters used to identify a variable or function.
|
||||
Number // Number is a series of numerical characters.
|
||||
Rune // Rune is a single unicode code point.
|
||||
String // String is an uninterpreted series of characters in the source code.
|
||||
Comment // Comment is a comment.
|
||||
Separator // ,
|
||||
|
|
|
@ -48,13 +48,14 @@ func Tokenize(buffer []byte) List {
|
|||
|
||||
continue
|
||||
|
||||
case '"':
|
||||
case '"', '\'':
|
||||
limiter := buffer[i]
|
||||
start := i
|
||||
end := Position(len(buffer))
|
||||
i++
|
||||
|
||||
for i < Position(len(buffer)) {
|
||||
if buffer[i] == '"' {
|
||||
if buffer[i] == limiter {
|
||||
end = i + 1
|
||||
i++
|
||||
break
|
||||
|
@ -63,7 +64,13 @@ func Tokenize(buffer []byte) List {
|
|||
i++
|
||||
}
|
||||
|
||||
tokens = append(tokens, Token{Kind: String, Position: start, Length: Length(end - start)})
|
||||
kind := String
|
||||
|
||||
if limiter == '\'' {
|
||||
kind = Rune
|
||||
}
|
||||
|
||||
tokens = append(tokens, Token{Kind: kind, Position: start, Length: Length(end - start)})
|
||||
continue
|
||||
|
||||
default:
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue