cli/q
1
0
Fork 0
Code Issues Pull requests Releases Activity

Improved tokenizer test coverage

Browse source
This commit is contained in:
Eduard Urbach 2024-07-16 23:32:39 +02:00
parent c2b42fe2ea
commit 055d628130
Signed by: eduard
GPG key ID: 49226B848C78F6C8
5 changed files with 489 additions and 414 deletions
Download patch file Download diff file Expand all files Collapse all files

402
src/build/token/Tokenize_test.go Normal file
View file

@ -0,0 +1,402 @@
package token_test
import (
"testing"
"git.akyoto.dev/cli/q/src/build/token"
"git.akyoto.dev/go/assert"
)
func TestFunction(t *testing.T) {
tokens := token.Tokenize([]byte("main(){}"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Identifier,
Bytes: []byte("main"),
Position: 0,
},
{
Kind: token.GroupStart,
Bytes: []byte("("),
Position: 4,
},
{
Kind: token.GroupEnd,
Bytes: []byte(")"),
Position: 5,
},
{
Kind: token.BlockStart,
Bytes: []byte("{"),
Position: 6,
},
{
Kind: token.BlockEnd,
Bytes: []byte("}"),
Position: 7,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 8,
},
})
}
func TestKeyword(t *testing.T) {
tokens := token.Tokenize([]byte("return x"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Keyword,
Bytes: []byte("return"),
Position: 0,
},
{
Kind: token.Identifier,
Bytes: []byte("x"),
Position: 7,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 8,
},
})
}
func TestArray(t *testing.T) {
tokens := token.Tokenize([]byte("array[i]"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Identifier,
Bytes: []byte("array"),
Position: 0,
},
{
Kind: token.ArrayStart,
Bytes: []byte("["),
Position: 5,
},
{
Kind: token.Identifier,
Bytes: []byte("i"),
Position: 6,
},
{
Kind: token.ArrayEnd,
Bytes: []byte("]"),
Position: 7,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 8,
},
})
}
func TestNewline(t *testing.T) {
tokens := token.Tokenize([]byte("\n\n"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.NewLine,
Bytes: []byte("\n"),
Position: 0,
},
{
Kind: token.NewLine,
Bytes: []byte("\n"),
Position: 1,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 2,
},
})
}
func TestNumber(t *testing.T) {
tokens := token.Tokenize([]byte(`123 456`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Number,
Bytes: []byte("123"),
Position: 0,
},
{
Kind: token.Number,
Bytes: []byte("456"),
Position: 4,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 7,
},
})
}
func TestOperator(t *testing.T) {
tokens := token.Tokenize([]byte(`+ - * /`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Operator,
Bytes: []byte("+"),
Position: 0,
},
{
Kind: token.Operator,
Bytes: []byte("-"),
Position: 2,
},
{
Kind: token.Operator,
Bytes: []byte("*"),
Position: 4,
},
{
Kind: token.Operator,
Bytes: []byte("/"),
Position: 6,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 7,
},
})
}
func TestOperatorAssign(t *testing.T) {
tokens := token.Tokenize([]byte(`+= -= *= /= ==`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Operator,
Bytes: []byte("+="),
Position: 0,
},
{
Kind: token.Operator,
Bytes: []byte("-="),
Position: 3,
},
{
Kind: token.Operator,
Bytes: []byte("*="),
Position: 6,
},
{
Kind: token.Operator,
Bytes: []byte("/="),
Position: 9,
},
{
Kind: token.Operator,
Bytes: []byte("=="),
Position: 12,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 14,
},
})
}
func TestSeparator(t *testing.T) {
tokens := token.Tokenize([]byte("a,b,c"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Identifier,
Bytes: []byte("a"),
Position: 0,
},
{
Kind: token.Separator,
Bytes: []byte(","),
Position: 1,
},
{
Kind: token.Identifier,
Bytes: []byte("b"),
Position: 2,
},
{
Kind: token.Separator,
Bytes: []byte(","),
Position: 3,
},
{
Kind: token.Identifier,
Bytes: []byte("c"),
Position: 4,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 5,
},
})
}
func TestComment(t *testing.T) {
tokens := token.Tokenize([]byte("// Hello\n// World"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Comment,
Bytes: []byte(`// Hello`),
Position: 0,
},
{
Kind: token.NewLine,
Bytes: []byte("\n"),
Position: 8,
},
{
Kind: token.Comment,
Bytes: []byte(`// World`),
Position: 9,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 17,
},
})
tokens = token.Tokenize([]byte("// Hello\n"))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Comment,
Bytes: []byte(`// Hello`),
Position: 0,
},
{
Kind: token.NewLine,
Bytes: []byte("\n"),
Position: 8,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 9,
},
})
tokens = token.Tokenize([]byte(`// Hello`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Comment,
Bytes: []byte(`// Hello`),
Position: 0,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 8,
},
})
tokens = token.Tokenize([]byte(`//`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Comment,
Bytes: []byte(`//`),
Position: 0,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 2,
},
})
tokens = token.Tokenize([]byte(`/`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Operator,
Bytes: []byte(`/`),
Position: 0,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 1,
},
})
}
func TestInvalid(t *testing.T) {
tokens := token.Tokenize([]byte(`@#`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.Invalid,
Bytes: []byte(`@`),
Position: 0,
},
{
Kind: token.Invalid,
Bytes: []byte(`#`),
Position: 1,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 2,
},
})
}
func TestString(t *testing.T) {
tokens := token.Tokenize([]byte(`"Hello" "World"`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.String,
Bytes: []byte(`"Hello"`),
Position: 0,
},
{
Kind: token.String,
Bytes: []byte(`"World"`),
Position: 8,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 15,
},
})
}
func TestStringMultiline(t *testing.T) {
tokens := token.Tokenize([]byte("\"Hello\nWorld\""))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.String,
Bytes: []byte("\"Hello\nWorld\""),
Position: 0,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 13,
},
})
}
func TestStringEOF(t *testing.T) {
tokens := token.Tokenize([]byte(`"EOF`))
assert.DeepEqual(t, tokens, token.List{
{
Kind: token.String,
Bytes: []byte(`"EOF`),
Position: 0,
},
{
Kind: token.EOF,
Bytes: nil,
Position: 4,
},
})
}