package token_test import ( "testing" "git.akyoto.dev/cli/q/src/build/token" "git.akyoto.dev/go/assert" ) func TestFunction(t *testing.T) { tokens := token.Tokenize([]byte("main(){}")) expected := []token.Kind{ token.Identifier, token.GroupStart, token.GroupEnd, token.BlockStart, token.BlockEnd, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestKeyword(t *testing.T) { tokens := token.Tokenize([]byte("return x")) expected := []token.Kind{ token.Return, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestArray(t *testing.T) { tokens := token.Tokenize([]byte("array[i]")) expected := []token.Kind{ token.Identifier, token.ArrayStart, token.Identifier, token.ArrayEnd, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNewline(t *testing.T) { tokens := token.Tokenize([]byte("\n\n")) expected := []token.Kind{ token.NewLine, token.NewLine, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNumber(t *testing.T) { tokens := token.Tokenize([]byte(`123 456`)) expected := []token.Kind{ token.Number, token.Number, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestOperator(t *testing.T) { tokens := token.Tokenize([]byte(`a + b - c * d / e`)) expected := []token.Kind{ token.Identifier, token.Add, token.Identifier, token.Sub, token.Identifier, token.Mul, token.Identifier, token.Div, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNegateFirstToken(t *testing.T) { tokens := token.Tokenize([]byte(`-a`)) expected := []token.Kind{ token.Negate, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNegateAfterGroupStart(t *testing.T) { tokens := token.Tokenize([]byte(`(-a)`)) expected := []token.Kind{ token.GroupStart, token.Negate, token.Identifier, token.GroupEnd, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNegateSub(t *testing.T) { tokens := token.Tokenize([]byte(`-a-b`)) expected := []token.Kind{ token.Negate, token.Identifier, token.Sub, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestNegateAfterOperator(t *testing.T) { tokens := token.Tokenize([]byte(`-a + -b`)) expected := []token.Kind{ token.Negate, token.Identifier, token.Add, token.Negate, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestOperatorAssign(t *testing.T) { tokens := token.Tokenize([]byte(`a += b -= c *= d /= e &= f |= g ^= h <<= i >>= j`)) expected := []token.Kind{ token.Identifier, token.AddAssign, token.Identifier, token.SubAssign, token.Identifier, token.MulAssign, token.Identifier, token.DivAssign, token.Identifier, token.AndAssign, token.Identifier, token.OrAssign, token.Identifier, token.XorAssign, token.Identifier, token.ShlAssign, token.Identifier, token.ShrAssign, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestSeparator(t *testing.T) { tokens := token.Tokenize([]byte("a,b,c")) expected := []token.Kind{ token.Identifier, token.Separator, token.Identifier, token.Separator, token.Identifier, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestComment(t *testing.T) { tokens := token.Tokenize([]byte("// Hello\n// World")) expected := []token.Kind{ token.Comment, token.NewLine, token.Comment, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } tokens = token.Tokenize([]byte("// Hello\n")) expected = []token.Kind{ token.Comment, token.NewLine, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } tokens = token.Tokenize([]byte(`// Hello`)) expected = []token.Kind{ token.Comment, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } tokens = token.Tokenize([]byte(`//`)) expected = []token.Kind{ token.Comment, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } tokens = token.Tokenize([]byte(`/`)) expected = []token.Kind{ token.Div, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestInvalid(t *testing.T) { tokens := token.Tokenize([]byte(`##`)) expected := []token.Kind{ token.Invalid, token.Invalid, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestString(t *testing.T) { tokens := token.Tokenize([]byte(`"Hello" "World"`)) expected := []token.Kind{ token.String, token.String, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestStringMultiline(t *testing.T) { tokens := token.Tokenize([]byte("\"Hello\nWorld\"")) expected := []token.Kind{ token.String, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } } func TestStringEOF(t *testing.T) { tokens := token.Tokenize([]byte(`"EOF`)) expected := []token.Kind{ token.String, token.EOF, } for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) } }