q/src/token/Tokenize_test.go

package token_test

import (
	"testing"

	"git.akyoto.dev/cli/q/src/token"
	"git.akyoto.dev/go/assert"
)

func TestFunction(t *testing.T) {
	tokens := token.Tokenize([]byte("main(){}"))

	expected := []token.Kind{
		token.Identifier,
		token.GroupStart,
		token.GroupEnd,
		token.BlockStart,
		token.BlockEnd,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestKeyword(t *testing.T) {
	tokens := token.Tokenize([]byte("assert if import else loop return"))

	expected := []token.Kind{
		token.Assert,
		token.If,
		token.Import,
		token.Else,
		token.Loop,
		token.Return,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestArray(t *testing.T) {
	tokens := token.Tokenize([]byte("array[i]"))

	expected := []token.Kind{
		token.Identifier,
		token.ArrayStart,
		token.Identifier,
		token.ArrayEnd,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestNewline(t *testing.T) {
	tokens := token.Tokenize([]byte("\n\n"))

	expected := []token.Kind{
		token.NewLine,
		token.NewLine,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestNumber(t *testing.T) {
	tokens := token.Tokenize([]byte(`123 456`))

	expected := []token.Kind{
		token.Number,
		token.Number,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestOperator(t *testing.T) {
	tokens := token.Tokenize([]byte(`a + b - c * d / e`))

	expected := []token.Kind{
		token.Identifier,
		token.Add,
		token.Identifier,
		token.Sub,
		token.Identifier,
		token.Mul,
		token.Identifier,
		token.Div,
		token.Identifier,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestOperatorAssign(t *testing.T) {
	tokens := token.Tokenize([]byte(`a += b -= c *= d /= e &= f |= g ^= h <<= i >>= j`))

	expected := []token.Kind{
		token.Identifier,
		token.AddAssign,
		token.Identifier,
		token.SubAssign,
		token.Identifier,
		token.MulAssign,
		token.Identifier,
		token.DivAssign,
		token.Identifier,
		token.AndAssign,
		token.Identifier,
		token.OrAssign,
		token.Identifier,
		token.XorAssign,
		token.Identifier,
		token.ShlAssign,
		token.Identifier,
		token.ShrAssign,
		token.Identifier,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestNegateFirstToken(t *testing.T) {
	tokens := token.Tokenize([]byte(`-a`))

	expected := []token.Kind{
		token.Negate,
		token.Identifier,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestNegateAfterGroupStart(t *testing.T) {
	tokens := token.Tokenize([]byte(`(-a)`))

	expected := []token.Kind{
		token.GroupStart,
		token.Negate,
		token.Identifier,
		token.GroupEnd,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestNegateSub(t *testing.T) {
	tokens := token.Tokenize([]byte(`-a-b`))

	expected := []token.Kind{
		token.Negate,
		token.Identifier,
		token.Sub,
		token.Identifier,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestNegateAfterOperator(t *testing.T) {
	tokens := token.Tokenize([]byte(`-a + -b`))

	expected := []token.Kind{
		token.Negate,
		token.Identifier,
		token.Add,
		token.Negate,
		token.Identifier,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestNegateNumber(t *testing.T) {
	tokens := token.Tokenize([]byte(`-1`))

	expected := []token.Kind{
		token.Number,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestBinaryNumber(t *testing.T) {
	tokens := token.Tokenize([]byte(`0b1010`))

	expected := []token.Kind{
		token.Number,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestOctalNumber(t *testing.T) {
	tokens := token.Tokenize([]byte(`0o755`))

	expected := []token.Kind{
		token.Number,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestHexadecimalNumber(t *testing.T) {
	tokens := token.Tokenize([]byte(`0xCAFE`))

	expected := []token.Kind{
		token.Number,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestStandaloneZero(t *testing.T) {
	tokens := token.Tokenize([]byte(`0`))

	expected := []token.Kind{
		token.Number,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestLeadingZero(t *testing.T) {
	tokens := token.Tokenize([]byte(`0123`))

	expected := []token.Kind{
		token.Number,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestSeparator(t *testing.T) {
	tokens := token.Tokenize([]byte("a,b,c"))

	expected := []token.Kind{
		token.Identifier,
		token.Separator,
		token.Identifier,
		token.Separator,
		token.Identifier,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestComment(t *testing.T) {
	tokens := token.Tokenize([]byte("// Hello\n// World"))

	expected := []token.Kind{
		token.Comment,
		token.NewLine,
		token.Comment,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}

	tokens = token.Tokenize([]byte("// Hello\n"))

	expected = []token.Kind{
		token.Comment,
		token.NewLine,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}

	tokens = token.Tokenize([]byte(`// Hello`))

	expected = []token.Kind{
		token.Comment,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}

	tokens = token.Tokenize([]byte(`//`))

	expected = []token.Kind{
		token.Comment,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}

	tokens = token.Tokenize([]byte(`/`))

	expected = []token.Kind{
		token.Div,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestInvalid(t *testing.T) {
	tokens := token.Tokenize([]byte(`##`))

	expected := []token.Kind{
		token.Invalid,
		token.Invalid,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestString(t *testing.T) {
	tokens := token.Tokenize([]byte(`"Hello" "World"`))

	expected := []token.Kind{
		token.String,
		token.String,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestStringMultiline(t *testing.T) {
	tokens := token.Tokenize([]byte("\"Hello\nWorld\""))

	expected := []token.Kind{
		token.String,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}

func TestStringEOF(t *testing.T) {
	tokens := token.Tokenize([]byte(`"EOF`))

	expected := []token.Kind{
		token.String,
		token.EOF,
	}

	for i, kind := range expected {
		assert.Equal(t, tokens[i].Kind, kind)
	}
}