Simplified file structure

src/token/Tokenize_test.go

@@ -0,0 +1,411 @@
+package token_test
+
+import (
+	"testing"
+
+	"git.akyoto.dev/cli/q/src/token"
+	"git.akyoto.dev/go/assert"
+)
+
+func TestFunction(t *testing.T) {
+	tokens := token.Tokenize([]byte("main(){}"))
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.GroupStart,
+		token.GroupEnd,
+		token.BlockStart,
+		token.BlockEnd,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestKeyword(t *testing.T) {
+	tokens := token.Tokenize([]byte("assert if import else loop return"))
+
+	expected := []token.Kind{
+		token.Assert,
+		token.If,
+		token.Import,
+		token.Else,
+		token.Loop,
+		token.Return,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestArray(t *testing.T) {
+	tokens := token.Tokenize([]byte("array[i]"))
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.ArrayStart,
+		token.Identifier,
+		token.ArrayEnd,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestNewline(t *testing.T) {
+	tokens := token.Tokenize([]byte("\n\n"))
+
+	expected := []token.Kind{
+		token.NewLine,
+		token.NewLine,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestNumber(t *testing.T) {
+	tokens := token.Tokenize([]byte(`123 456`))
+
+	expected := []token.Kind{
+		token.Number,
+		token.Number,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestOperator(t *testing.T) {
+	tokens := token.Tokenize([]byte(`a + b - c * d / e`))
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.Add,
+		token.Identifier,
+		token.Sub,
+		token.Identifier,
+		token.Mul,
+		token.Identifier,
+		token.Div,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestOperatorAssign(t *testing.T) {
+	tokens := token.Tokenize([]byte(`a += b -= c *= d /= e &= f |= g ^= h <<= i >>= j`))
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.AddAssign,
+		token.Identifier,
+		token.SubAssign,
+		token.Identifier,
+		token.MulAssign,
+		token.Identifier,
+		token.DivAssign,
+		token.Identifier,
+		token.AndAssign,
+		token.Identifier,
+		token.OrAssign,
+		token.Identifier,
+		token.XorAssign,
+		token.Identifier,
+		token.ShlAssign,
+		token.Identifier,
+		token.ShrAssign,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestNegateFirstToken(t *testing.T) {
+	tokens := token.Tokenize([]byte(`-a`))
+
+	expected := []token.Kind{
+		token.Negate,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestNegateAfterGroupStart(t *testing.T) {
+	tokens := token.Tokenize([]byte(`(-a)`))
+
+	expected := []token.Kind{
+		token.GroupStart,
+		token.Negate,
+		token.Identifier,
+		token.GroupEnd,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestNegateSub(t *testing.T) {
+	tokens := token.Tokenize([]byte(`-a-b`))
+
+	expected := []token.Kind{
+		token.Negate,
+		token.Identifier,
+		token.Sub,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestNegateAfterOperator(t *testing.T) {
+	tokens := token.Tokenize([]byte(`-a + -b`))
+
+	expected := []token.Kind{
+		token.Negate,
+		token.Identifier,
+		token.Add,
+		token.Negate,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestNegateNumber(t *testing.T) {
+	tokens := token.Tokenize([]byte(`-1`))
+
+	expected := []token.Kind{
+		token.Number,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestBinaryNumber(t *testing.T) {
+	tokens := token.Tokenize([]byte(`0b1010`))
+
+	expected := []token.Kind{
+		token.Number,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestOctalNumber(t *testing.T) {
+	tokens := token.Tokenize([]byte(`0o755`))
+
+	expected := []token.Kind{
+		token.Number,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestHexadecimalNumber(t *testing.T) {
+	tokens := token.Tokenize([]byte(`0xCAFE`))
+
+	expected := []token.Kind{
+		token.Number,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestStandaloneZero(t *testing.T) {
+	tokens := token.Tokenize([]byte(`0`))
+
+	expected := []token.Kind{
+		token.Number,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestLeadingZero(t *testing.T) {
+	tokens := token.Tokenize([]byte(`0123`))
+
+	expected := []token.Kind{
+		token.Number,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestSeparator(t *testing.T) {
+	tokens := token.Tokenize([]byte("a,b,c"))
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.Separator,
+		token.Identifier,
+		token.Separator,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestComment(t *testing.T) {
+	tokens := token.Tokenize([]byte("// Hello\n// World"))
+
+	expected := []token.Kind{
+		token.Comment,
+		token.NewLine,
+		token.Comment,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+
+	tokens = token.Tokenize([]byte("// Hello\n"))
+
+	expected = []token.Kind{
+		token.Comment,
+		token.NewLine,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+
+	tokens = token.Tokenize([]byte(`// Hello`))
+
+	expected = []token.Kind{
+		token.Comment,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+
+	tokens = token.Tokenize([]byte(`//`))
+
+	expected = []token.Kind{
+		token.Comment,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+
+	tokens = token.Tokenize([]byte(`/`))
+
+	expected = []token.Kind{
+		token.Div,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestInvalid(t *testing.T) {
+	tokens := token.Tokenize([]byte(`##`))
+
+	expected := []token.Kind{
+		token.Invalid,
+		token.Invalid,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestString(t *testing.T) {
+	tokens := token.Tokenize([]byte(`"Hello" "World"`))
+
+	expected := []token.Kind{
+		token.String,
+		token.String,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestStringMultiline(t *testing.T) {
+	tokens := token.Tokenize([]byte("\"Hello\nWorld\""))
+
+	expected := []token.Kind{
+		token.String,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
+func TestStringEOF(t *testing.T) {
+	tokens := token.Tokenize([]byte(`"EOF`))
+
+	expected := []token.Kind{
+		token.String,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}