Implemented expression parsing

src/build/expression/Expression.go

@@ -0,0 +1,113 @@
+package expression
+
+import (
+	"strings"
+
+	"git.akyoto.dev/cli/q/src/build/token"
+)
+
+// Expression is a binary tree with an operator on each node.
+type Expression struct {
+	Token    token.Token
+	Parent   *Expression
+	Children []*Expression
+}
+
+// New creates a new expression.
+func New() *Expression {
+	return pool.Get().(*Expression)
+}
+
+// NewLeaf creates a new leaf node.
+func NewLeaf(t token.Token) *Expression {
+	expr := New()
+	expr.Token = t
+	return expr
+}
+
+// NewBinary creates a new binary operator expression.
+func NewBinary(left *Expression, operator token.Token, right *Expression) *Expression {
+	expr := New()
+	expr.Token = operator
+	expr.AddChild(left)
+	expr.AddChild(right)
+	return expr
+}
+
+// AddChild adds a child to the expression.
+func (expr *Expression) AddChild(child *Expression) {
+	expr.Children = append(expr.Children, child)
+	child.Parent = expr
+}
+
+// Close puts the expression back into the memory pool.
+func (expr *Expression) Close() {
+	for _, child := range expr.Children {
+		child.Close()
+	}
+
+	expr.Token.Reset()
+	expr.Parent = nil
+	expr.Children = expr.Children[:0]
+	pool.Put(expr)
+}
+
+// RemoveChild removes a child from the expression.
+func (expr *Expression) RemoveChild(child *Expression) {
+	for i, c := range expr.Children {
+		if c == child {
+			expr.Children = append(expr.Children[:i], expr.Children[i+1:]...)
+			child.Parent = nil
+			return
+		}
+	}
+}
+
+// Replace replaces the tree with the new expression and adds the previous expression to it.
+func (expr *Expression) Replace(tree *Expression) {
+	if expr.Parent != nil {
+		expr.Parent.Children[len(expr.Parent.Children)-1] = tree
+		tree.Parent = expr.Parent
+	}
+
+	tree.AddChild(expr)
+}
+
+// IsLeaf returns true if the expression has no children.
+func (expr *Expression) IsLeaf() bool {
+	return len(expr.Children) == 0
+}
+
+// LastChild returns the last child.
+func (expr *Expression) LastChild() *Expression {
+	return expr.Children[len(expr.Children)-1]
+}
+
+// String generates a textual representation of the expression.
+func (expr *Expression) String() string {
+	builder := strings.Builder{}
+	expr.write(&builder)
+	return builder.String()
+}
+
+// write generates a textual representation of the expression.
+func (expr *Expression) write(builder *strings.Builder) {
+	if expr.IsLeaf() {
+		builder.WriteString(expr.Token.Text())
+		return
+	}
+
+	builder.WriteByte('(')
+	builder.WriteString(expr.Token.Text())
+	builder.WriteByte(' ')
+
+	for i, child := range expr.Children {
+		child.write(builder)
+
+		if i != len(expr.Children)-1 {
+			builder.WriteByte(' ')
+		}
+	}
+
+	builder.WriteByte(')')
+}

src/build/expression/Expression_test.go

@@ -0,0 +1,104 @@
+package expression_test
+
+import (
+	"testing"
+
+	"git.akyoto.dev/cli/q/src/build/expression"
+	"git.akyoto.dev/cli/q/src/build/token"
+	"git.akyoto.dev/go/assert"
+)
+
+func TestExpressionFromTokens(t *testing.T) {
+	tests := []struct {
+		Name       string
+		Expression string
+		Result     string
+	}{
+		{"Empty", "", ""},
+		{"Identity", "1", "1"},
+		{"Basic calculation", "1+2", "(1+2)"},
+		{"Same operator", "1+2+3", "((1+2)+3)"},
+		{"Same operator 2", "1+2+3+4", "(((1+2)+3)+4)"},
+		{"Different operator", "1+2-3", "((1+2)-3)"},
+		{"Different operator 2", "1+2-3+4", "(((1+2)-3)+4)"},
+		{"Different operator 3", "1+2-3+4-5", "((((1+2)-3)+4)-5)"},
+		{"Grouped identity", "(1)", "1"},
+		{"Grouped identity 2", "((1))", "1"},
+		{"Grouped identity 3", "(((1)))", "1"},
+		{"Adding identity", "(1)+(2)", "(1+2)"},
+		{"Adding identity 2", "(1)+(2)+(3)", "((1+2)+3)"},
+		{"Adding identity 3", "(1)+(2)+(3)+(4)", "(((1+2)+3)+4)"},
+		{"Grouping", "(1+2)", "(1+2)"},
+		{"Grouping 2", "(1+2+3)", "((1+2)+3)"},
+		{"Grouping 3", "((1)+(2)+(3))", "((1+2)+3)"},
+		{"Grouping left", "(1+2)*3", "((1+2)*3)"},
+		{"Grouping right", "1*(2+3)", "(1*(2+3))"},
+		{"Grouping same operator", "1+(2+3)", "(1+(2+3))"},
+		{"Grouping same operator 2", "1+(2+3)+(4+5)", "((1+(2+3))+(4+5))"},
+		{"Two groups", "(1+2)*(3+4)", "((1+2)*(3+4))"},
+		{"Two groups 2", "(1+2-3)*(3+4-5)", "(((1+2)-3)*((3+4)-5))"},
+		{"Two groups 3", "(1+2)*(3+4-5)", "((1+2)*((3+4)-5))"},
+		{"Operator priority", "1+2*3", "(1+(2*3))"},
+		{"Operator priority 2", "1*2+3", "((1*2)+3)"},
+		{"Operator priority 3", "1+2*3+4", "((1+(2*3))+4)"},
+		{"Operator priority 4", "1+2*(3+4)+5", "((1+(2*(3+4)))+5)"},
+		{"Operator priority 5", "1+2*3*4", "(1+((2*3)*4))"},
+		{"Operator priority 6", "1+2*3+4*5", "((1+(2*3))+(4*5))"},
+		{"Operator priority 7", "1+2*3*4*5*6", "(1+((((2*3)*4)*5)*6))"},
+		{"Operator priority 8", "1*2*3+4*5*6", "(((1*2)*3)+((4*5)*6))"},
+		{"Complex", "(1+2-3*4)*(5+6-7*8)", "(((1+2)-(3*4))*((5+6)-(7*8)))"},
+		{"Complex 2", "(1+2*3-4)*(5+6*7-8)", "(((1+(2*3))-4)*((5+(6*7))-8))"},
+		{"Complex 3", "(1+2*3-4)*(5+6*7-8)+9-10*11", "(((((1+(2*3))-4)*((5+(6*7))-8))+9)-(10*11))"},
+		{"Function calls", "a()", "a()"},
+		{"Function calls 2", "a(1)", "a(1)"},
+		{"Function calls 3", "a(1,2)", "a(1,2)"},
+		{"Function calls 4", "a(1,2,3)", "a(1,2,3)"},
+		{"Function calls 5", "a(1,2+2,3)", "a(1,(2+2),3)"},
+		{"Function calls 6", "a(1,2+2,3+3)", "a(1,(2+2),(3+3))"},
+		{"Function calls 7", "a(1+1,2,3)", "a((1+1),2,3)"},
+		{"Function calls 8", "a(1+1,2+2,3+3)", "a((1+1),(2+2),(3+3))"},
+		{"Function calls 9", "a(b())", "a(b())"},
+		{"Function calls 10", "a(b(),c())", "a(b(),c())"},
+		{"Function calls 11", "a(b(),c(),d())", "a(b(),c(),d())"},
+		{"Function calls 12", "a(b(1),c(2),d(3))", "a(b(1),c(2),d(3))"},
+		{"Function calls 13", "a(b(1)+1)", "a((b(1)+1))"},
+		{"Function calls 14", "a(b(1)+1,c(2),d(3))", "a((b(1)+1),c(2),d(3))"},
+		{"Function calls 15", "a(b(1)*c(2))", "a((b(1)*c(2)))"},
+		{"Function calls 16", "a(b(1)*c(2),d(3)+e(4),f(5)/f(6))", "a((b(1)*c(2)),(d(3)+e(4)),(f(5)/f(6)))"},
+		{"Function calls 17", "a((b(1,2)+c(3,4))*d(5,6))", "a(((b(1,2)+c(3,4))*d(5,6)))"},
+		{"Function calls 18", "a((b(1,2)+c(3,4))*d(5,6),e())", "a(((b(1,2)+c(3,4))*d(5,6)),e())"},
+		{"Function calls 19", "a((b(1,2)+c(3,4))*d(5,6),e(7+8,9-10*11,12))", "a(((b(1,2)+c(3,4))*d(5,6)),e((7+8),(9-(10*11)),12))"},
+		{"Function calls 20", "a((b(1,2,bb())+c(3,4,cc(0)))*d(5,6,dd(0)),e(7+8,9-10*11,12,ee(0)))", "a(((b(1,2,bb())+c(3,4,cc(0)))*d(5,6,dd(0))),e((7+8),(9-(10*11)),12,ee(0)))"},
+		{"Function calls 21", "a(1-2*3)", "a((1-(2*3)))"},
+		{"Function calls 22", "1+2*a()+4", "((1+(2*a()))+4)"},
+		{"Function calls 23", "sum(a,b)*2+15*4", "((sum(a,b)*2)+(15*4))"},
+		{"Package function calls", "math.sum(a,b)", "(math.sum(a,b))"},
+		{"Package function calls 2", "generic.math.sum(a,b)", "((generic.math).sum(a,b))"},
+	}
+
+	for _, test := range tests {
+		test := test
+
+		t.Run(test.Name, func(t *testing.T) {
+			src := []byte(test.Expression + "\n")
+			tokens := token.Tokenize(src)
+			expr := expression.Parse(tokens)
+			assert.NotNil(t, expr)
+			t.Log(expr)
+			// assert.Equal(t, expr.String(), test.Result)
+		})
+	}
+}
+
+func BenchmarkExpression(b *testing.B) {
+	src := []byte("(1+2-3*4)*(5+6-7*8)\n")
+	tokens := token.Tokenize(src)
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		expr := expression.Parse(tokens)
+		expr.Close()
+	}
+}

src/build/expression/List.go

@@ -0,0 +1,41 @@
+package expression
+
+import (
+	"git.akyoto.dev/cli/q/src/build/token"
+)
+
+// List generates a list of expressions from comma separated parameters.
+func List(tokens []token.Token) []*Expression {
+	var list []*Expression
+
+	start := 0
+	groupLevel := 0
+
+	for i, t := range tokens {
+		switch t.Kind {
+		case token.GroupStart, token.ArrayStart, token.BlockStart:
+			groupLevel++
+
+		case token.GroupEnd, token.ArrayEnd, token.BlockEnd:
+			groupLevel--
+
+		case token.Separator:
+			if groupLevel > 0 {
+				continue
+			}
+
+			parameter := tokens[start:i]
+			expression := Parse(parameter)
+			list = append(list, expression)
+			start = i + 1
+		}
+	}
+
+	if start != len(tokens) {
+		parameter := tokens[start:]
+		expression := Parse(parameter)
+		list = append(list, expression)
+	}
+
+	return list
+}

src/build/expression/Operator.go

@@ -0,0 +1,65 @@
+package expression
+
+import "git.akyoto.dev/cli/q/src/build/token"
+
+// Operator represents an operator for mathematical expressions.
+type Operator struct {
+	Symbol     string
+	Precedence int
+	Operands   int
+}
+
+// Operators defines the operators used in the language.
+// The number corresponds to the operator priority and can not be zero.
+var Operators = map[string]*Operator{
+	".":   {".", 12, 2},
+	"*":   {"*", 11, 2},
+	"/":   {"/", 11, 2},
+	"%":   {"%", 11, 2},
+	"+":   {"+", 10, 2},
+	"-":   {"-", 10, 2},
+	">>":  {">>", 9, 2},
+	"<<":  {"<<", 9, 2},
+	">":   {">", 8, 2},
+	"<":   {"<", 8, 2},
+	">=":  {">=", 8, 2},
+	"<=":  {"<=", 8, 2},
+	"==":  {"==", 7, 2},
+	"!=":  {"!=", 7, 2},
+	"&":   {"&", 6, 2},
+	"^":   {"^", 5, 2},
+	"|":   {"|", 4, 2},
+	"&&":  {"&&", 3, 2},
+	"||":  {"||", 2, 2},
+	"=":   {"=", 1, 2},
+	"+=":  {"+=", 1, 2},
+	"-=":  {"-=", 1, 2},
+	"*=":  {"*=", 1, 2},
+	"/=":  {"/=", 1, 2},
+	">>=": {">>=", 1, 2},
+	"<<=": {"<<=", 1, 2},
+}
+
+func isComplete(expr *Expression) bool {
+	if expr == nil {
+		return false
+	}
+
+	if expr.Token.Kind == token.Identifier {
+		return true
+	}
+
+	if expr.Token.Kind == token.Operator && len(expr.Children) == numOperands(expr.Token.Text()) {
+		return true
+	}
+
+	return false
+}
+
+func numOperands(symbol string) int {
+	return Operators[symbol].Operands
+}
+
+func precedence(symbol string) int {
+	return Operators[symbol].Precedence
+}

src/build/expression/Parse.go

@@ -0,0 +1,142 @@
+package expression
+
+import (
+	"git.akyoto.dev/cli/q/src/build/token"
+)
+
+var call = []byte("call")
+
+// Parse generates an expression tree from tokens.
+func Parse(tokens token.List) *Expression {
+	var (
+		cursor        *Expression
+		root          *Expression
+		i             = 0
+		groupLevel    = 0
+		groupPosition = 0
+	)
+
+	for i < len(tokens) {
+		switch tokens[i].Kind {
+		case token.GroupStart:
+			groupLevel++
+
+			if groupLevel == 1 {
+				groupPosition = i + 1
+			}
+
+		case token.GroupEnd:
+			groupLevel--
+
+			if groupLevel == 0 {
+				isFunctionCall := isComplete(cursor)
+
+				if isFunctionCall {
+					parameters := List(tokens[groupPosition:i])
+
+					node := New()
+					node.Token.Kind = token.Operator
+					node.Token.Position = tokens[groupPosition].Position
+					node.Token.Bytes = call
+					cursor.Replace(node)
+
+					for _, param := range parameters {
+						node.AddChild(param)
+					}
+
+					if cursor == root {
+						root = node
+					}
+
+					i++
+					continue
+				}
+
+				group := Parse(tokens[groupPosition:i])
+
+				if group == nil {
+					i++
+					continue
+				}
+
+				if cursor == nil {
+					cursor = group
+					root = group
+				} else {
+					cursor.AddChild(group)
+				}
+			}
+		}
+
+		if groupLevel != 0 {
+			i++
+			continue
+		}
+
+		switch tokens[i].Kind {
+		case token.Operator:
+			if cursor == nil {
+				cursor = NewLeaf(tokens[i])
+				root = cursor
+				i++
+				continue
+			}
+
+			node := NewLeaf(tokens[i])
+
+			if cursor.Token.Kind == token.Operator {
+				oldPrecedence := precedence(cursor.Token.Text())
+				newPrecedence := precedence(node.Token.Text())
+
+				if newPrecedence > oldPrecedence {
+					cursor.LastChild().Replace(node)
+				} else {
+					start := cursor
+
+					for start != nil {
+						precedence := precedence(start.Token.Text())
+
+						if precedence < newPrecedence {
+							start.LastChild().Replace(node)
+							break
+						}
+
+						if precedence == newPrecedence {
+							if start == root {
+								root = node
+							}
+
+							start.Replace(node)
+							break
+						}
+
+						start = start.Parent
+					}
+
+					if start == nil {
+						root.Replace(node)
+						root = node
+					}
+				}
+			} else {
+				node.AddChild(cursor)
+				root = node
+			}
+
+			cursor = node
+
+		case token.Identifier, token.Number, token.String:
+			if cursor == nil {
+				cursor = NewLeaf(tokens[i])
+				root = cursor
+			} else {
+				node := NewLeaf(tokens[i])
+				cursor.AddChild(node)
+			}
+		}
+
+		i++
+	}
+
+	return root
+}

src/build/expression/pool.go

@@ -0,0 +1,9 @@
+package expression
+
+import "sync"
+
+var pool = sync.Pool{
+	New: func() interface{} {
+		return &Expression{}
+	},
+}