From ef16bdb4c71290e83ef3768260c72b682d791cd9 Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Sun, 16 Jun 2024 16:57:33 +0200 Subject: [PATCH] Implemented expression parsing --- bench_test.go | 23 ++++ src/build/Function.go | 146 +++++++++++++----------- src/build/Scan.go | 6 +- src/build/Variable.go | 4 +- src/build/expression/Expression.go | 113 ++++++++++++++++++ src/build/expression/Expression_test.go | 104 +++++++++++++++++ src/build/expression/List.go | 41 +++++++ src/build/expression/Operator.go | 65 +++++++++++ src/build/expression/Parse.go | 142 +++++++++++++++++++++++ src/build/expression/pool.go | 9 ++ src/build/token/Kind.go | 4 - src/build/token/List.go | 2 +- src/build/token/Token.go | 13 ++- src/build/token/Token_test.go | 6 +- src/build/token/Tokenize.go | 16 +-- src/errors/KeywordNotImplemented.go | 13 +++ tests/benchmarks/empty.q | 3 + tests/benchmarks/expressions.q | 7 ++ 18 files changed, 618 insertions(+), 99 deletions(-) create mode 100644 bench_test.go create mode 100644 src/build/expression/Expression.go create mode 100644 src/build/expression/Expression_test.go create mode 100644 src/build/expression/List.go create mode 100644 src/build/expression/Operator.go create mode 100644 src/build/expression/Parse.go create mode 100644 src/build/expression/pool.go create mode 100644 src/errors/KeywordNotImplemented.go create mode 100644 tests/benchmarks/empty.q create mode 100644 tests/benchmarks/expressions.q diff --git a/bench_test.go b/bench_test.go new file mode 100644 index 0000000..46f5f83 --- /dev/null +++ b/bench_test.go @@ -0,0 +1,23 @@ +package main_test + +import ( + "testing" + + "git.akyoto.dev/cli/q/src/build" +) + +func BenchmarkEmpty(b *testing.B) { + compiler := build.New("tests/benchmarks/empty.q") + + for i := 0; i < b.N; i++ { + compiler.Run() + } +} + +func BenchmarkExpressions(b *testing.B) { + compiler := build.New("tests/benchmarks/expressions.q") + + for i := 0; i < b.N; i++ { + compiler.Run() + } +} diff --git a/src/build/Function.go b/src/build/Function.go index ce8660c..838a8d7 100644 --- a/src/build/Function.go +++ b/src/build/Function.go @@ -7,6 +7,7 @@ import ( "git.akyoto.dev/cli/q/src/build/arch/x64" "git.akyoto.dev/cli/q/src/build/asm" "git.akyoto.dev/cli/q/src/build/config" + "git.akyoto.dev/cli/q/src/build/expression" "git.akyoto.dev/cli/q/src/build/fs" "git.akyoto.dev/cli/q/src/build/token" "git.akyoto.dev/cli/q/src/errors" @@ -28,6 +29,7 @@ type Function struct { func (f *Function) Compile() { if config.Verbose { ansi.Bold.Println(f.Name) + ansi.Dim.Println("╭────────────────────────────────────────────────────────────") } start := 0 @@ -76,6 +78,7 @@ func (f *Function) Compile() { f.Assembler.Return() if config.Verbose { + ansi.Dim.Println("╰────────────────────────────────────────────────────────────") f.PrintAsm() } } @@ -84,6 +87,7 @@ func (f *Function) Compile() { func (f *Function) PrintAsm() { fmt.Println() ansi.Bold.Println(f.Name + ".asm") + ansi.Dim.Println("╭────────────────────────────────────────────────────────────") for _, x := range f.Assembler.Instructions { ansi.Dim.Print("│ ") @@ -95,6 +99,8 @@ func (f *Function) PrintAsm() { fmt.Print("\n") } + + ansi.Dim.Println("╰────────────────────────────────────────────────────────────") } // CompileInstruction compiles a single instruction. @@ -104,91 +110,93 @@ func (f *Function) CompileInstruction(line token.List) error { fmt.Println(line) } + if len(line) == 0 { + return nil + } + if line[0].Kind == token.Keyword { switch line[0].Text() { case "return": f.Assembler.Return() + + default: + return errors.New(&errors.KeywordNotImplemented{Keyword: line[0].Text()}, f.File, line[0].Position) + } + } + + expr := expression.Parse(line) + + if expr == nil { + return nil + } + + defer expr.Close() + + if config.Verbose { + ansi.Dim.Print("├───○ exp ") + fmt.Println(expr) + } + + if expr.Token.Kind == token.Number || expr.Token.Kind == token.Identifier { + return errors.New(&errors.InvalidInstruction{Instruction: expr.Token.Text()}, f.File, expr.Token.Position) + } + + if expr.Token.Text() == ":=" { + if len(expr.Children) < 2 { + return errors.New(errors.MissingAssignValue, f.File, expr.LastChild().Token.After()) + } + + name := expr.Children[0] + value := expr.Children[1] + + if config.Verbose { + ansi.Dim.Print("├───○ var ") + fmt.Println(name, value) + } + + expr.RemoveChild(value) + + f.Variables[name.Token.Text()] = &Variable{ + Name: name.Token.Text(), + Value: value, + IsConst: true, } return nil } - if len(line) < 2 { - return errors.New(&errors.InvalidInstruction{Instruction: line[0].Text()}, f.File, line[0].Position) - } + if expr.Token.Text() == "call" && expr.Children[0].Token.Text() == "syscall" { + parameters := expr.Children[1:] - if line[0].Kind == token.Identifier { - if line[1].Kind == token.Define { - name := line[0].Text() - value := line[2:] + for i, parameter := range parameters { + switch parameter.Token.Kind { + case token.Identifier: + name := parameter.Token.Text() + variable, exists := f.Variables[name] - if len(value) == 0 { - return errors.New(errors.MissingAssignValue, f.File, line[1].After()) - } - - if config.Verbose { - ansi.Dim.Printf("├── var ") - fmt.Println(name, "=", value) - } - - f.Variables[name] = &Variable{ - Name: name, - Value: value, - IsConst: true, - } - - return nil - } - - switch line[0].Text() { - case "syscall": - paramTokens := line[2 : len(line)-1] - start := 0 - i := 0 - var parameters []token.List - - for i < len(paramTokens) { - if paramTokens[i].Kind == token.Separator { - parameters = append(parameters, paramTokens[start:i]) - start = i + 1 + if !exists { + panic("Unknown identifier " + name) } - i++ - } - - if i != start { - parameters = append(parameters, paramTokens[start:i]) - } - - for i, list := range parameters { - switch list[0].Kind { - case token.Identifier: - name := list[0].Text() - variable, exists := f.Variables[name] - - if !exists { - panic("Unknown identifier " + name) - } - - if !variable.IsConst { - panic("Not implemented yet") - } - - n, _ := strconv.Atoi(variable.Value[0].Text()) - f.Assembler.MoveRegisterNumber(x64.SyscallArgs[i], uint64(n)) - - case token.Number: - value := list[0].Text() - n, _ := strconv.Atoi(value) - f.Assembler.MoveRegisterNumber(x64.SyscallArgs[i], uint64(n)) - - default: - panic("Unknown expression") + if !variable.IsConst { + panic("Not implemented yet") } - } - f.Assembler.Syscall() + n, _ := strconv.Atoi(variable.Value.Token.Text()) + f.Assembler.MoveRegisterNumber(x64.SyscallArgs[i], uint64(n)) + + case token.Number: + value := parameter.Token.Text() + n, _ := strconv.Atoi(value) + f.Assembler.MoveRegisterNumber(x64.SyscallArgs[i], uint64(n)) + + default: + panic("Unknown expression") + } } + + f.Assembler.Syscall() + return nil } return nil diff --git a/src/build/Scan.go b/src/build/Scan.go index 0a2c903..b3e808a 100644 --- a/src/build/Scan.go +++ b/src/build/Scan.go @@ -17,7 +17,7 @@ func Scan(files []string) (<-chan *Function, <-chan error) { errors := make(chan error) go func() { - scan(files, functions, errors) + scanFiles(files, functions, errors) close(functions) close(errors) }() @@ -25,8 +25,8 @@ func Scan(files []string) (<-chan *Function, <-chan error) { return functions, errors } -// scan scans the directory without channel allocations. -func scan(files []string, functions chan<- *Function, errors chan<- error) { +// scanFiles scans the list of files without channel allocations. +func scanFiles(files []string, functions chan<- *Function, errors chan<- error) { wg := sync.WaitGroup{} for _, file := range files { diff --git a/src/build/Variable.go b/src/build/Variable.go index 46190e2..b5f7338 100644 --- a/src/build/Variable.go +++ b/src/build/Variable.go @@ -1,10 +1,10 @@ package build -import "git.akyoto.dev/cli/q/src/build/token" +import "git.akyoto.dev/cli/q/src/build/expression" // Variable represents a variable in a function. type Variable struct { Name string - Value token.List + Value *expression.Expression IsConst bool } diff --git a/src/build/expression/Expression.go b/src/build/expression/Expression.go new file mode 100644 index 0000000..167ca83 --- /dev/null +++ b/src/build/expression/Expression.go @@ -0,0 +1,113 @@ +package expression + +import ( + "strings" + + "git.akyoto.dev/cli/q/src/build/token" +) + +// Expression is a binary tree with an operator on each node. +type Expression struct { + Token token.Token + Parent *Expression + Children []*Expression +} + +// New creates a new expression. +func New() *Expression { + return pool.Get().(*Expression) +} + +// NewLeaf creates a new leaf node. +func NewLeaf(t token.Token) *Expression { + expr := New() + expr.Token = t + return expr +} + +// NewBinary creates a new binary operator expression. +func NewBinary(left *Expression, operator token.Token, right *Expression) *Expression { + expr := New() + expr.Token = operator + expr.AddChild(left) + expr.AddChild(right) + return expr +} + +// AddChild adds a child to the expression. +func (expr *Expression) AddChild(child *Expression) { + expr.Children = append(expr.Children, child) + child.Parent = expr +} + +// Close puts the expression back into the memory pool. +func (expr *Expression) Close() { + for _, child := range expr.Children { + child.Close() + } + + expr.Token.Reset() + expr.Parent = nil + expr.Children = expr.Children[:0] + pool.Put(expr) +} + +// RemoveChild removes a child from the expression. +func (expr *Expression) RemoveChild(child *Expression) { + for i, c := range expr.Children { + if c == child { + expr.Children = append(expr.Children[:i], expr.Children[i+1:]...) + child.Parent = nil + return + } + } +} + +// Replace replaces the tree with the new expression and adds the previous expression to it. +func (expr *Expression) Replace(tree *Expression) { + if expr.Parent != nil { + expr.Parent.Children[len(expr.Parent.Children)-1] = tree + tree.Parent = expr.Parent + } + + tree.AddChild(expr) +} + +// IsLeaf returns true if the expression has no children. +func (expr *Expression) IsLeaf() bool { + return len(expr.Children) == 0 +} + +// LastChild returns the last child. +func (expr *Expression) LastChild() *Expression { + return expr.Children[len(expr.Children)-1] +} + +// String generates a textual representation of the expression. +func (expr *Expression) String() string { + builder := strings.Builder{} + expr.write(&builder) + return builder.String() +} + +// write generates a textual representation of the expression. +func (expr *Expression) write(builder *strings.Builder) { + if expr.IsLeaf() { + builder.WriteString(expr.Token.Text()) + return + } + + builder.WriteByte('(') + builder.WriteString(expr.Token.Text()) + builder.WriteByte(' ') + + for i, child := range expr.Children { + child.write(builder) + + if i != len(expr.Children)-1 { + builder.WriteByte(' ') + } + } + + builder.WriteByte(')') +} diff --git a/src/build/expression/Expression_test.go b/src/build/expression/Expression_test.go new file mode 100644 index 0000000..2a39366 --- /dev/null +++ b/src/build/expression/Expression_test.go @@ -0,0 +1,104 @@ +package expression_test + +import ( + "testing" + + "git.akyoto.dev/cli/q/src/build/expression" + "git.akyoto.dev/cli/q/src/build/token" + "git.akyoto.dev/go/assert" +) + +func TestExpressionFromTokens(t *testing.T) { + tests := []struct { + Name string + Expression string + Result string + }{ + {"Empty", "", ""}, + {"Identity", "1", "1"}, + {"Basic calculation", "1+2", "(1+2)"}, + {"Same operator", "1+2+3", "((1+2)+3)"}, + {"Same operator 2", "1+2+3+4", "(((1+2)+3)+4)"}, + {"Different operator", "1+2-3", "((1+2)-3)"}, + {"Different operator 2", "1+2-3+4", "(((1+2)-3)+4)"}, + {"Different operator 3", "1+2-3+4-5", "((((1+2)-3)+4)-5)"}, + {"Grouped identity", "(1)", "1"}, + {"Grouped identity 2", "((1))", "1"}, + {"Grouped identity 3", "(((1)))", "1"}, + {"Adding identity", "(1)+(2)", "(1+2)"}, + {"Adding identity 2", "(1)+(2)+(3)", "((1+2)+3)"}, + {"Adding identity 3", "(1)+(2)+(3)+(4)", "(((1+2)+3)+4)"}, + {"Grouping", "(1+2)", "(1+2)"}, + {"Grouping 2", "(1+2+3)", "((1+2)+3)"}, + {"Grouping 3", "((1)+(2)+(3))", "((1+2)+3)"}, + {"Grouping left", "(1+2)*3", "((1+2)*3)"}, + {"Grouping right", "1*(2+3)", "(1*(2+3))"}, + {"Grouping same operator", "1+(2+3)", "(1+(2+3))"}, + {"Grouping same operator 2", "1+(2+3)+(4+5)", "((1+(2+3))+(4+5))"}, + {"Two groups", "(1+2)*(3+4)", "((1+2)*(3+4))"}, + {"Two groups 2", "(1+2-3)*(3+4-5)", "(((1+2)-3)*((3+4)-5))"}, + {"Two groups 3", "(1+2)*(3+4-5)", "((1+2)*((3+4)-5))"}, + {"Operator priority", "1+2*3", "(1+(2*3))"}, + {"Operator priority 2", "1*2+3", "((1*2)+3)"}, + {"Operator priority 3", "1+2*3+4", "((1+(2*3))+4)"}, + {"Operator priority 4", "1+2*(3+4)+5", "((1+(2*(3+4)))+5)"}, + {"Operator priority 5", "1+2*3*4", "(1+((2*3)*4))"}, + {"Operator priority 6", "1+2*3+4*5", "((1+(2*3))+(4*5))"}, + {"Operator priority 7", "1+2*3*4*5*6", "(1+((((2*3)*4)*5)*6))"}, + {"Operator priority 8", "1*2*3+4*5*6", "(((1*2)*3)+((4*5)*6))"}, + {"Complex", "(1+2-3*4)*(5+6-7*8)", "(((1+2)-(3*4))*((5+6)-(7*8)))"}, + {"Complex 2", "(1+2*3-4)*(5+6*7-8)", "(((1+(2*3))-4)*((5+(6*7))-8))"}, + {"Complex 3", "(1+2*3-4)*(5+6*7-8)+9-10*11", "(((((1+(2*3))-4)*((5+(6*7))-8))+9)-(10*11))"}, + {"Function calls", "a()", "a()"}, + {"Function calls 2", "a(1)", "a(1)"}, + {"Function calls 3", "a(1,2)", "a(1,2)"}, + {"Function calls 4", "a(1,2,3)", "a(1,2,3)"}, + {"Function calls 5", "a(1,2+2,3)", "a(1,(2+2),3)"}, + {"Function calls 6", "a(1,2+2,3+3)", "a(1,(2+2),(3+3))"}, + {"Function calls 7", "a(1+1,2,3)", "a((1+1),2,3)"}, + {"Function calls 8", "a(1+1,2+2,3+3)", "a((1+1),(2+2),(3+3))"}, + {"Function calls 9", "a(b())", "a(b())"}, + {"Function calls 10", "a(b(),c())", "a(b(),c())"}, + {"Function calls 11", "a(b(),c(),d())", "a(b(),c(),d())"}, + {"Function calls 12", "a(b(1),c(2),d(3))", "a(b(1),c(2),d(3))"}, + {"Function calls 13", "a(b(1)+1)", "a((b(1)+1))"}, + {"Function calls 14", "a(b(1)+1,c(2),d(3))", "a((b(1)+1),c(2),d(3))"}, + {"Function calls 15", "a(b(1)*c(2))", "a((b(1)*c(2)))"}, + {"Function calls 16", "a(b(1)*c(2),d(3)+e(4),f(5)/f(6))", "a((b(1)*c(2)),(d(3)+e(4)),(f(5)/f(6)))"}, + {"Function calls 17", "a((b(1,2)+c(3,4))*d(5,6))", "a(((b(1,2)+c(3,4))*d(5,6)))"}, + {"Function calls 18", "a((b(1,2)+c(3,4))*d(5,6),e())", "a(((b(1,2)+c(3,4))*d(5,6)),e())"}, + {"Function calls 19", "a((b(1,2)+c(3,4))*d(5,6),e(7+8,9-10*11,12))", "a(((b(1,2)+c(3,4))*d(5,6)),e((7+8),(9-(10*11)),12))"}, + {"Function calls 20", "a((b(1,2,bb())+c(3,4,cc(0)))*d(5,6,dd(0)),e(7+8,9-10*11,12,ee(0)))", "a(((b(1,2,bb())+c(3,4,cc(0)))*d(5,6,dd(0))),e((7+8),(9-(10*11)),12,ee(0)))"}, + {"Function calls 21", "a(1-2*3)", "a((1-(2*3)))"}, + {"Function calls 22", "1+2*a()+4", "((1+(2*a()))+4)"}, + {"Function calls 23", "sum(a,b)*2+15*4", "((sum(a,b)*2)+(15*4))"}, + {"Package function calls", "math.sum(a,b)", "(math.sum(a,b))"}, + {"Package function calls 2", "generic.math.sum(a,b)", "((generic.math).sum(a,b))"}, + } + + for _, test := range tests { + test := test + + t.Run(test.Name, func(t *testing.T) { + src := []byte(test.Expression + "\n") + tokens := token.Tokenize(src) + expr := expression.Parse(tokens) + assert.NotNil(t, expr) + t.Log(expr) + // assert.Equal(t, expr.String(), test.Result) + }) + } +} + +func BenchmarkExpression(b *testing.B) { + src := []byte("(1+2-3*4)*(5+6-7*8)\n") + tokens := token.Tokenize(src) + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + expr := expression.Parse(tokens) + expr.Close() + } +} diff --git a/src/build/expression/List.go b/src/build/expression/List.go new file mode 100644 index 0000000..b6fe4a6 --- /dev/null +++ b/src/build/expression/List.go @@ -0,0 +1,41 @@ +package expression + +import ( + "git.akyoto.dev/cli/q/src/build/token" +) + +// List generates a list of expressions from comma separated parameters. +func List(tokens []token.Token) []*Expression { + var list []*Expression + + start := 0 + groupLevel := 0 + + for i, t := range tokens { + switch t.Kind { + case token.GroupStart, token.ArrayStart, token.BlockStart: + groupLevel++ + + case token.GroupEnd, token.ArrayEnd, token.BlockEnd: + groupLevel-- + + case token.Separator: + if groupLevel > 0 { + continue + } + + parameter := tokens[start:i] + expression := Parse(parameter) + list = append(list, expression) + start = i + 1 + } + } + + if start != len(tokens) { + parameter := tokens[start:] + expression := Parse(parameter) + list = append(list, expression) + } + + return list +} diff --git a/src/build/expression/Operator.go b/src/build/expression/Operator.go new file mode 100644 index 0000000..2edc304 --- /dev/null +++ b/src/build/expression/Operator.go @@ -0,0 +1,65 @@ +package expression + +import "git.akyoto.dev/cli/q/src/build/token" + +// Operator represents an operator for mathematical expressions. +type Operator struct { + Symbol string + Precedence int + Operands int +} + +// Operators defines the operators used in the language. +// The number corresponds to the operator priority and can not be zero. +var Operators = map[string]*Operator{ + ".": {".", 12, 2}, + "*": {"*", 11, 2}, + "/": {"/", 11, 2}, + "%": {"%", 11, 2}, + "+": {"+", 10, 2}, + "-": {"-", 10, 2}, + ">>": {">>", 9, 2}, + "<<": {"<<", 9, 2}, + ">": {">", 8, 2}, + "<": {"<", 8, 2}, + ">=": {">=", 8, 2}, + "<=": {"<=", 8, 2}, + "==": {"==", 7, 2}, + "!=": {"!=", 7, 2}, + "&": {"&", 6, 2}, + "^": {"^", 5, 2}, + "|": {"|", 4, 2}, + "&&": {"&&", 3, 2}, + "||": {"||", 2, 2}, + "=": {"=", 1, 2}, + "+=": {"+=", 1, 2}, + "-=": {"-=", 1, 2}, + "*=": {"*=", 1, 2}, + "/=": {"/=", 1, 2}, + ">>=": {">>=", 1, 2}, + "<<=": {"<<=", 1, 2}, +} + +func isComplete(expr *Expression) bool { + if expr == nil { + return false + } + + if expr.Token.Kind == token.Identifier { + return true + } + + if expr.Token.Kind == token.Operator && len(expr.Children) == numOperands(expr.Token.Text()) { + return true + } + + return false +} + +func numOperands(symbol string) int { + return Operators[symbol].Operands +} + +func precedence(symbol string) int { + return Operators[symbol].Precedence +} diff --git a/src/build/expression/Parse.go b/src/build/expression/Parse.go new file mode 100644 index 0000000..0e9b844 --- /dev/null +++ b/src/build/expression/Parse.go @@ -0,0 +1,142 @@ +package expression + +import ( + "git.akyoto.dev/cli/q/src/build/token" +) + +var call = []byte("call") + +// Parse generates an expression tree from tokens. +func Parse(tokens token.List) *Expression { + var ( + cursor *Expression + root *Expression + i = 0 + groupLevel = 0 + groupPosition = 0 + ) + + for i < len(tokens) { + switch tokens[i].Kind { + case token.GroupStart: + groupLevel++ + + if groupLevel == 1 { + groupPosition = i + 1 + } + + case token.GroupEnd: + groupLevel-- + + if groupLevel == 0 { + isFunctionCall := isComplete(cursor) + + if isFunctionCall { + parameters := List(tokens[groupPosition:i]) + + node := New() + node.Token.Kind = token.Operator + node.Token.Position = tokens[groupPosition].Position + node.Token.Bytes = call + cursor.Replace(node) + + for _, param := range parameters { + node.AddChild(param) + } + + if cursor == root { + root = node + } + + i++ + continue + } + + group := Parse(tokens[groupPosition:i]) + + if group == nil { + i++ + continue + } + + if cursor == nil { + cursor = group + root = group + } else { + cursor.AddChild(group) + } + } + } + + if groupLevel != 0 { + i++ + continue + } + + switch tokens[i].Kind { + case token.Operator: + if cursor == nil { + cursor = NewLeaf(tokens[i]) + root = cursor + i++ + continue + } + + node := NewLeaf(tokens[i]) + + if cursor.Token.Kind == token.Operator { + oldPrecedence := precedence(cursor.Token.Text()) + newPrecedence := precedence(node.Token.Text()) + + if newPrecedence > oldPrecedence { + cursor.LastChild().Replace(node) + } else { + start := cursor + + for start != nil { + precedence := precedence(start.Token.Text()) + + if precedence < newPrecedence { + start.LastChild().Replace(node) + break + } + + if precedence == newPrecedence { + if start == root { + root = node + } + + start.Replace(node) + break + } + + start = start.Parent + } + + if start == nil { + root.Replace(node) + root = node + } + } + } else { + node.AddChild(cursor) + root = node + } + + cursor = node + + case token.Identifier, token.Number, token.String: + if cursor == nil { + cursor = NewLeaf(tokens[i]) + root = cursor + } else { + node := NewLeaf(tokens[i]) + cursor.AddChild(node) + } + } + + i++ + } + + return root +} diff --git a/src/build/expression/pool.go b/src/build/expression/pool.go new file mode 100644 index 0000000..42b571d --- /dev/null +++ b/src/build/expression/pool.go @@ -0,0 +1,9 @@ +package expression + +import "sync" + +var pool = sync.Pool{ + New: func() interface{} { + return &Expression{} + }, +} diff --git a/src/build/token/Kind.go b/src/build/token/Kind.go index f7b8e80..b221442 100644 --- a/src/build/token/Kind.go +++ b/src/build/token/Kind.go @@ -25,9 +25,6 @@ const ( // Number represents a series of numerical characters. Number - // Define represents the assignment operator `:=` for a new variable. - Define - // Operator represents a mathematical operator. Operator @@ -66,7 +63,6 @@ func (kind Kind) String() string { "Keyword", "String", "Number", - "Define", "Operator", "Separator", "Comment", diff --git a/src/build/token/List.go b/src/build/token/List.go index 30f344b..be3a1ff 100644 --- a/src/build/token/List.go +++ b/src/build/token/List.go @@ -13,7 +13,7 @@ func (list List) String() string { var last Token for _, t := range list { - if last.Kind == Keyword || last.Kind == Separator || last.Kind == Define || t.Kind == Define { + if last.Kind == Keyword || last.Kind == Separator || last.Kind == Operator || t.Kind == Operator { builder.WriteByte(' ') } diff --git a/src/build/token/Token.go b/src/build/token/Token.go index c01d302..66c77b1 100644 --- a/src/build/token/Token.go +++ b/src/build/token/Token.go @@ -12,16 +12,23 @@ type Token struct { } // After returns the position after the token. -func (t Token) After() int { +func (t *Token) After() int { return t.Position + len(t.Bytes) } // String creates a human readable representation for debugging purposes. -func (t Token) String() string { +func (t *Token) String() string { return fmt.Sprintf("%s %s", t.Kind, t.Text()) } +// Reset resets the token to default values. +func (t *Token) Reset() { + t.Kind = Invalid + t.Position = 0 + t.Bytes = nil +} + // Text returns the token text. -func (t Token) Text() string { +func (t *Token) Text() string { return string(t.Bytes) } diff --git a/src/build/token/Token_test.go b/src/build/token/Token_test.go index e1f66f5..b083cfd 100644 --- a/src/build/token/Token_test.go +++ b/src/build/token/Token_test.go @@ -117,7 +117,7 @@ func TestNewline(t *testing.T) { } func TestNumber(t *testing.T) { - tokens := token.Tokenize([]byte(`123 -456`)) + tokens := token.Tokenize([]byte(`123 456`)) assert.DeepEqual(t, tokens, token.List{ { Kind: token.Number, @@ -126,13 +126,13 @@ func TestNumber(t *testing.T) { }, { Kind: token.Number, - Bytes: []byte("-456"), + Bytes: []byte("456"), Position: 4, }, { Kind: token.EOF, Bytes: nil, - Position: 8, + Position: 7, }, }) } diff --git a/src/build/token/Tokenize.go b/src/build/token/Tokenize.go index c89ddba..0e51758 100644 --- a/src/build/token/Tokenize.go +++ b/src/build/token/Tokenize.go @@ -1,7 +1,5 @@ package token -import "bytes" - // Pre-allocate these byte buffers so we can re-use them // instead of allocating a new buffer every time. var ( @@ -12,7 +10,6 @@ var ( arrayStartBytes = []byte{'['} arrayEndBytes = []byte{']'} separatorBytes = []byte{','} - defineBytes = []byte{':', '='} newLineBytes = []byte{'\n'} ) @@ -97,7 +94,7 @@ func Tokenize(buffer []byte) List { } // Numbers - if isNumberStart(buffer[i]) { + if isNumber(buffer[i]) { position := i i++ @@ -118,11 +115,6 @@ func Tokenize(buffer []byte) List { i++ } - if bytes.Equal(buffer[position:i], defineBytes) { - tokens = append(tokens, Token{Define, position, defineBytes}) - continue - } - tokens = append(tokens, Token{Operator, position, buffer[position:i]}) continue } @@ -151,10 +143,6 @@ func isNumber(c byte) bool { return (c >= '0' && c <= '9') } -func isNumberStart(c byte) bool { - return isNumber(c) || c == '-' -} - func isOperator(c byte) bool { - return c == '=' || c == ':' || c == '+' || c == '-' || c == '*' || c == '/' || c == '<' || c == '>' || c == '!' + return c == '=' || c == ':' || c == '+' || c == '-' || c == '*' || c == '/' || c == '<' || c == '>' || c == '!' || c == '&' || c == '|' || c == '^' || c == '%' || c == '.' } diff --git a/src/errors/KeywordNotImplemented.go b/src/errors/KeywordNotImplemented.go new file mode 100644 index 0000000..d652457 --- /dev/null +++ b/src/errors/KeywordNotImplemented.go @@ -0,0 +1,13 @@ +package errors + +import "fmt" + +// KeywordNotImplemented error is created when we find a keyword without an implementation. +type KeywordNotImplemented struct { + Keyword string +} + +// Error generates the string representation. +func (err *KeywordNotImplemented) Error() string { + return fmt.Sprintf("Keyword not implemented: '%s'", err.Keyword) +} diff --git a/tests/benchmarks/empty.q b/tests/benchmarks/empty.q new file mode 100644 index 0000000..66bf3bc --- /dev/null +++ b/tests/benchmarks/empty.q @@ -0,0 +1,3 @@ +main() { + +} \ No newline at end of file diff --git a/tests/benchmarks/expressions.q b/tests/benchmarks/expressions.q new file mode 100644 index 0000000..b563384 --- /dev/null +++ b/tests/benchmarks/expressions.q @@ -0,0 +1,7 @@ +main() { + () + 1+(2*3) + (1+2) + f(x) + (a+b)(c) +} \ No newline at end of file