cli/q
Code Issues 1 Pull Requests Activity

Reduced token size

Browse Source
This commit is contained in:
Eduard Urbach
2024-07-21 14:35:06 +02:00
parent ca36d34cb9
commit 04ba68a075
47 changed files with 543 additions and 764 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/build/expression/Expression.go
View File

@ -37,11 +37,11 @@ func (expr *Expression) AddChild(child *Expression) {
}
// Count counts how often the given token appears in the expression.
func (expr *Expression) Count(kind token.Kind, name string) int {
func (expr *Expression) Count(buffer []byte, kind token.Kind, name string) int {
count := 0
expr.EachLeaf(func(leaf *Expression) error {
if leaf.Token.Kind == kind && leaf.Token.Text() == name {
if leaf.Token.Kind == kind && leaf.Token.Text(buffer) == name {
count++
}
@ -112,25 +112,33 @@ func (expr *Expression) LastChild() *Expression {
}
// String generates a textual representation of the expression.
func (expr *Expression) String() string {
func (expr *Expression) String(data []byte) string {
builder := strings.Builder{}
expr.write(&builder)
expr.write(&builder, data)
return builder.String()
}
// write generates a textual representation of the expression.
func (expr *Expression) write(builder *strings.Builder) {
func (expr *Expression) write(builder *strings.Builder, data []byte) {
if expr.IsLeaf() {
builder.WriteString(expr.Token.Text())
builder.WriteString(expr.Token.Text(data))
return
}
builder.WriteByte('(')
builder.WriteString(expr.Token.Text())
switch expr.Token.Kind {
case token.Call:
builder.WriteString("λ")
case token.Array:
builder.WriteString("@")
default:
builder.WriteString(expr.Token.Text(data))
}
for _, child := range expr.Children {
builder.WriteByte(' ')
child.write(builder)
child.write(builder, data)
}
builder.WriteByte(')')

30
src/build/expression/Expression_test.go
View File

@ -95,7 +95,7 @@ func TestParse(t *testing.T) {
defer expr.Reset()
assert.NotNil(t, expr)
assert.Equal(t, expr.String(), test.Result)
assert.Equal(t, expr.String(src), test.Result)
})
}
}
@ -104,11 +104,11 @@ func TestCount(t *testing.T) {
src := []byte("(a+b-c*d)+(a*b-c+d)")
tokens := token.Tokenize(src)
expr := expression.Parse(tokens)
assert.Equal(t, expr.Count(token.Identifier, "a"), 2)
assert.Equal(t, expr.Count(token.Identifier, "b"), 2)
assert.Equal(t, expr.Count(token.Identifier, "c"), 2)
assert.Equal(t, expr.Count(token.Identifier, "d"), 2)
assert.Equal(t, expr.Count(token.Identifier, "e"), 0)
assert.Equal(t, expr.Count(src, token.Identifier, "a"), 2)
assert.Equal(t, expr.Count(src, token.Identifier, "b"), 2)
assert.Equal(t, expr.Count(src, token.Identifier, "c"), 2)
assert.Equal(t, expr.Count(src, token.Identifier, "d"), 2)
assert.Equal(t, expr.Count(src, token.Identifier, "e"), 0)
}
func TestEachLeaf(t *testing.T) {
@ -118,7 +118,7 @@ func TestEachLeaf(t *testing.T) {
leaves := []string{}
err := expr.EachLeaf(func(leaf *expression.Expression) error {
leaves = append(leaves, leaf.Token.Text())
leaves = append(leaves, leaf.Token.Text(src))
return nil
})
@ -140,7 +140,7 @@ func TestEachParameter(t *testing.T) {
err := expression.EachParameter(tokens, func(parameter token.List) error {
expr := expression.Parse(parameter)
parameters = append(parameters, expr.String())
parameters = append(parameters, expr.String(src))
return nil
})
@ -178,17 +178,3 @@ func TestNilGroup(t *testing.T) {
expr := expression.Parse(tokens)
assert.Nil(t, expr)
}
func TestInvalidOperator(t *testing.T) {
src := []byte("a +++ 2")
tokens := token.Tokenize(src)
expr := expression.Parse(tokens)
assert.Equal(t, expr.String(), "(+++ a 2)")
}
func TestInvalidOperatorCall(t *testing.T) {
src := []byte("+++()")
tokens := token.Tokenize(src)
expr := expression.Parse(tokens)
assert.NotNil(t, expr)
}

68
src/build/expression/Operator.go
View File

@ -15,39 +15,39 @@ type Operator struct {
// Operators defines the operators used in the language.
// The number corresponds to the operator priority and can not be zero.
var Operators = map[string]*Operator{
".": {".", 13, 2},
"λ": {"λ", 12, 1},
"@": {"@", 12, 2},
"!": {"!", 11, 1},
"*": {"*", 10, 2},
"/": {"/", 10, 2},
"%": {"%", 10, 2},
"+": {"+", 9, 2},
"-": {"-", 9, 2},
">>": {">>", 8, 2},
"<<": {"<<", 8, 2},
"&": {"&", 7, 2},
"^": {"^", 6, 2},
"|": {"|", 5, 2},
var Operators = map[token.Kind]*Operator{
token.Period: {".", 13, 2},
token.Call: {"λ", 12, 1},
token.Array: {"@", 12, 2},
token.Not: {"!", 11, 1},
token.Mul: {"*", 10, 2},
token.Div: {"/", 10, 2},
token.Mod: {"%", 10, 2},
token.Add: {"+", 9, 2},
token.Sub: {"-", 9, 2},
token.Shr: {">>", 8, 2},
token.Shl: {"<<", 8, 2},
token.And: {"&", 7, 2},
token.Xor: {"^", 6, 2},
token.Or: {"|", 5, 2},
">": {">", 4, 2},
"<": {"<", 4, 2},
">=": {">=", 4, 2},
"<=": {"<=", 4, 2},
"==": {"==", 3, 2},
"!=": {"!=", 3, 2},
"&&": {"&&", 2, 2},
"||": {"||", 1, 2},
token.Greater: {">", 4, 2},
token.Less: {"<", 4, 2},
token.GreaterEqual: {">=", 4, 2},
token.LessEqual: {"<=", 4, 2},
token.Equal: {"==", 3, 2},
token.NotEqual: {"!=", 3, 2},
token.LogicalAnd: {"&&", 2, 2},
token.LogicalOr: {"||", 1, 2},
"=": {"=", math.MinInt8, 2},
":=": {":=", math.MinInt8, 2},
"+=": {"+=", math.MinInt8, 2},
"-=": {"-=", math.MinInt8, 2},
"*=": {"*=", math.MinInt8, 2},
"/=": {"/=", math.MinInt8, 2},
">>=": {">>=", math.MinInt8, 2},
"<<=": {"<<=", math.MinInt8, 2},
token.Assign: {"=", math.MinInt8, 2},
token.Define: {":=", math.MinInt8, 2},
token.AddAssign: {"+=", math.MinInt8, 2},
token.SubAssign: {"-=", math.MinInt8, 2},
token.MulAssign: {"*=", math.MinInt8, 2},
token.DivAssign: {"/=", math.MinInt8, 2},
token.ShrAssign: {">>=", math.MinInt8, 2},
token.ShlAssign: {"<<=", math.MinInt8, 2},
}
func isComplete(expr *Expression) bool {
@ -59,14 +59,14 @@ func isComplete(expr *Expression) bool {
return true
}
if expr.Token.Kind == token.Operator && len(expr.Children) == numOperands(expr.Token.Text()) {
if expr.Token.IsOperator() && len(expr.Children) == numOperands(expr.Token.Kind) {
return true
}
return false
}
func numOperands(symbol string) int {
func numOperands(symbol token.Kind) int {
operator, exists := Operators[symbol]
if !exists {
@ -76,7 +76,7 @@ func numOperands(symbol string) int {
return operator.Operands
}
func precedence(symbol string) int8 {
func precedence(symbol token.Kind) int8 {
operator, exists := Operators[symbol]
if !exists {

27
src/build/expression/Parse.go
View File

@ -6,13 +6,8 @@ import (
"git.akyoto.dev/cli/q/src/build/token"
)
var (
call = []byte("λ")
array = []byte("@")
)
// Parse generates an expression tree from tokens.
func Parse(tokens token.List) *Expression {
func Parse(tokens []token.Token) *Expression {
var (
cursor *Expression
root *Expression
@ -43,20 +38,18 @@ func Parse(tokens token.List) *Expression {
parameters := NewList(tokens[groupPosition:i])
node := New()
node.Token.Kind = token.Operator
node.Token.Position = tokens[groupPosition].Position
switch t.Kind {
case token.GroupEnd:
node.Token.Bytes = call
node.Precedence = precedence("λ")
node.Token.Kind = token.Call
case token.ArrayEnd:
node.Token.Bytes = array
node.Precedence = precedence("@")
node.Token.Kind = token.Array
}
if cursor.Token.Kind == token.Operator && node.Precedence > cursor.Precedence {
node.Precedence = precedence(node.Token.Kind)
if cursor.Token.IsOperator() && node.Precedence > cursor.Precedence {
cursor.LastChild().Replace(node)
} else {
if cursor == root {
@ -108,18 +101,18 @@ func Parse(tokens token.List) *Expression {
continue
}
if t.Kind == token.Operator {
if t.IsOperator() {
if cursor == nil {
cursor = NewLeaf(t)
cursor.Precedence = precedence(t.Text())
cursor.Precedence = precedence(t.Kind)
root = cursor
continue
}
node := NewLeaf(t)
node.Precedence = precedence(t.Text())
node.Precedence = precedence(t.Kind)
if cursor.Token.Kind == token.Operator {
if cursor.Token.IsOperator() {
oldPrecedence := cursor.Precedence
newPrecedence := node.Precedence