From e78e212f83cfcad442fff2b15992ff305f07cd5a Mon Sep 17 00:00:00 2001 From: ning <710leo@gmail.com> Date: Tue, 20 Feb 2024 11:11:44 +0800 Subject: [PATCH] replace parser pkg --- go.mod | 1 + go.sum | 2 + pkg/parser/ast.go | 60 ----- pkg/parser/calc.go | 208 +++------------ pkg/parser/calc_test.go | 427 +++++++++++++++++++++++++++++++ pkg/parser/lexer.go | 423 ------------------------------- pkg/parser/paser.go | 547 ---------------------------------------- 7 files changed, 462 insertions(+), 1206 deletions(-) delete mode 100644 pkg/parser/ast.go create mode 100644 pkg/parser/calc_test.go delete mode 100644 pkg/parser/lexer.go delete mode 100644 pkg/parser/paser.go diff --git a/go.mod b/go.mod index 5999a531..c97ad608 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/expr-lang/expr v1.16.1 github.com/gin-contrib/pprof v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/go-ldap/ldap/v3 v3.4.4 diff --git a/go.sum b/go.sum index 62de545d..37b95fd8 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/expr-lang/expr v1.16.1 h1:Na8CUcMdyGbnNpShY7kzcHCU7WqxuL+hnxgHZ4vaz/A= +github.com/expr-lang/expr v1.16.1/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ= github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= diff --git a/pkg/parser/ast.go b/pkg/parser/ast.go deleted file mode 100644 index 488fb591..00000000 --- a/pkg/parser/ast.go +++ /dev/null @@ -1,60 +0,0 @@ -package parser - -import ( - "fmt" - "reflect" - "strings" -) - -type Node interface { -} - -type NumberNode struct { - Type TokenType - Lit string -} - -type IdentifierNode struct { - Lit string -} - -type BinaryNode struct { - Type TokenType - Left Node - Right Node -} - -func formatNode(node Node, field string, ident int) { - if arr, ok := node.([]Node); ok { - for _, v := range arr { - formatNode(v, "", ident) - } - return - } - typ := reflect.TypeOf(node) - val := reflect.ValueOf(node) - if typ.Kind() == reflect.Ptr { - typ = typ.Elem() - val = val.Elem() - } - if field != "" { - fmt.Printf("%s%s: %s {\n", formatIdent(ident), field, typ.Name()) - } else { - fmt.Printf("%s%s {\n", formatIdent(ident), typ.Name()) - } - for i := 0; i < typ.NumField(); i++ { - fieldTyp := typ.Field(i) - fieldVal := val.Field(i) - fieldKind := fieldTyp.Type.Kind() - if fieldKind != reflect.Interface && fieldKind != reflect.Ptr { - fmt.Printf("%s%s: %s\n", formatIdent(ident+1), fieldTyp.Name, fieldVal.Interface()) - } else { - formatNode(fieldVal.Interface(), fieldTyp.Name, ident+1) - } - } - fmt.Printf("%s}\n", formatIdent(ident)) -} - -func formatIdent(n int) string { - return strings.Repeat(". ", n) -} diff --git a/pkg/parser/calc.go b/pkg/parser/calc.go index f11c3d00..cca9d442 100644 --- a/pkg/parser/calc.go +++ b/pkg/parser/calc.go @@ -1,203 +1,59 @@ package parser import ( - "fmt" - "strconv" + "regexp" + "strings" + "github.com/expr-lang/expr" "github.com/toolkits/pkg/logger" ) func MathCalc(s string, data map[string]float64) (float64, error) { - var err error - p := NewParser([]rune(s)) - err = p.Parse() - if err != nil { - return 0, err - } - - for _, stat := range p.Stats() { - v, err := eval(stat, data) - if err != nil { - return 0, err - } - logger.Infof("exp:%s res:%v", s, v) - return v, nil - } - - return 0, err -} - -func Calc(s string, data map[string]float64) bool { - var err error - p := NewParser([]rune(s)) - err = p.Parse() - if err != nil { - logger.Errorf("parse err:%v", err) - return false - } - - for _, stat := range p.Stats() { - v, err := eval(stat, data) - if err != nil { - logger.Error("eval error:", err) - return false - } - logger.Infof("exp:%s res:%v", s, v) - if v > 0.0 { - return true - } - } - - return false -} - -func eval(stat Node, data map[string]float64) (float64, error) { - switch node := stat.(type) { - case *BinaryNode: - return evalBinary(node, data) - case *IdentifierNode: - return get(node.Lit, data) - case *NumberNode: - return evaluateNumber(node) - default: - return 0, fmt.Errorf("invalid node: %v", node) + m := make(map[string]float64) + for k, v := range data { + m[cleanStr(k)] = v } -} -func evaluateNumber(node *NumberNode) (float64, error) { - switch node.Type { - case IntLiteral: - v, err := strconv.ParseFloat(node.Lit, 64) - if err != nil { - return 0, err - } - return v, nil - } - return 0, fmt.Errorf("invalid type: %v", node.Type) -} - -func get(name string, data map[string]float64) (float64, error) { - value, exists := data[name] - if !exists { - return 0, fmt.Errorf("%s not found", name) - } - - return value, nil -} - -func evalBinary(node *BinaryNode, data map[string]float64) (float64, error) { - left, err := eval(node.Left, data) + program, err := expr.Compile(cleanStr(s), expr.Env(m)) if err != nil { return 0, err } - right, err := eval(node.Right, data) + + output, err := expr.Run(program, m) if err != nil { return 0, err } - switch node.Type { - case AND: - return and(left, right), nil - case OR: - return or(left, right), nil - case Plus: - return add(left, right), nil - case Minus: - return minus(left, right), nil - case Star: - return star(left, right), nil - case Slash: - return slash(left, right) - case GT: - return gt(left, right), nil - case GE: - return ge(left, right), nil - case LT: - return lt(left, right), nil - case LE: - return le(left, right), nil - case EQ: - return eq(left, right), nil - case NE: - return ne(left, right), nil - } - return 0, fmt.Errorf("invalid operator: %v", node.Type) -} - -// and -func and(left, right float64) float64 { - if left > 0.0 && right > 0.0 { - return 1 - } - return 0 -} - -// or -func or(left, right float64) float64 { - if left > 0.0 || right > 0.0 { - return 1 - } - return 0 -} - -func gt(left, right float64) float64 { - if left > right { - return 1 - } - return 0 -} - -func ge(left, right float64) float64 { - if left >= right { - return 1 - } - return 0 -} - -func lt(left, right float64) float64 { - if left < right { - return 1 - } - return 0 -} - -func le(left, right float64) float64 { - if left <= right { - return 1 - } - return 0 -} - -func eq(left, right float64) float64 { - if left == right { - return 1 + if result, ok := output.(float64); ok { + return result, nil + } else if result, ok := output.(bool); ok { + if result { + return 1, nil + } else { + return 0, nil + } + } else { + return 0, nil } - return 0 } -func ne(left, right float64) float64 { - if left != right { - return 1 +func Calc(s string, data map[string]float64) bool { + v, err := MathCalc(s, data) + if err != nil { + logger.Errorf("Calc exp:%s data:%v error: %v", s, data, err) + return false } - return 0 -} -func add(left, right float64) float64 { - return left + right + return v > 0 } -func minus(left, right float64) float64 { - return left - right +func cleanStr(s string) string { + s = replaceDollarSigns(s) + s = strings.ReplaceAll(s, "$.", "") + return s } -func star(left, right float64) float64 { - return left * right -} - -func slash(left, right float64) (float64, error) { - if right == 0 { - return 0, fmt.Errorf("right is zero") - } - res := left / right - return res, nil +func replaceDollarSigns(s string) string { + re := regexp.MustCompile(`\$([A-Z])\.`) + return re.ReplaceAllString(s, "${1}_") } diff --git a/pkg/parser/calc_test.go b/pkg/parser/calc_test.go new file mode 100644 index 00000000..71b85dda --- /dev/null +++ b/pkg/parser/calc_test.go @@ -0,0 +1,427 @@ +package parser + +import ( + "testing" +) + +func TestMathCalc(t *testing.T) { + tests := []struct { + name string + expr string + data map[string]float64 + expected float64 + wantErr bool + }{ + { + name: "Add and Subtract", + expr: "一个 + $.B - $.C", + data: map[string]float64{"一个": 1, "$.B": 2, "$.C": 3}, + expected: 0, + wantErr: false, + }, + { + name: "Multiply and Divide", + expr: "($A.err_count >0&& $A.err_count <=3)||($B.err_count>0 && $B.err_count <=5)", + data: map[string]float64{"$A.err_count": 4, "$B.err_count": 2}, + expected: 1, + wantErr: false, + }, + { + name: "Subtract and Add", + expr: "$.C - $.D + $.A", + data: map[string]float64{"$.A": 5, "$.C": 3, "$.D": 2}, + expected: 6, + wantErr: false, + }, + { + name: "Divide and Multiply", + expr: "$.B / $.C * $.D", + data: map[string]float64{"$.B": 6, "$.C": 2, "$.D": 3}, + expected: 9, + wantErr: false, + }, + { + name: "Divide and Multiply", + expr: "$.B / $.C * $.D", + data: map[string]float64{"$.B": 6, "$.C": 2, "$.D": 3}, + expected: 9, + wantErr: false, + }, + { + name: "Multiply and Add", + expr: "$.A * $.B + $.C", + data: map[string]float64{"$.A": 2, "$.B": 3, "$.C": 4}, + expected: 10, + wantErr: false, + }, + { + name: "Subtract and Divide", + expr: "$.D - $.A / $.B", + data: map[string]float64{"$.D": 10, "$.A": 4, "$.B": 2}, + expected: 8, + wantErr: false, + }, + { + name: "Add, Subtract and Subtract", + expr: "$.C + $.D - $.A", + data: map[string]float64{"$.C": 3, "$.D": 4, "$.A": 5}, + expected: 2, + wantErr: false, + }, + { + name: "Multiply and Subtract", + expr: "$.B * $.A - $.D", + data: map[string]float64{"$.B": 2, "$.A": 3, "$.D": 4}, + expected: 2, + wantErr: false, + }, + { + name: "Divide and Add", + expr: "$.A / $.B + $.C", + data: map[string]float64{"$.A": 4, "$.B": 2, "$.C": 3}, + expected: 5, + wantErr: false, + }, + { + name: "Add and Multiply", + expr: "$.D + $.A * $.B", + data: map[string]float64{"$.D": 1, "$.A": 2, "$.B": 3}, + expected: 7, + wantErr: false, + }, + { + name: "Divide and Add with Parentheses", + expr: "($A / $B) + ($C * $D)", + data: map[string]float64{"$A": 4, "$B": 2, "$C": 1, "$D": 3}, + expected: 5.0, + wantErr: false, + }, + { + name: "Divide with Parentheses", + expr: "($.A - $.B) / ($.C + $.D)", + data: map[string]float64{"$.A": 6, "$.B": 2, "$.C": 3, "$.D": 1}, + expected: 1.0, + wantErr: false, + }, + { + name: "Add and Multiply with Parentheses", + expr: "($.A + $.B) * ($.C - $.D)", + data: map[string]float64{"$.A": 8, "$.B": 2, "$.C": 4, "$.D": 2}, + expected: 20, + wantErr: false, + }, + { + name: "Divide and Multiply with Parentheses", + expr: "($.A * $.B) / ($.C - $.D)", + data: map[string]float64{"$.A": 8, "$.B": 2, "$.C": 4, "$.D": 2}, + expected: 8, + wantErr: false, + }, + { + name: "Add and Divide with Parentheses", + expr: "$.A + ($.B * $.C) / $.D", + data: map[string]float64{"$.A": 1, "$.B": 2, "$.C": 3, "$.D": 4}, + expected: 2.5, + wantErr: false, + }, + { + name: "Subtract and Multiply with Parentheses", + expr: "($.A + $.B) - ($.C * $.D)", + data: map[string]float64{"$.A": 5, "$.B": 2, "$.C": 3, "$.D": 1}, + expected: 4, + wantErr: false, + }, + { + name: "Multiply and Divide with Parentheses", + expr: "$.A / ($.B - $.C) * $.D", + data: map[string]float64{"$.A": 4, "$.B": 3, "$.C": 2, "$.D": 5}, + expected: 20.0, + wantErr: false, + }, + { + name: "Multiply and Divide with Parentheses 2", + expr: "($.A - $.B) * ($.C / $.D)", + data: map[string]float64{"$.A": 3, "$.B": 1, "$.C": 2, "$.D": 4}, + expected: 1.0, + wantErr: false, + }, + + { + name: "Complex expression", + expr: "$.A/$.B*$.D", + data: map[string]float64{"$.A": 1, "$.B": 2, "$.C": 3, "$.D": 4}, + expected: 2, + wantErr: false, + }, + { + name: "Complex expression", + expr: "$.A/$.B*$.C", + data: map[string]float64{"$.A": 2, "$.B": 2, "$.C": 2}, + expected: 2, + wantErr: false, + }, + { + name: "Complex expression", + expr: "$.A/($.B*$.C)", + data: map[string]float64{"$.A": 2, "$.B": 2, "$.C": 2}, + expected: 0.5, + wantErr: false, + }, + { + name: "Addition", + expr: "$.A + $.B", + data: map[string]float64{"$.A": 2, "$.B": 3}, + expected: 5, + wantErr: false, + }, + { + name: "Subtraction", + expr: "$.A - $.B", + data: map[string]float64{"$.A": 5, "$.B": 3}, + expected: 2, + wantErr: false, + }, + { + name: "Multiplication", + expr: "$.A * $.B", + data: map[string]float64{"$.A": 4, "$.B": 3}, + expected: 12, + wantErr: false, + }, + { + name: "Division", + expr: "$.A / $.B", + data: map[string]float64{"$.A": 10, "$.B": 2}, + expected: 5, + wantErr: false, + }, + { + name: "Mixed operations", + expr: "($.A + $.B) * ($.C - $.D)", + data: map[string]float64{"$.A": 1, "$.B": 2, "$.C": 5, "$.D": 3}, + expected: 6, // Corrected from 9 to 6 + wantErr: false, + }, + { + name: "Division by zero", + expr: "( $D/$A >= 0.1 || $D/$B <= 0.5 ) && $C >= 1000", + data: map[string]float64{"$A": 428382, "$B": 250218, "$C": 305578, "$D": 325028}, + expected: 1, + wantErr: true, + }, + { + name: "Parentheses", + expr: "($.A + $.B) / ($.C - $.D)", + data: map[string]float64{"$.A": 6, "$.B": 4, "$.C": 10, "$.D": 2}, + expected: 1.25, // Corrected from 2.5 to 1.25 + wantErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Run the MathCalc function + result, err := MathCalc(tc.expr, tc.data) + + // Check for expected errors + if tc.wantErr { + if err == nil { + t.Errorf("Expected an error for expr '%s', but got none:%v", tc.expr, result) + } + return + } + + // If an error is not expected, but occurs, fail the test + if err != nil { + t.Fatalf("Unexpected error for expr '%s' data:%v err:%v", tc.expr, tc.data, err) + } + + // Compare the expected result with the actual result + if result != tc.expected { + t.Errorf("Expected result for expr '%s' to be %v, got %v", tc.expr, tc.expected, result) + } + }) + } +} + +func TestCalc(t *testing.T) { + tests := []struct { + name string + expr string + data map[string]float64 + expected bool + }{ + { + name: "Greater than - true", + expr: "$.A > $.B", + data: map[string]float64{"$.A": 5, "$.B": 3}, + expected: true, + }, + { + name: "Multiply and Subtract with Parentheses", + expr: "$A.yesterday_rate > 0.1 && $A.last_week_rate>0.1 or ($A.今天 >300 || $A.昨天>300 || $A.上周今天 > 300)", + data: map[string]float64{"$A.yesterday_rate": 0.1, "$A.last_week_rate": 2, "$A.今天": 200.4, "$A.昨天": 200.4, "$A.上周今天": 200.4}, + expected: false, + }, + { + name: "Count Greater Than Zero with Code", + expr: "$A.count > 0", + data: map[string]float64{"$A.count": 197, "$A.code": 30000}, + expected: true, + }, + { + name: "Today, Yesterday, and Lastweek Rate Comparison", + expr: "$A.todayRate<0.3 && $A.yesterdayRate<0.3 && $A.lastweekRate<0.3", + data: map[string]float64{"$A.todayRate": 1.1, "$A.yesterdayRate": 0.8, "$A.lastweekRate": 1.2}, + expected: false, + }, + { + name: "Today, Yesterday, and Lastweek Rate Low Threshold", + expr: "$A.todayRate<0.1 && $A.yesterdayRate<0.1 && $A.lastweekRate<0.1", + data: map[string]float64{"$A.todayRate": 0.9, "$A.yesterdayRate": 0.8, "$A.lastweekRate": 0.9}, + expected: false, + }, + { + name: "Agent Specific Today, Yesterday, and Lastweek Rate Comparison", + expr: "$A.agent == 11 && $A.todayRate<0.3 && $A.yesterdayRate<0.3 && $A.lastweekRate<0.3", + data: map[string]float64{"$A.agent": 11, "$A.todayRate": 0.9, "$A.yesterdayRate": 0.9, "$A.lastweekRate": 1}, + expected: false, + }, + { + name: "Today, Yesterday, and Lastweek Rate Below 0.1 - Case 1", + expr: "$A<0.1 && $A.yesterdayRate<0.1 && $A.lastweekRate<0.1", + data: map[string]float64{"$A": 0.8, "$A.yesterdayRate": 0.9, "$A.lastweekRate": 0.9}, + expected: false, + }, + { + name: "Today, Yesterday, and Lastweek Rate Below 0.1 - Case 2", + expr: "$A.today_rate<0.1 && $A.yesterday_rate<0.1 && $A.lastweek_rate<0.1", + data: map[string]float64{"$A.today_rate": 0.9, "$A.yesterday_rate": 0.9, "$A.lastweek_rate": 0.9}, + expected: false, + }, + { + name: "Today, Yesterday, and Lastweek Rate Below 0.1 - Case 3", + expr: "$B.today_rate<0.1 && $A.yesterday_rate<0.1 && $A.lastweek_rate<0.1", + data: map[string]float64{"$B.today_rate": 0.5, "$A.yesterday_rate": 0.9, "$A.lastweek_rate": 0.8}, + expected: false, + }, + { + name: "Yesterday and Byesterday Rates Logical Conditions - Case 1", + expr: "($A.yesterday_rate > 2 && $A.byesterday_rate > 2) or ($A.yesterday_rate <= 0.7 && $A.byesterday_rate <= 0.7)", + data: map[string]float64{"$A.yesterday_rate": 3, "$A.byesterday_rate": 3}, + expected: true, + }, + { + name: "Yesterday and Byesterday Rates Higher Thresholds - Case 1", + expr: "($A.yesterday_rate > 1.5 && $A.byesterday_rate > 1.5) or ($A.yesterday_rate <= 0.8 && $A.byesterday_rate <= 0.8)", + data: map[string]float64{"$A.yesterday_rate": 1.08, "$A.byesterday_rate": 1.02}, + expected: false, + }, + { + name: "Greater than - false", + expr: "($A.yesterday_rate > 1.0 && $A.byesterday_rate > 1.0 ) or ($A.yesterday_rate <= 0.9 && $A.byesterday_rate <= 0.9)", + data: map[string]float64{"$A.byesterday_rate": 0.33, "$A.yesterday_rate": 2}, + expected: false, + }, + { + name: "Less than - true", + expr: "$A.count > 100 or $A.count2 > -3", + data: map[string]float64{"$A.count": 5, "$A.count2": -1, "$.D": 2}, + expected: true, + }, + { + name: "Less than - false", + expr: "$.A < $.B/$.B*4", + data: map[string]float64{"$.A": 5, "$.B": 3}, + expected: false, + }, + { + name: "Greater than or equal - true", + expr: "$.A >= $.B", + data: map[string]float64{"$.A": 3, "$.B": 3}, + expected: true, + }, + { + name: "Less than or equal - true", + expr: "$.A <= $.B", + data: map[string]float64{"$.A": 2, "$.B": 2}, + expected: true, + }, + { + name: "Not equal - true", + expr: "$.A != $.B", + data: map[string]float64{"$.A": 3, "$.B": 2}, + expected: true, + }, + { + name: "Not equal - false", + expr: "$.A != $.B", + data: map[string]float64{"$.A": 2, "$.B": 2}, + expected: false, + }, + { + name: "Addition resulting in true", + expr: "$.A + $.B > $.C", + data: map[string]float64{"$.A": 3, "$.B": 2, "$.C": 4}, + expected: true, + }, + { + name: "Subtraction resulting in false", + expr: "$.A - $.B < $.C", + data: map[string]float64{"$.A": 1, "$.B": 3, "$.C": 1}, + expected: true, + }, + { + name: "Multiplication resulting in true", + expr: "$.A * $.B > $.C", + data: map[string]float64{"$.A": 2, "$.B": 3, "$.C": 5}, + expected: true, + }, + { + name: "Division resulting in false", + expr: "$.A / $.B*$.C < $.C", + data: map[string]float64{"$.A": 4, "$.B": 2, "$.C": 2}, + expected: false, + }, + { + name: "Addition with parentheses resulting in true", + expr: "($.A + $.B) > $.C && $.A >0", + data: map[string]float64{"$.A": 1, "$.B": 4, "$.C": 4}, + expected: true, + }, + { + name: "Addition with parentheses resulting in true", + expr: "($.A + $.B) > $.C || $.A < 0", + data: map[string]float64{"$.A": 1, "$.B": 4, "$.C": 4}, + expected: true, + }, + { + name: "Complex expression with parentheses resulting in false", + expr: "($.A + $.B) * $.C < $.D", + data: map[string]float64{"$.A": 1, "$.B": 2, "$.C": 3, "$.D": 10}, + expected: true, + }, + { + name: "Nested parentheses resulting in true", + expr: "($.A + ($.B - $.C)) * $.D > $.E", + data: map[string]float64{"$.A": 2, "$.B": 5, "$.C": 2, "$.D": 2, "$.E": 8}, + expected: true, + }, + { + name: "Division with parentheses resulting in false", + expr: " ( true || false ) && true", + data: map[string]float64{"$A": 673601, "$A.": 673601, "$B": 250218, "$C": 456513, "$C.": 456513, "$D": 456513, "$D.": 456513}, + expected: true, + }, + // $A:673601.5 $A.:673601.5 $B:361520 $B.:361520 $C:456513 $C.:456513 $D:422634 $D.:422634] + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := Calc(tc.expr, tc.data) + if result != tc.expected { + t.Errorf("Expected result for expr '%s' to be %v, got %v", tc.expr, tc.expected, result) + } + }) + } +} diff --git a/pkg/parser/lexer.go b/pkg/parser/lexer.go deleted file mode 100644 index 7194bd3b..00000000 --- a/pkg/parser/lexer.go +++ /dev/null @@ -1,423 +0,0 @@ -package parser - -import ( - "errors" - "fmt" -) - -var ( - ErrorLexEOF = errors.New("lex source EOF") -) - -type TokenType int - -const ( - EOF TokenType = iota - AND - OR - EXP - - Identifier - - GT - GE - LT - LE - EQ - NE - - Plus - Minus - Star - Slash - LeftParen - RightParen - - IntLiteral - UintLiteral - FloatLiteral -) - -func (t TokenType) String() string { - switch t { - case Identifier: - return "Identifier" - case GT: - return "GT" - case GE: - return "GE" - case LT: - return "LT" - case LE: - return "LE" - case EQ: - return "EQ" - case NE: - return "NE" - case Plus: - return "Plus" - case Minus: - return "Minus" - case Star: - return "Star" - case Slash: - return "Slash" - case LeftParen: - return "leftParen" - case RightParen: - return "rightParen" - - case AND: - return "AND" - case OR: - return "OR" - case EXP: - return "expr" - - case IntLiteral: - return "IntLiteral" - case FloatLiteral: - return "FloatLiteral" - default: - return "Unknown TokenType" - } -} - -type Token struct { - typ TokenType - buf []rune -} - -func (t *Token) push(r rune) { - t.buf = append(t.buf, r) -} - -func (t *Token) String() string { - return fmt.Sprintf("token: {%v: '%s'}", t.typ, string(t.buf)) -} - -type Lexer struct { - buf []rune - idx int // always point to the next rune -} - -func newLexer(buf []rune) *Lexer { - return &Lexer{ - buf: buf, - idx: 0, - } -} - -func isAlpha(ch rune) bool { - return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '.' -} - -func isPrefix(ch rune) bool { - return ch == '$' -} - -func isWhitespace(ch rune) bool { - return ch == ' ' || ch == '\t' || ch == '\r' -} - -func isDigit(ch rune) bool { - return ch >= '0' && ch <= '9' || ch == '.' -} - -func (l *Lexer) lex() ([]*Token, error) { - toks := make([]*Token, 0) - for { - tok, err := l.lexToken() - if err != nil { - if err == ErrorLexEOF { - break - } - return nil, err - } - toks = append(toks, tok) - } - return toks, nil -} - -func (l *Lexer) lexToken() (*Token, error) { - l.skipWhitespace() - ch, err := l.next() - if err != nil { - return nil, err - } - switch { - case isPrefix(ch): - return l.lexIdentifier(ch), nil - case ch == '&': - return l.lexAnd(ch), nil - case ch == '|': - return l.lexOR(ch), nil - case ch == '>': - return l.lexGT(ch), nil - case ch == '<': - return l.lexLT(ch), nil - case ch == '!': - return l.lexNE(ch), nil - case ch == '=': - return l.lexEQ(ch), nil - case ch == '+': - return l.lexPlus(ch), nil - case ch == '-': - return l.lexMinus(ch), nil - case ch == '*': - return l.lexStar(ch), nil - case ch == '/': - return l.lexSlash(ch), nil - case ch == '(': - return l.lexLeftParen(ch), nil - case ch == ')': - return l.lexRightParen(ch), nil - case isDigit(ch): - return l.lexDigital(ch), nil - default: - return nil, errors.New("not supported rune: " + string(ch)) - } -} - -func (l *Lexer) lexIdentifier(ch rune) *Token { - tok := &Token{ - typ: Identifier, - buf: []rune{ch}, - } - for { - ch, err := l.peek() - if err != nil { - return tok - } - if isAlpha(ch) || isDigit(ch) { - l.mustNext() - tok.push(ch) - continue - } - return tok - } -} - -func (l *Lexer) lexGT(ch rune) *Token { - tok := &Token{ - typ: GT, - buf: []rune{ch}, - } - ch, err := l.peek() - if err != nil { - return tok - } - if ch == '=' { - tok.typ = GE - tok.buf = append(tok.buf, ch) - l.mustNext() - } - return tok -} - -func (l *Lexer) lexLT(ch rune) *Token { - tok := &Token{ - typ: LT, - buf: []rune{ch}, - } - ch, err := l.peek() - if err != nil { - return tok - } - if ch == '=' { - tok.typ = LE - tok.buf = append(tok.buf, ch) - l.mustNext() - } - return tok -} - -func (l *Lexer) lexEQ(ch rune) *Token { - tok := &Token{ - typ: 0, - buf: []rune{ch}, - } - - ch, err := l.peek() - if err != nil { - return tok - } - - if ch == '=' { - tok.typ = EQ - tok.buf = append(tok.buf, ch) - l.mustNext() - } - // 如果不是 == 处理报错 - return tok -} - -func (l *Lexer) lexNE(ch rune) *Token { - tok := &Token{ - typ: 0, - buf: []rune{ch}, - } - - ch, err := l.peek() - if err != nil { - return tok - } - - if ch == '=' { - tok.typ = NE - tok.buf = append(tok.buf, ch) - l.mustNext() - } - // 如果不是 == 处理报错 - return tok -} - -func (l *Lexer) lexPlus(ch rune) *Token { - tok := &Token{ - typ: Plus, - buf: []rune{ch}, - } - return tok -} - -func (l *Lexer) lexMinus(ch rune) *Token { - tok := &Token{ - typ: Minus, - buf: []rune{ch}, - } - return tok -} - -func (l *Lexer) lexStar(ch rune) *Token { - tok := &Token{ - typ: Star, - buf: []rune{ch}, - } - return tok -} - -func (l *Lexer) lexSlash(ch rune) *Token { - tok := &Token{ - typ: Slash, - buf: []rune{ch}, - } - return tok -} - -func (l *Lexer) lexAnd(ch rune) *Token { - tok := &Token{ - typ: 0, - buf: []rune{ch}, - } - - ch, err := l.peek() - if err != nil { - return tok - } - - if ch == '&' { - tok.typ = AND - tok.buf = append(tok.buf, ch) - l.mustNext() - } - - return tok -} - -func (l *Lexer) lexOR(ch rune) *Token { - tok := &Token{ - typ: 0, - buf: []rune{ch}, - } - - ch, err := l.peek() - if err != nil { - return tok - } - - if ch == '|' { - tok.typ = OR - tok.buf = append(tok.buf, ch) - l.mustNext() - } - - return tok -} - -func (l *Lexer) lexLeftParen(ch rune) *Token { - tok := &Token{ - typ: LeftParen, - buf: []rune{ch}, - } - return tok -} - -func (l *Lexer) lexRightParen(ch rune) *Token { - tok := &Token{ - typ: RightParen, - buf: []rune{ch}, - } - return tok -} - -func (l *Lexer) lexDigital(ch rune) *Token { - tok := &Token{ - typ: IntLiteral, - buf: []rune{ch}, - } - for { - ch, err := l.peek() - if err != nil { - return tok - } - if isDigit(ch) { - l.mustNext() - tok.push(ch) - continue - } - return tok - } -} - -func (l *Lexer) skipWhitespace() bool { - found := false - for { - ch, err := l.peek() - if err != nil { - return false - } - if isWhitespace(ch) { - l.mustNext() - found = true - continue - } - if ch == '\n' { - l.mustNext() - found = true - continue - } - break - } - return found -} - -func (l *Lexer) next() (rune, error) { - ch, err := l.peek() - if err != nil { - return 0, err - } - l.idx++ - return ch, nil -} - -func (l *Lexer) mustNext() rune { - l.idx++ - return l.buf[l.idx-1] -} - -func (l *Lexer) peek() (rune, error) { - if l.idx >= len(l.buf) { - return 0, ErrorLexEOF - } - return l.buf[l.idx], nil -} diff --git a/pkg/parser/paser.go b/pkg/parser/paser.go deleted file mode 100644 index e07717b3..00000000 --- a/pkg/parser/paser.go +++ /dev/null @@ -1,547 +0,0 @@ -package parser - -import ( - "fmt" - - "github.com/toolkits/pkg/logger" -) - -type Parser struct { - buf []rune - tokens []*Token - idx int - err error - isEOF bool - stats []Node -} - -/* -exp -> or | or = exp -or -> and | or || and -and -> equal | and && equal -equal -> rel | equal == rel | equal != rel -rel -> add | rel > add | rel < add | rel >= add | rel <= add -add -> mul | add + mul | add - mul -mul -> pri | mul * pri | mul / pri -pri -> Id | Literal | (exp) -*/ - -func NewParser(buf []rune) *Parser { - return &Parser{ - buf: buf, - } -} - -func (p *Parser) Parse() error { - lexer := newLexer(p.buf) - tokens, err := lexer.lex() - if err != nil { - return err - } - p.tokens = tokens - p.stats = make([]Node, 0) - - for { - node := p.parseStat() - if node == nil { - return nil - } - if p.hasError() { - return nil - } - nodes, ok := node.([]Node) - if ok { - p.stats = append(p.stats, nodes...) - } else { - p.stats = append(p.stats, node) - } - } -} - -func (p *Parser) PrintAST() { - for _, node := range p.Stats() { - formatNode(node, "", 0) - } -} - -func (p *Parser) Stats() []Node { - return p.stats -} - -func (p *Parser) Err() error { - return p.err -} - -func (p *Parser) parseStat() Node { - if p.hasError() { - return nil - } - tok, valid := p.peek() - if !valid { - return nil - } - switch tok.typ { - case Identifier, IntLiteral: - p.mustNext() - opTok, valid := p.peek() - if !valid { - p.back() - return p.parseExpr() - } - switch opTok.typ { - case Plus, Minus, Star, Slash, GE, GT, LE, LT, AND, OR, LeftParen: - p.back() - return p.parseExpr() - default: - p.reportError("invalid token: %v", tok) - return nil - } - default: - p.reportError("invalid token: %v", tok) - return nil - } -} - -func (p *Parser) mustNext() *Token { - p.idx++ - return p.tokens[p.idx-1] -} - -func (p *Parser) back() { - p.idx-- -} - -func (p *Parser) hasError() bool { - if p.err != nil { - logger.Errorf("parse err", p.err) - } - return p.err != nil -} - -func (p *Parser) peek() (*Token, bool) { - if p.idx >= len(p.tokens) { - p.isEOF = true - return nil, false - } - return p.tokens[p.idx], true -} - -// exp -> or | or = exp -func (p *Parser) parseExpr() Node { - if p.hasError() { - return nil - } - - leftNode := p.parseOr() - var binNode *BinaryNode - firstTime := true - - for { - opTok, valid := p.peek() - if !valid { - if firstTime { - return leftNode - } - break - } - - switch opTok.typ { - case AND: - p.mustNext() - default: - if firstTime { - return leftNode - } - return binNode - } - - if firstTime { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: leftNode, - Right: p.parseExpr(), - } - firstTime = false - } else { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: binNode, - Right: p.parseExpr(), - } - } - if p.hasError() { - return nil - } - } - - return binNode -} - -// or -> and | or || and -func (p *Parser) parseOr() Node { - if p.hasError() { - return nil - } - - leftNode := p.parseAnd() - var binNode *BinaryNode - firstTime := true - - for { - opTok, valid := p.peek() - if !valid { - if firstTime { - return leftNode - } - break - } - - switch opTok.typ { - case AND: - p.mustNext() - default: - if firstTime { - return leftNode - } - return binNode - } - - if firstTime { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: leftNode, - Right: p.parseOr(), - } - firstTime = false - } else { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: binNode, - Right: p.parseOr(), - } - } - if p.hasError() { - return nil - } - } - - return binNode -} - -// and -> equal | and && equal -func (p *Parser) parseAnd() Node { - if p.hasError() { - return nil - } - - leftNode := p.parseEqual() - var binNode *BinaryNode - firstTime := true - - for { - opTok, valid := p.peek() - if !valid { - if firstTime { - return leftNode - } - break - } - - switch opTok.typ { - case AND: - p.mustNext() - default: - if firstTime { - return leftNode - } - return binNode - } - - if firstTime { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: leftNode, - Right: p.parseAnd(), - } - firstTime = false - } else { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: binNode, - Right: p.parseAnd(), - } - } - if p.hasError() { - return nil - } - } - - return binNode -} - -// equal -> rel | equal == rel | equal != rel -func (p *Parser) parseEqual() Node { - if p.hasError() { - return nil - } - - leftNode := p.parseRel() - var binNode *BinaryNode - firstTime := true - - for { - opTok, valid := p.peek() - if !valid { - if firstTime { - return leftNode - } - break - } - - switch opTok.typ { - case EQ, NE: - p.mustNext() - default: - if firstTime { - return leftNode - } - return binNode - } - - if firstTime { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: leftNode, - Right: p.parseEqual(), - } - firstTime = false - } else { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: binNode, - Right: p.parseEqual(), - } - } - if p.hasError() { - return nil - } - } - - return binNode -} - -// rel -> add | rel > add | rel < add | rel >= add | rel <= add -func (p *Parser) parseRel() Node { - if p.hasError() { - return nil - } - - leftNode := p.parseAdd() - var binNode *BinaryNode - firstTime := true - - for { - opTok, valid := p.peek() - if !valid { - if firstTime { - return leftNode - } - break - } - - switch opTok.typ { - case GE, GT, LE, LT: - p.mustNext() - default: - if firstTime { - return leftNode - } - return binNode - } - - if firstTime { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: leftNode, - Right: p.parseRel(), - } - firstTime = false - } else { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: binNode, - Right: p.parseRel(), - } - } - if p.hasError() { - return nil - } - } - - return binNode -} - -// add -> mul ( + mul)* -func (p *Parser) parseAdd() Node { - if p.hasError() { - return nil - } - - leftNode := p.parseMul() - var binNode *BinaryNode - firstTime := true - - for { - opTok, valid := p.peek() - if !valid { - if firstTime { - return leftNode - } - break - } - - switch opTok.typ { - case Plus, Minus: - p.mustNext() - default: - if firstTime { - return leftNode - } - return binNode - } - - if firstTime { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: leftNode, - Right: p.parseAdd(), - } - firstTime = false - } else { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: binNode, - Right: p.parseAdd(), - } - } - if p.hasError() { - return nil - } - } - - return binNode -} - -// mul -> pri | mul * pri | mul / pri -func (p *Parser) parseMul() Node { - if p.hasError() { - return nil - } - - leftNode := p.parsePri() - var binNode *BinaryNode - firstTime := true - for { - opTok, valid := p.peek() - if !valid { - if firstTime { - return leftNode - } - break - } - - switch opTok.typ { - case Star, Slash: - p.mustNext() - default: - if firstTime { - return leftNode - } - return binNode - } - - if firstTime { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: leftNode, - Right: p.parseMul(), - } - firstTime = false - } else { - binNode = &BinaryNode{ - Type: opTok.typ, - Left: binNode, - Right: p.parseMul(), - } - } - if p.hasError() { - return nil - } - } - return binNode -} - -// pri -> Id | Literal | (exp) -func (p *Parser) parsePri() Node { - if p.hasError() { - return nil - } - - tok, valid := p.peek() - if !valid { - p.reportError("unexpected EOF") - return nil - } - - if tok.typ == IntLiteral { - p.mustNext() - return &NumberNode{ - Type: tok.typ, - Lit: string(tok.buf), - } - } - - if tok.typ == Identifier { - p.mustNext() - return &IdentifierNode{ - Lit: string(tok.buf), - } - } - - if tok.typ == LeftParen { - p.mustNext() - node := p.parseExpr() - if node != nil { - tk, valid := p.peek() - if !valid { - p.reportError("unexpected EOF") - return nil - } - - if tk.typ == RightParen { - p.mustNext() - } else { - p.reportError("expecting right parenthesis") - } - } else { - p.reportError("expecting an additive expression inside parenthesis") - } - } - - p.reportError("expect int Identifier but met %v", tok) - return nil -} - -func (p *Parser) reportError(args ...interface{}) { - if len(args) >= 1 { - i := args[0] - switch v := i.(type) { - case string: - p.err = fmt.Errorf(v, args[1:]...) - case error: - p.err = v - default: - panic(v) - } - } -}