ccfos
/
nightingale
Not watched
1
0
Fork 0
Code
Releases 20 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

replace parser pkg

main
ning 2 years ago
parent
cdc2d4c039
commit
e78e212f83
7 changed files with 462 additions and 1206 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -0
      go.mod
  2. +2
    -0
      go.sum
  3. +0
    -60
      pkg/parser/ast.go
  4. +32
    -176
      pkg/parser/calc.go
  5. +427
    -0
      pkg/parser/calc_test.go
  6. +0
    -423
      pkg/parser/lexer.go
  7. +0
    -547
      pkg/parser/paser.go

+ 1
- 0
go.mod View File

@@ -7,6 +7,7 @@ require (
github.com/coreos/go-oidc v2.2.1+incompatible
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/expr-lang/expr v1.16.1
github.com/gin-contrib/pprof v1.4.0
github.com/gin-gonic/gin v1.9.1
github.com/go-ldap/ldap/v3 v3.4.4


+ 2
- 0
go.sum View File

@@ -32,6 +32,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/expr-lang/expr v1.16.1 h1:Na8CUcMdyGbnNpShY7kzcHCU7WqxuL+hnxgHZ4vaz/A=
github.com/expr-lang/expr v1.16.1/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ=
github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=


+ 0
- 60
pkg/parser/ast.go View File

@@ -1,60 +0,0 @@
package parser

import (
"fmt"
"reflect"
"strings"
)

type Node interface {
}

type NumberNode struct {
Type TokenType
Lit string
}

type IdentifierNode struct {
Lit string
}

type BinaryNode struct {
Type TokenType
Left Node
Right Node
}

func formatNode(node Node, field string, ident int) {
if arr, ok := node.([]Node); ok {
for _, v := range arr {
formatNode(v, "", ident)
}
return
}
typ := reflect.TypeOf(node)
val := reflect.ValueOf(node)
if typ.Kind() == reflect.Ptr {
typ = typ.Elem()
val = val.Elem()
}
if field != "" {
fmt.Printf("%s%s: %s {\n", formatIdent(ident), field, typ.Name())
} else {
fmt.Printf("%s%s {\n", formatIdent(ident), typ.Name())
}
for i := 0; i < typ.NumField(); i++ {
fieldTyp := typ.Field(i)
fieldVal := val.Field(i)
fieldKind := fieldTyp.Type.Kind()
if fieldKind != reflect.Interface && fieldKind != reflect.Ptr {
fmt.Printf("%s%s: %s\n", formatIdent(ident+1), fieldTyp.Name, fieldVal.Interface())
} else {
formatNode(fieldVal.Interface(), fieldTyp.Name, ident+1)
}
}
fmt.Printf("%s}\n", formatIdent(ident))
}

func formatIdent(n int) string {
return strings.Repeat(". ", n)
}

+ 32
- 176
pkg/parser/calc.go View File

@@ -1,203 +1,59 @@
package parser

import (
"fmt"
"strconv"
"regexp"
"strings"

"github.com/expr-lang/expr"
"github.com/toolkits/pkg/logger"
)

func MathCalc(s string, data map[string]float64) (float64, error) {
var err error
p := NewParser([]rune(s))
err = p.Parse()
if err != nil {
return 0, err
}

for _, stat := range p.Stats() {
v, err := eval(stat, data)
if err != nil {
return 0, err
}
logger.Infof("exp:%s res:%v", s, v)
return v, nil
}

return 0, err
}

func Calc(s string, data map[string]float64) bool {
var err error
p := NewParser([]rune(s))
err = p.Parse()
if err != nil {
logger.Errorf("parse err:%v", err)
return false
}

for _, stat := range p.Stats() {
v, err := eval(stat, data)
if err != nil {
logger.Error("eval error:", err)
return false
}
logger.Infof("exp:%s res:%v", s, v)
if v > 0.0 {
return true
}
}

return false
}

func eval(stat Node, data map[string]float64) (float64, error) {
switch node := stat.(type) {
case *BinaryNode:
return evalBinary(node, data)
case *IdentifierNode:
return get(node.Lit, data)
case *NumberNode:
return evaluateNumber(node)
default:
return 0, fmt.Errorf("invalid node: %v", node)
m := make(map[string]float64)
for k, v := range data {
m[cleanStr(k)] = v
}
}

func evaluateNumber(node *NumberNode) (float64, error) {
switch node.Type {
case IntLiteral:
v, err := strconv.ParseFloat(node.Lit, 64)
if err != nil {
return 0, err
}
return v, nil
}
return 0, fmt.Errorf("invalid type: %v", node.Type)
}

func get(name string, data map[string]float64) (float64, error) {
value, exists := data[name]
if !exists {
return 0, fmt.Errorf("%s not found", name)
}

return value, nil
}

func evalBinary(node *BinaryNode, data map[string]float64) (float64, error) {
left, err := eval(node.Left, data)
program, err := expr.Compile(cleanStr(s), expr.Env(m))
if err != nil {
return 0, err
}
right, err := eval(node.Right, data)

output, err := expr.Run(program, m)
if err != nil {
return 0, err
}

switch node.Type {
case AND:
return and(left, right), nil
case OR:
return or(left, right), nil
case Plus:
return add(left, right), nil
case Minus:
return minus(left, right), nil
case Star:
return star(left, right), nil
case Slash:
return slash(left, right)
case GT:
return gt(left, right), nil
case GE:
return ge(left, right), nil
case LT:
return lt(left, right), nil
case LE:
return le(left, right), nil
case EQ:
return eq(left, right), nil
case NE:
return ne(left, right), nil
}
return 0, fmt.Errorf("invalid operator: %v", node.Type)
}

// and
func and(left, right float64) float64 {
if left > 0.0 && right > 0.0 {
return 1
}
return 0
}

// or
func or(left, right float64) float64 {
if left > 0.0 || right > 0.0 {
return 1
}
return 0
}

func gt(left, right float64) float64 {
if left > right {
return 1
}
return 0
}

func ge(left, right float64) float64 {
if left >= right {
return 1
}
return 0
}

func lt(left, right float64) float64 {
if left < right {
return 1
}
return 0
}

func le(left, right float64) float64 {
if left <= right {
return 1
}
return 0
}

func eq(left, right float64) float64 {
if left == right {
return 1
if result, ok := output.(float64); ok {
return result, nil
} else if result, ok := output.(bool); ok {
if result {
return 1, nil
} else {
return 0, nil
}
} else {
return 0, nil
}
return 0
}

func ne(left, right float64) float64 {
if left != right {
return 1
func Calc(s string, data map[string]float64) bool {
v, err := MathCalc(s, data)
if err != nil {
logger.Errorf("Calc exp:%s data:%v error: %v", s, data, err)
return false
}
return 0
}

func add(left, right float64) float64 {
return left + right
return v > 0
}

func minus(left, right float64) float64 {
return left - right
func cleanStr(s string) string {
s = replaceDollarSigns(s)
s = strings.ReplaceAll(s, "$.", "")
return s
}

func star(left, right float64) float64 {
return left * right
}

func slash(left, right float64) (float64, error) {
if right == 0 {
return 0, fmt.Errorf("right is zero")
}
res := left / right
return res, nil
func replaceDollarSigns(s string) string {
re := regexp.MustCompile(`\$([A-Z])\.`)
return re.ReplaceAllString(s, "${1}_")
}

+ 427
- 0
pkg/parser/calc_test.go View File

@@ -0,0 +1,427 @@
package parser

import (
"testing"
)

func TestMathCalc(t *testing.T) {
tests := []struct {
name string
expr string
data map[string]float64
expected float64
wantErr bool
}{
{
name: "Add and Subtract",
expr: "一个 + $.B - $.C",
data: map[string]float64{"一个": 1, "$.B": 2, "$.C": 3},
expected: 0,
wantErr: false,
},
{
name: "Multiply and Divide",
expr: "($A.err_count >0&& $A.err_count <=3)||($B.err_count>0 && $B.err_count <=5)",
data: map[string]float64{"$A.err_count": 4, "$B.err_count": 2},
expected: 1,
wantErr: false,
},
{
name: "Subtract and Add",
expr: "$.C - $.D + $.A",
data: map[string]float64{"$.A": 5, "$.C": 3, "$.D": 2},
expected: 6,
wantErr: false,
},
{
name: "Divide and Multiply",
expr: "$.B / $.C * $.D",
data: map[string]float64{"$.B": 6, "$.C": 2, "$.D": 3},
expected: 9,
wantErr: false,
},
{
name: "Divide and Multiply",
expr: "$.B / $.C * $.D",
data: map[string]float64{"$.B": 6, "$.C": 2, "$.D": 3},
expected: 9,
wantErr: false,
},
{
name: "Multiply and Add",
expr: "$.A * $.B + $.C",
data: map[string]float64{"$.A": 2, "$.B": 3, "$.C": 4},
expected: 10,
wantErr: false,
},
{
name: "Subtract and Divide",
expr: "$.D - $.A / $.B",
data: map[string]float64{"$.D": 10, "$.A": 4, "$.B": 2},
expected: 8,
wantErr: false,
},
{
name: "Add, Subtract and Subtract",
expr: "$.C + $.D - $.A",
data: map[string]float64{"$.C": 3, "$.D": 4, "$.A": 5},
expected: 2,
wantErr: false,
},
{
name: "Multiply and Subtract",
expr: "$.B * $.A - $.D",
data: map[string]float64{"$.B": 2, "$.A": 3, "$.D": 4},
expected: 2,
wantErr: false,
},
{
name: "Divide and Add",
expr: "$.A / $.B + $.C",
data: map[string]float64{"$.A": 4, "$.B": 2, "$.C": 3},
expected: 5,
wantErr: false,
},
{
name: "Add and Multiply",
expr: "$.D + $.A * $.B",
data: map[string]float64{"$.D": 1, "$.A": 2, "$.B": 3},
expected: 7,
wantErr: false,
},
{
name: "Divide and Add with Parentheses",
expr: "($A / $B) + ($C * $D)",
data: map[string]float64{"$A": 4, "$B": 2, "$C": 1, "$D": 3},
expected: 5.0,
wantErr: false,
},
{
name: "Divide with Parentheses",
expr: "($.A - $.B) / ($.C + $.D)",
data: map[string]float64{"$.A": 6, "$.B": 2, "$.C": 3, "$.D": 1},
expected: 1.0,
wantErr: false,
},
{
name: "Add and Multiply with Parentheses",
expr: "($.A + $.B) * ($.C - $.D)",
data: map[string]float64{"$.A": 8, "$.B": 2, "$.C": 4, "$.D": 2},
expected: 20,
wantErr: false,
},
{
name: "Divide and Multiply with Parentheses",
expr: "($.A * $.B) / ($.C - $.D)",
data: map[string]float64{"$.A": 8, "$.B": 2, "$.C": 4, "$.D": 2},
expected: 8,
wantErr: false,
},
{
name: "Add and Divide with Parentheses",
expr: "$.A + ($.B * $.C) / $.D",
data: map[string]float64{"$.A": 1, "$.B": 2, "$.C": 3, "$.D": 4},
expected: 2.5,
wantErr: false,
},
{
name: "Subtract and Multiply with Parentheses",
expr: "($.A + $.B) - ($.C * $.D)",
data: map[string]float64{"$.A": 5, "$.B": 2, "$.C": 3, "$.D": 1},
expected: 4,
wantErr: false,
},
{
name: "Multiply and Divide with Parentheses",
expr: "$.A / ($.B - $.C) * $.D",
data: map[string]float64{"$.A": 4, "$.B": 3, "$.C": 2, "$.D": 5},
expected: 20.0,
wantErr: false,
},
{
name: "Multiply and Divide with Parentheses 2",
expr: "($.A - $.B) * ($.C / $.D)",
data: map[string]float64{"$.A": 3, "$.B": 1, "$.C": 2, "$.D": 4},
expected: 1.0,
wantErr: false,
},

{
name: "Complex expression",
expr: "$.A/$.B*$.D",
data: map[string]float64{"$.A": 1, "$.B": 2, "$.C": 3, "$.D": 4},
expected: 2,
wantErr: false,
},
{
name: "Complex expression",
expr: "$.A/$.B*$.C",
data: map[string]float64{"$.A": 2, "$.B": 2, "$.C": 2},
expected: 2,
wantErr: false,
},
{
name: "Complex expression",
expr: "$.A/($.B*$.C)",
data: map[string]float64{"$.A": 2, "$.B": 2, "$.C": 2},
expected: 0.5,
wantErr: false,
},
{
name: "Addition",
expr: "$.A + $.B",
data: map[string]float64{"$.A": 2, "$.B": 3},
expected: 5,
wantErr: false,
},
{
name: "Subtraction",
expr: "$.A - $.B",
data: map[string]float64{"$.A": 5, "$.B": 3},
expected: 2,
wantErr: false,
},
{
name: "Multiplication",
expr: "$.A * $.B",
data: map[string]float64{"$.A": 4, "$.B": 3},
expected: 12,
wantErr: false,
},
{
name: "Division",
expr: "$.A / $.B",
data: map[string]float64{"$.A": 10, "$.B": 2},
expected: 5,
wantErr: false,
},
{
name: "Mixed operations",
expr: "($.A + $.B) * ($.C - $.D)",
data: map[string]float64{"$.A": 1, "$.B": 2, "$.C": 5, "$.D": 3},
expected: 6, // Corrected from 9 to 6
wantErr: false,
},
{
name: "Division by zero",
expr: "( $D/$A >= 0.1 || $D/$B <= 0.5 ) && $C >= 1000",
data: map[string]float64{"$A": 428382, "$B": 250218, "$C": 305578, "$D": 325028},
expected: 1,
wantErr: true,
},
{
name: "Parentheses",
expr: "($.A + $.B) / ($.C - $.D)",
data: map[string]float64{"$.A": 6, "$.B": 4, "$.C": 10, "$.D": 2},
expected: 1.25, // Corrected from 2.5 to 1.25
wantErr: false,
},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
// Run the MathCalc function
result, err := MathCalc(tc.expr, tc.data)

// Check for expected errors
if tc.wantErr {
if err == nil {
t.Errorf("Expected an error for expr '%s', but got none:%v", tc.expr, result)
}
return
}

// If an error is not expected, but occurs, fail the test
if err != nil {
t.Fatalf("Unexpected error for expr '%s' data:%v err:%v", tc.expr, tc.data, err)
}

// Compare the expected result with the actual result
if result != tc.expected {
t.Errorf("Expected result for expr '%s' to be %v, got %v", tc.expr, tc.expected, result)
}
})
}
}

func TestCalc(t *testing.T) {
tests := []struct {
name string
expr string
data map[string]float64
expected bool
}{
{
name: "Greater than - true",
expr: "$.A > $.B",
data: map[string]float64{"$.A": 5, "$.B": 3},
expected: true,
},
{
name: "Multiply and Subtract with Parentheses",
expr: "$A.yesterday_rate > 0.1 && $A.last_week_rate>0.1 or ($A.今天 >300 || $A.昨天>300 || $A.上周今天 > 300)",
data: map[string]float64{"$A.yesterday_rate": 0.1, "$A.last_week_rate": 2, "$A.今天": 200.4, "$A.昨天": 200.4, "$A.上周今天": 200.4},
expected: false,
},
{
name: "Count Greater Than Zero with Code",
expr: "$A.count > 0",
data: map[string]float64{"$A.count": 197, "$A.code": 30000},
expected: true,
},
{
name: "Today, Yesterday, and Lastweek Rate Comparison",
expr: "$A.todayRate<0.3 && $A.yesterdayRate<0.3 && $A.lastweekRate<0.3",
data: map[string]float64{"$A.todayRate": 1.1, "$A.yesterdayRate": 0.8, "$A.lastweekRate": 1.2},
expected: false,
},
{
name: "Today, Yesterday, and Lastweek Rate Low Threshold",
expr: "$A.todayRate<0.1 && $A.yesterdayRate<0.1 && $A.lastweekRate<0.1",
data: map[string]float64{"$A.todayRate": 0.9, "$A.yesterdayRate": 0.8, "$A.lastweekRate": 0.9},
expected: false,
},
{
name: "Agent Specific Today, Yesterday, and Lastweek Rate Comparison",
expr: "$A.agent == 11 && $A.todayRate<0.3 && $A.yesterdayRate<0.3 && $A.lastweekRate<0.3",
data: map[string]float64{"$A.agent": 11, "$A.todayRate": 0.9, "$A.yesterdayRate": 0.9, "$A.lastweekRate": 1},
expected: false,
},
{
name: "Today, Yesterday, and Lastweek Rate Below 0.1 - Case 1",
expr: "$A<0.1 && $A.yesterdayRate<0.1 && $A.lastweekRate<0.1",
data: map[string]float64{"$A": 0.8, "$A.yesterdayRate": 0.9, "$A.lastweekRate": 0.9},
expected: false,
},
{
name: "Today, Yesterday, and Lastweek Rate Below 0.1 - Case 2",
expr: "$A.today_rate<0.1 && $A.yesterday_rate<0.1 && $A.lastweek_rate<0.1",
data: map[string]float64{"$A.today_rate": 0.9, "$A.yesterday_rate": 0.9, "$A.lastweek_rate": 0.9},
expected: false,
},
{
name: "Today, Yesterday, and Lastweek Rate Below 0.1 - Case 3",
expr: "$B.today_rate<0.1 && $A.yesterday_rate<0.1 && $A.lastweek_rate<0.1",
data: map[string]float64{"$B.today_rate": 0.5, "$A.yesterday_rate": 0.9, "$A.lastweek_rate": 0.8},
expected: false,
},
{
name: "Yesterday and Byesterday Rates Logical Conditions - Case 1",
expr: "($A.yesterday_rate > 2 && $A.byesterday_rate > 2) or ($A.yesterday_rate <= 0.7 && $A.byesterday_rate <= 0.7)",
data: map[string]float64{"$A.yesterday_rate": 3, "$A.byesterday_rate": 3},
expected: true,
},
{
name: "Yesterday and Byesterday Rates Higher Thresholds - Case 1",
expr: "($A.yesterday_rate > 1.5 && $A.byesterday_rate > 1.5) or ($A.yesterday_rate <= 0.8 && $A.byesterday_rate <= 0.8)",
data: map[string]float64{"$A.yesterday_rate": 1.08, "$A.byesterday_rate": 1.02},
expected: false,
},
{
name: "Greater than - false",
expr: "($A.yesterday_rate > 1.0 && $A.byesterday_rate > 1.0 ) or ($A.yesterday_rate <= 0.9 && $A.byesterday_rate <= 0.9)",
data: map[string]float64{"$A.byesterday_rate": 0.33, "$A.yesterday_rate": 2},
expected: false,
},
{
name: "Less than - true",
expr: "$A.count > 100 or $A.count2 > -3",
data: map[string]float64{"$A.count": 5, "$A.count2": -1, "$.D": 2},
expected: true,
},
{
name: "Less than - false",
expr: "$.A < $.B/$.B*4",
data: map[string]float64{"$.A": 5, "$.B": 3},
expected: false,
},
{
name: "Greater than or equal - true",
expr: "$.A >= $.B",
data: map[string]float64{"$.A": 3, "$.B": 3},
expected: true,
},
{
name: "Less than or equal - true",
expr: "$.A <= $.B",
data: map[string]float64{"$.A": 2, "$.B": 2},
expected: true,
},
{
name: "Not equal - true",
expr: "$.A != $.B",
data: map[string]float64{"$.A": 3, "$.B": 2},
expected: true,
},
{
name: "Not equal - false",
expr: "$.A != $.B",
data: map[string]float64{"$.A": 2, "$.B": 2},
expected: false,
},
{
name: "Addition resulting in true",
expr: "$.A + $.B > $.C",
data: map[string]float64{"$.A": 3, "$.B": 2, "$.C": 4},
expected: true,
},
{
name: "Subtraction resulting in false",
expr: "$.A - $.B < $.C",
data: map[string]float64{"$.A": 1, "$.B": 3, "$.C": 1},
expected: true,
},
{
name: "Multiplication resulting in true",
expr: "$.A * $.B > $.C",
data: map[string]float64{"$.A": 2, "$.B": 3, "$.C": 5},
expected: true,
},
{
name: "Division resulting in false",
expr: "$.A / $.B*$.C < $.C",
data: map[string]float64{"$.A": 4, "$.B": 2, "$.C": 2},
expected: false,
},
{
name: "Addition with parentheses resulting in true",
expr: "($.A + $.B) > $.C && $.A >0",
data: map[string]float64{"$.A": 1, "$.B": 4, "$.C": 4},
expected: true,
},
{
name: "Addition with parentheses resulting in true",
expr: "($.A + $.B) > $.C || $.A < 0",
data: map[string]float64{"$.A": 1, "$.B": 4, "$.C": 4},
expected: true,
},
{
name: "Complex expression with parentheses resulting in false",
expr: "($.A + $.B) * $.C < $.D",
data: map[string]float64{"$.A": 1, "$.B": 2, "$.C": 3, "$.D": 10},
expected: true,
},
{
name: "Nested parentheses resulting in true",
expr: "($.A + ($.B - $.C)) * $.D > $.E",
data: map[string]float64{"$.A": 2, "$.B": 5, "$.C": 2, "$.D": 2, "$.E": 8},
expected: true,
},
{
name: "Division with parentheses resulting in false",
expr: " ( true || false ) && true",
data: map[string]float64{"$A": 673601, "$A.": 673601, "$B": 250218, "$C": 456513, "$C.": 456513, "$D": 456513, "$D.": 456513},
expected: true,
},
// $A:673601.5 $A.:673601.5 $B:361520 $B.:361520 $C:456513 $C.:456513 $D:422634 $D.:422634]
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
result := Calc(tc.expr, tc.data)
if result != tc.expected {
t.Errorf("Expected result for expr '%s' to be %v, got %v", tc.expr, tc.expected, result)
}
})
}
}

+ 0
- 423
pkg/parser/lexer.go View File

@@ -1,423 +0,0 @@
package parser

import (
"errors"
"fmt"
)

var (
ErrorLexEOF = errors.New("lex source EOF")
)

type TokenType int

const (
EOF TokenType = iota
AND
OR
EXP

Identifier

GT
GE
LT
LE
EQ
NE

Plus
Minus
Star
Slash
LeftParen
RightParen

IntLiteral
UintLiteral
FloatLiteral
)

func (t TokenType) String() string {
switch t {
case Identifier:
return "Identifier"
case GT:
return "GT"
case GE:
return "GE"
case LT:
return "LT"
case LE:
return "LE"
case EQ:
return "EQ"
case NE:
return "NE"
case Plus:
return "Plus"
case Minus:
return "Minus"
case Star:
return "Star"
case Slash:
return "Slash"
case LeftParen:
return "leftParen"
case RightParen:
return "rightParen"

case AND:
return "AND"
case OR:
return "OR"
case EXP:
return "expr"

case IntLiteral:
return "IntLiteral"
case FloatLiteral:
return "FloatLiteral"
default:
return "Unknown TokenType"
}
}

type Token struct {
typ TokenType
buf []rune
}

func (t *Token) push(r rune) {
t.buf = append(t.buf, r)
}

func (t *Token) String() string {
return fmt.Sprintf("token: {%v: '%s'}", t.typ, string(t.buf))
}

type Lexer struct {
buf []rune
idx int // always point to the next rune
}

func newLexer(buf []rune) *Lexer {
return &Lexer{
buf: buf,
idx: 0,
}
}

func isAlpha(ch rune) bool {
return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '.'
}

func isPrefix(ch rune) bool {
return ch == '$'
}

func isWhitespace(ch rune) bool {
return ch == ' ' || ch == '\t' || ch == '\r'
}

func isDigit(ch rune) bool {
return ch >= '0' && ch <= '9' || ch == '.'
}

func (l *Lexer) lex() ([]*Token, error) {
toks := make([]*Token, 0)
for {
tok, err := l.lexToken()
if err != nil {
if err == ErrorLexEOF {
break
}
return nil, err
}
toks = append(toks, tok)
}
return toks, nil
}

func (l *Lexer) lexToken() (*Token, error) {
l.skipWhitespace()
ch, err := l.next()
if err != nil {
return nil, err
}
switch {
case isPrefix(ch):
return l.lexIdentifier(ch), nil
case ch == '&':
return l.lexAnd(ch), nil
case ch == '|':
return l.lexOR(ch), nil
case ch == '>':
return l.lexGT(ch), nil
case ch == '<':
return l.lexLT(ch), nil
case ch == '!':
return l.lexNE(ch), nil
case ch == '=':
return l.lexEQ(ch), nil
case ch == '+':
return l.lexPlus(ch), nil
case ch == '-':
return l.lexMinus(ch), nil
case ch == '*':
return l.lexStar(ch), nil
case ch == '/':
return l.lexSlash(ch), nil
case ch == '(':
return l.lexLeftParen(ch), nil
case ch == ')':
return l.lexRightParen(ch), nil
case isDigit(ch):
return l.lexDigital(ch), nil
default:
return nil, errors.New("not supported rune: " + string(ch))
}
}

func (l *Lexer) lexIdentifier(ch rune) *Token {
tok := &Token{
typ: Identifier,
buf: []rune{ch},
}
for {
ch, err := l.peek()
if err != nil {
return tok
}
if isAlpha(ch) || isDigit(ch) {
l.mustNext()
tok.push(ch)
continue
}
return tok
}
}

func (l *Lexer) lexGT(ch rune) *Token {
tok := &Token{
typ: GT,
buf: []rune{ch},
}
ch, err := l.peek()
if err != nil {
return tok
}
if ch == '=' {
tok.typ = GE
tok.buf = append(tok.buf, ch)
l.mustNext()
}
return tok
}

func (l *Lexer) lexLT(ch rune) *Token {
tok := &Token{
typ: LT,
buf: []rune{ch},
}
ch, err := l.peek()
if err != nil {
return tok
}
if ch == '=' {
tok.typ = LE
tok.buf = append(tok.buf, ch)
l.mustNext()
}
return tok
}

func (l *Lexer) lexEQ(ch rune) *Token {
tok := &Token{
typ: 0,
buf: []rune{ch},
}

ch, err := l.peek()
if err != nil {
return tok
}

if ch == '=' {
tok.typ = EQ
tok.buf = append(tok.buf, ch)
l.mustNext()
}
// 如果不是 == 处理报错
return tok
}

func (l *Lexer) lexNE(ch rune) *Token {
tok := &Token{
typ: 0,
buf: []rune{ch},
}

ch, err := l.peek()
if err != nil {
return tok
}

if ch == '=' {
tok.typ = NE
tok.buf = append(tok.buf, ch)
l.mustNext()
}
// 如果不是 == 处理报错
return tok
}

func (l *Lexer) lexPlus(ch rune) *Token {
tok := &Token{
typ: Plus,
buf: []rune{ch},
}
return tok
}

func (l *Lexer) lexMinus(ch rune) *Token {
tok := &Token{
typ: Minus,
buf: []rune{ch},
}
return tok
}

func (l *Lexer) lexStar(ch rune) *Token {
tok := &Token{
typ: Star,
buf: []rune{ch},
}
return tok
}

func (l *Lexer) lexSlash(ch rune) *Token {
tok := &Token{
typ: Slash,
buf: []rune{ch},
}
return tok
}

func (l *Lexer) lexAnd(ch rune) *Token {
tok := &Token{
typ: 0,
buf: []rune{ch},
}

ch, err := l.peek()
if err != nil {
return tok
}

if ch == '&' {
tok.typ = AND
tok.buf = append(tok.buf, ch)
l.mustNext()
}

return tok
}

func (l *Lexer) lexOR(ch rune) *Token {
tok := &Token{
typ: 0,
buf: []rune{ch},
}

ch, err := l.peek()
if err != nil {
return tok
}

if ch == '|' {
tok.typ = OR
tok.buf = append(tok.buf, ch)
l.mustNext()
}

return tok
}

func (l *Lexer) lexLeftParen(ch rune) *Token {
tok := &Token{
typ: LeftParen,
buf: []rune{ch},
}
return tok
}

func (l *Lexer) lexRightParen(ch rune) *Token {
tok := &Token{
typ: RightParen,
buf: []rune{ch},
}
return tok
}

func (l *Lexer) lexDigital(ch rune) *Token {
tok := &Token{
typ: IntLiteral,
buf: []rune{ch},
}
for {
ch, err := l.peek()
if err != nil {
return tok
}
if isDigit(ch) {
l.mustNext()
tok.push(ch)
continue
}
return tok
}
}

func (l *Lexer) skipWhitespace() bool {
found := false
for {
ch, err := l.peek()
if err != nil {
return false
}
if isWhitespace(ch) {
l.mustNext()
found = true
continue
}
if ch == '\n' {
l.mustNext()
found = true
continue
}
break
}
return found
}

func (l *Lexer) next() (rune, error) {
ch, err := l.peek()
if err != nil {
return 0, err
}
l.idx++
return ch, nil
}

func (l *Lexer) mustNext() rune {
l.idx++
return l.buf[l.idx-1]
}

func (l *Lexer) peek() (rune, error) {
if l.idx >= len(l.buf) {
return 0, ErrorLexEOF
}
return l.buf[l.idx], nil
}

+ 0
- 547
pkg/parser/paser.go View File

@@ -1,547 +0,0 @@
package parser

import (
"fmt"

"github.com/toolkits/pkg/logger"
)

type Parser struct {
buf []rune
tokens []*Token
idx int
err error
isEOF bool
stats []Node
}

/*
exp -> or | or = exp
or -> and | or || and
and -> equal | and && equal
equal -> rel | equal == rel | equal != rel
rel -> add | rel > add | rel < add | rel >= add | rel <= add
add -> mul | add + mul | add - mul
mul -> pri | mul * pri | mul / pri
pri -> Id | Literal | (exp)
*/

func NewParser(buf []rune) *Parser {
return &Parser{
buf: buf,
}
}

func (p *Parser) Parse() error {
lexer := newLexer(p.buf)
tokens, err := lexer.lex()
if err != nil {
return err
}
p.tokens = tokens
p.stats = make([]Node, 0)

for {
node := p.parseStat()
if node == nil {
return nil
}
if p.hasError() {
return nil
}
nodes, ok := node.([]Node)
if ok {
p.stats = append(p.stats, nodes...)
} else {
p.stats = append(p.stats, node)
}
}
}

func (p *Parser) PrintAST() {
for _, node := range p.Stats() {
formatNode(node, "", 0)
}
}

func (p *Parser) Stats() []Node {
return p.stats
}

func (p *Parser) Err() error {
return p.err
}

func (p *Parser) parseStat() Node {
if p.hasError() {
return nil
}
tok, valid := p.peek()
if !valid {
return nil
}
switch tok.typ {
case Identifier, IntLiteral:
p.mustNext()
opTok, valid := p.peek()
if !valid {
p.back()
return p.parseExpr()
}
switch opTok.typ {
case Plus, Minus, Star, Slash, GE, GT, LE, LT, AND, OR, LeftParen:
p.back()
return p.parseExpr()
default:
p.reportError("invalid token: %v", tok)
return nil
}
default:
p.reportError("invalid token: %v", tok)
return nil
}
}

func (p *Parser) mustNext() *Token {
p.idx++
return p.tokens[p.idx-1]
}

func (p *Parser) back() {
p.idx--
}

func (p *Parser) hasError() bool {
if p.err != nil {
logger.Errorf("parse err", p.err)
}
return p.err != nil
}

func (p *Parser) peek() (*Token, bool) {
if p.idx >= len(p.tokens) {
p.isEOF = true
return nil, false
}
return p.tokens[p.idx], true
}

// exp -> or | or = exp
func (p *Parser) parseExpr() Node {
if p.hasError() {
return nil
}

leftNode := p.parseOr()
var binNode *BinaryNode
firstTime := true

for {
opTok, valid := p.peek()
if !valid {
if firstTime {
return leftNode
}
break
}

switch opTok.typ {
case AND:
p.mustNext()
default:
if firstTime {
return leftNode
}
return binNode
}

if firstTime {
binNode = &BinaryNode{
Type: opTok.typ,
Left: leftNode,
Right: p.parseExpr(),
}
firstTime = false
} else {
binNode = &BinaryNode{
Type: opTok.typ,
Left: binNode,
Right: p.parseExpr(),
}
}
if p.hasError() {
return nil
}
}

return binNode
}

// or -> and | or || and
func (p *Parser) parseOr() Node {
if p.hasError() {
return nil
}

leftNode := p.parseAnd()
var binNode *BinaryNode
firstTime := true

for {
opTok, valid := p.peek()
if !valid {
if firstTime {
return leftNode
}
break
}

switch opTok.typ {
case AND:
p.mustNext()
default:
if firstTime {
return leftNode
}
return binNode
}

if firstTime {
binNode = &BinaryNode{
Type: opTok.typ,
Left: leftNode,
Right: p.parseOr(),
}
firstTime = false
} else {
binNode = &BinaryNode{
Type: opTok.typ,
Left: binNode,
Right: p.parseOr(),
}
}
if p.hasError() {
return nil
}
}

return binNode
}

// and -> equal | and && equal
func (p *Parser) parseAnd() Node {
if p.hasError() {
return nil
}

leftNode := p.parseEqual()
var binNode *BinaryNode
firstTime := true

for {
opTok, valid := p.peek()
if !valid {
if firstTime {
return leftNode
}
break
}

switch opTok.typ {
case AND:
p.mustNext()
default:
if firstTime {
return leftNode
}
return binNode
}

if firstTime {
binNode = &BinaryNode{
Type: opTok.typ,
Left: leftNode,
Right: p.parseAnd(),
}
firstTime = false
} else {
binNode = &BinaryNode{
Type: opTok.typ,
Left: binNode,
Right: p.parseAnd(),
}
}
if p.hasError() {
return nil
}
}

return binNode
}

// equal -> rel | equal == rel | equal != rel
func (p *Parser) parseEqual() Node {
if p.hasError() {
return nil
}

leftNode := p.parseRel()
var binNode *BinaryNode
firstTime := true

for {
opTok, valid := p.peek()
if !valid {
if firstTime {
return leftNode
}
break
}

switch opTok.typ {
case EQ, NE:
p.mustNext()
default:
if firstTime {
return leftNode
}
return binNode
}

if firstTime {
binNode = &BinaryNode{
Type: opTok.typ,
Left: leftNode,
Right: p.parseEqual(),
}
firstTime = false
} else {
binNode = &BinaryNode{
Type: opTok.typ,
Left: binNode,
Right: p.parseEqual(),
}
}
if p.hasError() {
return nil
}
}

return binNode
}

// rel -> add | rel > add | rel < add | rel >= add | rel <= add
func (p *Parser) parseRel() Node {
if p.hasError() {
return nil
}

leftNode := p.parseAdd()
var binNode *BinaryNode
firstTime := true

for {
opTok, valid := p.peek()
if !valid {
if firstTime {
return leftNode
}
break
}

switch opTok.typ {
case GE, GT, LE, LT:
p.mustNext()
default:
if firstTime {
return leftNode
}
return binNode
}

if firstTime {
binNode = &BinaryNode{
Type: opTok.typ,
Left: leftNode,
Right: p.parseRel(),
}
firstTime = false
} else {
binNode = &BinaryNode{
Type: opTok.typ,
Left: binNode,
Right: p.parseRel(),
}
}
if p.hasError() {
return nil
}
}

return binNode
}

// add -> mul ( + mul)*
func (p *Parser) parseAdd() Node {
if p.hasError() {
return nil
}

leftNode := p.parseMul()
var binNode *BinaryNode
firstTime := true

for {
opTok, valid := p.peek()
if !valid {
if firstTime {
return leftNode
}
break
}

switch opTok.typ {
case Plus, Minus:
p.mustNext()
default:
if firstTime {
return leftNode
}
return binNode
}

if firstTime {
binNode = &BinaryNode{
Type: opTok.typ,
Left: leftNode,
Right: p.parseAdd(),
}
firstTime = false
} else {
binNode = &BinaryNode{
Type: opTok.typ,
Left: binNode,
Right: p.parseAdd(),
}
}
if p.hasError() {
return nil
}
}

return binNode
}

// mul -> pri | mul * pri | mul / pri
func (p *Parser) parseMul() Node {
if p.hasError() {
return nil
}

leftNode := p.parsePri()
var binNode *BinaryNode
firstTime := true
for {
opTok, valid := p.peek()
if !valid {
if firstTime {
return leftNode
}
break
}

switch opTok.typ {
case Star, Slash:
p.mustNext()
default:
if firstTime {
return leftNode
}
return binNode
}

if firstTime {
binNode = &BinaryNode{
Type: opTok.typ,
Left: leftNode,
Right: p.parseMul(),
}
firstTime = false
} else {
binNode = &BinaryNode{
Type: opTok.typ,
Left: binNode,
Right: p.parseMul(),
}
}
if p.hasError() {
return nil
}
}
return binNode
}

// pri -> Id | Literal | (exp)
func (p *Parser) parsePri() Node {
if p.hasError() {
return nil
}

tok, valid := p.peek()
if !valid {
p.reportError("unexpected EOF")
return nil
}

if tok.typ == IntLiteral {
p.mustNext()
return &NumberNode{
Type: tok.typ,
Lit: string(tok.buf),
}
}

if tok.typ == Identifier {
p.mustNext()
return &IdentifierNode{
Lit: string(tok.buf),
}
}

if tok.typ == LeftParen {
p.mustNext()
node := p.parseExpr()
if node != nil {
tk, valid := p.peek()
if !valid {
p.reportError("unexpected EOF")
return nil
}

if tk.typ == RightParen {
p.mustNext()
} else {
p.reportError("expecting right parenthesis")
}
} else {
p.reportError("expecting an additive expression inside parenthesis")
}
}

p.reportError("expect int Identifier but met %v", tok)
return nil
}

func (p *Parser) reportError(args ...interface{}) {
if len(args) >= 1 {
i := args[0]
switch v := i.(type) {
case string:
p.err = fmt.Errorf(v, args[1:]...)
case error:
p.err = v
default:
panic(v)
}
}
}

Write Preview
Loading…
Cancel
Save