├── .travis.yml ├── LICENSE ├── README.md ├── core ├── assignment.go ├── assignment_ext.go ├── assignment_ext_test.go ├── assignment_test.go ├── cache.go ├── condition.go ├── condition_test.go ├── context.go ├── context_test.go ├── executor.go ├── executor_test.go ├── logger.go ├── operation.go ├── operation_test.go ├── trace.go ├── utils.go ├── utils_test.go ├── variable.go ├── variable_test.go ├── variables.go └── variables_test.go ├── ext ├── location │ ├── variables.go │ └── variables_test.go └── request │ ├── doc.go │ ├── variables.go │ └── variables_test.go ├── filter.go ├── filter_example_test.go ├── filter_test.go ├── go.mod ├── go.sum ├── utils.go └── utils_test.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.x 5 | - 1.15.x 6 | 7 | os: 8 | - linux 9 | - osx 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 techxmind 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # filter 2 | Simple and extensible data filtering framework. You can simply create filters from json/yaml data, filter definition can immbeded in business data. e.g. 3 | 4 | *DATA* 5 | ``` 6 | { 7 | "banner" : { 8 | "type" : "image", 9 | "src" : "https://example.com/working-hard.png", 10 | "link" : "https://example.com/activity.html" 11 | }, 12 | //... other config 13 | 14 | "filter" : [ 15 | ["time", "between", "18:00,23:00"], 16 | ["ctx.user.group", "=", "programer"], 17 | ["banner", "+", { 18 | "src" : "https://example.com/chat-with-beaty.png", 19 | "link" : "https://chat.com" 20 | }] 21 | ] 22 | } 23 | ``` 24 | 25 | *CODE* 26 | ``` 27 | import ( 28 | "encoding/json" 29 | 30 | "github.com/techxmind/filter" 31 | "github.com/techxmind/filter/core" 32 | ) 33 | 34 | func main() { 35 | var dataStr = `...` // data above 36 | 37 | var data map[string]interface{} 38 | 39 | json.Unmarshal([]byte(dataStr), &data) 40 | 41 | // your business context 42 | ctx := context.Background() 43 | 44 | filterCtx := core.WithContext(ctx) 45 | 46 | // you can also set context value in your business ctx with context.WithValue("group", ...) 47 | filterCtx.Set("user", map[string]interface{}{"group": "programer"}) 48 | 49 | f, _ := filter.New(data["filter"].([]interface{})) 50 | f.Run(filterCtx, data) 51 | 52 | // if current time is between 18:00 ~ 23:00, output: 53 | // map[link:https://chat.com src:https://example.com/chat-with-beaty.png type:image] 54 | fmt.Println(data["banner"]) 55 | } 56 | ``` 57 | 58 | ## Variables 59 | 60 | Register your custom variable: 61 | 62 | ``` 63 | import ( 64 | "github.com/techxmind/filter/core" 65 | ) 66 | 67 | // Register variable "username" that fetch value from context 68 | core.GetVariableFactory().Register( 69 | core.SingletonVariableCreator(core.NewSimpleVariable("username", core.Cacheable, &ContextValue{USER_AGENT})), 70 | "username" 71 | ) 72 | ``` 73 | 74 | Check `ext` folder to see more examples. 75 | 76 | ## Operations 77 | 78 | Register your custom operation: 79 | 80 | ``` 81 | import ( 82 | "errors" 83 | "strings" 84 | 85 | "github.com/techxmind/filter/core" 86 | "github.com/techxmind/go-utils/itype" 87 | ) 88 | 89 | // Register operation "contains" to check if variable contains specified substring 90 | // ["url", "contains", "something"] 91 | core.GetOperationFactory().Register(&ContainsOperation{}, "contains") 92 | 93 | type ContainsOperation struct {} 94 | 95 | func (o *ContainsOperation) String() string { 96 | return "contains" 97 | } 98 | 99 | func (o *ContainsOperation) Run(ctx *core.Context, variable core.Variable, value interface{}) bool { 100 | v := core.GetVariableValue(ctx, variable) 101 | 102 | return strings.Contains(itype.String(v), itype.String(value)) 103 | } 104 | 105 | func (o *ContainsOperation) PrepareValue(v interface{}) (interface{}, error) { 106 | if str, ok := v.(string); ok { 107 | return v, nil 108 | } 109 | 110 | return nil, errors.New("[contains] operation require value of type string") 111 | } 112 | ``` 113 | 114 | ## Assignments 115 | ... 116 | 117 | ## Trace 118 | 119 | Trace each step of filter's execution. 120 | 121 | ``` 122 | import ( 123 | "context" 124 | "os" 125 | 126 | "github.com/techxmind/filter/core" 127 | "github.com/techxmind/filter/filter" 128 | ) 129 | 130 | //other code... 131 | 132 | // your business context 133 | ctx := context.Background() 134 | 135 | // Initialize filter context with trace option 136 | // You can use your custom Trace instaed of the default implementtion 137 | filterCtx := core.WithContext(ctx, WithTrace(core.NewTrace(os.Stderr))) 138 | 139 | youfilter.Run(filterCtx, data) 140 | ``` 141 | 142 | -------------------------------------------------------------------------------- /core/assignment.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "errors" 5 | "strconv" 6 | "strings" 7 | 8 | "github.com/techxmind/go-utils/object" 9 | ) 10 | 11 | var _assignmentFactory AssignmentFactory 12 | 13 | func init() { 14 | _assignmentFactory = &stdAssignmentFactory{ 15 | assignments: map[string]Assignment{ 16 | "=": &EqualAssignment{}, 17 | "+": &MergeAssignment{}, 18 | "-": &DeleteAssignment{}, 19 | }, 20 | } 21 | } 22 | 23 | type Assignment interface { 24 | Run(ctx *Context, data interface{}, key string, val interface{}) 25 | PrepareValue(value interface{}) (interface{}, error) 26 | } 27 | 28 | type BaseAssignmentPrepareValue struct{} 29 | 30 | func (self *BaseAssignmentPrepareValue) PrepareValue(value interface{}) (interface{}, error) { 31 | return value, nil 32 | } 33 | 34 | type AssignmentFactory interface { 35 | Get(string) Assignment 36 | Register(Assignment, ...string) 37 | } 38 | 39 | // GetAssignmentFactory return AssignmentFactory for registering new Assigment 40 | // 41 | func GetAssignmentFactory() AssignmentFactory { 42 | return _assignmentFactory 43 | } 44 | 45 | type stdAssignmentFactory struct { 46 | assignments map[string]Assignment 47 | } 48 | 49 | func (self *stdAssignmentFactory) Get(name string) Assignment { 50 | if assignment, ok := self.assignments[name]; ok { 51 | return assignment 52 | } 53 | 54 | return nil 55 | } 56 | 57 | func (self *stdAssignmentFactory) Register(op Assignment, names ...string) { 58 | for _, name := range names { 59 | self.assignments[name] = op 60 | } 61 | } 62 | 63 | //["key", "=", "val"] 64 | type Setter interface { 65 | AssignmentSet(key string, value interface{}) bool 66 | } 67 | 68 | type EqualAssignment struct{ BaseAssignmentPrepareValue } 69 | 70 | func (self *EqualAssignment) Run(_ *Context, data interface{}, key string, value interface{}) { 71 | if v, ok := data.(Setter); ok { 72 | if v.AssignmentSet(key, value) { 73 | return 74 | } 75 | } 76 | 77 | keys := strings.Split(key, ".") 78 | lastKey := keys[len(keys)-1] 79 | 80 | var obj interface{} 81 | if len(keys) > 1 { 82 | obj, _ = object.GetObject(data, strings.Join(keys[:len(keys)-1], "."), true) 83 | } else { 84 | obj = data 85 | } 86 | 87 | if obj == nil { 88 | return 89 | } 90 | 91 | switch v := obj.(type) { 92 | case map[string]interface{}: 93 | if IsScalar(value) { 94 | v[lastKey] = value 95 | } else { 96 | v[lastKey] = Clone(value) 97 | } 98 | case []interface{}: 99 | if index, err := strconv.ParseInt(lastKey, 10, 32); err == nil { 100 | if int(index) >= len(v) { 101 | return 102 | } 103 | if IsScalar(value) { 104 | v[int(index)] = value 105 | } else { 106 | v[int(index)] = Clone(value) 107 | } 108 | } 109 | } 110 | } 111 | 112 | //["key", "+", {}] 113 | type Merger interface { 114 | AssignmentMerge(key string, value interface{}) bool 115 | } 116 | type MergeAssignment struct{} 117 | 118 | func (self *MergeAssignment) Run(_ *Context, data interface{}, key string, value interface{}) { 119 | if v, ok := data.(Merger); ok { 120 | if v.AssignmentMerge(key, value) { 121 | return 122 | } 123 | } 124 | 125 | keys := strings.Split(key, ".") 126 | lastKey := keys[len(keys)-1] 127 | 128 | var obj interface{} 129 | 130 | if len(keys) > 1 { 131 | obj, _ = object.GetObject(data, strings.Join(keys[:len(keys)-1], "."), true) 132 | } else { 133 | obj = data 134 | } 135 | 136 | if obj == nil { 137 | return 138 | } 139 | 140 | switch v := obj.(type) { 141 | case map[string]interface{}: 142 | if robj, ok := v[lastKey]; !ok { 143 | v[lastKey] = value 144 | } else { 145 | if robj2, ok := robj.(map[string]interface{}); ok { 146 | for ikey, ivalue := range value.(map[string]interface{}) { 147 | if IsScalar(ivalue) { 148 | robj2[ikey] = ivalue 149 | } else { 150 | robj2[ikey] = Clone(ivalue) 151 | } 152 | } 153 | } 154 | } 155 | } 156 | } 157 | 158 | func (self *MergeAssignment) PrepareValue(value interface{}) (interface{}, error) { 159 | if value == nil { 160 | return nil, errors.New("assignment[+] value must not be nil") 161 | } 162 | 163 | if _, ok := value.(map[string]interface{}); !ok { 164 | return nil, errors.New("assignment[+] value must be map[string]interface{}") 165 | } 166 | 167 | return value, nil 168 | } 169 | 170 | //["key", "-", ["key"]] 171 | type Deleter interface { 172 | AssignmentDelete(key string, value interface{}) bool 173 | } 174 | type DeleteAssignment struct{} 175 | 176 | func (self *DeleteAssignment) Run(_ *Context, data interface{}, key string, value interface{}) { 177 | // can use '$' or '.' to specify root path 178 | // ["$", "-", "key1,key2.."] 179 | // [".", "-", "key1,key2.."] 180 | key = strings.TrimLeft(strings.TrimLeft(key, "$"), ".") 181 | 182 | if v, ok := data.(Deleter); ok { 183 | if v.AssignmentDelete(key, value) { 184 | return 185 | } 186 | } 187 | 188 | obj, _ := object.GetObject(data, key, false) 189 | 190 | if obj == nil { 191 | return 192 | } 193 | 194 | if v, ok := obj.(map[string]interface{}); ok { 195 | for _, key := range value.([]interface{}) { 196 | if _, ok := v[key.(string)]; ok { 197 | delete(v, key.(string)) 198 | } 199 | } 200 | } 201 | } 202 | 203 | func (self *DeleteAssignment) PrepareValue(value interface{}) (interface{}, error) { 204 | if value == nil { 205 | return nil, errors.New("assignment[-] value must be list") 206 | } 207 | 208 | list := ToArray(value) 209 | 210 | if len(list) == 0 { 211 | return nil, errors.New("assignment[-] value must be list") 212 | } 213 | 214 | for _, item := range list { 215 | if _, ok := item.(string); !ok { 216 | return nil, errors.New("assignment[-] value must be string or []string") 217 | } 218 | } 219 | 220 | return list, nil 221 | } 222 | -------------------------------------------------------------------------------- /core/assignment_ext.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "math" 5 | "math/rand" 6 | 7 | "github.com/pkg/errors" 8 | 9 | "github.com/techxmind/go-utils/itype" 10 | ) 11 | 12 | func init() { 13 | _assignmentFactory.Register(&GroupAssign{}, "=>") 14 | _assignmentFactory.Register(&ProbabilitySet{}, "*=") 15 | } 16 | 17 | // ProbabilitySet set value with specified probability. 18 | // e.g. : 19 | // ["key", "*=", [ [10, "value1"], [10, "value2"], [10, "value3"] ]] 20 | // value1,value2,value3 has the same weight 10, each of them has a probability 10/(10+10+10) = 1/3 to been chosen. 21 | // 22 | type ProbabilitySet struct { 23 | } 24 | 25 | type probabilityItem struct { 26 | linePoint int64 27 | value interface{} 28 | } 29 | 30 | func (a *ProbabilitySet) PrepareValue(value interface{}) (val interface{}, err error) { 31 | if !IsArray(value) { 32 | err = errors.Errorf("assignment[*=] value must be array.") 33 | return 34 | } 35 | 36 | var ( 37 | linePoint int64 = 0 38 | setter = _assignmentFactory.Get("=") 39 | items = make([]*probabilityItem, 0) 40 | ) 41 | 42 | for _, item := range ToArray(value) { 43 | if !IsArray(item) { 44 | err = errors.Errorf("assignment[*=] value element must be array [#weight, #value].") 45 | return 46 | } 47 | 48 | pitem := ToArray(item) 49 | 50 | if len(pitem) != 2 { 51 | err = errors.Errorf("assignment[*=] value element must be array [#weight, #value].") 52 | return 53 | } 54 | 55 | if itype.GetType(pitem[0]) != itype.NUMBER { 56 | err = errors.Errorf("assignment[*=] value element 1st item(weight) must be number and gt 0") 57 | } 58 | 59 | weight := int64(math.Round(itype.Float(pitem[0]) * 1000)) 60 | if weight < 0 { 61 | err = errors.Errorf("assignment[*=] value element 1st item(weight) must be number and gt 0") 62 | return 63 | } 64 | 65 | linePoint += weight 66 | 67 | pvalue, err := setter.PrepareValue(pitem[1]) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | items = append(items, &probabilityItem{ 73 | linePoint: linePoint, 74 | value: pvalue, 75 | }) 76 | } 77 | 78 | return items, nil 79 | } 80 | 81 | func (a *ProbabilitySet) Run(ctx *Context, data interface{}, key string, value interface{}) { 82 | items, ok := value.([]*probabilityItem) 83 | if !ok { 84 | return 85 | } 86 | n := len(items) 87 | if n == 0 { 88 | return 89 | } 90 | max := items[n-1].linePoint 91 | choose := rand.Intn(int(max)) + 1 92 | for _, item := range items { 93 | if choose <= int(item.linePoint) { 94 | _assignmentFactory.Get("=").Run(ctx, data, key, item.value) 95 | break 96 | } 97 | } 98 | } 99 | 100 | type GroupAssign struct{} 101 | 102 | func (a *GroupAssign) PrepareValue(value interface{}) (val interface{}, err error) { 103 | if !IsArray(value) { 104 | return nil, errors.New("assignment[=>] value must be array") 105 | } 106 | 107 | executor, err := NewExecutor(ToArray(value)) 108 | if err != nil { 109 | return nil, errors.Wrap(err, "assignment[=>]") 110 | } 111 | 112 | return executor, nil 113 | } 114 | 115 | func (a *GroupAssign) Run(ctx *Context, data interface{}, key string, value interface{}) { 116 | executor, ok := value.(Executor) 117 | 118 | if !ok { 119 | return 120 | } 121 | 122 | executor.Execute(ctx, data) 123 | } 124 | -------------------------------------------------------------------------------- /core/assignment_ext_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/techxmind/go-utils/itype" 10 | "github.com/techxmind/go-utils/object" 11 | ) 12 | 13 | func TestProbabilitySet(t *testing.T) { 14 | ctx := NewContext() 15 | a := _assignmentFactory.Get("*=") 16 | v := []interface{}{ 17 | []interface{}{10, 10}, 18 | []interface{}{30, 30}, 19 | []interface{}{60, 60}, 20 | } 21 | data := make(map[string]interface{}) 22 | hit := make(map[int]int) 23 | val, err := a.PrepareValue(v) 24 | require.NoError(t, err) 25 | for i := 0; i < 10000; i++ { 26 | a.Run(ctx, data, "a.b", val) 27 | bi, _ := object.GetValue(data, "a.b") 28 | b := int(itype.Int(bi)) 29 | assert.Contains(t, []int{10, 30, 60}, b) 30 | hit[b] += 1 31 | } 32 | t.Log("hits:", hit) 33 | assert.Equal(t, 3, len(hit), "hits.size = 3") 34 | assert.True(t, hit[60] > hit[30], "hits.60 > hits.30") 35 | assert.True(t, hit[30] > hit[10], "hits.60 > hits.30") 36 | } 37 | 38 | func TestGroupAssign(t *testing.T) { 39 | ctx := NewContext() 40 | data := make(map[string]interface{}) 41 | a := _assignmentFactory.Get("=>") 42 | 43 | val1, err := a.PrepareValue([]interface{}{"a", "=", "a"}) 44 | require.NoError(t, err) 45 | 46 | val2, err := a.PrepareValue([]interface{}{ 47 | []interface{}{"b", "=", "b"}, 48 | []interface{}{"c", "=", "c"}, 49 | }) 50 | require.NoError(t, err) 51 | 52 | a.Run(ctx, data, "set", val1) 53 | assert.Equal(t, map[string]interface{}{"a": "a"}, data) 54 | 55 | a.Run(ctx, data, "set", val2) 56 | assert.Equal(t, map[string]interface{}{"a": "a", "b": "b", "c": "c"}, data) 57 | } 58 | -------------------------------------------------------------------------------- /core/assignment_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/stretchr/testify/suite" 8 | 9 | "github.com/techxmind/go-utils/object" 10 | ) 11 | 12 | type AssignmentTestSuite struct { 13 | suite.Suite 14 | ctx *Context 15 | } 16 | 17 | type assignTestCase struct { 18 | input []interface{} 19 | expected []interface{} 20 | prepareError bool 21 | } 22 | 23 | func (s *AssignmentTestSuite) SetupTest() { 24 | ctx := NewContext() 25 | data := map[string]interface{}{ 26 | "area": map[string]interface{}{ 27 | "zipcode": 200211, 28 | "city": "shanghai", 29 | }, 30 | "birthday": "1985-11-21", 31 | "height": "178", 32 | "age": 25, 33 | "fav_books": "book1,book2,book3", 34 | "pets": []interface{}{"dog", "cat"}, 35 | } 36 | ctx = WithData(ctx, data) 37 | s.ctx = ctx 38 | } 39 | 40 | func (s *AssignmentTestSuite) TestEqualAssignment() { 41 | tests := []assignTestCase{ 42 | { 43 | input: []interface{}{"area.province", "=", "shanghai"}, 44 | expected: []interface{}{"area", map[string]interface{}{ 45 | "zipcode": 200211, 46 | "city": "shanghai", 47 | "province": "shanghai", 48 | }}, 49 | }, 50 | { 51 | input: []interface{}{"pets.0", "=", "rabbit"}, 52 | expected: []interface{}{"pets", []interface{}{ 53 | "rabbit", 54 | "cat", 55 | }}, 56 | }, 57 | { 58 | input: []interface{}{"assets.house.area", "=", 100.4}, 59 | expected: []interface{}{"assets", map[string]interface{}{ 60 | "house": map[string]interface{}{ 61 | "area": 100.4, 62 | }, 63 | }}, 64 | }, 65 | } 66 | 67 | s.testCases(tests) 68 | } 69 | 70 | func (s *AssignmentTestSuite) TestDeleteAssignment() { 71 | tests := []assignTestCase{ 72 | { 73 | input: []interface{}{"area", "-", "city,zipcode"}, 74 | expected: []interface{}{"area", map[string]interface{}{}}, 75 | }, 76 | { 77 | input: []interface{}{"$", "-", []interface{}{"height"}}, 78 | expected: []interface{}{"height", nil}, 79 | }, 80 | { 81 | input: []interface{}{"area", "-", 0}, 82 | prepareError: true, 83 | }, 84 | } 85 | 86 | s.testCases(tests) 87 | } 88 | 89 | func (s *AssignmentTestSuite) TestMergeAssignment() { 90 | tests := []assignTestCase{ 91 | { 92 | input: []interface{}{"area", "+", map[string]interface{}{ 93 | "province": "shanghai", 94 | }}, 95 | expected: []interface{}{"area", map[string]interface{}{ 96 | "zipcode": 200211, 97 | "city": "shanghai", 98 | "province": "shanghai", 99 | }}, 100 | }, 101 | { 102 | input: []interface{}{"assets.house", "+", map[string]interface{}{ 103 | "area": 100, 104 | }}, 105 | expected: []interface{}{"assets", map[string]interface{}{ 106 | "house": map[string]interface{}{ 107 | "area": 100, 108 | }, 109 | }}, 110 | }, 111 | { 112 | input: []interface{}{"assets.car", "+", "lexus"}, 113 | prepareError: true, 114 | }, 115 | } 116 | 117 | s.testCases(tests) 118 | } 119 | 120 | func (s *AssignmentTestSuite) testCases(tests []assignTestCase) { 121 | for i, c := range tests { 122 | a := _assignmentFactory.Get(c.input[1].(string)) 123 | v, err := a.PrepareValue(c.input[2]) 124 | if c.prepareError { 125 | s.Error(err) 126 | continue 127 | } 128 | require.NoError(s.T(), err) 129 | a.Run(s.ctx, s.ctx.Data(), c.input[0].(string), v) 130 | 131 | cv, _ := object.GetValue(s.ctx.Data(), c.expected[0].(string)) 132 | 133 | s.Equal(c.expected[1], cv, "case %d: %v", i, c.input) 134 | } 135 | } 136 | 137 | func TestAssignmentTestSuite(t *testing.T) { 138 | suite.Run(t, new(AssignmentTestSuite)) 139 | } 140 | -------------------------------------------------------------------------------- /core/cache.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type Cache interface { 8 | Load(key interface{}) (interface{}, bool) 9 | Store(key, value interface{}) 10 | Delete(key interface{}) 11 | } 12 | 13 | func NewCache() Cache { 14 | return new(sync.Map) 15 | } 16 | -------------------------------------------------------------------------------- /core/condition.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | //Condition interface 11 | type Condition interface { 12 | Success(ctx *Context) bool 13 | String() string 14 | } 15 | 16 | //StdCondition 17 | type StdCondition struct { 18 | expr string 19 | variable Variable 20 | operation Operation 21 | value interface{} 22 | } 23 | 24 | func (c *StdCondition) Success(ctx *Context) bool { 25 | ok := c.operation.Run(ctx, c.variable, c.value) 26 | 27 | if trace := ctx.Trace(); trace != nil { 28 | trace.Log( 29 | c.String(), 30 | " => ", 31 | GetVariableValue(ctx, c.variable), 32 | c.operation.String(), 33 | c.value, 34 | " => ", 35 | ok, 36 | ) 37 | } 38 | 39 | return ok 40 | } 41 | 42 | func (c *StdCondition) String() string { 43 | return c.expr 44 | } 45 | 46 | //ConditionGroup 47 | type GROUP_LOGIC int 48 | 49 | const ( 50 | LOGIC_ALL GROUP_LOGIC = iota 51 | LOGIC_ANY 52 | LOGIC_NONE 53 | LOGIC_ANY_NOT 54 | ) 55 | 56 | type ConditionGroup struct { 57 | logic GROUP_LOGIC 58 | conditions []Condition 59 | } 60 | 61 | func (c *ConditionGroup) Success(ctx *Context) bool { 62 | result := c.logic != LOGIC_ANY_NOT 63 | for _, condition := range c.conditions { 64 | if ok := condition.Success(ctx); ok { 65 | if c.logic == LOGIC_ANY { 66 | result = true 67 | break 68 | } 69 | if c.logic == LOGIC_NONE { 70 | result = false 71 | break 72 | } 73 | } else { 74 | if c.logic == LOGIC_ALL { 75 | result = false 76 | break 77 | } else if c.logic == LOGIC_ANY_NOT { 78 | result = true 79 | break 80 | } else if c.logic == LOGIC_ANY { 81 | result = false 82 | } 83 | } 84 | } 85 | 86 | return result 87 | } 88 | 89 | func (c *ConditionGroup) String() string { 90 | var b strings.Builder 91 | for k, v := range groupConditionKeys { 92 | if v == c.logic { 93 | b.WriteString(strings.ToUpper(k)) 94 | break 95 | } 96 | } 97 | b.WriteString("{") 98 | for _, c := range c.conditions { 99 | b.WriteString(c.String()) 100 | //b.WriteString(", ") 101 | } 102 | b.WriteString("}") 103 | 104 | return b.String() 105 | } 106 | 107 | func (c *ConditionGroup) add(condition Condition) { 108 | c.conditions = append(c.conditions, condition) 109 | } 110 | 111 | func NewConditionGroup(logic GROUP_LOGIC) *ConditionGroup { 112 | return &ConditionGroup{ 113 | logic: logic, 114 | conditions: make([]Condition, 0), 115 | } 116 | } 117 | 118 | var groupConditionKeys = map[string]GROUP_LOGIC{ 119 | "any?": LOGIC_ANY, 120 | "not?": LOGIC_ANY_NOT, 121 | "all?": LOGIC_ALL, 122 | "none?": LOGIC_NONE, 123 | } 124 | 125 | func NewCondition(item []interface{}, groupLogic GROUP_LOGIC) (Condition, error) { 126 | if len(item) == 0 { 127 | return nil, errors.New("Empty") 128 | } 129 | 130 | if IsArray(item[0]) { 131 | group := NewConditionGroup(groupLogic) 132 | 133 | for _, subitem := range item { 134 | if !IsArray(subitem) { 135 | return nil, errors.Errorf("Sub item must be an array. -> %s", jstr(subitem)) 136 | } 137 | if subCondition, err := NewCondition(ToArray(subitem), LOGIC_ALL); err != nil { 138 | return nil, err 139 | } else { 140 | group.add(subCondition) 141 | } 142 | } 143 | 144 | return group, nil 145 | } 146 | 147 | if len(item) != 3 { 148 | return nil, errors.Errorf("Item must contains 3 elements. -> %s", jstr(item)) 149 | } 150 | 151 | key, ok := item[0].(string) 152 | if !ok { 153 | return nil, errors.Errorf("Item 1st element[%g] is not string. -> %s", item[0], jstr(item)) 154 | } 155 | 156 | if logic, ok := groupConditionKeys[key]; ok { 157 | list, ok := item[2].([]interface{}) 158 | if !ok { 159 | return nil, errors.Errorf("Logic[%s] item 3rd element must be an array. -> %s", key, jstr(item)) 160 | } 161 | if group, err := NewCondition(list, logic); err != nil { 162 | return nil, err 163 | } else { 164 | return group, nil 165 | } 166 | } 167 | 168 | variable := _variableFactory.Create(key) 169 | if variable == nil { 170 | return nil, errors.Errorf("Unknown var[%s]. -> %s", key, jstr(item)) 171 | } 172 | 173 | operationName, ok := item[1].(string) 174 | if !ok { 175 | return nil, errors.Errorf("Item 2nd element(operation) is not string. -> %s", jstr(item)) 176 | } 177 | 178 | operation := _operationFactory.Get(operationName) 179 | 180 | if operation == nil { 181 | return nil, errors.Errorf("Unknown operation[%s]. -> %s", operationName, jstr(item)) 182 | } 183 | 184 | pvalue, err := operation.PrepareValue(item[2]) 185 | 186 | if err != nil { 187 | return nil, err 188 | } 189 | 190 | expr := fmt.Sprintf("%s %s %s", key, operationName, jstr(pvalue)) 191 | 192 | condition := &StdCondition{ 193 | expr: expr, 194 | variable: variable, 195 | operation: operation, 196 | value: pvalue, 197 | } 198 | 199 | return condition, nil 200 | } 201 | -------------------------------------------------------------------------------- /core/condition_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestCondition(t *testing.T) { 11 | ctx := NewContext() 12 | ctx.Set("a", map[string]interface{}{ 13 | "b": 1, 14 | "c": []interface{}{1, 2}, 15 | "d": 5, 16 | }) 17 | 18 | tests := []struct { 19 | item []interface{} 20 | logic GROUP_LOGIC 21 | expected bool 22 | hasError bool 23 | }{ 24 | { 25 | item: []interface{}{"ctx.a.b", "=", 1}, 26 | logic: LOGIC_ALL, 27 | expected: true, 28 | }, 29 | { 30 | item: []interface{}{"ctx.a.b", "=", 2}, 31 | logic: LOGIC_ALL, 32 | expected: false, 33 | }, 34 | { 35 | item: []interface{}{ 36 | []interface{}{"ctx.a.b", "=", 1}, 37 | []interface{}{"ctx.a.c.0", "=", 1}, 38 | []interface{}{"ctx.a.d", "=", 5}, 39 | }, 40 | logic: LOGIC_ALL, 41 | expected: true, 42 | }, 43 | { 44 | item: []interface{}{ 45 | []interface{}{"ctx.a.b", "=", 2}, 46 | []interface{}{"ctx.a.c.0", "=", 1}, 47 | []interface{}{"ctx.a.d", "=", 5}, 48 | }, 49 | logic: LOGIC_ALL, 50 | expected: false, 51 | }, 52 | { 53 | item: []interface{}{ 54 | []interface{}{"ctx.a.b", "=", 2}, 55 | []interface{}{"ctx.a.c.0", "=", 1}, 56 | []interface{}{"ctx.a.d", "=", 5}, 57 | }, 58 | logic: LOGIC_ANY, 59 | expected: true, 60 | }, 61 | { 62 | item: []interface{}{ 63 | []interface{}{"ctx.a.b", "=", 2}, 64 | []interface{}{"ctx.a.c.0", "=", 22}, 65 | []interface{}{"ctx.a.d", "=", 55}, 66 | }, 67 | logic: LOGIC_ANY, 68 | expected: false, 69 | }, 70 | { 71 | item: []interface{}{ 72 | []interface{}{"ctx.a.b", "=", 1}, 73 | []interface{}{"ctx.a.c.0", "=", 1}, 74 | }, 75 | logic: LOGIC_NONE, 76 | expected: false, 77 | }, 78 | { 79 | item: []interface{}{ 80 | []interface{}{"ctx.a.b", "=", 2}, 81 | }, 82 | logic: LOGIC_NONE, 83 | expected: true, 84 | }, 85 | { 86 | item: []interface{}{ 87 | []interface{}{"ctx.a.b", "=", 2}, 88 | []interface{}{"ctx.a.c.0", "=", 1}, 89 | }, 90 | logic: LOGIC_NONE, 91 | expected: false, 92 | }, 93 | { 94 | item: []interface{}{ 95 | []interface{}{"ctx.a.b", "=", 2}, 96 | []interface{}{"ctx.a.c.0", "=", 1}, 97 | }, 98 | logic: LOGIC_ANY_NOT, 99 | expected: true, 100 | }, 101 | { 102 | item: []interface{}{ 103 | []interface{}{"ctx.a.b", "=", 1}, 104 | []interface{}{"ctx.a.c.0", "=", 1}, 105 | }, 106 | logic: LOGIC_ANY_NOT, 107 | expected: false, 108 | }, 109 | { 110 | item: []interface{}{ 111 | "any?", "=>", []interface{}{ 112 | []interface{}{"ctx.a.b", "=", 2}, 113 | []interface{}{ 114 | "none?", "=>", []interface{}{ 115 | []interface{}{"ctx.a.c.0", "=", 2}, 116 | }, 117 | }, 118 | }, 119 | }, 120 | logic: LOGIC_ALL, 121 | expected: true, 122 | }, 123 | } 124 | 125 | for i, c := range tests { 126 | cond, err := NewCondition(c.item, c.logic) 127 | if c.hasError { 128 | assert.Error(t, err) 129 | continue 130 | } 131 | require.NoError(t, err, err) 132 | assert.Equal(t, c.expected, cond.Success(ctx), "case %d: %s", i, cond.String()) 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /core/context.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "sync/atomic" 7 | "time" 8 | ) 9 | 10 | type ctxKey string 11 | 12 | const ( 13 | filterDataCtxKey ctxKey = "data" 14 | cacheCtxKey ctxKey = "cache" 15 | ctxDataCtxKey ctxKey = "ctx" 16 | traceCtxKey ctxKey = "trace" 17 | ) 18 | 19 | type Context struct { 20 | ctx context.Context 21 | mu sync.Mutex 22 | } 23 | 24 | func NewContext() *Context { 25 | return WithContext(context.Background()) 26 | } 27 | 28 | type ContextOption interface { 29 | apply(*Context) 30 | } 31 | 32 | type ContextOptionFunc func(*Context) 33 | 34 | func (f ContextOptionFunc) apply(c *Context) { 35 | f(c) 36 | } 37 | 38 | // WithCache ContextOption 39 | func WithCache(cache Cache) ContextOption { 40 | return ContextOptionFunc(func(c *Context) { 41 | c.ctx = context.WithValue(c.ctx, cacheCtxKey, cache) 42 | }) 43 | } 44 | 45 | // WithTrace ContextOption 46 | func WithTrace(trace Trace) ContextOption { 47 | return ContextOptionFunc(func(c *Context) { 48 | c.ctx = context.WithValue(c.ctx, traceCtxKey, trace) 49 | }) 50 | } 51 | 52 | func WithContext(ctx context.Context, opts ...ContextOption) *Context { 53 | if ctx == nil { 54 | ctx = context.Background() 55 | } 56 | 57 | // check context data 58 | if ctxData := ctx.Value(ctxDataCtxKey); ctxData == nil { 59 | ctx = context.WithValue(ctx, ctxDataCtxKey, newContextData()) 60 | } 61 | 62 | c := &Context{ 63 | ctx: ctx, 64 | } 65 | 66 | for _, opt := range opts { 67 | opt.apply(c) 68 | } 69 | 70 | return c 71 | } 72 | 73 | // WithData return new *Context contains data. 74 | // call every time the filter runs, to make data thread-safe 75 | func WithData(ctx context.Context, data interface{}) *Context { 76 | ctx = context.WithValue(ctx, filterDataCtxKey, data) 77 | 78 | return WithContext(ctx) 79 | } 80 | 81 | // Data return filter data 82 | func (c *Context) Data() interface{} { 83 | var data interface{} 84 | if data = c.ctx.Value(filterDataCtxKey); data == nil { 85 | c.mu.Lock() 86 | data = c.ctx.Value(filterDataCtxKey) 87 | if data == nil { 88 | data = make(map[string]interface{}) 89 | c.ctx = context.WithValue(c.ctx, filterDataCtxKey, data) 90 | } 91 | c.mu.Unlock() 92 | } 93 | 94 | return data 95 | } 96 | 97 | // Cache return filter Cache object 98 | func (c *Context) Cache() Cache { 99 | var cache interface{} 100 | 101 | if cache = c.ctx.Value(cacheCtxKey); cache == nil { 102 | c.mu.Lock() 103 | cache = c.ctx.Value(cacheCtxKey) 104 | if cache == nil { 105 | cache = NewCache() 106 | c.ctx = context.WithValue(c.ctx, cacheCtxKey, cache) 107 | } 108 | c.mu.Unlock() 109 | } 110 | 111 | return cache.(Cache) 112 | } 113 | 114 | // Trace return Trace 115 | func (c *Context) Trace() Trace { 116 | if t := c.ctx.Value(traceCtxKey); t != nil { 117 | return t.(Trace) 118 | } 119 | 120 | return nil 121 | } 122 | 123 | // Set set context data 124 | func (c *Context) Set(key string, value interface{}) { 125 | if data := c.ctx.Value(ctxDataCtxKey); data != nil { 126 | data.(*contextData).Set(key, value) 127 | } 128 | } 129 | 130 | // Delete delte context data 131 | func (c *Context) Delete(key string) { 132 | if data := c.ctx.Value(ctxDataCtxKey); data != nil { 133 | data.(*contextData).Delete(key) 134 | } 135 | } 136 | 137 | // Get get context data 138 | func (c *Context) Get(key string) (value interface{}, exists bool) { 139 | if data := c.ctx.Value(ctxDataCtxKey); data != nil { 140 | return data.(*contextData).Get(key) 141 | } 142 | 143 | return nil, false 144 | } 145 | 146 | // GetAll return a map contains all context data 147 | func (c *Context) GetAll() map[string]interface{} { 148 | if data := c.ctx.Value(ctxDataCtxKey); data != nil { 149 | return data.(*contextData).GetAll() 150 | } 151 | 152 | return nil 153 | } 154 | 155 | // Implements context.Context interface 156 | func (c *Context) Deadline() (deadline time.Time, ok bool) { 157 | return c.ctx.Deadline() 158 | } 159 | 160 | func (c *Context) Done() <-chan struct{} { 161 | return c.ctx.Done() 162 | } 163 | 164 | func (c *Context) Err() error { 165 | return c.ctx.Err() 166 | } 167 | 168 | func (c *Context) Value(key interface{}) interface{} { 169 | return c.ctx.Value(key) 170 | } 171 | 172 | // contextData 173 | type contextData struct { 174 | mu sync.Mutex 175 | 176 | // context data 177 | m *sync.Map 178 | // for checking if readonly is old 179 | amended int32 180 | // context data readonly 181 | readonly map[string]interface{} 182 | } 183 | 184 | func newContextData() *contextData { 185 | return &contextData{ 186 | m: new(sync.Map), 187 | readonly: make(map[string]interface{}), 188 | } 189 | } 190 | 191 | func (d *contextData) Set(key string, value interface{}) { 192 | d.m.Store(key, value) 193 | atomic.AddInt32(&d.amended, 1) 194 | } 195 | 196 | func (d *contextData) Delete(key string) { 197 | d.m.Delete(key) 198 | atomic.AddInt32(&d.amended, 1) 199 | } 200 | 201 | func (d *contextData) Get(key string) (value interface{}, exists bool) { 202 | return d.m.Load(key) 203 | } 204 | 205 | func (d *contextData) GetAll() map[string]interface{} { 206 | if atomic.LoadInt32(&d.amended) == 0 { 207 | return d.readonly 208 | } 209 | 210 | d.mu.Lock() 211 | defer d.mu.Unlock() 212 | 213 | old := atomic.LoadInt32(&d.amended) 214 | if old == 0 { 215 | return d.readonly 216 | } 217 | 218 | readonly := make(map[string]interface{}) 219 | d.m.Range(func(key, val interface{}) bool { 220 | readonly[key.(string)] = val 221 | return true 222 | }) 223 | atomic.CompareAndSwapInt32(&d.amended, old, 0) 224 | d.readonly = readonly 225 | 226 | return d.readonly 227 | } 228 | -------------------------------------------------------------------------------- /core/context_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestNewContext(t *testing.T) { 11 | assert.Implements(t, (*context.Context)(nil), NewContext()) 12 | } 13 | 14 | func TestWithContext(t *testing.T) { 15 | pctx := context.WithValue(context.Background(), "foo", "bar") 16 | ctx := WithContext(pctx) 17 | 18 | assert.Equal(t, "bar", ctx.Value("foo")) 19 | } 20 | 21 | func TestData(t *testing.T) { 22 | assert.NotNil(t, NewContext().Data()) 23 | } 24 | 25 | func TestCache(t *testing.T) { 26 | assert.NotNil(t, NewContext().Cache()) 27 | } 28 | 29 | func TestGetSet(t *testing.T) { 30 | ctx := NewContext() 31 | 32 | ctx.Set("foo", "bar") 33 | v, exists := ctx.Get("foo") 34 | assert.Equal(t, "bar", v) 35 | assert.True(t, exists) 36 | v, exists = ctx.Get("bar") 37 | assert.Nil(t, v) 38 | assert.False(t, exists) 39 | assert.Equal(t, map[string]interface{}{"foo": "bar"}, ctx.GetAll()) 40 | 41 | ctx.Set("bar", "zap") 42 | assert.Equal(t, map[string]interface{}{"foo": "bar", "bar": "zap"}, ctx.GetAll()) 43 | 44 | ctx.Delete("foo") 45 | assert.Equal(t, map[string]interface{}{"bar": "zap"}, ctx.GetAll()) 46 | } 47 | 48 | func Benchmark_Context(b *testing.B) { 49 | ctx := NewContext() 50 | b.RunParallel(func(pb *testing.PB) { 51 | for pb.Next() { 52 | ctx.Data() 53 | ctx.Cache() 54 | ctx.Set("foo", "bar") 55 | ctx.Get("foo") 56 | ctx.Get("bar") 57 | } 58 | }) 59 | } 60 | -------------------------------------------------------------------------------- /core/executor.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | //Executor 10 | type Executor interface { 11 | Execute(*Context, interface{}) 12 | } 13 | 14 | //StdExecutor 15 | type StdExecutor struct { 16 | expr string 17 | key string 18 | assignment Assignment 19 | value interface{} 20 | } 21 | 22 | func (e *StdExecutor) Execute(ctx *Context, data interface{}) { 23 | if trace := ctx.Trace(); trace != nil { 24 | trace.Log( 25 | e.expr, 26 | ) 27 | } 28 | 29 | e.assignment.Run(ctx, data, e.key, e.value) 30 | } 31 | 32 | //ExecutorGroup 33 | type ExecutorGroup struct { 34 | executors []Executor 35 | } 36 | 37 | func (e *ExecutorGroup) Execute(ctx *Context, data interface{}) { 38 | for _, executor := range e.executors { 39 | executor.Execute(ctx, data) 40 | } 41 | } 42 | 43 | func (e *ExecutorGroup) add(executor Executor) { 44 | e.executors = append(e.executors, executor) 45 | } 46 | 47 | func NewExecutorGroup() *ExecutorGroup { 48 | return &ExecutorGroup{ 49 | executors: make([]Executor, 0), 50 | } 51 | } 52 | 53 | func NewExecutor(item []interface{}) (Executor, error) { 54 | if len(item) == 0 { 55 | return nil, errors.New("Executor is empty") 56 | } 57 | 58 | if IsArray(item[0]) { 59 | group := NewExecutorGroup() 60 | 61 | for _, subitem := range item { 62 | if !IsArray(subitem) { 63 | return nil, errors.Errorf("Executor child item is not array. -> %s", jstr(item)) 64 | } 65 | if subExecutor, err := NewExecutor(ToArray(subitem)); err != nil { 66 | return nil, err 67 | } else { 68 | group.add(subExecutor) 69 | } 70 | } 71 | 72 | return group, nil 73 | } 74 | 75 | if len(item) != 3 { 76 | return nil, errors.Errorf("Executor item must contains 3 elements. -> %s", jstr(item)) 77 | } 78 | 79 | key, ok := item[0].(string) 80 | if !ok { 81 | return nil, errors.Errorf("Executor item 1st element (%v) is not string. -> %s", item[0], jstr(item)) 82 | } 83 | 84 | assignmentName, ok := item[1].(string) 85 | if !ok { 86 | return nil, errors.Errorf("Executor item 2nd element (%v) is not string. -> %s", item[1], jstr(item)) 87 | } 88 | 89 | assignment := _assignmentFactory.Get(assignmentName) 90 | if assignment == nil { 91 | return nil, errors.Errorf("Executor with invalid assignment[%s]", assignmentName) 92 | } 93 | 94 | value, err := assignment.PrepareValue(item[2]) 95 | if err != nil { 96 | return nil, errors.Errorf("Executor assignment[%s] prepare value err:%s", assignmentName, err) 97 | } 98 | 99 | expr := fmt.Sprintf("%s %s %s", key, assignmentName, jstr(value)) 100 | 101 | executor := &StdExecutor{ 102 | expr: expr, 103 | key: key, 104 | assignment: assignment, 105 | value: value, 106 | } 107 | 108 | return executor, nil 109 | } 110 | -------------------------------------------------------------------------------- /core/executor_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func _e(k, op, val interface{}) []interface{} { 10 | return []interface{}{k, op, val} 11 | } 12 | 13 | func TestExecutor(t *testing.T) { 14 | data := map[string]interface{}{ 15 | "foo": map[string]interface{}{ 16 | "bar": []interface{}{1, 2}, 17 | }, 18 | } 19 | 20 | tests := []struct { 21 | input []interface{} 22 | expected interface{} 23 | hasError bool 24 | }{ 25 | { 26 | _e("foo.bar", "=", 1), 27 | map[string]interface{}{ 28 | "foo": map[string]interface{}{ 29 | "bar": 1, 30 | }, 31 | }, 32 | false, 33 | }, 34 | { 35 | []interface{}{ 36 | _e("foo.bar.0", "=", 2), 37 | _e("foo.zap", "=", 3), 38 | }, 39 | map[string]interface{}{ 40 | "foo": map[string]interface{}{ 41 | "bar": []interface{}{2, 2}, 42 | "zap": 3, 43 | }, 44 | }, 45 | false, 46 | }, 47 | { 48 | []interface{}{ 49 | "a", "*", "b", 50 | }, 51 | nil, 52 | true, 53 | }, 54 | { 55 | []interface{}{ 56 | "a", "*=", "b", 57 | }, 58 | nil, 59 | true, 60 | }, 61 | } 62 | 63 | ctx := NewContext() 64 | 65 | for i, c := range tests { 66 | e, err := NewExecutor(c.input) 67 | if c.hasError { 68 | t.Log(err) 69 | assert.Error(t, err) 70 | continue 71 | } 72 | d := Clone(data) 73 | e.Execute(ctx, d) 74 | assert.Equal(t, c.expected, d, "case %d: %v", i, c) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /core/logger.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | ) 7 | 8 | type StdLogger interface { 9 | Print(v ...interface{}) 10 | Printf(format string, v ...interface{}) 11 | Println(v ...interface{}) 12 | } 13 | 14 | var ( 15 | Logger StdLogger = log.New(ioutil.Discard, "[filter]", log.LstdFlags) 16 | ) 17 | -------------------------------------------------------------------------------- /core/operation.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "regexp" 7 | "strings" 8 | 9 | "github.com/techxmind/go-utils/compare" 10 | "github.com/techxmind/go-utils/itype" 11 | ) 12 | 13 | var _operationFactory OperationFactory 14 | 15 | func init() { 16 | _operationFactory = &stdOperationFactory{ 17 | operations: map[string]Operation{ 18 | "=": &EqualOperation{stringer: stringer("=")}, 19 | "!=": &NotEqualOperation{stringer: stringer("!=")}, 20 | ">": &GtOperation{stringer: stringer(">")}, 21 | ">=": &GeOperation{stringer: stringer(">=")}, 22 | "<": &LtOperation{stringer: stringer("<")}, 23 | "<=": &LeOperation{stringer: stringer("<=")}, 24 | "between": &BetweenOperation{stringer: stringer("between")}, 25 | "in": &InOperation{stringer: stringer("in")}, 26 | "not in": &NotInOperation{stringer: stringer("not in")}, 27 | "~": &MatchOperation{stringer: stringer("~")}, 28 | "!~": &NotMatchOperation{stringer: stringer("!~")}, 29 | "any": &AnyOperation{stringer: stringer("any")}, 30 | "has": &HasOperation{stringer: stringer("has")}, 31 | "none": &NoneOperation{stringer: stringer("none")}, 32 | }, 33 | } 34 | } 35 | 36 | type Operation interface { 37 | Run(ctx *Context, variable Variable, value interface{}) bool 38 | PrepareValue(value interface{}) (interface{}, error) 39 | String() string 40 | } 41 | 42 | type OperationFactory interface { 43 | Get(string) Operation 44 | Register(Operation, ...string) 45 | } 46 | 47 | func GetOperationFactory() OperationFactory { 48 | return _operationFactory 49 | } 50 | 51 | // stdOperationFactory is default OperationFactory 52 | type stdOperationFactory struct { 53 | operations map[string]Operation 54 | } 55 | 56 | func (f *stdOperationFactory) Get(name string) Operation { 57 | if value, ok := f.operations[name]; ok { 58 | return value 59 | } 60 | 61 | return nil 62 | } 63 | 64 | func (f *stdOperationFactory) Register(op Operation, names ...string) { 65 | for _, name := range names { 66 | f.operations[name] = op 67 | } 68 | } 69 | 70 | //---------------------------------------------------------------------------------- 71 | // helper functions 72 | //---------------------------------------------------------------------------------- 73 | 74 | type operationBase struct{} 75 | 76 | func (o operationBase) PrepareValue(value interface{}) (interface{}, error) { 77 | return value, nil 78 | } 79 | 80 | //---------------------------------------------------------------------------------- 81 | // core operations define 82 | //---------------------------------------------------------------------------------- 83 | 84 | //---------------------------------------------------------------------------------- 85 | type EqualOperation struct { 86 | stringer 87 | operationBase 88 | } 89 | 90 | func (o *EqualOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 91 | cmpValue := GetVariableValue(ctx, variable) 92 | 93 | if b, ok := value.(bool); ok { 94 | return itype.Bool(cmpValue) == b 95 | } 96 | 97 | return compare.Object(cmpValue, value) == 0 98 | } 99 | 100 | //---------------------------------------------------------------------------------- 101 | type NotEqualOperation struct { 102 | stringer 103 | EqualOperation 104 | } 105 | 106 | func (o *NotEqualOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 107 | return !o.EqualOperation.Run(ctx, variable, value) 108 | } 109 | 110 | //---------------------------------------------------------------------------------- 111 | type GtOperation struct { 112 | stringer 113 | operationBase 114 | } 115 | 116 | func (o *GtOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 117 | cmpValue := GetVariableValue(ctx, variable) 118 | return compare.Object(cmpValue, value) == 1 119 | } 120 | 121 | //---------------------------------------------------------------------------------- 122 | type LeOperation struct { 123 | stringer 124 | GtOperation 125 | } 126 | 127 | func (o *LeOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 128 | return !o.GtOperation.Run(ctx, variable, value) 129 | } 130 | 131 | //---------------------------------------------------------------------------------- 132 | type LtOperation struct { 133 | stringer 134 | operationBase 135 | } 136 | 137 | func (o *LtOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 138 | cmpValue := GetVariableValue(ctx, variable) 139 | 140 | return compare.Object(cmpValue, value) == -1 141 | } 142 | 143 | //---------------------------------------------------------------------------------- 144 | type GeOperation struct { 145 | stringer 146 | LtOperation 147 | } 148 | 149 | func (o *GeOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 150 | return !o.LtOperation.Run(ctx, variable, value) 151 | } 152 | 153 | //---------------------------------------------------------------------------------- 154 | type BetweenOperation struct{ stringer } 155 | 156 | func (o *BetweenOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 157 | cmpValue := GetVariableValue(ctx, variable) 158 | startAndEnd := value.([]interface{}) 159 | return compare.Object(cmpValue, startAndEnd[0]) >= 0 && compare.Object(cmpValue, startAndEnd[1]) <= 0 160 | } 161 | func (o *BetweenOperation) PrepareValue(value interface{}) (interface{}, error) { 162 | startAndEnd := ToArray(value) 163 | 164 | if len(startAndEnd) != 2 { 165 | return nil, errors.New(fmt.Sprintf("[between] operation value must be a list with 2 elements")) 166 | } 167 | 168 | return startAndEnd, nil 169 | } 170 | 171 | //---------------------------------------------------------------------------------- 172 | type InOperation struct{ stringer } 173 | 174 | func (o *InOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 175 | cmpValue := GetVariableValue(ctx, variable) 176 | 177 | for _, elem := range value.([]interface{}) { 178 | if compare.Object(elem, cmpValue) == 0 { 179 | return true 180 | } 181 | } 182 | 183 | return false 184 | } 185 | 186 | func (o *InOperation) PrepareValue(value interface{}) (interface{}, error) { 187 | elems := ToArray(value) 188 | 189 | if len(elems) == 0 { 190 | return nil, errors.New(fmt.Sprintf("[in/not in] operation value must be a list")) 191 | } 192 | 193 | return elems, nil 194 | } 195 | 196 | //---------------------------------------------------------------------------------- 197 | type NotInOperation struct { 198 | stringer 199 | InOperation 200 | } 201 | 202 | func (o *NotInOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 203 | return !o.InOperation.Run(ctx, variable, value) 204 | } 205 | 206 | //---------------------------------------------------------------------------------- 207 | type MatchOperation struct{ stringer } 208 | 209 | func (o *MatchOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 210 | cmpValue := GetVariableValue(ctx, variable) 211 | 212 | cmpValueStr, ok := cmpValue.(string) 213 | 214 | if !ok { 215 | return false 216 | } 217 | 218 | if regexpObj, ok := value.(*regexp.Regexp); ok { 219 | 220 | return regexpObj.MatchString(cmpValueStr) 221 | } else if strObj, ok := value.(string); ok { 222 | 223 | return strings.Contains(strings.ToLower(cmpValueStr), strObj) 224 | } 225 | 226 | return false 227 | } 228 | 229 | func (o *MatchOperation) PrepareValue(value interface{}) (interface{}, error) { 230 | str, ok := value.(string) 231 | if !ok || str == "" { 232 | return nil, errors.New(fmt.Sprintf("[match] operation value must be a string")) 233 | } 234 | 235 | if !(strings.HasPrefix(str, "/") && strings.HasSuffix(str, "/")) { 236 | return strings.ToLower(str), nil 237 | } 238 | 239 | str = strings.Trim(str, "/") 240 | if str == "" { 241 | return nil, errors.New(fmt.Sprintf("[match] operation value is not a valid regexp expression[%s]", str)) 242 | } 243 | 244 | if robj, err := regexp.Compile("(?i)" + str); err != nil { 245 | return nil, errors.New(fmt.Sprintf("[match] operation value is not a valid regexp expression[%s].err:%s", str, err)) 246 | } else { 247 | return robj, nil 248 | } 249 | } 250 | 251 | //---------------------------------------------------------------------------------- 252 | type NotMatchOperation struct { 253 | stringer 254 | MatchOperation 255 | } 256 | 257 | func (o *NotMatchOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 258 | return !o.MatchOperation.Run(ctx, variable, value) 259 | } 260 | 261 | //---------------------------------------------------------------------------------- 262 | type AnyOperation struct{ stringer } 263 | 264 | func (o *AnyOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 265 | cmpValue := GetVariableValue(ctx, variable) 266 | 267 | cmpElems := ToArray(cmpValue) 268 | elems := ToArray(value) 269 | 270 | if len(elems) == 0 || len(cmpElems) == 0 { 271 | return false 272 | } 273 | 274 | for _, elem := range elems { 275 | for _, cmpElem := range cmpElems { 276 | if compare.Object(elem, cmpElem) == 0 { 277 | return true 278 | } 279 | } 280 | } 281 | 282 | return false 283 | } 284 | 285 | func (o *AnyOperation) PrepareValue(value interface{}) (interface{}, error) { 286 | elems := ToArray(value) 287 | 288 | if len(elems) == 0 { 289 | return nil, errors.New(fmt.Sprintf("[any] operation value must be a list")) 290 | } 291 | 292 | return elems, nil 293 | } 294 | 295 | //---------------------------------------------------------------------------------- 296 | type HasOperation struct{ stringer } 297 | 298 | func (o *HasOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 299 | cmpValue := GetVariableValue(ctx, variable) 300 | 301 | cmpElems := ToArray(cmpValue) 302 | elems := ToArray(value) 303 | 304 | if len(elems) == 0 || len(cmpElems) == 0 { 305 | return false 306 | } 307 | 308 | for _, elem := range elems { 309 | ok := false 310 | for _, cmpElem := range cmpElems { 311 | if compare.Object(elem, cmpElem) == 0 { 312 | ok = true 313 | break 314 | } 315 | } 316 | if !ok { 317 | return false 318 | } 319 | } 320 | 321 | return true 322 | } 323 | func (o *HasOperation) PrepareValue(value interface{}) (interface{}, error) { 324 | elems := ToArray(value) 325 | 326 | if len(elems) == 0 { 327 | return nil, errors.New(fmt.Sprintf("[has] operation value must be a list")) 328 | } 329 | 330 | return elems, nil 331 | } 332 | 333 | //---------------------------------------------------------------------------------- 334 | type NoneOperation struct { 335 | stringer 336 | AnyOperation 337 | } 338 | 339 | func (o *NoneOperation) Run(ctx *Context, variable Variable, value interface{}) bool { 340 | return !o.AnyOperation.Run(ctx, variable, value) 341 | } 342 | -------------------------------------------------------------------------------- /core/operation_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/stretchr/testify/suite" 8 | ) 9 | 10 | type OperationTestSuite struct { 11 | suite.Suite 12 | ctx *Context 13 | } 14 | 15 | type opTestCase struct { 16 | input []interface{} 17 | expected bool 18 | expectedError bool 19 | } 20 | 21 | func (s *OperationTestSuite) SetupTest() { 22 | ctx := NewContext() 23 | data := map[string]interface{}{ 24 | "area": map[string]interface{}{ 25 | "zipcode": 200211, 26 | "city": "shanghai", 27 | }, 28 | "birthday": "1985-11-21", 29 | "height": "178", 30 | "age": 25, 31 | "fav_books": "book1,book2,book3", 32 | "pets": []interface{}{"dog", "cat"}, 33 | } 34 | ctx = WithData(ctx, data) 35 | s.ctx = ctx 36 | } 37 | 38 | func (s *OperationTestSuite) TestEqual() { 39 | tests := []opTestCase{ 40 | {[]interface{}{"succ", "=", true}, true, false}, 41 | {[]interface{}{"succ", "=", 1}, true, false}, 42 | {[]interface{}{"data.area.zipcode", "=", 200211}, true, false}, 43 | {[]interface{}{"data.area.zipcode", "=", "200211"}, true, false}, 44 | {[]interface{}{"data.age", "=", 25}, true, false}, 45 | {[]interface{}{"data.age", "=", 24}, false, false}, 46 | {[]interface{}{"data.age", "=", "foo"}, false, false}, 47 | {[]interface{}{"data.area", "=", true}, true, false}, 48 | } 49 | 50 | s.testCases(tests) 51 | 52 | s.testCases(s.getOppositeCases(tests, map[string]string{"=": "!="})) 53 | } 54 | 55 | func (s *OperationTestSuite) TestCompare() { 56 | tests := []opTestCase{ 57 | {[]interface{}{"data.age", ">", 24}, true, false}, 58 | {[]interface{}{"data.age", ">", 25}, false, false}, 59 | {[]interface{}{"data.age", ">=", 25}, true, false}, 60 | {[]interface{}{"data.age", "<", 26}, true, false}, 61 | {[]interface{}{"data.age", "<=", 25}, true, false}, 62 | {[]interface{}{"data.age", "<", 25}, false, false}, 63 | {[]interface{}{"data.age", "<", 25}, false, false}, 64 | {[]interface{}{"data.age", "<", "foo"}, false, false}, 65 | {[]interface{}{"data.height", "<", 177}, false, false}, 66 | {[]interface{}{"data.height", ">", 179}, false, false}, 67 | } 68 | 69 | s.testCases(tests) 70 | 71 | s.testCases(s.getOppositeCases(tests, map[string]string{ 72 | ">": "<=", 73 | ">=": "<", 74 | "<": ">=", 75 | "<=": ">", 76 | })) 77 | } 78 | 79 | func (s *OperationTestSuite) TestBetween() { 80 | tests := []opTestCase{ 81 | {[]interface{}{"data.age", "between", "24,26"}, true, false}, 82 | {[]interface{}{"data.age", "between", []interface{}{24, 26}}, true, false}, 83 | {[]interface{}{"data.age", "between", "24,26,27"}, false, true}, 84 | {[]interface{}{"data.birthday", "between", "1984-11-11,1986-11-11"}, true, false}, 85 | } 86 | 87 | s.testCases(tests) 88 | } 89 | 90 | func (s *OperationTestSuite) TestIn() { 91 | tests := []opTestCase{ 92 | {[]interface{}{"data.age", "in", "24,25 ,26"}, true, false}, 93 | {[]interface{}{"data.age", "in", "24,26"}, false, false}, 94 | {[]interface{}{"data.age", "in", []interface{}{24, 25, 26}}, true, false}, 95 | {[]interface{}{"data.age", "in", []interface{}{24, 26}}, false, false}, 96 | } 97 | 98 | s.testCases(tests) 99 | s.testCases(s.getOppositeCases(tests, map[string]string{"in": "not in"})) 100 | } 101 | 102 | func (s *OperationTestSuite) TestMatch() { 103 | tests := []opTestCase{ 104 | {[]interface{}{"data.area.city", "~", "shang"}, true, false}, 105 | {[]interface{}{"data.area.city", "~", "hai"}, true, false}, 106 | {[]interface{}{"data.area.city", "~", "h.i"}, false, false}, 107 | {[]interface{}{"data.area.city", "~", "/h.i/"}, true, false}, 108 | {[]interface{}{"data.area.city", "~", "/^s.+i$/"}, true, false}, 109 | {[]interface{}{"data.area.city", "~", "/^(s.+i$/"}, false, true}, 110 | } 111 | 112 | s.testCases(tests) 113 | s.testCases(s.getOppositeCases(tests, map[string]string{"~": "!~"})) 114 | } 115 | 116 | func (s *OperationTestSuite) TestListMatch() { 117 | tests := []opTestCase{ 118 | {[]interface{}{"data.fav_books", "has", "book1,book3"}, true, false}, 119 | {[]interface{}{"data.fav_books", "has", "book1,book3,book4"}, false, false}, 120 | {[]interface{}{"data.pets", "any", "cat,pig"}, true, false}, 121 | {[]interface{}{"data.pets", "any", "rabbit,fly"}, false, false}, 122 | {[]interface{}{"data.pets", "none", []interface{}{"pig", "rabbit"}}, true, false}, 123 | {[]interface{}{"data.pets", "none", []interface{}{"dog", "rabbit"}}, false, false}, 124 | } 125 | 126 | s.testCases(tests) 127 | s.testCases(s.getOppositeCases(tests, map[string]string{"any": "none", "none": "any"})) 128 | } 129 | 130 | func (s *OperationTestSuite) getOppositeCases(tests []opTestCase, oppositeOpMap map[string]string) []opTestCase { 131 | cases := make([]opTestCase, 0, len(tests)) 132 | for _, c := range tests { 133 | op, ok := oppositeOpMap[c.input[1].(string)] 134 | if !ok { 135 | continue 136 | } 137 | c.input[1] = op 138 | c.expected = !c.expected 139 | cases = append(cases, c) 140 | } 141 | return cases 142 | } 143 | 144 | func (s *OperationTestSuite) testCases(tests []opTestCase) { 145 | for _, c := range tests { 146 | op := _operationFactory.Get(c.input[1].(string)) 147 | require.NotNil(s.T(), op) 148 | v := _variableFactory.Create(c.input[0].(string)) 149 | require.NotNil(s.T(), v) 150 | pv, err := op.PrepareValue(c.input[2]) 151 | if c.expectedError { 152 | s.Error(err) 153 | continue 154 | } 155 | require.NoError(s.T(), err) 156 | s.Equal(c.expected, op.Run(s.ctx, v, pv), c.input) 157 | } 158 | } 159 | 160 | func TestOperationTestSuite(t *testing.T) { 161 | suite.Run(t, new(OperationTestSuite)) 162 | } 163 | -------------------------------------------------------------------------------- /core/trace.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | ) 8 | 9 | type Trace interface { 10 | Enter(name string) Trace 11 | Leave(name string) Trace 12 | Log(a ...interface{}) Trace 13 | } 14 | 15 | type stdTrace struct { 16 | level int 17 | levelPadding string 18 | w io.Writer 19 | } 20 | 21 | func NewTrace(w io.Writer) Trace { 22 | return &stdTrace{ 23 | w: w, 24 | } 25 | } 26 | 27 | func (t *stdTrace) Enter(name string) Trace { 28 | t.Log(name) 29 | t.level++ 30 | t.setPadding() 31 | 32 | return t 33 | } 34 | 35 | func (t *stdTrace) Leave(name string) Trace { 36 | t.level-- 37 | t.setPadding() 38 | return t 39 | } 40 | 41 | func (t *stdTrace) setPadding() { 42 | if t.level <= 0 { 43 | t.levelPadding = "" 44 | return 45 | } 46 | 47 | t.levelPadding = strings.Repeat(" ", t.level) 48 | } 49 | 50 | func (t *stdTrace) Log(a ...interface{}) Trace { 51 | if t.levelPadding != "" { 52 | t.w.Write([]byte(t.levelPadding)) 53 | } 54 | 55 | // color bool vars 56 | for i := 0; i < len(a); i++ { 57 | if b, ok := a[i].(bool); ok { 58 | if b { 59 | a[i] = "\033[0;32mtrue\033[0m" 60 | } else { 61 | a[i] = "\033[0;31mfalse\033[0m" 62 | } 63 | } 64 | } 65 | 66 | t.w.Write([]byte(fmt.Sprintln(a...))) 67 | return t 68 | } 69 | -------------------------------------------------------------------------------- /core/utils.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "encoding/json" 5 | "reflect" 6 | "strings" 7 | 8 | "github.com/mohae/deepcopy" 9 | "github.com/spaolacci/murmur3" 10 | "github.com/techxmind/go-utils/itype" 11 | ) 12 | 13 | // ToArray convert value to []interface{} 14 | func ToArray(value interface{}) []interface{} { 15 | var ret []interface{} 16 | 17 | if value == nil { 18 | return ret 19 | } 20 | 21 | if v, ok := value.([]interface{}); ok { 22 | return v 23 | } 24 | 25 | tp := itype.GetType(value) 26 | 27 | // split string with seperator ',' 28 | if tp == itype.STRING { 29 | v := value.(string) 30 | if v == "" { 31 | return []interface{}{} 32 | } 33 | sarr := strings.Split(v, ",") 34 | ret = make([]interface{}, 0, len(sarr)) 35 | for _, e := range sarr { 36 | e = strings.TrimSpace(e) 37 | if e != "" { 38 | ret = append(ret, e) 39 | } 40 | } 41 | return ret 42 | } 43 | 44 | if tp != itype.ARRAY { 45 | return []interface{}{value} 46 | } 47 | 48 | va := reflect.ValueOf(value) 49 | 50 | ret = make([]interface{}, va.Len()) 51 | for i := 0; i < va.Len(); i++ { 52 | ret[i] = va.Index(i).Interface() 53 | } 54 | 55 | return ret 56 | } 57 | 58 | func jstr(v interface{}) string { 59 | var s strings.Builder 60 | encoder := json.NewEncoder(&s) 61 | encoder.SetEscapeHTML(false) 62 | encoder.SetIndent("", "") 63 | encoder.Encode(v) 64 | 65 | return strings.TrimSpace(s.String()) 66 | } 67 | 68 | func IsArray(v interface{}) bool { 69 | return itype.GetType(v) == itype.ARRAY 70 | } 71 | 72 | func IsScalar(v interface{}) bool { 73 | tp := itype.GetType(v) 74 | 75 | return tp == itype.NUMBER || tp == itype.BOOL || tp == itype.STRING 76 | } 77 | 78 | func Clone(v interface{}) interface{} { 79 | return deepcopy.Copy(v) 80 | } 81 | 82 | func HashID(v string) uint64 { 83 | return murmur3.Sum64([]byte(v)) 84 | } 85 | 86 | func CacheID(v string) uint64 { 87 | return HashID(v) 88 | } 89 | 90 | // help struct for String() method 91 | type stringer string 92 | 93 | func (s stringer) String() string { 94 | return string(s) 95 | } 96 | -------------------------------------------------------------------------------- /core/utils_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestToArray(t *testing.T) { 10 | tests := []struct { 11 | in interface{} 12 | check []interface{} 13 | }{ 14 | { 15 | 1, 16 | []interface{}{1}, 17 | }, 18 | { 19 | "a", 20 | []interface{}{"a"}, 21 | }, 22 | { 23 | "a,b ,c,", 24 | []interface{}{"a", "b", "c"}, 25 | }, 26 | { 27 | []interface{}{"a", "b"}, 28 | []interface{}{"a", "b"}, 29 | }, 30 | { 31 | []int{1, 2, 3}, 32 | []interface{}{1, 2, 3}, 33 | }, 34 | { 35 | //invalid 36 | map[int]int{1: 1}, 37 | []interface{}{map[int]int{1: 1}}, 38 | }, 39 | } 40 | 41 | for _, test := range tests { 42 | a := ToArray(test.in) 43 | assert.Equal(t, test.check, a) 44 | } 45 | } 46 | 47 | func TestIsArray(t *testing.T) { 48 | tests := []struct { 49 | in interface{} 50 | check bool 51 | }{ 52 | {1, false}, 53 | {"a,b,c", false}, 54 | {[]int{1}, true}, 55 | {[1]int{1}, true}, 56 | {[]interface{}{1}, true}, 57 | {map[string]string{"a": "b"}, false}, 58 | {nil, false}, 59 | } 60 | 61 | for i, test := range tests { 62 | assert.Equal(t, test.check, IsArray(test.in), "case %d : %v", i, test.in) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /core/variable.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | var ( 8 | Cacheable = true 9 | Uncacheable = false 10 | 11 | _variableFactory = &stdVariableFactory{ 12 | creators: make(map[string]VariableCreator), 13 | } 14 | ) 15 | 16 | type Variable interface { 17 | Cacheable() bool 18 | Name() string 19 | Valuer 20 | } 21 | 22 | type Valuer interface { 23 | Value(*Context) interface{} 24 | } 25 | 26 | type VariableCreator interface { 27 | Create(string) Variable 28 | } 29 | 30 | type VariableCreatorFunc func(string) Variable 31 | 32 | func (f VariableCreatorFunc) Create(name string) Variable { 33 | return f(name) 34 | } 35 | 36 | type VariableFactory interface { 37 | VariableCreator 38 | Register(VariableCreator, ...string) 39 | } 40 | 41 | // GetVariableFactory return VariableFactory 42 | func GetVariableFactory() VariableFactory { 43 | return _variableFactory 44 | } 45 | 46 | // stdVariableFactory default VariableFactory 47 | type stdVariableFactory struct { 48 | creators map[string]VariableCreator 49 | } 50 | 51 | func (self *stdVariableFactory) Register(creator VariableCreator, names ...string) { 52 | for _, name := range names { 53 | self.creators[name] = creator 54 | } 55 | } 56 | 57 | func (f *stdVariableFactory) Create(name string) Variable { 58 | if creator, ok := f.creators[name]; ok { 59 | 60 | return creator.Create(name) 61 | } else { 62 | segments := strings.Split(name, ".") 63 | if len(segments) > 1 { 64 | if creator, ok := f.creators[segments[0]+"."]; ok { 65 | return creator.Create(name) 66 | } 67 | } 68 | } 69 | 70 | return nil 71 | } 72 | 73 | // GetVariableValue get value of variable, also handlers variable cacheing, hooking logic 74 | // 75 | func GetVariableValue(ctx *Context, v Variable) interface{} { 76 | if v == nil { 77 | return "" 78 | } 79 | 80 | if v.Cacheable() { 81 | if value, ok := ctx.Cache().Load(v.Name()); ok { 82 | return value 83 | } 84 | } 85 | 86 | value := v.Value(ctx) 87 | 88 | if v.Cacheable() { 89 | ctx.Cache().Store(v.Name(), value) 90 | } 91 | 92 | return value 93 | } 94 | 95 | // ValueFunc implements Valuer interface 96 | type ValueFunc func(*Context) interface{} 97 | 98 | func (f ValueFunc) Value(ctx *Context) interface{} { 99 | return f(ctx) 100 | } 101 | 102 | // StaticValue static value implements Valuer interface 103 | type StaticValue struct { 104 | Val interface{} 105 | } 106 | 107 | func (v StaticValue) Value(_ *Context) interface{} { 108 | return v.Val 109 | } 110 | 111 | // SimpleVariable implements Variable interface 112 | type SimpleVariable struct { 113 | cacheable bool 114 | name string 115 | value Valuer 116 | } 117 | 118 | func NewSimpleVariable(name string, cacheable bool, value Valuer) Variable { 119 | return &SimpleVariable{ 120 | name: name, 121 | cacheable: cacheable, 122 | value: value, 123 | } 124 | } 125 | 126 | func (v *SimpleVariable) Name() string { 127 | return v.name 128 | } 129 | 130 | func (v *SimpleVariable) Cacheable() bool { 131 | return v.cacheable 132 | } 133 | 134 | func (v *SimpleVariable) Value(ctx *Context) interface{} { 135 | return v.value.Value(ctx) 136 | } 137 | 138 | func SingletonVariableCreator(instance Variable) VariableCreatorFunc { 139 | return func(name string) Variable { 140 | return instance 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /core/variable_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestVariable(t *testing.T) { 10 | ctx := NewContext() 11 | fooVar := NewSimpleVariable("foo", Cacheable, StaticValue{"foo"}) 12 | assert.NotNil(t, fooVar) 13 | assert.Equal(t, "foo", fooVar.Name()) 14 | assert.Equal(t, Cacheable, fooVar.Cacheable()) 15 | assert.Equal(t, "foo", fooVar.Value(ctx)) 16 | 17 | f := GetVariableFactory() 18 | f.Register(SingletonVariableCreator(fooVar), "foo", "my-foo") 19 | v1 := f.Create("foo") 20 | assert.Equal(t, fooVar, v1) 21 | v2 := f.Create("my-foo") 22 | assert.Equal(t, fooVar, v2) 23 | v3 := f.Create("your-foo") 24 | assert.Nil(t, v3) 25 | } 26 | 27 | func TestGetVariableValue(t *testing.T) { 28 | ctx := NewContext() 29 | var1Val := &StaticValue{"var1"} 30 | var1 := NewSimpleVariable("var1", Uncacheable, var1Val) 31 | assert.Equal(t, "var1", GetVariableValue(ctx, var1)) 32 | var1Val.Val = "var1-changed" 33 | assert.Equal(t, "var1-changed", GetVariableValue(ctx, var1)) 34 | 35 | var2Val := &StaticValue{"var2"} 36 | var2 := NewSimpleVariable("var2", Cacheable, var2Val) 37 | assert.Equal(t, "var2", GetVariableValue(ctx, var2)) 38 | var2Val.Val = "var2-channged" 39 | assert.Equal(t, "var2", GetVariableValue(ctx, var2)) 40 | } 41 | -------------------------------------------------------------------------------- /core/variables.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "math/rand" 5 | "strings" 6 | "time" 7 | 8 | "github.com/techxmind/go-utils/object" 9 | ) 10 | 11 | // register core varaiables 12 | // succ : bool, true 13 | // rand : int, random value in range 1 ~ 100 14 | // datetime : string, current date time with format 2006-01-02 15:04: 05 15 | // date : string, current date with format 2006-01-02 16 | // time : string, current time with format 15:04:05 17 | // year : int, current year, e.g. 2020 18 | // month : int, current month in range 1 ~ 12 19 | // day : int, current day in range 1 ~ 31 20 | // hour : int, current hour in range 0 ~ 23 21 | // minute : int, current minute in range 0 ~ 59 22 | // second : int, current second in range 0 ~ 59 23 | // unixtime : int, number of seconds since the Epoch 24 | // wday : int, the day of the week, range 1 ~ 7, Monday = 1 ... 25 | // data.xx : mixed, xx is key path to the value in data being filtered. e.g. data.foo.bar means data['foo']['bar'] 26 | // ctx.xx : mixed, like data.xx. Search value in data["ctx"] or context 27 | // e.g. ctx.foo.bar search in order: 28 | // 1. check data["ctx"]["foo"]["bar"] 29 | // 2. check context data setted by ctx.Set("foo", fooValue), check fooValue["bar"] 30 | // 3. check context data setted by context.WithValue("foo", fooValue); check fooValue["bar"] 31 | // 32 | func init() { 33 | // variable: succ 34 | _variableFactory.Register( 35 | SingletonVariableCreator(NewSimpleVariable("succ", Cacheable, &StaticValue{true})), 36 | "succ", 37 | ) 38 | 39 | rand.Seed(time.Now().UnixNano()) 40 | // variable: rand 41 | _variableFactory.Register( 42 | SingletonVariableCreator( 43 | NewSimpleVariable("rand", Uncacheable, ValueFunc(func(ctx *Context) interface{} { 44 | return rand.Intn(100) + 1 45 | })), 46 | ), 47 | "rand", 48 | ) 49 | 50 | // variable: time group... 51 | names := []string{ 52 | "datetime", "date", "time", "year", "month", "day", 53 | "hour", "minute", "second", "unixtime", "wday", 54 | } 55 | for _, name := range names { 56 | _variableFactory.Register(SingletonVariableCreator(&variableTime{name}), name) 57 | } 58 | 59 | // variable: data.xx 60 | _variableFactory.Register(VariableCreatorFunc(variableDataCreator), "data.") 61 | 62 | // variable: ctx.xx 63 | _variableFactory.Register(VariableCreatorFunc(variableCtxCreator), "ctx.") 64 | } 65 | 66 | var ( 67 | // Mock it for testing 68 | _currentTime = time.Now 69 | ) 70 | 71 | // variable: time 72 | type variableTime struct { 73 | name string 74 | } 75 | 76 | func (v *variableTime) Cacheable() bool { return false } 77 | func (v *variableTime) Name() string { return v.name } 78 | func (v *variableTime) Value(ctx *Context) interface{} { 79 | now := _currentTime() 80 | 81 | switch v.name { 82 | case "unixtime": 83 | return now.Unix() 84 | case "hour": 85 | return now.Hour() 86 | case "minute": 87 | return now.Minute() 88 | case "second": 89 | return now.Second() 90 | case "year": 91 | return now.Year() 92 | case "month": 93 | return int(now.Month()) 94 | case "day": 95 | return now.Day() 96 | case "wday": 97 | wday := int(now.Weekday()) 98 | // Set Sunday to 7 99 | if wday == 0 { 100 | wday = 7 101 | } 102 | return wday 103 | case "date": 104 | return now.Format("2006-01-02") 105 | case "time": 106 | return now.Format("15:04:05") 107 | case "datetime": 108 | fallthrough 109 | default: 110 | return now.Format("2006-01-02 15:04:05") 111 | } 112 | } 113 | 114 | // variableData access the data being filtered 115 | type variableData struct { 116 | name string 117 | key string 118 | } 119 | 120 | func (self *variableData) Cacheable() bool { return false } 121 | func (self *variableData) Name() string { return self.name } 122 | func (self *variableData) Value(ctx *Context) interface{} { 123 | if v, ok := object.GetValue(ctx.Data(), self.key); ok { 124 | return v 125 | } 126 | 127 | return nil 128 | } 129 | 130 | func variableDataCreator(name string) Variable { 131 | key := strings.TrimPrefix(name, "data.") 132 | 133 | if key == "" { 134 | return nil 135 | } 136 | 137 | return &variableData{ 138 | name: name, 139 | key: key, 140 | } 141 | } 142 | 143 | // variableCtx access context value 144 | type variableCtx struct { 145 | name string 146 | key string 147 | } 148 | 149 | func (self *variableCtx) Cacheable() bool { return false } 150 | 151 | func (self *variableCtx) Name() string { 152 | return self.name 153 | } 154 | 155 | // Context variable search value with order: 156 | // 1. check data["ctx"] data map 157 | // 2. check context data, both setted by WithValue or Set method 158 | // 159 | func (self *variableCtx) Value(ctx *Context) interface{} { 160 | 161 | // First priority: data["ctx"][key...] 162 | if value, ok := object.GetValue(ctx.Data(), "ctx."+self.key); ok { 163 | return value 164 | } 165 | 166 | // Secondary priority: from Context.Set(topKey, value) 167 | if value, ok := object.GetValue(ctx.GetAll(), self.key); ok { 168 | return value 169 | } 170 | 171 | paths := strings.Split(self.key, ".") 172 | 173 | // Default from Context.WithValue(topKey, value) 174 | if v := ctx.Value(paths[0]); v != nil { 175 | if len(paths) == 1 { 176 | return v 177 | } 178 | 179 | v, _ := object.GetValue(v, strings.Join(paths[1:], ".")) 180 | 181 | return v 182 | } 183 | 184 | return nil 185 | } 186 | 187 | func variableCtxCreator(name string) Variable { 188 | key := strings.TrimPrefix(name, "ctx.") 189 | if key == "" { 190 | return nil 191 | } 192 | return &variableCtx{ 193 | name: name, 194 | key: key, 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /core/variables_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/techxmind/go-utils/itype" 12 | ) 13 | 14 | func TestVariableSucc(t *testing.T) { 15 | v := _variableFactory.Create("succ") 16 | require.NotNil(t, v) 17 | 18 | val := GetVariableValue(NewContext(), v) 19 | assert.Equal(t, true, val) 20 | } 21 | 22 | func TestVariableRand(t *testing.T) { 23 | ctx := NewContext() 24 | v := _variableFactory.Create("rand") 25 | require.NotNil(t, v) 26 | hit := map[int64]bool{} 27 | for i := 0; i < 100; i++ { 28 | val := GetVariableValue(ctx, v) 29 | assert.GreaterOrEqual(t, val, 1) 30 | assert.LessOrEqual(t, val, 100) 31 | hit[itype.Int(val)] = true 32 | } 33 | assert.Greater(t, len(hit), 1) 34 | } 35 | 36 | func TestVariableTime(t *testing.T) { 37 | tm, _ := time.Parse(time.RFC3339, "2020-11-11T18:59:59Z") 38 | _currentTime = func() time.Time { 39 | return tm 40 | } 41 | defer func() { 42 | _currentTime = time.Now 43 | }() 44 | 45 | ctx := NewContext() 46 | tests := []struct { 47 | input string 48 | expected interface{} 49 | }{ 50 | {"datetime", "2020-11-11 18:59:59"}, 51 | {"date", "2020-11-11"}, 52 | {"year", 2020}, 53 | {"month", 11}, 54 | {"day", 11}, 55 | {"hour", 18}, 56 | {"minute", 59}, 57 | {"second", 59}, 58 | {"unixtime", tm.Unix()}, 59 | {"wday", 3}, 60 | } 61 | 62 | for i, test := range tests { 63 | v := _variableFactory.Create(test.input) 64 | require.NotNil(t, v) 65 | assert.Equal(t, test.expected, GetVariableValue(ctx, v), "case %d: %s", i, test.input) 66 | } 67 | } 68 | 69 | func TestVariableData(t *testing.T) { 70 | ctx := NewContext() 71 | ctx = WithData(ctx, map[string]interface{}{ 72 | "foo": map[string]interface{}{ 73 | "bar": []interface{}{1, "2", map[string]interface{}{ 74 | "zap": true, 75 | }}, 76 | }, 77 | }) 78 | 79 | tests := []struct { 80 | input string 81 | expected interface{} 82 | }{ 83 | {"data.foo.bar.0", 1}, 84 | {"data.foo.bar.2.zap", true}, 85 | {"data.baz", nil}, 86 | } 87 | 88 | for i, test := range tests { 89 | v := _variableFactory.Create(test.input) 90 | require.NotNil(t, v, test.input) 91 | assert.Equal(t, test.expected, GetVariableValue(ctx, v), "case %d: %s", i, test.input) 92 | } 93 | } 94 | 95 | func TestVariableCtx(t *testing.T) { 96 | ctx := WithContext(context.WithValue(context.Background(), "zap", "zap-ctx-value")) 97 | ctx = WithData(ctx, map[string]interface{}{ 98 | "ctx": map[string]interface{}{ 99 | "baz": "baz-in-data", 100 | }, 101 | }) 102 | ctx.Set("foo", map[string]interface{}{ 103 | "bar": "bar-ctx", 104 | }) 105 | ctx.Set("baz", "baz-ctx") 106 | 107 | tests := []struct { 108 | input string 109 | expected interface{} 110 | }{ 111 | {"ctx.foo.bar", "bar-ctx"}, 112 | {"ctx.baz", "baz-in-data"}, 113 | {"ctx.zap", "zap-ctx-value"}, 114 | {"ctx.other", nil}, 115 | } 116 | 117 | for i, c := range tests { 118 | v := _variableFactory.Create(c.input) 119 | require.NotNil(t, v, c.input) 120 | assert.Equal(t, c.expected, GetVariableValue(ctx, v), "case %d : %s", i, c.input) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /ext/location/variables.go: -------------------------------------------------------------------------------- 1 | // Define location variables from client ip 2 | // So you should import package that defines ip var, e.g. "github.com/techxmind/vars/request" 3 | // Variables: 4 | // country, province, city 5 | package location 6 | 7 | import ( 8 | "github.com/techxmind/filter/core" 9 | "github.com/techxmind/ip2location" 10 | ) 11 | 12 | func init() { 13 | f := core.GetVariableFactory() 14 | 15 | // variabe: country 16 | f.Register( 17 | core.SingletonVariableCreator(core.NewSimpleVariable("country", core.Cacheable, &VariableLocation{"country"})), 18 | "country", 19 | ) 20 | 21 | // variabe: province 22 | f.Register( 23 | core.SingletonVariableCreator(core.NewSimpleVariable("province", core.Cacheable, &VariableLocation{"province"})), 24 | "province", 25 | ) 26 | 27 | // variabe: city 28 | f.Register( 29 | core.SingletonVariableCreator(core.NewSimpleVariable("city", core.Cacheable, &VariableLocation{"city"})), 30 | "city", 31 | ) 32 | } 33 | 34 | var ( 35 | // mock it in unit test 36 | _getLocation = ip2location.Get 37 | _getIpVar = func() core.Variable { 38 | return core.GetVariableFactory().Create("ip") 39 | } 40 | ) 41 | 42 | type VariableLocation struct { 43 | name string 44 | } 45 | 46 | func (v *VariableLocation) Cacheable() bool { return true } 47 | func (v *VariableLocation) Name() string { return v.name } 48 | func (v *VariableLocation) Value(ctx *core.Context) interface{} { 49 | ipVar := _getIpVar() 50 | if ipVar == nil { 51 | return nil 52 | } 53 | ip, ok := core.GetVariableValue(ctx, ipVar).(string) 54 | if !ok { 55 | return nil 56 | } 57 | loc, err := _getLocation(ip) 58 | if err != nil { 59 | return nil 60 | } 61 | 62 | if v.name == "country" { 63 | return loc.Country 64 | } else if v.name == "province" { 65 | return loc.Province 66 | } else { 67 | return loc.City 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /ext/location/variables_test.go: -------------------------------------------------------------------------------- 1 | package location 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/techxmind/filter/core" 10 | "github.com/techxmind/ip2location" 11 | ) 12 | 13 | func TestLocation(t *testing.T) { 14 | f := core.GetVariableFactory() 15 | originalGetLocation := _getLocation 16 | _getLocation = func(_ string) (*ip2location.Location, error) { 17 | return &ip2location.Location{ 18 | Country: "中国", 19 | Province: "江苏省", 20 | City: "南京市", 21 | }, nil 22 | } 23 | originalGetIpVar := _getIpVar 24 | _getIpVar = func() core.Variable { 25 | return core.NewSimpleVariable("ip", core.Cacheable, &core.StaticValue{"8.8.8.8"}) 26 | } 27 | defer func() { 28 | _getLocation = originalGetLocation 29 | _getIpVar = originalGetIpVar 30 | }() 31 | 32 | ctx := core.NewContext() 33 | v := f.Create("country") 34 | require.NotNil(t, v) 35 | assert.Equal(t, "中国", core.GetVariableValue(ctx, v)) 36 | 37 | v = f.Create("province") 38 | require.NotNil(t, v) 39 | assert.Equal(t, "江苏省", core.GetVariableValue(ctx, v)) 40 | 41 | v = f.Create("city") 42 | require.NotNil(t, v) 43 | assert.Equal(t, "南京市", core.GetVariableValue(ctx, v)) 44 | } 45 | -------------------------------------------------------------------------------- /ext/request/doc.go: -------------------------------------------------------------------------------- 1 | // Variables in client-server enviroment 2 | // url : request url, setted with context.WithValue("request-url", "....") 3 | // ua : client user-agent, setted with context.WithValue("user-agent", "...") 4 | // ip : client_ip, setted with context.WithValue("client-ip", "...") 5 | // get.xxx : query value from url. e.g. url: xxx/a=1&b=1,2,3&c={"foo":[{"bar":1}]} 6 | // get.a => "1" 7 | // get.b => "1,2,3" 8 | // get.b[0] => "1" // list string or array element access 9 | // get.b[2] => "3" 10 | // get.c{foo.0.bar} => "1" // json string access, pass json path 11 | // 12 | package request 13 | -------------------------------------------------------------------------------- /ext/request/variables.go: -------------------------------------------------------------------------------- 1 | package request 2 | 3 | import ( 4 | "encoding/json" 5 | "net/url" 6 | "regexp" 7 | "strconv" 8 | 9 | "github.com/techxmind/filter/core" 10 | "github.com/techxmind/go-utils/itype" 11 | "github.com/techxmind/go-utils/object" 12 | ) 13 | 14 | const ( 15 | REQUEST_URL = "request-url" 16 | USER_AGENT = "user-agent" 17 | CLIENT_IP = "client-ip" 18 | ) 19 | 20 | func init() { 21 | f := core.GetVariableFactory() 22 | 23 | // variable: url 24 | // request url from context 25 | f.Register( 26 | core.SingletonVariableCreator(&VariableURL{REQUEST_URL}), 27 | "url", "request-url", 28 | ) 29 | 30 | // variabe: ua 31 | // user-agent from context 32 | f.Register( 33 | core.SingletonVariableCreator(core.NewSimpleVariable("ua", core.Cacheable, &ContextValue{USER_AGENT})), 34 | "ua", "user-agent", 35 | ) 36 | 37 | // variabe: ip 38 | // user-agent from context 39 | f.Register( 40 | core.SingletonVariableCreator(core.NewSimpleVariable("ip", core.Cacheable, &ContextValue{CLIENT_IP})), 41 | "ip", "client-ip", 42 | ) 43 | 44 | // variable: get.xx 45 | // value from request-url query parameters 46 | f.Register( 47 | core.VariableCreatorFunc(VariableGetCreator), 48 | "get.", "query.", 49 | ) 50 | } 51 | 52 | // ContextValue implements Valuer, get value from context 53 | type ContextValue struct { 54 | name interface{} 55 | } 56 | 57 | func (v *ContextValue) Value(ctx *core.Context) interface{} { 58 | return ctx.Value(v.name) 59 | } 60 | 61 | // VariableURL 62 | type VariableURL struct { 63 | name interface{} 64 | } 65 | 66 | func (v *VariableURL) Cacheable() bool { return true } 67 | func (v *VariableURL) Name() string { return itype.String(v.name) } 68 | func (v *VariableURL) Value(ctx *core.Context) interface{} { 69 | return ctx.Value(v.name) 70 | } 71 | func (v *VariableURL) Query(ctx *core.Context) url.Values { 72 | us, ok := v.Value(ctx).(string) 73 | 74 | if !ok || us == "" { 75 | return nil 76 | } 77 | 78 | cache := ctx.Cache() 79 | cacheID := core.CacheID("values:" + us) 80 | val, ok := cache.Load(cacheID) 81 | if ok { 82 | if values, ok := val.(url.Values); ok { 83 | return values 84 | } 85 | } 86 | 87 | u, err := url.Parse(us) 88 | if err != nil { 89 | core.Logger.Printf("Parse url err:%s %v\n", us, err) 90 | return nil 91 | } 92 | 93 | values := u.Query() 94 | cache.Store(cacheID, values) 95 | 96 | return values 97 | } 98 | 99 | // VariableQueryStr get value from url query 100 | type VariableQueryStr struct { 101 | name string 102 | paramName string 103 | queryValueGetter func(*core.Context, string) string 104 | listMode bool 105 | listIndex int 106 | jsonMode bool 107 | jsonKey string 108 | } 109 | 110 | func (self *VariableQueryStr) Cacheable() bool { return true } 111 | func (self *VariableQueryStr) Name() string { return self.name } 112 | func (self *VariableQueryStr) Value(ctx *core.Context) interface{} { 113 | value := self.queryValueGetter(ctx, self.paramName) 114 | 115 | if value == "" || (!self.listMode && !self.jsonMode) { 116 | return value 117 | } 118 | 119 | var ( 120 | ivalue interface{} = value 121 | cache = ctx.Cache() 122 | ) 123 | 124 | if self.jsonMode { 125 | var data interface{} 126 | if cacheData, ok := cache.Load("json." + self.name); ok { 127 | data = cacheData 128 | } else { 129 | if err := json.Unmarshal([]byte(value), &data); err != nil { 130 | core.Logger.Printf("json.Unmarshal url query variable[%s] err. value=%s err=%v\n", self.paramName, value, err) 131 | return "" 132 | } 133 | } 134 | if data == nil { 135 | core.Logger.Printf("Query variable[%s] json.Unmarshal get nil. value=%s", self.paramName, value) 136 | return nil 137 | } 138 | if v, ok := object.GetValue(data, self.jsonKey); ok { 139 | ivalue = v 140 | } else { 141 | return nil 142 | } 143 | } 144 | 145 | if self.listMode { 146 | arr := core.ToArray(ivalue) 147 | if self.listIndex < 0 || self.listIndex >= len(arr) { 148 | return nil 149 | } 150 | return arr[self.listIndex] 151 | } 152 | 153 | return ivalue 154 | } 155 | 156 | var _getRegexp = regexp.MustCompile("^(?:get|query).(.+?)(?:\\{([^\\}]+)\\})?(?:\\[(\\d+)\\])?$") 157 | 158 | func queryValueGetter(ctx *core.Context, name string) string { 159 | urlVar := core.GetVariableFactory().Create("url") 160 | if urlVar == nil { 161 | return "" 162 | } 163 | 164 | url, ok := urlVar.(*VariableURL) 165 | if !ok { 166 | return "" 167 | } 168 | 169 | values := url.Query(ctx) 170 | 171 | if values == nil { 172 | return "" 173 | } 174 | 175 | return values.Get(name) 176 | } 177 | 178 | func VariableGetCreator(name string) core.Variable { 179 | if ma := _getRegexp.FindStringSubmatch(name); len(ma) == 4 { 180 | obj := &VariableQueryStr{ 181 | name: ma[0], 182 | paramName: ma[1], 183 | queryValueGetter: queryValueGetter, 184 | listMode: false, 185 | listIndex: 0, 186 | jsonMode: false, 187 | jsonKey: "", 188 | } 189 | if ma[2] != "" { 190 | obj.jsonMode = true 191 | obj.jsonKey = ma[2] 192 | } 193 | if ma[3] != "" { 194 | obj.listMode = true 195 | obj.listIndex, _ = strconv.Atoi(ma[3]) 196 | } 197 | 198 | return obj 199 | } 200 | 201 | return nil 202 | } 203 | -------------------------------------------------------------------------------- /ext/request/variables_test.go: -------------------------------------------------------------------------------- 1 | package request 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/techxmind/filter/core" 11 | ) 12 | 13 | func TestAll(t *testing.T) { 14 | url := `http://www.techxmind.com/config?a=1&b=1,2,3&c={"d":[{"e":{"f":4}}, 5]}` 15 | ua := "test" 16 | ip := "8.8.8.8" 17 | 18 | c := context.WithValue( 19 | context.Background(), 20 | REQUEST_URL, 21 | url, 22 | ) 23 | c = context.WithValue( 24 | c, 25 | USER_AGENT, 26 | ua, 27 | ) 28 | c = context.WithValue( 29 | c, 30 | CLIENT_IP, 31 | ip, 32 | ) 33 | ctx := core.WithContext(c) 34 | f := core.GetVariableFactory() 35 | 36 | tests := []struct { 37 | input string 38 | expected interface{} 39 | }{ 40 | {"url", url}, 41 | {"request-url", url}, 42 | {"ua", ua}, 43 | {"user-agent", ua}, 44 | {"ip", ip}, 45 | {"client-ip", ip}, 46 | {"get.a", "1"}, 47 | {"get.b", "1,2,3"}, 48 | {"get.b[0]", "1"}, 49 | {"get.c{d.0.e.f}", 4}, 50 | } 51 | 52 | for i, c := range tests { 53 | v := f.Create(c.input) 54 | require.NotNil(t, v) 55 | assert.EqualValues(t, c.expected, core.GetVariableValue(ctx, v), "case %d: %s = %v", i, c.input, c.expected) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /filter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sort" 7 | 8 | "github.com/pkg/errors" 9 | 10 | "github.com/techxmind/filter/core" 11 | ) 12 | 13 | var ( 14 | _ = fmt.Println 15 | ) 16 | 17 | // Filter interface 18 | type Filter interface { 19 | Name() string 20 | Run(ctx context.Context, data interface{}) bool 21 | } 22 | 23 | //singleFilter contains single filter 24 | type singleFilter struct { 25 | name string 26 | condition core.Condition 27 | executor core.Executor 28 | } 29 | 30 | func (f *singleFilter) Name() string { return f.name } 31 | func (f *singleFilter) Run(pctx context.Context, data interface{}) bool { 32 | ctx := core.WithData(pctx, data) 33 | trace := ctx.Trace() 34 | 35 | if trace != nil { 36 | trace.Enter("COND") 37 | } 38 | 39 | ok := f.condition.Success(ctx) 40 | 41 | if trace != nil { 42 | trace.Leave("COND").Log("RET", ok) 43 | } 44 | 45 | if !ok { 46 | return false 47 | } 48 | 49 | if f.executor != nil { 50 | if trace != nil { 51 | trace.Enter("EXEC") 52 | } 53 | 54 | f.executor.Execute(ctx, data) 55 | 56 | if trace != nil { 57 | trace.Leave("EXEC") 58 | } 59 | } 60 | return true 61 | } 62 | 63 | // FilterGroup contains multiple filters 64 | type FilterGroup struct { 65 | name string 66 | filters []Filter 67 | // when shortMode = true, Run method will return immediately when find the first filter that return true. 68 | shortMode bool 69 | 70 | // when enableRank = true, run filters with rank order and usually is running in shortMode. 71 | enableRank bool 72 | ranks []*rank 73 | rankBoundary []*rankBoundary 74 | } 75 | 76 | func NewFilterGroup(options ...Option) *FilterGroup { 77 | opts := getFilterOpts(options) 78 | 79 | return &FilterGroup{ 80 | name: opts.name, 81 | filters: make([]Filter, 0), 82 | shortMode: opts.shortMode, 83 | enableRank: opts.enableRank, 84 | ranks: make([]*rank, 0), 85 | rankBoundary: make([]*rankBoundary, 0), 86 | } 87 | } 88 | 89 | type rank struct { 90 | idx int // index of filter 91 | weight int64 // weight of filter 92 | priority int64 // priority of filter 93 | } 94 | 95 | func (r rank) Weight() int64 { 96 | return r.weight 97 | } 98 | 99 | type rankBoundary struct { 100 | boundary int // priority boundary of rank 101 | totalWeight int64 // total weight of priority 102 | } 103 | 104 | func (f *FilterGroup) Name() string { return f.name } 105 | 106 | func (f *FilterGroup) Run(pctx context.Context, data interface{}) (succ bool) { 107 | ctx := core.WithData(pctx, data) 108 | trace := ctx.Trace() 109 | 110 | idxes := make([]int, len(f.filters)) 111 | for i, _ := range f.filters { 112 | idxes[i] = i 113 | } 114 | 115 | if trace != nil { 116 | trace.Enter("FILTER " + f.Name()) 117 | } 118 | 119 | if f.enableRank { 120 | // sort filter by priority desc 121 | for i, rank := range f.ranks { 122 | idxes[i] = rank.idx 123 | } 124 | // shuffle each priority group items by probability(weight) 125 | lastIdx := 0 126 | for _, b := range f.rankBoundary { 127 | itemCount := b.boundary - lastIdx 128 | if itemCount <= 1 { 129 | lastIdx = b.boundary 130 | continue 131 | } 132 | totalWeight := b.totalWeight 133 | items := make([]Weighter, itemCount) 134 | for i := lastIdx; i < b.boundary; i++ { 135 | items[i] = f.ranks[i] 136 | } 137 | 138 | for i := 0; i < len(items); i++ { 139 | idx := i + PickIndexByWeight(items[i:], totalWeight) 140 | items[idx], items[i] = items[i], items[idx] 141 | totalWeight -= items[i].Weight() 142 | } 143 | partIdxes := make([]int, len(items)) 144 | for i, item := range items { 145 | partIdxes[i] = item.(*rank).idx 146 | } 147 | copy(idxes[lastIdx:b.boundary], partIdxes) 148 | lastIdx = b.boundary 149 | } 150 | 151 | if trace != nil { 152 | trace.Log("RANK ", idxes) 153 | } 154 | } 155 | 156 | for _, idx := range idxes { 157 | filter := f.filters[idx] 158 | if trace != nil { 159 | trace.Enter("FILTER " + filter.Name()) 160 | } 161 | 162 | isucc := filter.Run(ctx, data) 163 | 164 | if trace != nil { 165 | trace.Leave("FILTER "+filter.Name()).Log("RET", isucc) 166 | } 167 | if isucc { 168 | succ = isucc 169 | if f.shortMode { 170 | if trace != nil { 171 | trace.Leave("END "+filter.Name()).Log("RET", succ) 172 | } 173 | return 174 | } 175 | } 176 | } 177 | 178 | if trace != nil { 179 | trace.Leave("END "+f.Name()).Log("RET", succ) 180 | } 181 | return 182 | } 183 | 184 | func (f *FilterGroup) Add(filter Filter, options ...Option) { 185 | opts := getFilterOpts(options) 186 | 187 | f.filters = append(f.filters, filter) 188 | 189 | if !f.enableRank { 190 | return 191 | } 192 | 193 | f.ranks = append(f.ranks, &rank{ 194 | idx: len(f.filters) - 1, 195 | weight: opts.weight, 196 | priority: opts.priority, 197 | }) 198 | 199 | // sort by priority desc 200 | sort.Slice(f.ranks, func(i, j int) bool { 201 | return f.ranks[i].priority > f.ranks[j].priority 202 | }) 203 | 204 | // group by priority, set group boundary and total weight for later probability calculation 205 | f.rankBoundary = f.rankBoundary[:0] 206 | lastPriority := int64(-1) 207 | totalWeight := int64(0) 208 | for i, rank := range f.ranks { 209 | if i != 0 && rank.priority != lastPriority { 210 | f.rankBoundary = append(f.rankBoundary, &rankBoundary{ 211 | boundary: i, 212 | totalWeight: totalWeight, 213 | }) 214 | totalWeight = 0 215 | } 216 | totalWeight += rank.weight 217 | lastPriority = rank.priority 218 | } 219 | f.rankBoundary = append(f.rankBoundary, &rankBoundary{ 220 | boundary: len(f.ranks), 221 | totalWeight: totalWeight, 222 | }) 223 | } 224 | 225 | type Options struct { 226 | weight int64 227 | priority int64 228 | shortMode bool 229 | enableRank bool 230 | name string 231 | namePrefix string 232 | } 233 | 234 | type Option interface { 235 | apply(*Options) 236 | } 237 | 238 | // Weight Option 239 | type Weight uint64 240 | 241 | func (o Weight) apply(opts *Options) { 242 | opts.weight = int64(o) 243 | } 244 | 245 | // Priority Option 246 | type Priority uint64 247 | 248 | func (o Priority) apply(opts *Options) { 249 | opts.priority = int64(o) 250 | } 251 | 252 | // ShortMode Option 253 | type ShortMode bool 254 | 255 | func (o ShortMode) apply(opts *Options) { 256 | opts.shortMode = bool(o) 257 | } 258 | 259 | // EnableRank Option 260 | type EnableRank bool 261 | 262 | func (o EnableRank) apply(opts *Options) { 263 | opts.enableRank = bool(o) 264 | // auto set short mode 265 | opts.shortMode = true 266 | } 267 | 268 | // Name Option 269 | type Name string 270 | 271 | func (o Name) apply(opts *Options) { 272 | opts.name = string(o) 273 | } 274 | 275 | // NamePrefix Option 276 | type NamePrefix string 277 | 278 | func (o NamePrefix) apply(opts *Options) { 279 | opts.namePrefix = string(o) 280 | } 281 | 282 | // getFilterOpts return *Options 283 | func getFilterOpts(opts []Option) *Options { 284 | o := &Options{} 285 | 286 | for _, opt := range opts { 287 | opt.apply(o) 288 | } 289 | 290 | return o 291 | } 292 | 293 | // New build filter with specified data struct. 294 | // items: 295 | // single filter: 296 | // [ 297 | // "$filter-name" // filter name, first item, optional 298 | // ["$var-name", "$op", "$op-value"], // condition 299 | // ["$var-name", "$op", "$op-value"], // condition 300 | // ["$data-key", "$assign", "$assign-value"] // executor, last item 301 | // ] 302 | // 303 | // filter group: 304 | // [ 305 | // // filter 306 | // [ 307 | // "$filter-name" // filter name, first item, optional 308 | // ["$var-name", "$op", "$op-value"], // condition 309 | // ["$var-name", "$op", "$op-value"], // condition 310 | // ["$data-key", "$assign", "$assign-value"] // executor, last item 311 | // ], 312 | // // filter 313 | // [ 314 | // "$filter-name" // filter name, first item, optional 315 | // ["$var-name", "$op", "$op-value"], // condition 316 | // ["$var-name", "$op", "$op-value"], // condition 317 | // ["$data-key", "$assign", "$assign-value"] // executor, last item 318 | // ] 319 | // ] 320 | // 321 | // options: 322 | // ShortMode(true) // enable short mode, only active in group filter 323 | // EnableRank(true) // enable rank mode, and set short mode only active in group filter 324 | // Name("filter-name") // specify filter name 325 | // 326 | func New(items []interface{}, options ...Option) (Filter, error) { 327 | if len(items) == 0 { 328 | return nil, errors.New("Empty filter") 329 | } 330 | 331 | if !core.IsArray(items[0]) { 332 | return nil, errors.New("Filter data error,first element is not array") 333 | } 334 | 335 | item := core.ToArray(items[0]) 336 | if len(item) == 0 { 337 | return nil, errors.New("Filter data error,first element is empty array") 338 | } 339 | 340 | // single filter 341 | if !core.IsArray(item[0]) { 342 | return buildFilter(items, options...) 343 | } 344 | 345 | group := NewFilterGroup(options...) 346 | 347 | if group.name == "" { 348 | group.name = generateFilterName(items) 349 | } 350 | 351 | for _, item := range items { 352 | if !core.IsArray(item) { 353 | return nil, errors.New("Filter group data error,element must be array") 354 | } 355 | if filter, err := buildFilter(core.ToArray(item), NamePrefix(group.name+".")); err != nil { 356 | return nil, err 357 | } else { 358 | group.Add(filter) 359 | } 360 | } 361 | 362 | return group, nil 363 | } 364 | 365 | // buildFilter build filter with data. 366 | // [ 367 | // "$filter-name" // filter name, first item, optional 368 | // ["$var-name", "$op", "$op-value"], // condition 369 | // ["$var-name", "$op", "$op-value"], // condition 370 | // ["$data-key", "$assign", "$assign-value"] // executor, last item 371 | // ] 372 | // 373 | func buildFilter(data []interface{}, options ...Option) (Filter, error) { 374 | if len(data) == 0 { 375 | return nil, errors.New("Filter struct is empty") 376 | } 377 | 378 | opts := getFilterOpts(options) 379 | 380 | // filter name 381 | name, ok := data[0].(string) 382 | if ok { 383 | data = data[1:] 384 | } else { 385 | if opts.name != "" { 386 | name = opts.name 387 | } else { 388 | name = generateFilterName(data) 389 | } 390 | } 391 | 392 | if opts.namePrefix != "" { 393 | name = opts.namePrefix + name 394 | } 395 | 396 | if len(data) < 2 { 397 | return nil, errors.New("Filter struct must contain conditions and assigment") 398 | } 399 | 400 | filter := &singleFilter{ 401 | name: name, 402 | } 403 | 404 | if condition, err := core.NewCondition(data[:len(data)-1], core.LOGIC_ALL); err != nil { 405 | return nil, errors.Wrap(err, "condition") 406 | } else { 407 | filter.condition = condition 408 | } 409 | 410 | if data[len(data)-1] != nil { 411 | if executor, err := core.NewExecutor(data[len(data)-1:]); err != nil { 412 | return nil, errors.Wrap(err, "executor") 413 | } else { 414 | filter.executor = executor 415 | } 416 | } 417 | 418 | return filter, nil 419 | } 420 | -------------------------------------------------------------------------------- /filter_example_test.go: -------------------------------------------------------------------------------- 1 | package filter_test 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | 9 | "github.com/techxmind/filter" 10 | "github.com/techxmind/filter/core" 11 | ) 12 | 13 | func ExampleFilterTrace() { 14 | // your business context 15 | ctx := context.Background() 16 | 17 | filterJson := ` 18 | [ 19 | [ 20 | ["ctx.foo", "in", "a,b,c"], 21 | ["ctx.bar", ">", 10], 22 | ["result-1", "=", "result-1"] 23 | ], 24 | [ 25 | ["any?", "=>", [ 26 | ["ctx.foo", "in", "a,b,c"], 27 | ["ctx.bar", ">", 10] 28 | ]], 29 | ["result-2", "=", "result-2"] 30 | ] 31 | ] 32 | ` 33 | 34 | fctx := core.WithContext(ctx, core.WithTrace(core.NewTrace(os.Stderr))) 35 | fctx.Set("bar", 11) 36 | 37 | var filterData []interface{} 38 | if err := json.Unmarshal([]byte(filterJson), &filterData); err == nil { 39 | if f, err := filter.New(filterData); err == nil { 40 | data := make(map[string]interface{}) 41 | f.Run(fctx, data) 42 | fmt.Printf("%v", data) 43 | // Output: map[result-2:result-2] 44 | } 45 | } 46 | } 47 | 48 | func ExampleDoc() { 49 | 50 | dataStr := ` 51 | { 52 | "banner" : { 53 | "type" : "image", 54 | "src" : "https://example.com/working-hard.png", 55 | "link" : "https://example.com/activity.html" 56 | }, 57 | 58 | "filter" : [ 59 | ["time", "between", "18:00,23:00"], 60 | ["ctx.user.group", "=", "programer"], 61 | ["banner", "+", { 62 | "src" : "https://example.com/chat-with-beaty.png", 63 | "link" : "https://chat.com" 64 | }] 65 | ] 66 | } 67 | ` 68 | 69 | var data map[string]interface{} 70 | json.Unmarshal([]byte(dataStr), &data) 71 | 72 | // your business context 73 | ctx := context.Background() 74 | 75 | filterCtx := core.WithContext(ctx) 76 | 77 | // you can also set context value in your business ctx with context.WithValue("group", ...) 78 | filterCtx.Set("user", map[string]interface{}{"group": "programer"}) 79 | 80 | f, _ := filter.New(data["filter"].([]interface{})) 81 | f.Run(filterCtx, data) 82 | 83 | fmt.Println(data["banner"]) 84 | } 85 | -------------------------------------------------------------------------------- /filter_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/techxmind/filter/core" 10 | "github.com/techxmind/go-utils/itype" 11 | ) 12 | 13 | func TestBuildFilter(t *testing.T) { 14 | f, err := buildFilter(arr( 15 | arr("time", "between", "10:00:00,19:00:00"), 16 | arr("hit", "=", true), 17 | )) 18 | assert.NoError(t, err) 19 | fl, ok := f.(*singleFilter) 20 | require.True(t, ok) 21 | t.Logf("filter name:%s", fl.name) 22 | assert.NotNil(t, fl.condition) 23 | assert.NotNil(t, fl.executor) 24 | 25 | f, err = buildFilter(arr( 26 | "filter-name", 27 | arr("time", "between", "10:00:00,19:00:00"), 28 | arr("hit", "=", true), 29 | )) 30 | require.NoError(t, err) 31 | assert.Equal(t, "filter-name", f.Name()) 32 | 33 | f, err = buildFilter(arr( 34 | arr("time", "between", "10:00:00,19:00:00"), 35 | )) 36 | assert.Error(t, err) 37 | 38 | f, err = buildFilter(arr( 39 | arr("time", "between", "10:00:00,19:00:00"), 40 | nil, 41 | )) 42 | require.NoError(t, err) 43 | assert.Nil(t, f.(*singleFilter).executor) 44 | 45 | f, err = buildFilter(arr( 46 | arr("time", "between", "10:00:00,19:00:00"), 47 | nil, 48 | ), Name("filter-name")) 49 | require.NoError(t, err) 50 | assert.Equal(t, "filter-name", f.Name()) 51 | 52 | f, err = buildFilter(arr( 53 | arr("ctx.foo", "=", "bar"), 54 | arr("hello", "=", "world"), 55 | ), Name("filter-name"), NamePrefix("my-")) 56 | require.NoError(t, err) 57 | assert.Equal(t, "my-filter-name", f.Name()) 58 | 59 | ctx := core.NewContext() 60 | ctx.Set("foo", "bar") 61 | data := make(map[string]interface{}) 62 | ok = f.Run(ctx, data) 63 | assert.True(t, ok) 64 | assert.Equal(t, "world", data["hello"]) 65 | } 66 | 67 | func TestNew(t *testing.T) { 68 | ctx := core.NewContext() 69 | ctx.Set("foo", "bar") 70 | ctx.Set("bar", "zap") 71 | ctx.Set("zap", "baz") 72 | data := make(map[string]interface{}) 73 | 74 | f, err := New(arr( 75 | arr("ctx.foo", "=", "bar"), 76 | arr("a", "=", 1), 77 | )) 78 | require.NoError(t, err) 79 | require.True(t, f.Run(ctx, data)) 80 | assert.Equal(t, 1, data["a"]) 81 | 82 | getItems := func(v interface{}) []interface{} { 83 | return arr( 84 | arr( 85 | arr("ctx.foo", "=", "bar"), 86 | arr("a", "=", v), 87 | ), 88 | 89 | arr( 90 | arr("ctx.bar", "=", "zap"), 91 | arr("b", "=", v), 92 | ), 93 | ) 94 | } 95 | 96 | f, err = New(getItems(2)) 97 | require.NoError(t, err) 98 | require.True(t, f.Run(ctx, data)) 99 | assert.Equal(t, 2, data["a"]) 100 | assert.Equal(t, 2, data["b"]) 101 | 102 | f, err = New(getItems(3), ShortMode(true)) 103 | require.NoError(t, err) 104 | require.True(t, f.Run(ctx, data)) 105 | assert.Equal(t, 3, data["a"]) 106 | assert.Equal(t, 2, data["b"]) 107 | 108 | f1, err := New(arr( 109 | arr("succ", "=", true), 110 | arr("a", "=", 1), 111 | )) 112 | f2, err := New(arr( 113 | arr("succ", "=", true), 114 | arr("a", "=", 2), 115 | )) 116 | f3, err := New(arr( 117 | arr("succ", "=", true), 118 | arr("a", "=", 3), 119 | )) 120 | g := NewFilterGroup(EnableRank(true)) 121 | g.Add(f1, Weight(10), Priority(3)) 122 | g.Add(f2, Weight(5), Priority(3)) 123 | g.Add(f3, Weight(100), Priority(1)) 124 | 125 | hit := make(map[int]int) 126 | for i := 1; i < 10000; i++ { 127 | data := make(map[string]interface{}) 128 | g.Run(ctx, data) 129 | v := int(itype.Int(data["a"])) 130 | hit[v] += 1 131 | } 132 | 133 | require.Equal(t, 2, len(hit), "hit.size = 2") 134 | require.True(t, hit[2] < hit[1], "hit.2 < hit.1") 135 | t.Log("hit:", hit) 136 | } 137 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/techxmind/filter 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 7 | github.com/pkg/errors v0.9.1 8 | github.com/spaolacci/murmur3 v1.1.0 9 | github.com/stretchr/testify v1.6.1 10 | github.com/techxmind/go-utils v0.0.0-20201127043211-03b94e0bd51e 11 | github.com/techxmind/ip2location v0.0.0-20201016120605-9ca74285b024 12 | ) 13 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= 4 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= 5 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 6 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 7 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 8 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 9 | github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= 10 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 11 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 12 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 13 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 14 | github.com/techxmind/go-utils v0.0.0-20201127043211-03b94e0bd51e h1:1mDmWymk9uDjI5ofH2J9stlZoNUYWWWFDqcV2ksMxRk= 15 | github.com/techxmind/go-utils v0.0.0-20201127043211-03b94e0bd51e/go.mod h1:F8DtfFXxjw9sDArW6D/kbA83e0tjkCayyOSRCPEX+YQ= 16 | github.com/techxmind/ip2location v0.0.0-20201016120605-9ca74285b024 h1:kj2r55FtGEUQ4ixRefHAwf2aNm7+9gSFk/4zRMeVyEo= 17 | github.com/techxmind/ip2location v0.0.0-20201016120605-9ca74285b024/go.mod h1:RGFFgdoTk3ygSslSF5fqGfGC+3jlIGyzl/ScewTFoBM= 18 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 19 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 20 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 21 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 22 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "encoding/json" 5 | "math/rand" 6 | "strconv" 7 | 8 | "github.com/techxmind/filter/core" 9 | ) 10 | 11 | func generateFilterName(filter interface{}) string { 12 | v, _ := json.Marshal(filter) 13 | return strconv.FormatUint(core.HashID(string(v)), 36) 14 | } 15 | 16 | type Weighter interface { 17 | Weight() int64 18 | } 19 | 20 | func PickIndexByWeight(items []Weighter, totalWeight int64) int { 21 | if totalWeight == 0 { 22 | for _, item := range items { 23 | totalWeight += item.Weight() 24 | } 25 | } 26 | 27 | if totalWeight == 0 { 28 | return 0 29 | } 30 | 31 | choose := rand.Int63n(totalWeight) + 1 32 | line := int64(0) 33 | 34 | for i, b := range items { 35 | line += b.Weight() 36 | if choose <= line { 37 | return i 38 | } 39 | } 40 | 41 | return 0 42 | } 43 | 44 | // help func to create []interface{} for unit test 45 | func arr(vals ...interface{}) []interface{} { 46 | return core.ToArray(vals) 47 | } 48 | -------------------------------------------------------------------------------- /utils_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestPickIndexByWeight(t *testing.T) { 10 | items := []Weighter{ 11 | rank{weight: 10}, 12 | rank{weight: 30}, 13 | rank{weight: 60}, 14 | } 15 | 16 | hit := make(map[int64]int) 17 | 18 | for i := 0; i < 10000; i++ { 19 | index := PickIndexByWeight(items, 0) 20 | hit[items[index].Weight()] += 1 21 | } 22 | 23 | assert.Equal(t, 3, len(hit), "hit.size = 3") 24 | assert.True(t, hit[60] > hit[30], "hit.60 > hit.30") 25 | assert.True(t, hit[30] > hit[10], "hit.30 > hit.10") 26 | t.Log("hit:", hit) 27 | } 28 | --------------------------------------------------------------------------------