├── test ├── load_conf │ ├── flow │ │ ├── flow-StuAvg.yml │ │ ├── flow-FlowName2.yml │ │ ├── flow-FlowName5.yml │ │ ├── flow-FlowName3.yml │ │ ├── flow-FlowName4.yml │ │ ├── flow-FlowFork1.yml │ │ └── flow-FlowName1.yml │ ├── func │ │ ├── func-AvgStuScore.yml │ │ ├── func-jumpFunc.yml │ │ ├── func-AbortFunc.yml │ │ ├── func-NoResultFunc.yml │ │ ├── func-PrintStuAvgScore.yml │ │ ├── func-dataReuseFunc.yml │ │ ├── func-FuncName1.yml │ │ ├── func-FuncName3.yml │ │ └── func-FuncName2.yml │ ├── conn │ │ └── conn-ConnName1.yml │ └── kis-flow.yml ├── export_conf │ ├── flow-flowName1.yaml │ ├── conn-ConnName1.yaml │ ├── func-funcName1.yaml │ ├── func-funcName2.yaml │ └── func-funcName3.yaml ├── prometheus_server_test.go ├── proto │ └── stu_score.go ├── caas │ ├── caas_init1.go │ └── caas_demo1.go ├── kis_log_test.go ├── faas │ ├── faas_abort.go │ ├── faas_jump.go │ ├── faas_no_result.go │ ├── faas_demo3.go │ ├── faas_stu_score_avg_print.go │ ├── faas_demo4.go │ ├── faas_data_reuse.go │ ├── faas_demo1.go │ ├── faas_stu_score_avg.go │ └── faas_demo2.go ├── kis_params_test.go ├── kis_config_export_test.go ├── kis_metrics_test.go ├── kis_config_import_test.go ├── kis_flow_commit_batch_test.go ├── init.go ├── kis_function_test.go ├── kis_pool_test.go ├── kis_fork_test.go ├── kis_config_test.go ├── kis_connector_test.go ├── kis_action_test.go ├── kis_flow_test.go └── kis_auto_inject_param_test.go ├── common ├── data_type.go └── const.go ├── config ├── kis_global_config.go ├── kis_flow_config.go ├── kis_conn_config.go └── kis_func_config.go ├── go.mod ├── id └── kis_id.go ├── function ├── kis_function_c.go ├── kis_function_e.go ├── kis_function_l.go ├── kis_function_s.go ├── kis_function_v.go └── kis_base_function.go ├── .github └── FUNDING.yml ├── kis ├── serialize.go ├── connector.go ├── router.go ├── action.go ├── function.go ├── flow.go ├── faas.go └── pool.go ├── LICENSE ├── log ├── kis_log.go └── kis_default_log.go ├── file ├── config_export.go └── config_import.go ├── flow ├── kis_flow_action.go ├── kis_flow_data.go └── kis_flow.go ├── conn └── kis_connector.go ├── metrics └── kis_metrics.go ├── serialize └── serialize_default.go ├── README-CN.md └── README.md /test/load_conf/flow/flow-StuAvg.yml: -------------------------------------------------------------------------------- 1 | kistype: flow 2 | status: 1 3 | flow_name: StuAvg 4 | flows: 5 | - fname: AvgStuScore 6 | - fname: PrintStuAvgScore 7 | -------------------------------------------------------------------------------- /test/load_conf/flow/flow-FlowName2.yml: -------------------------------------------------------------------------------- 1 | kistype: flow 2 | status: 1 3 | flow_name: flowName2 4 | flows: 5 | - fname: funcName1 6 | - fname: abortFunc 7 | - fname: funcName3 8 | -------------------------------------------------------------------------------- /test/load_conf/flow/flow-FlowName5.yml: -------------------------------------------------------------------------------- 1 | kistype: flow 2 | status: 1 3 | flow_name: flowName5 4 | flows: 5 | - fname: funcName1 6 | - fname: funcName2 7 | - fname: jumpFunc 8 | -------------------------------------------------------------------------------- /test/load_conf/func/func-AvgStuScore.yml: -------------------------------------------------------------------------------- 1 | kistype: func 2 | fname: AvgStuScore 3 | fmode: Calculate 4 | source: 5 | name: StudentAverageScore 6 | must: 7 | - stu_id 8 | -------------------------------------------------------------------------------- /test/load_conf/func/func-jumpFunc.yml: -------------------------------------------------------------------------------- 1 | kistype: func 2 | fname: jumpFunc 3 | fmode: Calculate 4 | source: 5 | name: UserOrderErrorRate 6 | must: 7 | - order_id 8 | - user_id -------------------------------------------------------------------------------- /test/load_conf/flow/flow-FlowName3.yml: -------------------------------------------------------------------------------- 1 | kistype: flow 2 | status: 1 3 | flow_name: flowName3 4 | flows: 5 | - fname: funcName1 6 | - fname: dataReuseFunc 7 | - fname: funcName3 8 | -------------------------------------------------------------------------------- /test/load_conf/flow/flow-FlowName4.yml: -------------------------------------------------------------------------------- 1 | kistype: flow 2 | status: 1 3 | flow_name: flowName4 4 | flows: 5 | - fname: funcName1 6 | - fname: noResultFunc 7 | - fname: funcName3 8 | -------------------------------------------------------------------------------- /test/load_conf/func/func-AbortFunc.yml: -------------------------------------------------------------------------------- 1 | kistype: func 2 | fname: abortFunc 3 | fmode: Calculate 4 | source: 5 | name: UserOrderErrorRate 6 | must: 7 | - order_id 8 | - user_id -------------------------------------------------------------------------------- /test/load_conf/func/func-NoResultFunc.yml: -------------------------------------------------------------------------------- 1 | kistype: func 2 | fname: noResultFunc 3 | fmode: Calculate 4 | source: 5 | name: UserOrderErrorRate 6 | must: 7 | - order_id 8 | - user_id -------------------------------------------------------------------------------- /test/load_conf/func/func-PrintStuAvgScore.yml: -------------------------------------------------------------------------------- 1 | kistype: func 2 | fname: PrintStuAvgScore 3 | fmode: Expand 4 | source: 5 | name: StudentAverageScore 6 | must: 7 | - stu_id 8 | -------------------------------------------------------------------------------- /test/load_conf/func/func-dataReuseFunc.yml: -------------------------------------------------------------------------------- 1 | kistype: func 2 | fname: dataReuseFunc 3 | fmode: Calculate 4 | source: 5 | name: UserOrderErrorRate 6 | must: 7 | - order_id 8 | - user_id -------------------------------------------------------------------------------- /test/export_conf/flow-flowName1.yaml: -------------------------------------------------------------------------------- 1 | kistype: flow 2 | status: 1 3 | flow_name: flowName1 4 | flows: 5 | - fname: funcName1 6 | params: {} 7 | - fname: funcName2 8 | params: {} 9 | - fname: funcName3 10 | params: {} 11 | -------------------------------------------------------------------------------- /test/load_conf/flow/flow-FlowFork1.yml: -------------------------------------------------------------------------------- 1 | kistype: flow 2 | status: 1 3 | flow_name: flowFork1 4 | flows: 5 | - fname: funcName1 6 | params: 7 | myKey1: flowValue1-1 8 | - fname: funcName3 9 | params: 10 | myKey1: flowValue3-1 11 | -------------------------------------------------------------------------------- /test/load_conf/conn/conn-ConnName1.yml: -------------------------------------------------------------------------------- 1 | kistype: conn 2 | cname: ConnName1 3 | addrs: '0.0.0.0:9988,0.0.0.0:9999,0.0.0.0:9990' 4 | type: redis 5 | key: redis-key 6 | params: 7 | args1: value1 8 | args2: value2 9 | load: null 10 | save: 11 | - funcName2 -------------------------------------------------------------------------------- /test/export_conf/conn-ConnName1.yaml: -------------------------------------------------------------------------------- 1 | kistype: conn 2 | cname: ConnName1 3 | addrs: 0.0.0.0:9988,0.0.0.0:9999,0.0.0.0:9990 4 | type: redis 5 | key: redis-key 6 | params: 7 | args1: value1 8 | args2: value2 9 | load: [] 10 | save: 11 | - funcName2 12 | - funcName2 13 | -------------------------------------------------------------------------------- /test/load_conf/func/func-FuncName1.yml: -------------------------------------------------------------------------------- 1 | kistype: func 2 | fname: funcName1 3 | fmode: Verify 4 | source: 5 | name: TiktokOrder 6 | must: 7 | - order_id 8 | - user_id 9 | option: 10 | default_params: 11 | default1: funcName1_param1 12 | default2: funcName1_param2 13 | -------------------------------------------------------------------------------- /test/load_conf/func/func-FuncName3.yml: -------------------------------------------------------------------------------- 1 | kistype: func 2 | fname: funcName3 3 | fmode: Calculate 4 | source: 5 | name: UserOrderErrorRate 6 | must: 7 | - order_id 8 | - user_id 9 | option: 10 | default_params: 11 | default1: funcName3_param1 12 | default2: funcName3_param2 13 | -------------------------------------------------------------------------------- /test/export_conf/func-funcName1.yaml: -------------------------------------------------------------------------------- 1 | kistype: func 2 | fname: funcName1 3 | fmode: Verify 4 | source: 5 | name: TikTok-Order 6 | must: 7 | - order_id 8 | - user_id 9 | option: 10 | cname: "" 11 | retry_times: 0 12 | return_duration: 0 13 | default_params: {} 14 | -------------------------------------------------------------------------------- /test/prometheus_server_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aceld/kis-flow/metrics" 7 | ) 8 | 9 | func TestPrometheusServer(t *testing.T) { 10 | 11 | err := metrics.RunMetricsService("0.0.0.0:20004") 12 | if err != nil { 13 | panic(err) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /test/export_conf/func-funcName2.yaml: -------------------------------------------------------------------------------- 1 | kistype: func 2 | fname: funcName2 3 | fmode: Save 4 | source: 5 | name: UserOrderErrorRate 6 | must: 7 | - order_id 8 | - user_id 9 | option: 10 | cname: ConnName1 11 | retry_times: 0 12 | return_duration: 0 13 | default_params: {} 14 | -------------------------------------------------------------------------------- /test/export_conf/func-funcName3.yaml: -------------------------------------------------------------------------------- 1 | kistype: func 2 | fname: funcName3 3 | fmode: Calculate 4 | source: 5 | name: UserOrderErrorRate 6 | must: 7 | - order_id 8 | - user_id 9 | option: 10 | cname: "" 11 | retry_times: 0 12 | return_duration: 0 13 | default_params: {} 14 | -------------------------------------------------------------------------------- /test/load_conf/func/func-FuncName2.yml: -------------------------------------------------------------------------------- 1 | kistype: func 2 | fname: funcName2 3 | fmode: Save 4 | source: 5 | name: UserOrderErrorRate 6 | must: 7 | - order_id 8 | - user_id 9 | option: 10 | cname: ConnName1 11 | default_params: 12 | default1: funcName2_param1 13 | default2: funcName2_param2 14 | -------------------------------------------------------------------------------- /test/load_conf/kis-flow.yml: -------------------------------------------------------------------------------- 1 | # kistype Global is the global configuration for kisflow 2 | kistype: global 3 | # Whether to enable prometheus monitoring 4 | prometheus_enable: true 5 | # Whether kisflow needs to listen on a separate port 6 | prometheus_listen: true 7 | # Prometheus scrape address 8 | prometheus_serve: 0.0.0.0:20004 -------------------------------------------------------------------------------- /test/proto/stu_score.go: -------------------------------------------------------------------------------- 1 | package proto 2 | 3 | type StuScores struct { 4 | StuId int `json:"stu_id"` 5 | Score1 int `json:"score_1"` 6 | Score2 int `json:"score_2"` 7 | Score3 int `json:"score_3"` 8 | } 9 | 10 | type StuAvgScore struct { 11 | StuId int `json:"stu_id"` 12 | AvgScore float64 `json:"avg_score"` 13 | } 14 | -------------------------------------------------------------------------------- /test/load_conf/flow/flow-FlowName1.yml: -------------------------------------------------------------------------------- 1 | kistype: flow 2 | status: 1 3 | flow_name: flowName1 4 | flows: 5 | - fname: funcName1 6 | params: 7 | myKey1: flowValue1-1 8 | myKey2: flowValue1-2 9 | - fname: funcName2 10 | params: 11 | myKey1: flowValue2-1 12 | myKey2: flowValue2-2 13 | - fname: funcName3 14 | params: 15 | myKey1: flowValue3-1 16 | myKey2: flowValue3-2 17 | -------------------------------------------------------------------------------- /common/data_type.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | // KisRow represents a single row of data 4 | type KisRow interface{} 5 | 6 | // KisRowArr represents a batch of data for a single business operation 7 | type KisRowArr []KisRow 8 | 9 | // KisDataMap contains all the data carried by the current Flow 10 | // key : Function ID where the data resides 11 | // value: Corresponding KisRow 12 | type KisDataMap map[string]KisRowArr 13 | -------------------------------------------------------------------------------- /test/caas/caas_init1.go: -------------------------------------------------------------------------------- 1 | package caas 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/aceld/kis-flow/kis" 7 | ) 8 | 9 | // type ConnInit func(conn Connector) error 10 | 11 | func InitConnDemo1(connector kis.Connector) error { 12 | fmt.Println("===> Call Connector InitDemo1") 13 | //config info 14 | connConf := connector.GetConfig() 15 | 16 | fmt.Println(connConf) 17 | 18 | // init connector 19 | 20 | return nil 21 | } 22 | -------------------------------------------------------------------------------- /test/kis_log_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aceld/kis-flow/log" 8 | ) 9 | 10 | func TestKisLogger(t *testing.T) { 11 | ctx := context.Background() 12 | 13 | log.Logger().InfoFX(ctx, "TestKisLogger InfoFX") 14 | log.Logger().ErrorFX(ctx, "TestKisLogger ErrorFX") 15 | log.Logger().DebugFX(ctx, "TestKisLogger DebugFX") 16 | 17 | log.Logger().InfoF("TestKisLogger InfoF") 18 | log.Logger().ErrorF("TestKisLogger ErrorF") 19 | log.Logger().DebugF("TestKisLogger DebugF") 20 | } 21 | -------------------------------------------------------------------------------- /test/faas/faas_abort.go: -------------------------------------------------------------------------------- 1 | package faas 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/aceld/kis-flow/kis" 8 | ) 9 | 10 | // type FaaS func(context.Context, Flow) error 11 | 12 | func AbortFuncHandler(ctx context.Context, flow kis.Flow) error { 13 | fmt.Println("---> Call AbortFuncHandler ----") 14 | 15 | for _, row := range flow.Input() { 16 | str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row) 17 | fmt.Println(str) 18 | } 19 | 20 | return flow.Next(kis.ActionAbort) 21 | } 22 | -------------------------------------------------------------------------------- /test/faas/faas_jump.go: -------------------------------------------------------------------------------- 1 | package faas 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/aceld/kis-flow/kis" 8 | ) 9 | 10 | // type FaaS func(context.Context, Flow) error 11 | 12 | func JumpFuncHandler(ctx context.Context, flow kis.Flow) error { 13 | fmt.Println("---> Call JumpFuncHandler ----") 14 | 15 | for _, row := range flow.Input() { 16 | str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row) 17 | fmt.Println(str) 18 | } 19 | 20 | return flow.Next(kis.ActionJumpFunc("funcName1")) 21 | } 22 | -------------------------------------------------------------------------------- /test/faas/faas_no_result.go: -------------------------------------------------------------------------------- 1 | package faas 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/aceld/kis-flow/kis" 8 | ) 9 | 10 | // type FaaS func(context.Context, Flow) error 11 | 12 | func NoResultFuncHandler(ctx context.Context, flow kis.Flow) error { 13 | fmt.Println("---> Call NoResultFuncHandler ----") 14 | 15 | for _, row := range flow.Input() { 16 | str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row) 17 | fmt.Println(str) 18 | } 19 | 20 | return flow.Next(kis.ActionForceEntryNext) 21 | } 22 | -------------------------------------------------------------------------------- /test/faas/faas_demo3.go: -------------------------------------------------------------------------------- 1 | package faas 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/aceld/kis-flow/kis" 8 | ) 9 | 10 | // type FaaS func(context.Context, Flow) error 11 | 12 | func FuncDemo3Handler(ctx context.Context, flow kis.Flow) error { 13 | fmt.Println("---> Call funcName3Handler ----") 14 | fmt.Printf("Params = %+v\n", flow.GetFuncParamAll()) 15 | 16 | for _, row := range flow.Input() { 17 | str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row) 18 | fmt.Println(str) 19 | } 20 | 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /test/kis_params_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aceld/kis-flow/file" 8 | "github.com/aceld/kis-flow/kis" 9 | ) 10 | 11 | func TestParams(t *testing.T) { 12 | ctx := context.Background() 13 | 14 | if err := file.ConfigImportYaml("load_conf/"); err != nil { 15 | panic(err) 16 | } 17 | 18 | flow1 := kis.Pool().GetFlow("flowName1") 19 | 20 | _ = flow1.CommitRow("This is Data1 from Test") 21 | _ = flow1.CommitRow("This is Data2 from Test") 22 | _ = flow1.CommitRow("This is Data3 from Test") 23 | 24 | if err := flow1.Run(ctx); err != nil { 25 | panic(err) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /test/faas/faas_stu_score_avg_print.go: -------------------------------------------------------------------------------- 1 | package faas 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/aceld/kis-flow/kis" 8 | "github.com/aceld/kis-flow/serialize" 9 | "github.com/aceld/kis-flow/test/proto" 10 | ) 11 | 12 | type PrintStuAvgScoreIn struct { 13 | serialize.DefaultSerialize 14 | proto.StuAvgScore 15 | } 16 | 17 | type PrintStuAvgScoreOut struct { 18 | serialize.DefaultSerialize 19 | } 20 | 21 | func PrintStuAvgScore(ctx context.Context, flow kis.Flow, rows []*PrintStuAvgScoreIn) error { 22 | 23 | for _, row := range rows { 24 | fmt.Printf("stuid: [%+v], avg score: [%+v]\n", row.StuId, row.AvgScore) 25 | } 26 | 27 | return nil 28 | } 29 | -------------------------------------------------------------------------------- /test/faas/faas_demo4.go: -------------------------------------------------------------------------------- 1 | package faas 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/aceld/kis-flow/kis" 8 | ) 9 | 10 | // type FaaS func(context.Context, Flow) error 11 | 12 | func FuncDemo4Handler(ctx context.Context, flow kis.Flow) error { 13 | fmt.Println("---> Call FuncDemo4Handler ----") 14 | 15 | for index, row := range flow.Input() { 16 | str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row) 17 | fmt.Println(str) 18 | 19 | resultStr := fmt.Sprintf("data from funcName[%s], index = %d", flow.GetThisFuncConf().FName, index) 20 | 21 | _ = flow.CommitRow(resultStr) 22 | } 23 | 24 | return nil 25 | } 26 | -------------------------------------------------------------------------------- /test/kis_config_export_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/aceld/kis-flow/file" 8 | "github.com/aceld/kis-flow/kis" 9 | ) 10 | 11 | func TestConfigExportYaml(t *testing.T) { 12 | 13 | // 1. Load the configuration file and build the Flow 14 | if err := file.ConfigImportYaml("load_conf/"); err != nil { 15 | fmt.Println("Wrong Config Yaml Path!") 16 | panic(err) 17 | } 18 | 19 | // 2. Export the built memory KisFlow structure configuration to files 20 | flows := kis.Pool().GetFlows() 21 | for _, flow := range flows { 22 | if err := file.ConfigExportYaml(flow, "/Users/Aceld/go/src/kis-flow/test/export_conf/"); err != nil { 23 | panic(err) 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /test/caas/caas_demo1.go: -------------------------------------------------------------------------------- 1 | package caas 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/aceld/kis-flow/kis" 8 | ) 9 | 10 | // type CaaS func(context.Context, Connector, Function, Flow, interface{}) (interface{}, error) 11 | 12 | func CaasDemoHanler1(ctx context.Context, conn kis.Connector, fn kis.Function, flow kis.Flow, args interface{}) (interface{}, error) { 13 | fmt.Printf("===> In CaasDemoHanler1: flowName: %s, cName:%s, fnName:%s, mode:%s\n", 14 | flow.GetName(), conn.GetName(), fn.GetConfig().FName, fn.GetConfig().FMode) 15 | 16 | fmt.Printf("Params = %+v\n", conn.GetConfig().Params) 17 | 18 | fmt.Printf("===> Call Connector CaasDemoHanler1, args from funciton: %s\n", args) 19 | 20 | return nil, nil 21 | } 22 | -------------------------------------------------------------------------------- /test/kis_metrics_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | "github.com/aceld/kis-flow/file" 10 | "github.com/aceld/kis-flow/kis" 11 | ) 12 | 13 | func TestMetricsDataTotal(t *testing.T) { 14 | ctx := context.Background() 15 | 16 | if err := file.ConfigImportYaml("load_conf/"); err != nil { 17 | fmt.Println("Wrong Config Yaml Path!") 18 | panic(err) 19 | } 20 | 21 | flow1 := kis.Pool().GetFlow("flowName1") 22 | 23 | n := 0 24 | 25 | for n < 10 { 26 | _ = flow1.CommitRow("This is Data1 from Test") 27 | 28 | if err := flow1.Run(ctx); err != nil { 29 | panic(err) 30 | } 31 | 32 | time.Sleep(1 * time.Second) 33 | n++ 34 | } 35 | 36 | select {} 37 | } 38 | -------------------------------------------------------------------------------- /test/faas/faas_data_reuse.go: -------------------------------------------------------------------------------- 1 | package faas 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/aceld/kis-flow/kis" 8 | ) 9 | 10 | // type FaaS func(context.Context, Flow) error 11 | 12 | func DataReuseFuncHandler(ctx context.Context, flow kis.Flow) error { 13 | fmt.Println("---> Call DataReuseFuncHandler ----") 14 | 15 | for index, row := range flow.Input() { 16 | str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row) 17 | fmt.Println(str) 18 | 19 | resultStr := fmt.Sprintf("data from funcName[%s], index = %d", flow.GetThisFuncConf().FName, index) 20 | 21 | _ = flow.CommitRow(resultStr) 22 | } 23 | 24 | return flow.Next(kis.ActionDataReuse) 25 | } 26 | -------------------------------------------------------------------------------- /config/kis_global_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // KisGlobalConfig represents the global configuration for KisFlow 4 | type KisGlobalConfig struct { 5 | // KisType Global is the global configuration for kisflow 6 | KisType string `yaml:"kistype"` 7 | // EnableProm indicates whether to start Prometheus monitoring 8 | EnableProm bool `yaml:"prometheus_enable"` 9 | // PrometheusListen indicates whether kisflow needs to start a separate port for listening 10 | PrometheusListen bool `yaml:"prometheus_listen"` 11 | // PrometheusServe is the address for Prometheus scraping 12 | PrometheusServe string `yaml:"prometheus_serve"` 13 | } 14 | 15 | // GlobalConfig is the default global configuration, all are set to off 16 | var GlobalConfig = new(KisGlobalConfig) 17 | -------------------------------------------------------------------------------- /test/faas/faas_demo1.go: -------------------------------------------------------------------------------- 1 | package faas 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/aceld/kis-flow/kis" 8 | ) 9 | 10 | // type FaaS func(context.Context, Flow) error 11 | 12 | func FuncDemo1Handler(ctx context.Context, flow kis.Flow) error { 13 | fmt.Println("---> Call funcName1Handler ----") 14 | fmt.Printf("Params = %+v\n", flow.GetFuncParamAll()) 15 | 16 | for index, row := range flow.Input() { 17 | str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row) 18 | fmt.Println(str) 19 | 20 | resultStr := fmt.Sprintf("data from funcName[%s], index = %d", flow.GetThisFuncConf().FName, index) 21 | 22 | _ = flow.CommitRow(resultStr) 23 | } 24 | 25 | return nil 26 | } 27 | -------------------------------------------------------------------------------- /test/kis_config_import_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aceld/kis-flow/file" 8 | "github.com/aceld/kis-flow/kis" 9 | ) 10 | 11 | func TestConfigImportYaml(t *testing.T) { 12 | ctx := context.Background() 13 | 14 | // 1. Load the configuration file and build the Flow 15 | if err := file.ConfigImportYaml("load_conf/"); err != nil { 16 | panic(err) 17 | } 18 | 19 | // 2. Get the Flow 20 | flow1 := kis.Pool().GetFlow("flowName1") 21 | 22 | // 3. Commit the raw data 23 | _ = flow1.CommitRow("This is Data1 from Test") 24 | _ = flow1.CommitRow("This is Data2 from Test") 25 | _ = flow1.CommitRow("This is Data3 from Test") 26 | 27 | // 4. Execute flow1 28 | if err := flow1.Run(ctx); err != nil { 29 | panic(err) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/aceld/kis-flow 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/google/uuid v1.6.0 7 | github.com/patrickmn/go-cache v2.1.0+incompatible 8 | github.com/prometheus/client_golang v1.14.0 9 | gopkg.in/yaml.v3 v3.0.1 10 | ) 11 | 12 | require ( 13 | github.com/beorn7/perks v1.0.1 // indirect 14 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 15 | github.com/golang/protobuf v1.5.2 // indirect 16 | github.com/google/go-cmp v0.6.0 // indirect 17 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 18 | github.com/prometheus/client_model v0.3.0 // indirect 19 | github.com/prometheus/common v0.37.0 // indirect 20 | github.com/prometheus/procfs v0.8.0 // indirect 21 | golang.org/x/sys v0.19.0 // indirect 22 | google.golang.org/protobuf v1.28.1 // indirect 23 | ) 24 | -------------------------------------------------------------------------------- /id/kis_id.go: -------------------------------------------------------------------------------- 1 | package id 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/aceld/kis-flow/common" 7 | "github.com/google/uuid" 8 | ) 9 | 10 | // KisID generates a random instance ID. 11 | // The format is "prefix1-[prefix2-][prefix3-]ID" 12 | // Example: flow-1234567890 13 | // Example: func-1234567890 14 | // Example: conn-1234567890 15 | // Example: func-1-1234567890 16 | func KisID(prefix ...string) (kisId string) { 17 | 18 | idStr := strings.Replace(uuid.New().String(), "-", "", -1) 19 | kisId = formatKisID(idStr, prefix...) 20 | 21 | return 22 | } 23 | 24 | func formatKisID(idStr string, prefix ...string) string { 25 | var kisId string 26 | 27 | for _, fix := range prefix { 28 | kisId += fix 29 | kisId += common.KisIDJoinChar 30 | } 31 | 32 | kisId += idStr 33 | 34 | return kisId 35 | } 36 | -------------------------------------------------------------------------------- /function/kis_function_c.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aceld/kis-flow/kis" 7 | "github.com/aceld/kis-flow/log" 8 | ) 9 | 10 | type KisFunctionC struct { 11 | BaseFunction 12 | } 13 | 14 | func NewKisFunctionC() kis.Function { 15 | f := new(KisFunctionC) 16 | 17 | // Initialize metaData 18 | f.metaData = make(map[string]interface{}) 19 | 20 | return f 21 | } 22 | 23 | func (f *KisFunctionC) Call(ctx context.Context, flow kis.Flow) error { 24 | log.Logger().DebugF("KisFunctionC, flow = %+v\n", flow) 25 | 26 | // Route to the specific computing Function through KisPool 27 | if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil { 28 | log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err) 29 | return err 30 | } 31 | 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /function/kis_function_e.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aceld/kis-flow/kis" 7 | "github.com/aceld/kis-flow/log" 8 | ) 9 | 10 | type KisFunctionE struct { 11 | BaseFunction 12 | } 13 | 14 | func NewKisFunctionE() kis.Function { 15 | f := new(KisFunctionE) 16 | 17 | // Initialize metaData 18 | f.metaData = make(map[string]interface{}) 19 | 20 | return f 21 | } 22 | 23 | func (f *KisFunctionE) Call(ctx context.Context, flow kis.Flow) error { 24 | log.Logger().DebugF("KisFunctionE, flow = %+v\n", flow) 25 | 26 | // Route to the specific computing Function through KisPool 27 | if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil { 28 | log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err) 29 | return err 30 | } 31 | 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /function/kis_function_l.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aceld/kis-flow/kis" 7 | "github.com/aceld/kis-flow/log" 8 | ) 9 | 10 | type KisFunctionL struct { 11 | BaseFunction 12 | } 13 | 14 | func NewKisFunctionL() kis.Function { 15 | f := new(KisFunctionL) 16 | 17 | // Initialize metaData 18 | f.metaData = make(map[string]interface{}) 19 | 20 | return f 21 | } 22 | 23 | func (f *KisFunctionL) Call(ctx context.Context, flow kis.Flow) error { 24 | log.Logger().DebugF("KisFunctionL, flow = %+v\n", flow) 25 | 26 | // Route to the specific computing Function through KisPool 27 | if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil { 28 | log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err) 29 | return err 30 | } 31 | 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /function/kis_function_s.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aceld/kis-flow/kis" 7 | "github.com/aceld/kis-flow/log" 8 | ) 9 | 10 | type KisFunctionS struct { 11 | BaseFunction 12 | } 13 | 14 | func NewKisFunctionS() kis.Function { 15 | f := new(KisFunctionS) 16 | 17 | // Initialize metaData 18 | f.metaData = make(map[string]interface{}) 19 | 20 | return f 21 | } 22 | 23 | func (f *KisFunctionS) Call(ctx context.Context, flow kis.Flow) error { 24 | log.Logger().DebugF("KisFunctionS, flow = %+v\n", flow) 25 | 26 | // Route to the specific computing Function through KisPool 27 | if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil { 28 | log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err) 29 | return err 30 | } 31 | 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /function/kis_function_v.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aceld/kis-flow/kis" 7 | "github.com/aceld/kis-flow/log" 8 | ) 9 | 10 | type KisFunctionV struct { 11 | BaseFunction 12 | } 13 | 14 | func NewKisFunctionV() kis.Function { 15 | f := new(KisFunctionV) 16 | 17 | // Initialize metaData 18 | f.metaData = make(map[string]interface{}) 19 | 20 | return f 21 | } 22 | 23 | func (f *KisFunctionV) Call(ctx context.Context, flow kis.Flow) error { 24 | log.Logger().DebugF("KisFunctionV, flow = %+v\n", flow) 25 | 26 | // Route to the specific computing Function through KisPool 27 | if err := kis.Pool().CallFunction(ctx, f.Config.FName, flow); err != nil { 28 | log.Logger().ErrorFX(ctx, "Function Called Error err = %s\n", err) 29 | return err 30 | } 31 | 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /test/faas/faas_stu_score_avg.go: -------------------------------------------------------------------------------- 1 | package faas 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aceld/kis-flow/kis" 7 | "github.com/aceld/kis-flow/serialize" 8 | "github.com/aceld/kis-flow/test/proto" 9 | ) 10 | 11 | type AvgStuScoreIn struct { 12 | serialize.DefaultSerialize 13 | proto.StuScores 14 | } 15 | 16 | type AvgStuScoreOut struct { 17 | serialize.DefaultSerialize 18 | proto.StuAvgScore 19 | } 20 | 21 | // AvgStuScore(FaaS) calculates the average score of students 22 | func AvgStuScore(ctx context.Context, flow kis.Flow, rows []*AvgStuScoreIn) error { 23 | for _, row := range rows { 24 | avgScore := proto.StuAvgScore{ 25 | StuId: row.StuId, 26 | AvgScore: float64(row.Score1+row.Score2+row.Score3) / 3, 27 | } 28 | 29 | // Submit result data 30 | _ = flow.CommitRow(AvgStuScoreOut{StuAvgScore: avgScore}) 31 | } 32 | 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [aceld] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 13 | polar: # Replace with a single Polar username 14 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 15 | -------------------------------------------------------------------------------- /kis/serialize.go: -------------------------------------------------------------------------------- 1 | package kis 2 | 3 | import ( 4 | "reflect" 5 | 6 | "github.com/aceld/kis-flow/common" 7 | "github.com/aceld/kis-flow/serialize" 8 | ) 9 | 10 | // Serialize Data serialization interface 11 | type Serialize interface { 12 | // UnMarshal is used to deserialize KisRowArr to a value of the specified type. 13 | UnMarshal(common.KisRowArr, reflect.Type) (reflect.Value, error) 14 | // Marshal is used to serialize a value of the specified type to KisRowArr. 15 | Marshal(interface{}) (common.KisRowArr, error) 16 | } 17 | 18 | // defaultSerialize Default serialization implementation provided by KisFlow (developers can customize) 19 | var defaultSerialize = &serialize.DefaultSerialize{} 20 | 21 | // isSerialize checks if the provided paramType implements the Serialize interface 22 | func isSerialize(paramType reflect.Type) bool { 23 | return paramType.Implements(reflect.TypeOf((*Serialize)(nil)).Elem()) 24 | } 25 | -------------------------------------------------------------------------------- /test/kis_flow_commit_batch_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aceld/kis-flow/file" 8 | "github.com/aceld/kis-flow/kis" 9 | "github.com/aceld/kis-flow/log" 10 | ) 11 | 12 | func TestForkFlowCommitBatch(t *testing.T) { 13 | ctx := context.Background() 14 | 15 | // 1. Load the configuration file and build the Flow 16 | if err := file.ConfigImportYaml("load_conf/"); err != nil { 17 | panic(err) 18 | } 19 | 20 | // 2. Get the Flow 21 | flow1 := kis.Pool().GetFlow("flowName1") 22 | 23 | stringRows := []string{ 24 | "This is Data1 from Test", 25 | "This is Data2 from Test", 26 | "This is Data3 from Test", 27 | } 28 | 29 | // 3. Commit raw data 30 | if err := flow1.CommitRowBatch(stringRows); err != nil { 31 | log.Logger().ErrorF("CommitRowBatch Error, err = %+v", err) 32 | panic(err) 33 | } 34 | 35 | // 4. Execute flow1 36 | if err := flow1.Run(ctx); err != nil { 37 | panic(err) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /kis/connector.go: -------------------------------------------------------------------------------- 1 | package kis 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aceld/kis-flow/config" 7 | ) 8 | 9 | // Connector defines the interface for connectors associated with external storage. 10 | type Connector interface { 11 | // Init initializes the connection to the storage engine associated with the Connector. 12 | Init() error 13 | // Call invokes the read-write operations of the external storage logic. 14 | Call(ctx context.Context, flow Flow, args interface{}) (interface{}, error) 15 | // GetID returns the ID of the Connector. 16 | GetID() string 17 | // GetName returns the name of the Connector. 18 | GetName() string 19 | // GetConfig returns the configuration information of the Connector. 20 | GetConfig() *config.KisConnConfig 21 | // GetMetaData gets the temporary data of the current Connector. 22 | GetMetaData(key string) interface{} 23 | // SetMetaData sets the temporary data of the current Connector. 24 | SetMetaData(key string, value interface{}) 25 | } 26 | -------------------------------------------------------------------------------- /test/init.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "github.com/aceld/kis-flow/common" 5 | "github.com/aceld/kis-flow/kis" 6 | "github.com/aceld/kis-flow/test/caas" 7 | "github.com/aceld/kis-flow/test/faas" 8 | ) 9 | 10 | func init() { 11 | // Register Function callback business 12 | kis.Pool().FaaS("funcName1", faas.FuncDemo1Handler) 13 | kis.Pool().FaaS("funcName2", faas.FuncDemo2Handler) 14 | kis.Pool().FaaS("funcName3", faas.FuncDemo3Handler) 15 | kis.Pool().FaaS("funcName4", faas.FuncDemo4Handler) 16 | kis.Pool().FaaS("abortFunc", faas.AbortFuncHandler) // abortFunc business 17 | kis.Pool().FaaS("dataReuseFunc", faas.DataReuseFuncHandler) // dataReuseFunc business 18 | kis.Pool().FaaS("noResultFunc", faas.NoResultFuncHandler) // noResultFunc business 19 | kis.Pool().FaaS("jumpFunc", faas.JumpFuncHandler) // jumpFunc business 20 | 21 | // Register ConnectorInit and Connector callback business 22 | kis.Pool().CaaSInit("ConnName1", caas.InitConnDemo1) 23 | kis.Pool().CaaS("ConnName1", "funcName2", common.S, caas.CaasDemoHanler1) 24 | } 25 | -------------------------------------------------------------------------------- /test/kis_function_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aceld/kis-flow/common" 8 | "github.com/aceld/kis-flow/config" 9 | "github.com/aceld/kis-flow/flow" 10 | "github.com/aceld/kis-flow/function" 11 | ) 12 | 13 | func TestNewKisFunction(t *testing.T) { 14 | ctx := context.Background() 15 | 16 | // 1. Create a KisFunction configuration instance 17 | source := config.KisSource{ 18 | Name: "TikTokOrder", 19 | Must: []string{"order_id", "user_id"}, 20 | } 21 | 22 | myFuncConfig1 := config.NewFuncConfig("funcName1", common.C, &source, nil) 23 | if myFuncConfig1 == nil { 24 | panic("myFuncConfig1 is nil") 25 | } 26 | 27 | // 2. Create a KisFlow configuration instance 28 | myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable) 29 | 30 | // 3. Create a KisFlow object 31 | flow1 := flow.NewKisFlow(myFlowConfig1) 32 | 33 | // 4. Create a KisFunction object 34 | func1 := function.NewKisFunction(flow1, myFuncConfig1) 35 | 36 | if err := func1.Call(ctx, flow1); err != nil { 37 | t.Errorf("func1.Call() error = %v", err) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 刘丹冰(Aceld) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test/faas/faas_demo2.go: -------------------------------------------------------------------------------- 1 | package faas 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/aceld/kis-flow/kis" 8 | "github.com/aceld/kis-flow/log" 9 | ) 10 | 11 | // type FaaS func(context.Context, Flow) error 12 | 13 | func FuncDemo2Handler(ctx context.Context, flow kis.Flow) error { 14 | fmt.Println("---> Call funcName2Handler ----") 15 | fmt.Printf("Params = %+v\n", flow.GetFuncParamAll()) 16 | 17 | for index, row := range flow.Input() { 18 | str := fmt.Sprintf("In FuncName = %s, FuncId = %s, row = %s", flow.GetThisFuncConf().FName, flow.GetThisFunction().GetID(), row) 19 | fmt.Println(str) 20 | 21 | conn, err := flow.GetConnector() 22 | if err != nil { 23 | log.Logger().ErrorFX(ctx, "FuncDemo2Handler(): GetConnector err = %s\n", err.Error()) 24 | return err 25 | } 26 | 27 | if _, err := conn.Call(ctx, flow, row); err != nil { 28 | log.Logger().ErrorFX(ctx, "FuncDemo2Handler(): Call err = %s\n", err.Error()) 29 | return err 30 | } 31 | 32 | resultStr := fmt.Sprintf("data from funcName[%s], index = %d", flow.GetThisFuncConf().FName, index) 33 | 34 | _ = flow.CommitRow(resultStr) 35 | } 36 | 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /log/kis_log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import "context" 4 | 5 | type KisLogger interface { 6 | // InfoFX with context Info-level log interface, format string format 7 | InfoFX(ctx context.Context, str string, v ...interface{}) 8 | // ErrorFX with context Error-level log interface, format string format 9 | ErrorFX(ctx context.Context, str string, v ...interface{}) 10 | // DebugFX with context Debug-level log interface, format string format 11 | DebugFX(ctx context.Context, str string, v ...interface{}) 12 | 13 | // InfoF without context Info-level log interface, format string format 14 | InfoF(str string, v ...interface{}) 15 | // ErrorF without context Error-level log interface, format string format 16 | ErrorF(str string, v ...interface{}) 17 | // DebugF without context Debug-level log interface, format string format 18 | DebugF(str string, v ...interface{}) 19 | 20 | // SetDebugMode set Debug mode 21 | SetDebugMode(enable bool) 22 | } 23 | 24 | // kisLog Default KisLog object, providing default log printing methods, all of which print to standard output. 25 | var kisLog KisLogger 26 | 27 | // SetLogger set KisLog object, can be a user-defined Logger object 28 | func SetLogger(newlog KisLogger) { 29 | kisLog = newlog 30 | } 31 | 32 | // Logger get the kisLog object 33 | func Logger() KisLogger { 34 | return kisLog 35 | } 36 | -------------------------------------------------------------------------------- /config/kis_flow_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "github.com/aceld/kis-flow/common" 4 | 5 | // KisFlowFunctionParam represents the Id of a Function and carries fixed configuration parameters in a Flow configuration 6 | type KisFlowFunctionParam struct { 7 | FuncName string `yaml:"fname"` // Required 8 | Params FParam `yaml:"params"` // Optional, custom fixed configuration parameters for the Function in the current Flow 9 | } 10 | 11 | // KisFlowConfig represents the object that spans the entire stream computing context environment 12 | type KisFlowConfig struct { 13 | KisType string `yaml:"kistype"` 14 | Status int `yaml:"status"` 15 | FlowName string `yaml:"flow_name"` 16 | Flows []KisFlowFunctionParam `yaml:"flows"` 17 | } 18 | 19 | // NewFlowConfig creates a Flow strategy configuration object, used to describe a KisFlow information 20 | func NewFlowConfig(flowName string, enable common.KisOnOff) *KisFlowConfig { 21 | config := new(KisFlowConfig) 22 | config.FlowName = flowName 23 | config.Flows = make([]KisFlowFunctionParam, 0) 24 | 25 | config.Status = int(enable) 26 | 27 | return config 28 | } 29 | 30 | // AppendFunctionConfig adds a Function Config to the current Flow 31 | func (fConfig *KisFlowConfig) AppendFunctionConfig(params KisFlowFunctionParam) { 32 | fConfig.Flows = append(fConfig.Flows, params) 33 | } 34 | -------------------------------------------------------------------------------- /kis/router.go: -------------------------------------------------------------------------------- 1 | package kis 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aceld/kis-flow/common" 7 | ) 8 | 9 | /* 10 | Function Call 11 | */ 12 | // funcRouter 13 | // key: Function Name 14 | // value: FaaSDesc callback description for custom business 15 | type funcRouter map[string]*FaaSDesc 16 | 17 | // flowRouter 18 | // key: Flow Name 19 | // value: Flow 20 | type flowRouter map[string]Flow 21 | 22 | /* 23 | Connector Init 24 | */ 25 | // ConnInit Connector third-party storage initialization 26 | type ConnInit func(conn Connector) error 27 | 28 | // connInitRouter 29 | // key: Connector Name 30 | // value: ConnInit 31 | type connInitRouter map[string]ConnInit 32 | 33 | /* 34 | Connector Call 35 | */ 36 | // CaaS Connector storage read/write business implementation 37 | type CaaS func(context.Context, Connector, Function, Flow, interface{}) (interface{}, error) 38 | 39 | // connFuncRouter Maps CaaS callback storage business to FunctionName 40 | // key: Function Name 41 | // value: Connector storage read/write business implementation 42 | type connFuncRouter map[string]CaaS 43 | 44 | // connSL Splits connFuncRouter into two subtrees based on KisMode 45 | // key: Function KisMode S/L 46 | // value: connFuncRouter 47 | type connSL map[common.KisMode]connFuncRouter 48 | 49 | // connTree 50 | // key: Connector Name 51 | // value: connSL second-level tree 52 | type connTree map[string]connSL 53 | -------------------------------------------------------------------------------- /file/config_export.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/aceld/kis-flow/common" 8 | "github.com/aceld/kis-flow/kis" 9 | 10 | yaml "gopkg.in/yaml.v3" 11 | ) 12 | 13 | // ConfigExportYaml exports the flow configuration and saves it locally 14 | func ConfigExportYaml(flow kis.Flow, savePath string) error { 15 | 16 | var data []byte 17 | var err error 18 | 19 | data, err = yaml.Marshal(flow.GetConfig()) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | // flow 25 | err = os.WriteFile(savePath+common.KisIDTypeFlow+"-"+flow.GetName()+".yaml", data, 0644) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | // function 31 | for _, fp := range flow.GetConfig().Flows { 32 | fConf := flow.GetFuncConfigByName(fp.FuncName) 33 | if fConf == nil { 34 | return fmt.Errorf("function name = %s config is nil ", fp.FuncName) 35 | } 36 | 37 | fData, err := yaml.Marshal(fConf) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | if err := os.WriteFile(savePath+common.KisIDTypeFunction+"-"+fp.FuncName+".yaml", fData, 0644); err != nil { 43 | return err 44 | } 45 | 46 | // Connector 47 | if fConf.Option.CName != "" { 48 | cConf, err := fConf.GetConnConfig() 49 | if err != nil { 50 | return err 51 | } 52 | 53 | cdata, err := yaml.Marshal(cConf) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | if err := os.WriteFile(savePath+common.KisIDTypeConnector+"-"+cConf.CName+".yaml", cdata, 0644); err != nil { 59 | return err 60 | } 61 | } 62 | } 63 | 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /log/kis_default_log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | ) 8 | 9 | // kisDefaultLog Default provided log object 10 | type kisDefaultLog struct { 11 | debugMode bool 12 | mu sync.Mutex 13 | } 14 | 15 | func (log *kisDefaultLog) SetDebugMode(enable bool) { 16 | log.mu.Lock() 17 | defer log.mu.Unlock() 18 | log.debugMode = enable 19 | } 20 | 21 | func (log *kisDefaultLog) InfoF(str string, v ...interface{}) { 22 | fmt.Printf(str, v...) 23 | fmt.Printf("\n") 24 | } 25 | 26 | func (log *kisDefaultLog) ErrorF(str string, v ...interface{}) { 27 | fmt.Printf(str, v...) 28 | fmt.Printf("\n") 29 | } 30 | 31 | func (log *kisDefaultLog) DebugF(str string, v ...interface{}) { 32 | log.mu.Lock() 33 | defer log.mu.Unlock() 34 | if log.debugMode { 35 | fmt.Printf(str, v...) 36 | fmt.Printf("\n") 37 | } 38 | } 39 | 40 | func (log *kisDefaultLog) InfoFX(ctx context.Context, str string, v ...interface{}) { 41 | fmt.Println(ctx) 42 | fmt.Printf(str, v...) 43 | fmt.Printf("\n") 44 | } 45 | 46 | func (log *kisDefaultLog) ErrorFX(ctx context.Context, str string, v ...interface{}) { 47 | fmt.Println(ctx) 48 | fmt.Printf(str, v...) 49 | fmt.Printf("\n") 50 | } 51 | 52 | func (log *kisDefaultLog) DebugFX(ctx context.Context, str string, v ...interface{}) { 53 | log.mu.Lock() 54 | defer log.mu.Unlock() 55 | if log.debugMode { 56 | fmt.Println(ctx) 57 | fmt.Printf(str, v...) 58 | fmt.Printf("\n") 59 | } 60 | } 61 | 62 | func init() { 63 | // If no logger is set, use the default kisDefaultLog object at startup 64 | if Logger() == nil { 65 | SetLogger(&kisDefaultLog{}) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /test/kis_pool_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aceld/kis-flow/common" 8 | "github.com/aceld/kis-flow/config" 9 | "github.com/aceld/kis-flow/flow" 10 | ) 11 | 12 | func TestNewKisPool(t *testing.T) { 13 | 14 | ctx := context.Background() 15 | 16 | // 1. Create 2 KisFunction configuration instances 17 | source1 := config.KisSource{ 18 | Name: "TickTokOrder", 19 | Must: []string{"order_id", "user_id"}, 20 | } 21 | 22 | source2 := config.KisSource{ 23 | Name: "UserOrderErrorRate", 24 | Must: []string{"order_id", "user_id"}, 25 | } 26 | 27 | myFuncConfig1 := config.NewFuncConfig("funcName1", common.C, &source1, nil) 28 | if myFuncConfig1 == nil { 29 | panic("myFuncConfig1 is nil") 30 | } 31 | 32 | myFuncConfig2 := config.NewFuncConfig("funcName4", common.E, &source2, nil) 33 | if myFuncConfig2 == nil { 34 | panic("myFuncConfig4 is nil") 35 | } 36 | 37 | // 2. Create a KisFlow configuration instance 38 | myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable) 39 | 40 | // 3. Create a KisFlow object 41 | flow1 := flow.NewKisFlow(myFlowConfig1) 42 | 43 | // 4. Link Functions to Flow 44 | if err := flow1.Link(myFuncConfig1, nil); err != nil { 45 | panic(err) 46 | } 47 | if err := flow1.Link(myFuncConfig2, nil); err != nil { 48 | panic(err) 49 | } 50 | 51 | // 5. Commit raw data 52 | _ = flow1.CommitRow("This is Data1 from Test") 53 | _ = flow1.CommitRow("This is Data2 from Test") 54 | _ = flow1.CommitRow("This is Data3 from Test") 55 | 56 | // 6. Execute flow1 57 | if err := flow1.Run(ctx); err != nil { 58 | panic(err) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /flow/kis_flow_action.go: -------------------------------------------------------------------------------- 1 | package flow 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/aceld/kis-flow/kis" 9 | ) 10 | 11 | // dealAction handles Action to determine the next direction of the Flow. 12 | func (flow *KisFlow) dealAction(ctx context.Context, fn kis.Function) (kis.Function, error) { 13 | 14 | // DataReuse Action 15 | if flow.action.DataReuse { 16 | if err := flow.commitReuseData(ctx); err != nil { 17 | return nil, err 18 | } 19 | } else { 20 | if err := flow.commitCurData(ctx); err != nil { 21 | return nil, err 22 | } 23 | } 24 | 25 | // ForceEntryNext Action 26 | if flow.action.ForceEntryNext { 27 | if err := flow.commitVoidData(ctx); err != nil { 28 | return nil, err 29 | } 30 | flow.abort = false 31 | } 32 | 33 | // JumpFunc Action 34 | if flow.action.JumpFunc != "" { 35 | if _, ok := flow.Funcs[flow.action.JumpFunc]; !ok { 36 | // The current JumpFunc is not in the flow 37 | return nil, errors.New(fmt.Sprintf("Flow Jump -> %s is not in Flow", flow.action.JumpFunc)) 38 | } 39 | 40 | jumpFunction := flow.Funcs[flow.action.JumpFunc] 41 | // Update the upper layer Function 42 | flow.PrevFunctionId = jumpFunction.GetPrevId() 43 | fn = jumpFunction 44 | 45 | // If set to jump, force the jump 46 | flow.abort = false 47 | 48 | } else { 49 | 50 | // Update the upper layer FunctionId cursor 51 | flow.PrevFunctionId = flow.ThisFunctionId 52 | fn = fn.Next() 53 | } 54 | 55 | // Abort Action force termination 56 | if flow.action.Abort { 57 | flow.abort = true 58 | } 59 | 60 | // Clear Action 61 | flow.action = kis.Action{} 62 | 63 | return fn, nil 64 | } 65 | -------------------------------------------------------------------------------- /config/kis_conn_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/aceld/kis-flow/common" 7 | ) 8 | 9 | // KisConnConfig describes the KisConnector strategy configuration 10 | type KisConnConfig struct { 11 | KisType string `yaml:"kistype"` // Configuration type 12 | CName string `yaml:"cname"` // Unique descriptive identifier 13 | AddrString string `yaml:"addrs"` // Base storage medium address 14 | Type common.KisConnType `yaml:"type"` // Storage medium engine type: "Mysql", "Redis", "Kafka", etc. 15 | Key string `yaml:"key"` // Identifier for a single storage: Key name for Redis, Table name for Mysql, Topic name for Kafka, etc. 16 | Params map[string]string `yaml:"params"` // Custom parameters in the configuration information 17 | 18 | // NsFuncionID bound to storage reading 19 | Load []string `yaml:"load"` 20 | Save []string `yaml:"save"` 21 | } 22 | 23 | // NewConnConfig creates a KisConnector strategy configuration object, used to describe a KisConnector information 24 | func NewConnConfig(cName string, addr string, t common.KisConnType, key string, param map[string]string) *KisConnConfig { 25 | strategy := new(KisConnConfig) 26 | strategy.CName = cName 27 | strategy.AddrString = addr 28 | strategy.Type = t 29 | strategy.Key = key 30 | strategy.Params = param 31 | 32 | return strategy 33 | } 34 | 35 | // WithFunc binds Connector to Function 36 | func (cConfig *KisConnConfig) WithFunc(fConfig *KisFuncConfig) error { 37 | 38 | switch common.KisMode(fConfig.FMode) { 39 | case common.S: 40 | cConfig.Save = append(cConfig.Save, fConfig.FName) 41 | case common.L: 42 | cConfig.Load = append(cConfig.Load, fConfig.FName) 43 | default: 44 | return fmt.Errorf("Wrong KisMode %s", fConfig.FMode) 45 | } 46 | 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /kis/action.go: -------------------------------------------------------------------------------- 1 | package kis 2 | 3 | // Action defines the actions for KisFlow execution. 4 | type Action struct { 5 | // DataReuse indicates whether to reuse data from the upper Function. 6 | DataReuse bool 7 | 8 | // ForceEntryNext overrides the default rule, where if the current Function's calculation result is 0 data entries, 9 | // subsequent Functions will not continue execution. 10 | // With ForceEntryNext set to true, the next Function will be entered regardless of the data. 11 | ForceEntryNext bool 12 | 13 | // JumpFunc specifies the Function to jump to for continued execution. 14 | // (Note: This can easily lead to Flow loop calls, causing an infinite loop.) 15 | JumpFunc string 16 | 17 | // Abort terminates the execution of the Flow. 18 | Abort bool 19 | } 20 | 21 | // ActionFunc is the type for KisFlow Functional Option. 22 | type ActionFunc func(ops *Action) 23 | 24 | // LoadActions loads Actions and sequentially executes the ActionFunc operations. 25 | func LoadActions(acts []ActionFunc) Action { 26 | action := Action{} 27 | 28 | if acts == nil { 29 | return action 30 | } 31 | 32 | for _, act := range acts { 33 | act(&action) 34 | } 35 | 36 | return action 37 | } 38 | 39 | // ActionDataReuse sets the option for reusing data from the upper Function. 40 | func ActionDataReuse(act *Action) { 41 | act.DataReuse = true 42 | } 43 | 44 | // ActionForceEntryNext sets the option to forcefully enter the next layer. 45 | func ActionForceEntryNext(act *Action) { 46 | act.ForceEntryNext = true 47 | } 48 | 49 | // ActionJumpFunc returns an ActionFunc function and sets the funcName to Action.JumpFunc. 50 | // (Note: This can easily lead to Flow loop calls, causing an infinite loop.) 51 | func ActionJumpFunc(funcName string) ActionFunc { 52 | return func(act *Action) { 53 | act.JumpFunc = funcName 54 | } 55 | } 56 | 57 | // ActionAbort terminates the execution of the Flow. 58 | func ActionAbort(action *Action) { 59 | action.Abort = true 60 | } 61 | -------------------------------------------------------------------------------- /test/kis_fork_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/aceld/kis-flow/common" 9 | "github.com/aceld/kis-flow/config" 10 | "github.com/aceld/kis-flow/file" 11 | "github.com/aceld/kis-flow/flow" 12 | "github.com/aceld/kis-flow/kis" 13 | ) 14 | 15 | func TestForkFlow(t *testing.T) { 16 | ctx := context.Background() 17 | 18 | // 1. Load configuration file and build Flow 19 | if err := file.ConfigImportYaml("load_conf/"); err != nil { 20 | panic(err) 21 | } 22 | 23 | // 2. Get Flow 24 | flow1 := kis.Pool().GetFlow("flowFork1") 25 | 26 | fmt.Println("----> flow1: ", flow1.GetFuncParamsAllFuncs()) 27 | 28 | flow1Clone1 := flow1.Fork(ctx) 29 | fmt.Println("----> flow1Clone1: ", flow1Clone1.GetFuncParamsAllFuncs()) 30 | 31 | // 3. Commit raw data 32 | _ = flow1Clone1.CommitRow("This is Data1 from Test") 33 | 34 | // 4. Execute flow1 35 | if err := flow1Clone1.Run(ctx); err != nil { 36 | panic(err) 37 | } 38 | } 39 | 40 | func TestForkFlowWithLink(t *testing.T) { 41 | ctx := context.Background() 42 | 43 | // Create a new flow configuration 44 | myFlowConfig1 := config.NewFlowConfig("flowFork1", common.FlowEnable) 45 | 46 | // Create new function configuration 47 | func1Config := config.NewFuncConfig("funcName1", common.V, nil, nil) 48 | func3Config := config.NewFuncConfig("funcName3", common.E, nil, nil) 49 | 50 | // Create a new flow 51 | flow1 := flow.NewKisFlow(myFlowConfig1) 52 | 53 | _ = flow1.Link(func1Config, config.FParam{"school": "TsingHua1", "country": "China1"}) 54 | _ = flow1.Link(func3Config, config.FParam{"school": "TsingHua3", "country": "China3"}) 55 | 56 | fmt.Println("----> flow1: ", flow1.GetFuncParamsAllFuncs()) 57 | 58 | flow1Clone1 := flow1.Fork(ctx) 59 | 60 | fmt.Println("----> flow1Clone1: ", flow1Clone1.GetFuncParamsAllFuncs()) 61 | 62 | // 3. Commit raw data 63 | _ = flow1Clone1.CommitRow("This is Data1 from Test") 64 | 65 | // 4. Execute flow1 66 | if err := flow1Clone1.Run(ctx); err != nil { 67 | panic(err) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /kis/function.go: -------------------------------------------------------------------------------- 1 | package kis 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aceld/kis-flow/config" 7 | ) 8 | 9 | // Function is the basic computation unit of streaming computation. KisFunction is a basic logical unit of streaming computation, any number of KisFunctions can be combined into a KisFlow 10 | type Function interface { 11 | // Call executes the streaming computation logic 12 | Call(ctx context.Context, flow Flow) error 13 | 14 | // SetConfig configures the current Function instance 15 | SetConfig(s *config.KisFuncConfig) error 16 | // GetConfig retrieves the configuration of the current Function instance 17 | GetConfig() *config.KisFuncConfig 18 | 19 | // SetFlow sets the Flow instance that the current Function instance depends on 20 | SetFlow(f Flow) error 21 | // GetFlow retrieves the Flow instance that the current Function instance depends on 22 | GetFlow() Flow 23 | 24 | // AddConnector adds a Connector to the current Function instance 25 | AddConnector(conn Connector) error 26 | // GetConnector retrieves the Connector associated with the current Function instance 27 | GetConnector() Connector 28 | 29 | // CreateId generates a random KisID for the current Function instance 30 | CreateId() 31 | // GetID retrieves the FID of the current Function 32 | GetID() string 33 | // GetPrevId retrieves the FID of the previous Function node of the current Function 34 | GetPrevId() string 35 | // GetNextId retrieves the FID of the next Function node of the current Function 36 | GetNextId() string 37 | 38 | // Next returns the next layer of the computation flow Function. If the current layer is the last layer, it returns nil 39 | Next() Function 40 | // Prev returns the previous layer of the computation flow Function. If the current layer is the last layer, it returns nil 41 | Prev() Function 42 | // SetN sets the next Function instance 43 | SetN(f Function) 44 | // SetP sets the previous Function instance 45 | SetP(f Function) 46 | // GetMetaData retrieves the temporary data of the current Function 47 | GetMetaData(key string) interface{} 48 | // SetMetaData sets the temporary data of the current Function 49 | SetMetaData(key string, value interface{}) 50 | } 51 | -------------------------------------------------------------------------------- /test/kis_config_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aceld/kis-flow/common" 7 | "github.com/aceld/kis-flow/config" 8 | "github.com/aceld/kis-flow/log" 9 | ) 10 | 11 | func TestNewFuncConfig(t *testing.T) { 12 | source := config.KisSource{ 13 | Name: "TikTokOrder", 14 | Must: []string{"order_id", "user_id"}, 15 | } 16 | 17 | option := config.KisFuncOption{ 18 | CName: "connectorName1", 19 | RetryTimes: 3, 20 | RetryDuration: 300, 21 | 22 | Params: config.FParam{ 23 | "param1": "value1", 24 | "param2": "value2", 25 | }, 26 | } 27 | 28 | myFunc1 := config.NewFuncConfig("funcName1", common.S, &source, &option) 29 | 30 | log.Logger().InfoF("funcName1: %+v\n", myFunc1) 31 | } 32 | 33 | func TestNewFlowConfig(t *testing.T) { 34 | 35 | flowFuncParams1 := config.KisFlowFunctionParam{ 36 | FuncName: "funcName1", 37 | Params: config.FParam{ 38 | "flowSetFunParam1": "value1", 39 | "flowSetFunParam2": "value2", 40 | }, 41 | } 42 | 43 | flowFuncParams2 := config.KisFlowFunctionParam{ 44 | FuncName: "funcName2", 45 | Params: config.FParam{ 46 | "default": "value1", 47 | }, 48 | } 49 | 50 | myFlow1 := config.NewFlowConfig("flowName1", common.FlowEnable) 51 | myFlow1.AppendFunctionConfig(flowFuncParams1) 52 | myFlow1.AppendFunctionConfig(flowFuncParams2) 53 | 54 | log.Logger().InfoF("myFlow1: %+v\n", myFlow1) 55 | } 56 | 57 | func TestNewConnConfig(t *testing.T) { 58 | 59 | source := config.KisSource{ 60 | Name: "TikTokOrder", 61 | Must: []string{"order_id", "user_id"}, 62 | } 63 | 64 | option := config.KisFuncOption{ 65 | CName: "connectorName1", 66 | RetryTimes: 3, 67 | RetryDuration: 300, 68 | 69 | Params: config.FParam{ 70 | "param1": "value1", 71 | "param2": "value2", 72 | }, 73 | } 74 | 75 | myFunc1 := config.NewFuncConfig("funcName1", common.S, &source, &option) 76 | 77 | connParams := config.FParam{ 78 | "param1": "value1", 79 | "param2": "value2", 80 | } 81 | 82 | myConnector1 := config.NewConnConfig("connectorName1", "0.0.0.0:9987,0.0.0.0:9997", common.REDIS, "key", connParams) 83 | 84 | if err := myConnector1.WithFunc(myFunc1); err != nil { 85 | log.Logger().ErrorF("WithFunc err: %s\n", err.Error()) 86 | } 87 | 88 | log.Logger().InfoF("myConnector1: %+v\n", myConnector1) 89 | } 90 | -------------------------------------------------------------------------------- /test/kis_connector_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aceld/kis-flow/common" 8 | "github.com/aceld/kis-flow/config" 9 | "github.com/aceld/kis-flow/flow" 10 | ) 11 | 12 | func TestNewKisConnector(t *testing.T) { 13 | 14 | ctx := context.Background() 15 | 16 | // 1. Create three KisFunction configuration instances, with myFuncConfig2 having a Connector configuration 17 | source1 := config.KisSource{ 18 | Name: "TikTokOrder", 19 | Must: []string{"order_id", "user_id"}, 20 | } 21 | 22 | source2 := config.KisSource{ 23 | Name: "UserOrderErrorRate", 24 | Must: []string{"order_id", "user_id"}, 25 | } 26 | 27 | myFuncConfig1 := config.NewFuncConfig("funcName1", common.C, &source1, nil) 28 | if myFuncConfig1 == nil { 29 | panic("myFuncConfig1 is nil") 30 | } 31 | 32 | option := config.KisFuncOption{ 33 | CName: "ConnName1", 34 | } 35 | 36 | myFuncConfig2 := config.NewFuncConfig("funcName2", common.S, &source2, &option) 37 | if myFuncConfig2 == nil { 38 | panic("myFuncConfig2 is nil") 39 | } 40 | 41 | myFuncConfig3 := config.NewFuncConfig("funcName3", common.E, &source2, nil) 42 | if myFuncConfig3 == nil { 43 | panic("myFuncConfig3 is nil") 44 | } 45 | 46 | // 2. Create a KisConnector configuration instance 47 | myConnConfig1 := config.NewConnConfig("ConnName1", "0.0.0.0:9998", common.REDIS, "redis-key", nil) 48 | if myConnConfig1 == nil { 49 | panic("myConnConfig1 is nil") 50 | } 51 | 52 | // 3. Bind the KisConnector configuration instance to the KisFunction configuration instance 53 | _ = myFuncConfig2.AddConnConfig(myConnConfig1) 54 | 55 | // 4. Create a KisFlow configuration instance 56 | myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable) 57 | 58 | // 5. Create a KisFlow object 59 | flow1 := flow.NewKisFlow(myFlowConfig1) 60 | 61 | // 6. Link Functions to the Flow 62 | if err := flow1.Link(myFuncConfig1, nil); err != nil { 63 | panic(err) 64 | } 65 | if err := flow1.Link(myFuncConfig2, nil); err != nil { 66 | panic(err) 67 | } 68 | if err := flow1.Link(myFuncConfig3, nil); err != nil { 69 | panic(err) 70 | } 71 | 72 | // 7. Commit raw data 73 | _ = flow1.CommitRow("This is Data1 from Test") 74 | _ = flow1.CommitRow("This is Data2 from Test") 75 | _ = flow1.CommitRow("This is Data3 from Test") 76 | 77 | // 8. Execute flow1 78 | if err := flow1.Run(ctx); err != nil { 79 | panic(err) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /conn/kis_connector.go: -------------------------------------------------------------------------------- 1 | package conn 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/aceld/kis-flow/common" 8 | "github.com/aceld/kis-flow/config" 9 | "github.com/aceld/kis-flow/id" 10 | "github.com/aceld/kis-flow/kis" 11 | ) 12 | 13 | // KisConnector represents a KisConnector instance 14 | type KisConnector struct { 15 | // Connector ID 16 | CId string 17 | // Connector Name 18 | CName string 19 | // Connector Config 20 | Conf *config.KisConnConfig 21 | 22 | // Connector Init 23 | onceInit sync.Once 24 | 25 | // KisConnector's custom temporary data 26 | metaData map[string]interface{} 27 | // Lock for reading and writing metaData 28 | mLock sync.RWMutex 29 | } 30 | 31 | // NewKisConnector creates a KisConnector based on the given configuration 32 | func NewKisConnector(config *config.KisConnConfig) *KisConnector { 33 | conn := new(KisConnector) 34 | conn.CId = id.KisID(common.KisIDTypeConnector) 35 | conn.CName = config.CName 36 | conn.Conf = config 37 | conn.metaData = make(map[string]interface{}) 38 | 39 | return conn 40 | } 41 | 42 | // Init initializes the connection to the associated storage engine of the Connector 43 | func (conn *KisConnector) Init() error { 44 | var err error 45 | 46 | // The initialization business of a Connector can only be executed once 47 | conn.onceInit.Do(func() { 48 | err = kis.Pool().CallConnInit(conn) 49 | }) 50 | 51 | return err 52 | } 53 | 54 | // Call invokes the read-write operations of the external storage logic through the Connector 55 | func (conn *KisConnector) Call(ctx context.Context, flow kis.Flow, args interface{}) (interface{}, error) { 56 | var result interface{} 57 | var err error 58 | 59 | result, err = kis.Pool().CallConnector(ctx, flow, conn, args) 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | return result, nil 65 | } 66 | 67 | // GetName returns the name of the Connector 68 | func (conn *KisConnector) GetName() string { 69 | return conn.CName 70 | } 71 | 72 | // GetConfig returns the configuration of the Connector 73 | func (conn *KisConnector) GetConfig() *config.KisConnConfig { 74 | return conn.Conf 75 | } 76 | 77 | // GetID returns the ID of the Connector 78 | func (conn *KisConnector) GetID() string { 79 | return conn.CId 80 | } 81 | 82 | // GetMetaData gets the temporary data of the current Connector 83 | func (conn *KisConnector) GetMetaData(key string) interface{} { 84 | conn.mLock.RLock() 85 | defer conn.mLock.RUnlock() 86 | 87 | data, ok := conn.metaData[key] 88 | if !ok { 89 | return nil 90 | } 91 | 92 | return data 93 | } 94 | 95 | // SetMetaData sets the temporary data of the current Connector 96 | func (conn *KisConnector) SetMetaData(key string, value interface{}) { 97 | conn.mLock.Lock() 98 | defer conn.mLock.Unlock() 99 | 100 | conn.metaData[key] = value 101 | } 102 | -------------------------------------------------------------------------------- /test/kis_action_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/aceld/kis-flow/file" 9 | "github.com/aceld/kis-flow/kis" 10 | ) 11 | 12 | func TestActionAbort(t *testing.T) { 13 | ctx := context.Background() 14 | 15 | // 1. Load the configuration file and build the Flow 16 | if err := file.ConfigImportYaml("load_conf/"); err != nil { 17 | fmt.Println("Wrong Config Yaml Path!") 18 | panic(err) 19 | } 20 | 21 | // 2. Get the Flow 22 | flow1 := kis.Pool().GetFlow("flowName2") 23 | 24 | // 3. Commit original data 25 | _ = flow1.CommitRow("This is Data1 from Test") 26 | _ = flow1.CommitRow("This is Data2 from Test") 27 | _ = flow1.CommitRow("This is Data3 from Test") 28 | 29 | // 4. Execute flow1 30 | if err := flow1.Run(ctx); err != nil { 31 | panic(err) 32 | } 33 | } 34 | 35 | func TestActionDataReuse(t *testing.T) { 36 | ctx := context.Background() 37 | 38 | // 1. Load the configuration file and build the Flow 39 | if err := file.ConfigImportYaml("load_conf/"); err != nil { 40 | fmt.Println("Wrong Config Yaml Path!") 41 | panic(err) 42 | } 43 | 44 | // 2. Get the Flow 45 | flow1 := kis.Pool().GetFlow("flowName3") 46 | 47 | // 3. Commit original data 48 | _ = flow1.CommitRow("This is Data1 from Test") 49 | _ = flow1.CommitRow("This is Data2 from Test") 50 | _ = flow1.CommitRow("This is Data3 from Test") 51 | 52 | // 4. Execute flow1 53 | if err := flow1.Run(ctx); err != nil { 54 | panic(err) 55 | } 56 | } 57 | 58 | func TestActionForceEntry(t *testing.T) { 59 | ctx := context.Background() 60 | 61 | // 1. Load the configuration file and build the Flow 62 | if err := file.ConfigImportYaml("load_conf/"); err != nil { 63 | fmt.Println("Wrong Config Yaml Path!") 64 | panic(err) 65 | } 66 | 67 | // 2. Get the Flow 68 | flow1 := kis.Pool().GetFlow("flowName4") 69 | 70 | // 3. Commit original data 71 | _ = flow1.CommitRow("This is Data1 from Test") 72 | _ = flow1.CommitRow("This is Data2 from Test") 73 | _ = flow1.CommitRow("This is Data3 from Test") 74 | 75 | // 4. Execute flow1 76 | if err := flow1.Run(ctx); err != nil { 77 | panic(err) 78 | } 79 | } 80 | 81 | func TestActionJumpFunc(t *testing.T) { 82 | ctx := context.Background() 83 | 84 | // 1. Load the configuration file and build the Flow 85 | if err := file.ConfigImportYaml("load_conf/"); err != nil { 86 | fmt.Println("Wrong Config Yaml Path!") 87 | panic(err) 88 | } 89 | 90 | // 2. Get the Flow 91 | flow1 := kis.Pool().GetFlow("flowName5") 92 | 93 | // 3. Commit original data 94 | _ = flow1.CommitRow("This is Data1 from Test") 95 | _ = flow1.CommitRow("This is Data2 from Test") 96 | _ = flow1.CommitRow("This is Data3 from Test") 97 | 98 | // 4. Execute flow1 99 | if err := flow1.Run(ctx); err != nil { 100 | panic(err) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /test/kis_flow_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aceld/kis-flow/common" 8 | "github.com/aceld/kis-flow/config" 9 | "github.com/aceld/kis-flow/flow" 10 | ) 11 | 12 | func TestNewKisFlow(t *testing.T) { 13 | ctx := context.Background() 14 | 15 | // 1. Create 2 KisFunction configuration instances 16 | source1 := config.KisSource{ 17 | Name: "TikTokOrder", 18 | Must: []string{"order_id", "user_id"}, 19 | } 20 | 21 | source2 := config.KisSource{ 22 | Name: "UserOrderErrorRate", 23 | Must: []string{"order_id", "user_id"}, 24 | } 25 | 26 | myFuncConfig1 := config.NewFuncConfig("funcName1", common.C, &source1, nil) 27 | if myFuncConfig1 == nil { 28 | panic("myFuncConfig1 is nil") 29 | } 30 | 31 | myFuncConfig2 := config.NewFuncConfig("funcName2", common.V, &source2, nil) 32 | if myFuncConfig2 == nil { 33 | panic("myFuncConfig2 is nil") 34 | } 35 | 36 | // 2. Create a KisFlow configuration instance 37 | myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable) 38 | 39 | // 3. Create a KisFlow object 40 | flow1 := flow.NewKisFlow(myFlowConfig1) 41 | 42 | // 4. Link functions to the Flow 43 | if err := flow1.Link(myFuncConfig1, nil); err != nil { 44 | panic(err) 45 | } 46 | if err := flow1.Link(myFuncConfig2, nil); err != nil { 47 | panic(err) 48 | } 49 | 50 | // 5. Execute flow1 51 | if err := flow1.Run(ctx); err != nil { 52 | panic(err) 53 | } 54 | } 55 | 56 | func TestNewKisFlowData(t *testing.T) { 57 | ctx := context.Background() 58 | 59 | // 1. Create 2 KisFunction configuration instances 60 | source1 := config.KisSource{ 61 | Name: "TikTokOrder", 62 | Must: []string{"order_id", "user_id"}, 63 | } 64 | 65 | source2 := config.KisSource{ 66 | Name: "UserOrderErrorRate", 67 | Must: []string{"order_id", "user_id"}, 68 | } 69 | 70 | myFuncConfig1 := config.NewFuncConfig("funcName1", common.C, &source1, nil) 71 | if myFuncConfig1 == nil { 72 | panic("myFuncConfig1 is nil") 73 | } 74 | 75 | myFuncConfig2 := config.NewFuncConfig("funcName4", common.E, &source2, nil) 76 | if myFuncConfig2 == nil { 77 | panic("myFuncConfig4 is nil") 78 | } 79 | 80 | // 2. Create a KisFlow configuration instance 81 | myFlowConfig1 := config.NewFlowConfig("flowName1", common.FlowEnable) 82 | 83 | // 3. Create a KisFlow object 84 | flow1 := flow.NewKisFlow(myFlowConfig1) 85 | 86 | // 4. Link Function to the Flow 87 | if err := flow1.Link(myFuncConfig1, nil); err != nil { 88 | panic(err) 89 | } 90 | if err := flow1.Link(myFuncConfig2, nil); err != nil { 91 | panic(err) 92 | } 93 | 94 | // 5. Commit raw data 95 | _ = flow1.CommitRow("This is Data1 from Test") 96 | _ = flow1.CommitRow("This is Data2 from Test") 97 | _ = flow1.CommitRow("This is Data3 from Test") 98 | 99 | // 6. Execute flow1 100 | if err := flow1.Run(ctx); err != nil { 101 | panic(err) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /kis/flow.go: -------------------------------------------------------------------------------- 1 | package kis 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/aceld/kis-flow/common" 8 | "github.com/aceld/kis-flow/config" 9 | ) 10 | 11 | type Flow interface { 12 | // Run schedules the Flow, sequentially dispatching and executing Functions in the Flow 13 | Run(ctx context.Context) error 14 | // Link connects the Functions in the Flow according to the configuration in the config file, and the Flow's configuration will also be updated 15 | Link(fConf *config.KisFuncConfig, fParams config.FParam) error 16 | // AppendNewFunction appends a new Function to the Flow 17 | AppendNewFunction(fConf *config.KisFuncConfig, fParams config.FParam) error 18 | // CommitRow submits Flow data to the upcoming Function layer 19 | CommitRow(row interface{}) error 20 | // CommitRowBatch submits Flow data to the upcoming Function layer (batch submission) 21 | // row: Must be a slice 22 | CommitRowBatch(row interface{}) error 23 | // Input gets the input source data of the currently executing Function in the Flow 24 | Input() common.KisRowArr 25 | // GetName gets the name of the Flow 26 | GetName() string 27 | // GetThisFunction gets the currently executing Function 28 | GetThisFunction() Function 29 | // GetThisFuncConf gets the configuration of the currently executing Function 30 | GetThisFuncConf() *config.KisFuncConfig 31 | // GetConnector gets the Connector of the currently executing Function 32 | GetConnector() (Connector, error) 33 | // GetConnConf gets the configuration of the Connector of the currently executing Function 34 | GetConnConf() (*config.KisConnConfig, error) 35 | // GetConfig gets the configuration of the current Flow 36 | GetConfig() *config.KisFlowConfig 37 | // GetFuncConfigByName gets the configuration of the current Flow by Function name 38 | GetFuncConfigByName(funcName string) *config.KisFuncConfig 39 | // Next carries the Action actions of the next layer Function that the current Flow is executing 40 | Next(acts ...ActionFunc) error 41 | // GetCacheData gets the cached data of the current Flow 42 | GetCacheData(key string) interface{} 43 | // SetCacheData sets the cached data of the current Flow 44 | SetCacheData(key string, value interface{}, Exp time.Duration) 45 | // GetMetaData gets the temporary data of the current Flow 46 | GetMetaData(key string) interface{} 47 | // SetMetaData sets the temporary data of the current Flow 48 | SetMetaData(key string, value interface{}) 49 | // GetFuncParam gets the default parameters of the current Flow's currently executing Function, retrieving a key-value pair 50 | GetFuncParam(key string) string 51 | // GetFuncParamAll gets the default parameters of the current Flow's currently executing Function, retrieving all Key-Value pairs 52 | GetFuncParamAll() config.FParam 53 | // GetFuncParamsAllFuncs gets the FuncParams of all Functions in the Flow, retrieving all Key-Value pairs 54 | GetFuncParamsAllFuncs() map[string]config.FParam 55 | // Fork gets a copy of the Flow (deep copy) 56 | Fork(ctx context.Context) Flow 57 | // GetID gets the Id of the Flow 58 | GetID() string 59 | } 60 | -------------------------------------------------------------------------------- /test/kis_auto_inject_param_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aceld/kis-flow/common" 8 | "github.com/aceld/kis-flow/config" 9 | "github.com/aceld/kis-flow/file" 10 | "github.com/aceld/kis-flow/flow" 11 | "github.com/aceld/kis-flow/kis" 12 | "github.com/aceld/kis-flow/test/faas" 13 | "github.com/aceld/kis-flow/test/proto" 14 | ) 15 | 16 | func TestAutoInjectParamWithConfig(t *testing.T) { 17 | ctx := context.Background() 18 | 19 | kis.Pool().FaaS("AvgStuScore", faas.AvgStuScore) 20 | kis.Pool().FaaS("PrintStuAvgScore", faas.PrintStuAvgScore) 21 | 22 | // 1. Load the configuration file and build the Flow 23 | if err := file.ConfigImportYaml("load_conf/"); err != nil { 24 | panic(err) 25 | } 26 | 27 | // 2. Get the Flow 28 | flow1 := kis.Pool().GetFlow("StuAvg") 29 | if flow1 == nil { 30 | panic("flow1 is nil") 31 | } 32 | 33 | // 3. Commit original data 34 | _ = flow1.CommitRow(&faas.AvgStuScoreIn{ 35 | StuScores: proto.StuScores{ 36 | StuId: 100, 37 | Score1: 1, 38 | Score2: 2, 39 | Score3: 3, 40 | }, 41 | }) 42 | _ = flow1.CommitRow(faas.AvgStuScoreIn{ 43 | StuScores: proto.StuScores{ 44 | StuId: 100, 45 | Score1: 1, 46 | Score2: 2, 47 | Score3: 3, 48 | }, 49 | }) 50 | 51 | // Commit original data (as JSON string) 52 | _ = flow1.CommitRow(`{"stu_id":101}`) 53 | 54 | // 4. Execute flow1 55 | if err := flow1.Run(ctx); err != nil { 56 | panic(err) 57 | } 58 | } 59 | 60 | func TestAutoInjectParam(t *testing.T) { 61 | ctx := context.Background() 62 | 63 | kis.Pool().FaaS("AvgStuScore", faas.AvgStuScore) 64 | kis.Pool().FaaS("PrintStuAvgScore", faas.PrintStuAvgScore) 65 | 66 | source1 := config.KisSource{ 67 | Name: "Test", 68 | Must: []string{}, 69 | } 70 | 71 | avgStuScoreConfig := config.NewFuncConfig("AvgStuScore", common.C, &source1, nil) 72 | if avgStuScoreConfig == nil { 73 | panic("AvgStuScore is nil") 74 | } 75 | 76 | printStuAvgScoreConfig := config.NewFuncConfig("PrintStuAvgScore", common.C, &source1, nil) 77 | if printStuAvgScoreConfig == nil { 78 | panic("printStuAvgScoreConfig is nil") 79 | } 80 | 81 | myFlowConfig1 := config.NewFlowConfig("cal_stu_avg_score", common.FlowEnable) 82 | 83 | flow1 := flow.NewKisFlow(myFlowConfig1) 84 | 85 | // 4. Link Functions to Flow 86 | if err := flow1.Link(avgStuScoreConfig, nil); err != nil { 87 | panic(err) 88 | } 89 | if err := flow1.Link(printStuAvgScoreConfig, nil); err != nil { 90 | panic(err) 91 | } 92 | 93 | // 3. Commit original data 94 | _ = flow1.CommitRow(&faas.AvgStuScoreIn{ 95 | StuScores: proto.StuScores{ 96 | StuId: 100, 97 | Score1: 1, 98 | Score2: 2, 99 | Score3: 3, 100 | }, 101 | }) 102 | _ = flow1.CommitRow(`{"stu_id":101}`) 103 | _ = flow1.CommitRow(faas.AvgStuScoreIn{ 104 | StuScores: proto.StuScores{ 105 | StuId: 100, 106 | Score1: 1, 107 | Score2: 2, 108 | Score3: 3, 109 | }, 110 | }) 111 | 112 | // 4. Execute flow1 113 | if err := flow1.Run(ctx); err != nil { 114 | panic(err) 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /config/kis_func_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/aceld/kis-flow/common" 7 | "github.com/aceld/kis-flow/log" 8 | ) 9 | 10 | // FParam represents the type for custom fixed configuration parameters for the Function in the current Flow 11 | type FParam map[string]string 12 | 13 | // KisSource represents the business source of the current Function 14 | type KisSource struct { 15 | Name string `yaml:"name"` // Description of the data source for this layer Function 16 | Must []string `yaml:"must"` // Required fields for the source 17 | } 18 | 19 | // KisFuncOption represents optional configurations 20 | type KisFuncOption struct { 21 | CName string `yaml:"cname"` // Connector name 22 | RetryTimes int `yaml:"retry_times"` // Optional, maximum retry times for Function scheduling (excluding normal scheduling) 23 | RetryDuration int `yaml:"return_duration"` // Optional, maximum time interval for each retry in Function scheduling (unit: ms) 24 | Params FParam `yaml:"default_params"` // Optional, custom fixed configuration parameters for the Function in the current Flow 25 | } 26 | 27 | // KisFuncConfig represents a KisFunction strategy configuration 28 | type KisFuncConfig struct { 29 | KisType string `yaml:"kistype"` 30 | FName string `yaml:"fname"` 31 | FMode string `yaml:"fmode"` 32 | Source KisSource `yaml:"source"` 33 | Option KisFuncOption `yaml:"option"` 34 | connConf *KisConnConfig 35 | } 36 | 37 | // NewFuncConfig creates a Function strategy configuration object, used to describe a KisFunction information 38 | func NewFuncConfig( 39 | funcName string, mode common.KisMode, 40 | source *KisSource, option *KisFuncOption) *KisFuncConfig { 41 | 42 | config := new(KisFuncConfig) 43 | config.FName = funcName 44 | 45 | if source == nil { 46 | defaultSource := KisSource{ 47 | Name: "unNamedSource", 48 | } 49 | source = &defaultSource 50 | log.Logger().InfoF("funcName NewConfig source is nil, funcName = %s, use default unNamed Source.", funcName) 51 | } 52 | config.Source = *source 53 | 54 | config.FMode = string(mode) 55 | 56 | /* 57 | // Functions S and L require the KisConnector parameters to be passed as they need to establish streaming relationships through Connector 58 | if mode == common.S || mode == common.L { 59 | if option == nil { 60 | log.Logger().ErrorF("Function S/L needs option->Cid\n") 61 | return nil 62 | } else if option.CName == "" { 63 | log.Logger().ErrorF("Function S/L needs option->Cid\n") 64 | return nil 65 | } 66 | } 67 | */ 68 | 69 | if option != nil { 70 | config.Option = *option 71 | } 72 | 73 | return config 74 | } 75 | 76 | // AddConnConfig WithConn binds Function to Connector 77 | func (fConf *KisFuncConfig) AddConnConfig(cConf *KisConnConfig) error { 78 | if cConf == nil { 79 | return fmt.Errorf("KisConnConfig is nil") 80 | } 81 | 82 | // Function needs to be associated with Connector 83 | fConf.connConf = cConf 84 | 85 | // Connector needs to be associated with Function 86 | _ = cConf.WithFunc(fConf) 87 | 88 | // Update CName in Function configuration 89 | fConf.Option.CName = cConf.CName 90 | 91 | return nil 92 | } 93 | 94 | // GetConnConfig gets the Connector configuration 95 | func (fConf *KisFuncConfig) GetConnConfig() (*KisConnConfig, error) { 96 | if fConf.connConf == nil { 97 | return nil, fmt.Errorf("KisFuncConfig.connConf not set") 98 | } 99 | 100 | return fConf.connConf, nil 101 | } 102 | -------------------------------------------------------------------------------- /common/const.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "time" 4 | 5 | // Prefix string for generating KisId by users 6 | const ( 7 | KisIDTypeFlow = "flow" // KisId type for Flow 8 | KisIDTypeConnector = "conn" // KisId type for Connector 9 | KisIDTypeFunction = "func" // KisId type for Function 10 | KisIDTypeGlobal = "global" // KisId type for Global 11 | KisIDJoinChar = "-" // Joining character for KisId 12 | ) 13 | 14 | const ( 15 | // FunctionIDFirstVirtual is the virtual Function ID for the first node Function 16 | FunctionIDFirstVirtual = "FunctionIDFirstVirtual" 17 | // FunctionIDLastVirtual is the virtual Function ID for the last node Function 18 | FunctionIDLastVirtual = "FunctionIDLastVirtual" 19 | ) 20 | 21 | // KisMode represents the mode of KisFunction 22 | type KisMode string 23 | 24 | const ( 25 | // V is for Verify, which mainly performs data filtering, validation, field sorting, idempotence, etc. 26 | V KisMode = "Verify" 27 | 28 | // S is for Save, S Function will store data through KisConnector. Functions with the same Connector can logically merge. 29 | S KisMode = "Save" 30 | 31 | // L is for Load, L Function will load data through KisConnector. Functions with the same Connector can logically merge with corresponding S Function. 32 | L KisMode = "Load" 33 | 34 | // C is for Calculate, which can generate new fields, calculate new values, and perform data aggregation, analysis, etc. 35 | C KisMode = "Calculate" 36 | 37 | // E is for Expand, which serves as a custom feature Function for stream computing and is also the last Function in the current KisFlow, similar to Sink. 38 | E KisMode = "Expand" 39 | ) 40 | 41 | // KisOnOff Whether to enable the Flow 42 | type KisOnOff int 43 | 44 | const ( 45 | // FlowEnable Enabled 46 | FlowEnable KisOnOff = 1 47 | // FlowDisable Disabled 48 | FlowDisable KisOnOff = 0 49 | ) 50 | 51 | // KisConnType represents the type of KisConnector 52 | type KisConnType string 53 | 54 | const ( 55 | // REDIS is the type of Redis 56 | REDIS KisConnType = "redis" 57 | // MYSQL is the type of MySQL 58 | MYSQL KisConnType = "mysql" 59 | // KAFKA is the type of Kafka 60 | KAFKA KisConnType = "kafka" 61 | // TIDB is the type of TiDB 62 | TIDB KisConnType = "tidb" 63 | // ES is the type of Elasticsearch 64 | ES KisConnType = "es" 65 | ) 66 | 67 | // cache 68 | const ( 69 | // DeFaultFlowCacheCleanUp is the default cleanup time for Cache in KisFlow's Flow object Cache 70 | DeFaultFlowCacheCleanUp = 5 // unit: min 71 | // DefaultExpiration is the default time for GoCahce, permanent storage 72 | DefaultExpiration time.Duration = 0 73 | ) 74 | 75 | // metrics 76 | const ( 77 | MetricsRoute string = "/metrics" 78 | 79 | LabelFlowName string = "flow_name" 80 | LabelFlowID string = "flow_id" 81 | LabelFunctionName string = "func_name" 82 | LabelFunctionMode string = "func_mode" 83 | 84 | CounterKisflowDataTotalName string = "kisflow_data_total" 85 | CounterKisflowDataTotalHelp string = "Total data volume of all KisFlow Flows" 86 | 87 | GamgeFlowDataTotalName string = "flow_data_total" 88 | GamgeFlowDataTotalHelp string = "Total data volume of each KisFlow FlowID data stream" 89 | 90 | GangeFlowScheCntsName string = "flow_schedule_cnts" 91 | GangeFlowScheCntsHelp string = "Number of times each KisFlow FlowID is scheduled" 92 | 93 | GangeFuncScheCntsName string = "func_schedule_cnts" 94 | GangeFuncScheCntsHelp string = "Number of times each KisFlow Function is scheduled" 95 | 96 | HistogramFunctionDurationName string = "func_run_duration" 97 | HistogramFunctionDurationHelp string = "Function execution time" 98 | 99 | HistogramFlowDurationName string = "flow_run_duration" 100 | HistogramFlowDurationHelp string = "Flow execution time" 101 | ) 102 | -------------------------------------------------------------------------------- /metrics/kis_metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/aceld/kis-flow/common" 7 | "github.com/aceld/kis-flow/config" 8 | "github.com/aceld/kis-flow/log" 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/prometheus/client_golang/prometheus/promhttp" 11 | ) 12 | 13 | // KisMetrics kisFlow's Prometheus monitoring metrics 14 | type KisMetrics struct { 15 | // Total data quantity 16 | DataTotal prometheus.Counter 17 | // Total data processed by each Flow 18 | FlowDataTotal *prometheus.GaugeVec 19 | // Flow scheduling counts 20 | FlowScheduleCntsToTal *prometheus.GaugeVec 21 | // Function scheduling counts 22 | FuncScheduleCntsTotal *prometheus.GaugeVec 23 | // Function execution time 24 | FunctionDuration *prometheus.HistogramVec 25 | // Flow execution time 26 | FlowDuration *prometheus.HistogramVec 27 | } 28 | 29 | var Metrics *KisMetrics 30 | 31 | // RunMetricsService starts the Prometheus monitoring service 32 | func RunMetricsService(serverAddr string) error { 33 | 34 | // Register Prometheus monitoring route path 35 | http.Handle(common.MetricsRoute, promhttp.Handler()) 36 | 37 | // Start HttpServer 38 | err := http.ListenAndServe(serverAddr, nil) // Multiple processes cannot listen on the same port 39 | if err != nil { 40 | log.Logger().ErrorF("RunMetricsService err = %s\n", err) 41 | } 42 | 43 | return err 44 | } 45 | 46 | func InitMetrics() { 47 | Metrics = new(KisMetrics) 48 | 49 | // Initialize DataTotal Counter 50 | Metrics.DataTotal = prometheus.NewCounter(prometheus.CounterOpts{ 51 | Name: common.CounterKisflowDataTotalName, 52 | Help: common.CounterKisflowDataTotalHelp, 53 | }) 54 | 55 | // Initialize FlowDataTotal GaugeVec 56 | Metrics.FlowDataTotal = prometheus.NewGaugeVec( 57 | prometheus.GaugeOpts{ 58 | Name: common.GamgeFlowDataTotalName, 59 | Help: common.GamgeFlowDataTotalHelp, 60 | }, 61 | // Label names 62 | []string{common.LabelFlowName}, 63 | ) 64 | 65 | // Initialize FlowScheduleCntsToTal GaugeVec 66 | Metrics.FlowScheduleCntsToTal = prometheus.NewGaugeVec( 67 | prometheus.GaugeOpts{ 68 | Name: common.GangeFlowScheCntsName, 69 | Help: common.GangeFlowScheCntsHelp, 70 | }, 71 | // Label names 72 | []string{common.LabelFlowName}, 73 | ) 74 | 75 | // Initialize FuncScheduleCntsTotal GaugeVec 76 | Metrics.FuncScheduleCntsTotal = prometheus.NewGaugeVec( 77 | prometheus.GaugeOpts{ 78 | Name: common.GangeFuncScheCntsName, 79 | Help: common.GangeFuncScheCntsHelp, 80 | }, 81 | // Label names 82 | []string{common.LabelFunctionName, common.LabelFunctionMode}, 83 | ) 84 | 85 | // Initialize FunctionDuration HistogramVec 86 | Metrics.FunctionDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ 87 | Name: common.HistogramFunctionDurationName, 88 | Help: common.HistogramFunctionDurationHelp, 89 | Buckets: []float64{0.005, 0.01, 0.03, 0.08, 0.1, 0.5, 1.0, 5.0, 10, 100, 1000, 5000, 30000}, // Unit: ms, maximum half a minute 90 | }, 91 | []string{common.LabelFunctionName, common.LabelFunctionMode}, 92 | ) 93 | 94 | // Initialize FlowDuration HistogramVec 95 | Metrics.FlowDuration = prometheus.NewHistogramVec( 96 | prometheus.HistogramOpts{ 97 | Name: common.HistogramFlowDurationName, 98 | Help: common.HistogramFlowDurationHelp, 99 | Buckets: []float64{0.005, 0.01, 0.03, 0.08, 0.1, 0.5, 1.0, 5.0, 10, 100, 1000, 5000, 30000, 60000}, // Unit: ms, maximum 1 minute 100 | }, 101 | []string{common.LabelFlowName}, 102 | ) 103 | 104 | // Register Metrics 105 | prometheus.MustRegister(Metrics.DataTotal) 106 | prometheus.MustRegister(Metrics.FlowDataTotal) 107 | prometheus.MustRegister(Metrics.FlowScheduleCntsToTal) 108 | prometheus.MustRegister(Metrics.FuncScheduleCntsTotal) 109 | prometheus.MustRegister(Metrics.FunctionDuration) 110 | prometheus.MustRegister(Metrics.FlowDuration) 111 | } 112 | 113 | // RunMetrics starts the Prometheus metrics service 114 | func RunMetrics() { 115 | // Initialize Prometheus metrics 116 | InitMetrics() 117 | 118 | if config.GlobalConfig.EnableProm == true && config.GlobalConfig.PrometheusListen == true { 119 | // Start Prometheus metrics service 120 | go RunMetricsService(config.GlobalConfig.PrometheusServe) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /function/kis_base_function.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "sync" 7 | 8 | "github.com/aceld/kis-flow/common" 9 | "github.com/aceld/kis-flow/config" 10 | "github.com/aceld/kis-flow/id" 11 | "github.com/aceld/kis-flow/kis" 12 | ) 13 | 14 | type BaseFunction struct { 15 | // Id, the instance ID of KisFunction, used to differentiate different instance objects within KisFlow 16 | Id string 17 | Config *config.KisFuncConfig 18 | 19 | // flow 20 | flow kis.Flow // Context environment KisFlow 21 | 22 | // connector 23 | connector kis.Connector 24 | 25 | // Custom temporary data of Function 26 | metaData map[string]interface{} 27 | // Manage the read-write lock of metaData 28 | mLock sync.RWMutex 29 | 30 | // link 31 | N kis.Function // Next flow computing Function 32 | P kis.Function // Previous flow computing Function 33 | } 34 | 35 | // Call 36 | // BaseFunction is an empty implementation, designed to allow other specific types of KisFunction, 37 | // such as KisFunction_V, to inherit BaseFuncion and override this method 38 | func (base *BaseFunction) Call(ctx context.Context, flow kis.Flow) error { return nil } 39 | 40 | func (base *BaseFunction) Next() kis.Function { 41 | return base.N 42 | } 43 | 44 | func (base *BaseFunction) Prev() kis.Function { 45 | return base.P 46 | } 47 | 48 | func (base *BaseFunction) SetN(f kis.Function) { 49 | base.N = f 50 | } 51 | 52 | func (base *BaseFunction) SetP(f kis.Function) { 53 | base.P = f 54 | } 55 | 56 | func (base *BaseFunction) SetConfig(s *config.KisFuncConfig) error { 57 | if s == nil { 58 | return errors.New("KisFuncConfig is nil") 59 | } 60 | 61 | base.Config = s 62 | 63 | return nil 64 | } 65 | 66 | func (base *BaseFunction) GetID() string { 67 | return base.Id 68 | } 69 | 70 | func (base *BaseFunction) GetPrevId() string { 71 | if base.P == nil { 72 | // Function is the first node 73 | return common.FunctionIDFirstVirtual 74 | } 75 | return base.P.GetID() 76 | } 77 | 78 | func (base *BaseFunction) GetNextId() string { 79 | if base.N == nil { 80 | // Function is the last node 81 | return common.FunctionIDLastVirtual 82 | } 83 | return base.N.GetID() 84 | } 85 | 86 | func (base *BaseFunction) GetConfig() *config.KisFuncConfig { 87 | return base.Config 88 | } 89 | 90 | func (base *BaseFunction) SetFlow(f kis.Flow) error { 91 | if f == nil { 92 | return errors.New("KisFlow is nil") 93 | } 94 | base.flow = f 95 | return nil 96 | } 97 | 98 | func (base *BaseFunction) GetFlow() kis.Flow { 99 | return base.flow 100 | } 101 | 102 | // AddConnector adds a Connector to the current Function instance 103 | func (base *BaseFunction) AddConnector(conn kis.Connector) error { 104 | if conn == nil { 105 | return errors.New("conn is nil") 106 | } 107 | 108 | base.connector = conn 109 | 110 | return nil 111 | } 112 | 113 | // GetConnector gets the Connector associated with the current Function instance 114 | func (base *BaseFunction) GetConnector() kis.Connector { 115 | return base.connector 116 | } 117 | 118 | func (base *BaseFunction) CreateId() { 119 | base.Id = id.KisID(common.KisIDTypeFunction) 120 | } 121 | 122 | // NewKisFunction creates a new NsFunction 123 | // flow: the current belonging flow instance 124 | // s: the configuration strategy of the current function 125 | func NewKisFunction(flow kis.Flow, config *config.KisFuncConfig) kis.Function { 126 | var f kis.Function 127 | 128 | // Factory produces generic objects 129 | switch common.KisMode(config.FMode) { 130 | case common.V: 131 | f = NewKisFunctionV() 132 | case common.S: 133 | f = NewKisFunctionS() 134 | case common.L: 135 | f = NewKisFunctionL() 136 | case common.C: 137 | f = NewKisFunctionC() 138 | case common.E: 139 | f = NewKisFunctionE() 140 | default: 141 | // LOG ERROR 142 | return nil 143 | } 144 | 145 | // Generate a random unique instance ID 146 | f.CreateId() 147 | 148 | // Set basic information attributes 149 | if err := f.SetConfig(config); err != nil { 150 | panic(err) 151 | } 152 | 153 | // Set Flow 154 | if err := f.SetFlow(flow); err != nil { 155 | panic(err) 156 | } 157 | 158 | return f 159 | } 160 | 161 | // GetMetaData gets the temporary data of the current Function 162 | func (base *BaseFunction) GetMetaData(key string) interface{} { 163 | base.mLock.RLock() 164 | defer base.mLock.RUnlock() 165 | 166 | data, ok := base.metaData[key] 167 | if !ok { 168 | return nil 169 | } 170 | 171 | return data 172 | } 173 | 174 | // SetMetaData sets the temporary data of the current Function 175 | func (base *BaseFunction) SetMetaData(key string, value interface{}) { 176 | base.mLock.Lock() 177 | defer base.mLock.Unlock() 178 | 179 | base.metaData[key] = value 180 | } 181 | -------------------------------------------------------------------------------- /serialize/serialize_default.go: -------------------------------------------------------------------------------- 1 | package serialize 2 | 3 | /* 4 | DefaultSerialize implements the Serialize interface, 5 | which is used to serialize KisRowArr into a specified type, or serialize a specified type into KisRowArr. 6 | 7 | This section is the default serialization method provided by KisFlow, and it defaults to json serialization. 8 | 9 | Developers can implement their own serialization methods according to their needs. 10 | */ 11 | 12 | import ( 13 | "encoding/json" 14 | "fmt" 15 | "reflect" 16 | 17 | "github.com/aceld/kis-flow/common" 18 | ) 19 | 20 | type DefaultSerialize struct{} 21 | 22 | // UnMarshal is used to deserialize KisRowArr into a specified type. 23 | func (f *DefaultSerialize) UnMarshal(arr common.KisRowArr, r reflect.Type) (reflect.Value, error) { 24 | // Ensure the passed-in type is a slice 25 | if r.Kind() != reflect.Slice { 26 | return reflect.Value{}, fmt.Errorf("r must be a slice") 27 | } 28 | 29 | slice := reflect.MakeSlice(r, 0, len(arr)) 30 | 31 | // Iterate through each element and attempt deserialization 32 | for _, row := range arr { 33 | var elem reflect.Value 34 | var err error 35 | 36 | // Try to assert as a struct or pointer 37 | elem, err = unMarshalStruct(row, r.Elem()) 38 | if err == nil { 39 | slice = reflect.Append(slice, elem) 40 | continue 41 | } 42 | 43 | // Try to directly deserialize the string 44 | elem, err = unMarshalJsonString(row, r.Elem()) 45 | if err == nil { 46 | slice = reflect.Append(slice, elem) 47 | continue 48 | } 49 | 50 | // Try to serialize to JSON first and then deserialize 51 | elem, err = unMarshalJsonStruct(row, r.Elem()) 52 | if err == nil { 53 | slice = reflect.Append(slice, elem) 54 | } else { 55 | return reflect.Value{}, fmt.Errorf("failed to decode row: %v", err) 56 | } 57 | } 58 | 59 | return slice, nil 60 | } 61 | 62 | // Try to assert as a struct or pointer 63 | func unMarshalStruct(row common.KisRow, elemType reflect.Type) (reflect.Value, error) { 64 | // Check if row is of struct or struct pointer type 65 | rowType := reflect.TypeOf(row) 66 | if rowType == nil { 67 | return reflect.Value{}, fmt.Errorf("row is nil pointer") 68 | } 69 | if rowType.Kind() != reflect.Struct && rowType.Kind() != reflect.Ptr { 70 | return reflect.Value{}, fmt.Errorf("row must be a struct or struct pointer type") 71 | } 72 | 73 | // If row is a pointer type, get the type it points to 74 | if rowType.Kind() == reflect.Ptr { 75 | // Nil pointer 76 | if reflect.ValueOf(row).IsNil() { 77 | return reflect.Value{}, fmt.Errorf("row is nil pointer") 78 | } 79 | 80 | // Dereference 81 | row = reflect.ValueOf(row).Elem().Interface() 82 | 83 | // Get the type after dereferencing 84 | rowType = reflect.TypeOf(row) 85 | } 86 | 87 | // Check if row can be asserted to elemType(target type) 88 | if !rowType.AssignableTo(elemType) { 89 | return reflect.Value{}, fmt.Errorf("row type cannot be asserted to elemType") 90 | } 91 | 92 | // Convert row to reflect.Value and return 93 | return reflect.ValueOf(row), nil 94 | } 95 | 96 | // Try to directly deserialize the string(Deserialize the Json string into a struct) 97 | func unMarshalJsonString(row common.KisRow, elemType reflect.Type) (reflect.Value, error) { 98 | // Check if the source data can be asserted as a string 99 | str, ok := row.(string) 100 | if !ok { 101 | return reflect.Value{}, fmt.Errorf("not a string") 102 | } 103 | 104 | // Create a new struct instance to store the deserialized value 105 | elem := reflect.New(elemType).Elem() 106 | 107 | // Try to deserialize the json string into a struct. 108 | if err := json.Unmarshal([]byte(str), elem.Addr().Interface()); err != nil { 109 | return reflect.Value{}, fmt.Errorf("failed to unmarshal string to struct: %v", err) 110 | } 111 | 112 | return elem, nil 113 | } 114 | 115 | // Try to serialize to JSON first and then deserialize(Serialize the struct to Json string, and then deserialize the Json string into a struct) 116 | func unMarshalJsonStruct(row common.KisRow, elemType reflect.Type) (reflect.Value, error) { 117 | // Serialize row to JSON string 118 | jsonBytes, err := json.Marshal(row) 119 | if err != nil { 120 | return reflect.Value{}, fmt.Errorf("failed to marshal row to JSON: %v ", err) 121 | } 122 | 123 | // Create a new struct instance to store the deserialized value 124 | elem := reflect.New(elemType).Interface() 125 | 126 | // Deserialize the JSON string into a struct 127 | if err := json.Unmarshal(jsonBytes, elem); err != nil { 128 | return reflect.Value{}, fmt.Errorf("failed to unmarshal JSON to element: %v ", err) 129 | } 130 | 131 | return reflect.ValueOf(elem).Elem(), nil 132 | } 133 | 134 | // Marshal is used to serialize a specified type into KisRowArr(json serialization). 135 | func (f *DefaultSerialize) Marshal(i interface{}) (common.KisRowArr, error) { 136 | var arr common.KisRowArr 137 | 138 | switch reflect.TypeOf(i).Kind() { 139 | case reflect.Slice, reflect.Array: 140 | slice := reflect.ValueOf(i) 141 | for i := 0; i < slice.Len(); i++ { 142 | // Serialize each element to a JSON string and append it to the slice. 143 | jsonBytes, err := json.Marshal(slice.Index(i).Interface()) 144 | if err != nil { 145 | return nil, fmt.Errorf("failed to marshal element to JSON: %v ", err) 146 | } 147 | arr = append(arr, string(jsonBytes)) 148 | } 149 | default: 150 | // If it's not a slice or array type, serialize the entire struct to a JSON string directly. 151 | jsonBytes, err := json.Marshal(i) 152 | if err != nil { 153 | return nil, fmt.Errorf("failed to marshal element to JSON: %v ", err) 154 | } 155 | arr = append(arr, string(jsonBytes)) 156 | } 157 | 158 | return arr, nil 159 | } 160 | -------------------------------------------------------------------------------- /kis/faas.go: -------------------------------------------------------------------------------- 1 | package kis 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "reflect" 7 | "runtime" 8 | "strings" 9 | ) 10 | 11 | // FaaS Function as a Service 12 | 13 | // Change the type definition from: 14 | // type FaaS func(context.Context, Flow) error 15 | // to: 16 | // type FaaS func(context.Context, Flow, ...interface{}) error 17 | // This allows passing data through variadic parameters of any type. 18 | type FaaS interface{} 19 | 20 | // FaaSDesc describes the FaaS callback computation function. 21 | type FaaSDesc struct { 22 | Serialize // Serialization implementation for the current Function's data input and output 23 | FnName string // Function name 24 | f interface{} // FaaS function 25 | fName string // Function name 26 | ArgsType []reflect.Type // Function parameter types (collection) 27 | ArgNum int // Number of function parameters 28 | FuncType reflect.Type // Function type 29 | FuncValue reflect.Value // Function value (function address) 30 | } 31 | 32 | // NewFaaSDesc creates an instance of FaaSDesc description based on the registered FnName and FaaS callback function. 33 | func NewFaaSDesc(fnName string, f FaaS) (*FaaSDesc, error) { 34 | 35 | // Serialization instance 36 | var serializeImpl Serialize 37 | 38 | // Callback function value (function address) 39 | funcValue := reflect.ValueOf(f) 40 | 41 | // Callback function type 42 | funcType := funcValue.Type() 43 | 44 | // Check if the provided FaaS pointer is a function type 45 | if !isFuncType(funcType) { 46 | return nil, fmt.Errorf("provided FaaS type is %s, not a function", funcType.Name()) 47 | } 48 | 49 | // Check if the FaaS function has a return value that only includes (error) 50 | if funcType.NumOut() != 1 || funcType.Out(0) != reflect.TypeOf((*error)(nil)).Elem() { 51 | return nil, errors.New("function must have exactly one return value of type error") 52 | } 53 | 54 | // FaaS function parameter types 55 | argsType := make([]reflect.Type, funcType.NumIn()) 56 | 57 | // Get the FaaS function name 58 | fullName := runtime.FuncForPC(funcValue.Pointer()).Name() 59 | 60 | // Ensure that the FaaS function parameter list contains context.Context and kis.Flow 61 | // Check if the function contains a parameter of type kis.Flow 62 | containsKisFlow := false 63 | // Check if the function contains a parameter of type context.Context 64 | containsCtx := false 65 | 66 | // Iterate over the FaaS function parameter types 67 | for i := 0; i < funcType.NumIn(); i++ { 68 | 69 | // Get the i-th formal parameter type 70 | paramType := funcType.In(i) 71 | 72 | if isFlowType(paramType) { 73 | // Check if the function contains a parameter of type kis.Flow 74 | containsKisFlow = true 75 | 76 | } else if isContextType(paramType) { 77 | // Check if the function contains a parameter of type context.Context 78 | containsCtx = true 79 | 80 | } else if isSliceType(paramType) { 81 | 82 | // Get the element type of the current parameter Slice 83 | itemType := paramType.Elem() 84 | 85 | // If the current parameter is a pointer type, get the struct type that the pointer points to 86 | if itemType.Kind() == reflect.Ptr { 87 | itemType = itemType.Elem() // Get the struct type that the pointer points to 88 | } 89 | 90 | // Check if f implements Serialize interface 91 | if isSerialize(itemType) { 92 | // If the current parameter implements the Serialize interface, use the serialization implementation of the current parameter 93 | serializeImpl = reflect.New(itemType).Interface().(Serialize) 94 | 95 | } else { 96 | // If the current parameter does not implement the Serialize interface, use the default serialization implementation 97 | serializeImpl = defaultSerialize // Use global default implementation 98 | } 99 | } 100 | 101 | // Append the current parameter type to the argsType collection 102 | argsType[i] = paramType 103 | } 104 | 105 | if !containsKisFlow { 106 | // If the function parameter list does not contain a parameter of type kis.Flow, return an error 107 | return nil, errors.New("function parameters must have kis.Flow param, please use FaaS type like: [type FaaS func(context.Context, Flow, ...interface{}) error]") 108 | } 109 | 110 | if !containsCtx { 111 | // If the function parameter list does not contain a parameter of type context.Context, return an error 112 | return nil, errors.New("function parameters must have context, please use FaaS type like: [type FaaS func(context.Context, Flow, ...interface{}) error]") 113 | } 114 | 115 | // Return the FaaSDesc description instance 116 | return &FaaSDesc{ 117 | Serialize: serializeImpl, 118 | FnName: fnName, 119 | f: f, 120 | fName: fullName, 121 | ArgsType: argsType, 122 | ArgNum: len(argsType), 123 | FuncType: funcType, 124 | FuncValue: funcValue, 125 | }, nil 126 | } 127 | 128 | // isFuncType checks whether the provided paramType is a function type 129 | func isFuncType(paramType reflect.Type) bool { 130 | return paramType.Kind() == reflect.Func 131 | } 132 | 133 | // isFlowType checks whether the provided paramType is of type kis.Flow 134 | func isFlowType(paramType reflect.Type) bool { 135 | var flowInterfaceType = reflect.TypeOf((*Flow)(nil)).Elem() 136 | 137 | return paramType.Implements(flowInterfaceType) 138 | } 139 | 140 | // isContextType checks whether the provided paramType is of type context.Context 141 | func isContextType(paramType reflect.Type) bool { 142 | typeName := paramType.Name() 143 | 144 | return strings.Contains(typeName, "Context") 145 | } 146 | 147 | // isSliceType checks whether the provided paramType is a slice type 148 | func isSliceType(paramType reflect.Type) bool { 149 | return paramType.Kind() == reflect.Slice 150 | } 151 | -------------------------------------------------------------------------------- /file/config_import.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | "path/filepath" 8 | 9 | "github.com/aceld/kis-flow/common" 10 | "github.com/aceld/kis-flow/config" 11 | "github.com/aceld/kis-flow/flow" 12 | "github.com/aceld/kis-flow/kis" 13 | "github.com/aceld/kis-flow/metrics" 14 | "gopkg.in/yaml.v3" 15 | ) 16 | 17 | type allConfig struct { 18 | Flows map[string]*config.KisFlowConfig 19 | Funcs map[string]*config.KisFuncConfig 20 | Conns map[string]*config.KisConnConfig 21 | } 22 | 23 | // kisTypeFlowConfigure parses Flow configuration file in yaml format 24 | func kisTypeFlowConfigure(all *allConfig, confData []byte, fileName string, kisType interface{}) error { 25 | flowCfg := new(config.KisFlowConfig) 26 | if ok := yaml.Unmarshal(confData, flowCfg); ok != nil { 27 | return fmt.Errorf("%s has wrong format kisType = %s", fileName, kisType) 28 | } 29 | 30 | // Skip the configuration loading if the Flow status is disabled 31 | if common.KisOnOff(flowCfg.Status) == common.FlowDisable { 32 | return nil 33 | } 34 | 35 | if _, ok := all.Flows[flowCfg.FlowName]; ok { 36 | return fmt.Errorf("%s set repeat flow_id:%s", fileName, flowCfg.FlowName) 37 | } 38 | 39 | // Add to the configuration set 40 | all.Flows[flowCfg.FlowName] = flowCfg 41 | 42 | return nil 43 | } 44 | 45 | // kisTypeFuncConfigure parses Function configuration file in yaml format 46 | func kisTypeFuncConfigure(all *allConfig, confData []byte, fileName string, kisType interface{}) error { 47 | function := new(config.KisFuncConfig) 48 | if ok := yaml.Unmarshal(confData, function); ok != nil { 49 | return fmt.Errorf("%s has wrong format kisType = %s", fileName, kisType) 50 | } 51 | if _, ok := all.Funcs[function.FName]; ok { 52 | return fmt.Errorf("%s set repeat function_id:%s", fileName, function.FName) 53 | } 54 | 55 | // Add to the configuration set 56 | all.Funcs[function.FName] = function 57 | 58 | return nil 59 | } 60 | 61 | // kisTypeConnConfigure parses Connector configuration file in yaml format 62 | func kisTypeConnConfigure(all *allConfig, confData []byte, fileName string, kisType interface{}) error { 63 | conn := new(config.KisConnConfig) 64 | if ok := yaml.Unmarshal(confData, conn); ok != nil { 65 | return fmt.Errorf("%s has wrong format kisType = %s", fileName, kisType) 66 | } 67 | 68 | if _, ok := all.Conns[conn.CName]; ok { 69 | return fmt.Errorf("%s set repeat conn_id:%s", fileName, conn.CName) 70 | } 71 | 72 | // Add to the configuration set 73 | all.Conns[conn.CName] = conn 74 | 75 | return nil 76 | } 77 | 78 | // kisTypeGlobalConfigure parses Global configuration file in yaml format 79 | func kisTypeGlobalConfigure(confData []byte, fileName string, kisType interface{}) error { 80 | // Global configuration 81 | if ok := yaml.Unmarshal(confData, config.GlobalConfig); ok != nil { 82 | return fmt.Errorf("%s is wrong format kisType = %s", fileName, kisType) 83 | } 84 | 85 | // Start Metrics service 86 | metrics.RunMetrics() 87 | 88 | return nil 89 | } 90 | 91 | // parseConfigWalkYaml recursively parses all configuration files in yaml format and stores the configuration information in allConfig 92 | func parseConfigWalkYaml(loadPath string) (*allConfig, error) { 93 | 94 | all := new(allConfig) 95 | 96 | all.Flows = make(map[string]*config.KisFlowConfig) 97 | all.Funcs = make(map[string]*config.KisFuncConfig) 98 | all.Conns = make(map[string]*config.KisConnConfig) 99 | 100 | err := filepath.Walk(loadPath, func(filePath string, info os.FileInfo, err error) error { 101 | // Validate the file extension 102 | if suffix := path.Ext(filePath); suffix != ".yml" && suffix != ".yaml" { 103 | return nil 104 | } 105 | 106 | // Read file content 107 | confData, err := os.ReadFile(filePath) 108 | if err != nil { 109 | return err 110 | } 111 | 112 | confMap := make(map[string]interface{}) 113 | 114 | // Validate yaml format 115 | if err := yaml.Unmarshal(confData, confMap); err != nil { 116 | return err 117 | } 118 | 119 | // Check if kisType exists 120 | var kisType interface{} 121 | 122 | kisType, ok := confMap["kistype"] 123 | if !ok { 124 | return fmt.Errorf("%s has no field [kistype]", filePath) 125 | } 126 | 127 | switch kisType { 128 | case common.KisIDTypeFlow: 129 | return kisTypeFlowConfigure(all, confData, filePath, kisType) 130 | 131 | case common.KisIDTypeFunction: 132 | return kisTypeFuncConfigure(all, confData, filePath, kisType) 133 | 134 | case common.KisIDTypeConnector: 135 | return kisTypeConnConfigure(all, confData, filePath, kisType) 136 | 137 | case common.KisIDTypeGlobal: 138 | return kisTypeGlobalConfigure(confData, filePath, kisType) 139 | 140 | default: 141 | return fmt.Errorf("%s set wrong kistype %s", filePath, kisType) 142 | } 143 | }) 144 | 145 | if err != nil { 146 | return nil, err 147 | } 148 | 149 | return all, nil 150 | } 151 | 152 | func buildFlow(all *allConfig, fp config.KisFlowFunctionParam, newFlow kis.Flow, flowName string) error { 153 | // Load the Functions that the current Flow depends on 154 | if funcConfig, ok := all.Funcs[fp.FuncName]; !ok { 155 | return fmt.Errorf("FlowName [%s] need FuncName [%s], But has No This FuncName Config", flowName, fp.FuncName) 156 | } else { 157 | // flow add connector 158 | if funcConfig.Option.CName != "" { 159 | // Load the Connectors that the current Function depends on 160 | if connConf, ok := all.Conns[funcConfig.Option.CName]; !ok { 161 | return fmt.Errorf("FuncName [%s] need ConnName [%s], But has No This ConnName Config", fp.FuncName, funcConfig.Option.CName) 162 | } else { 163 | // Function Config associates with Connector Config 164 | _ = funcConfig.AddConnConfig(connConf) 165 | } 166 | } 167 | 168 | // flow add function 169 | if err := newFlow.AppendNewFunction(funcConfig, fp.Params); err != nil { 170 | return err 171 | } 172 | } 173 | 174 | return nil 175 | } 176 | 177 | // ConfigImportYaml recursively parses all configuration files in yaml format 178 | func ConfigImportYaml(loadPath string) error { 179 | 180 | all, err := parseConfigWalkYaml(loadPath) 181 | if err != nil { 182 | return err 183 | } 184 | 185 | for flowName, flowConfig := range all.Flows { 186 | 187 | // Build a new Flow 188 | newFlow := flow.NewKisFlow(flowConfig) 189 | 190 | for _, fp := range flowConfig.Flows { 191 | if err := buildFlow(all, fp, newFlow, flowName); err != nil { 192 | return err 193 | } 194 | } 195 | 196 | // Add the flow to FlowPool 197 | kis.Pool().AddFlow(flowName, newFlow) 198 | } 199 | 200 | return nil 201 | } 202 | -------------------------------------------------------------------------------- /kis/pool.go: -------------------------------------------------------------------------------- 1 | package kis 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "reflect" 8 | "sync" 9 | 10 | "github.com/aceld/kis-flow/common" 11 | "github.com/aceld/kis-flow/log" 12 | ) 13 | 14 | var _poolOnce sync.Once 15 | 16 | // KisPool manages all Function and Flow configurations 17 | type KisPool struct { 18 | fnRouter funcRouter // All Function management routes 19 | fnLock sync.RWMutex // fnRouter lock 20 | 21 | flowRouter flowRouter // All flow objects 22 | flowLock sync.RWMutex // flowRouter lock 23 | 24 | cInitRouter connInitRouter // All Connector initialization routes 25 | ciLock sync.RWMutex // cInitRouter lock 26 | 27 | cTree connTree // All Connector management routes 28 | cLock sync.RWMutex // cTree lock 29 | } 30 | 31 | // Singleton 32 | var _pool *KisPool 33 | 34 | // Pool Singleton constructor 35 | func Pool() *KisPool { 36 | _poolOnce.Do(func() { 37 | // Create KisPool object 38 | _pool = &KisPool{} 39 | 40 | // Initialize fnRouter 41 | _pool.fnRouter = make(funcRouter) 42 | 43 | // Initialize flowRouter 44 | _pool.flowRouter = make(flowRouter) 45 | 46 | // Initialize connTree 47 | _pool.cTree = make(connTree) 48 | _pool.cInitRouter = make(connInitRouter) 49 | }) 50 | 51 | return _pool 52 | } 53 | 54 | func (pool *KisPool) AddFlow(name string, flow Flow) { 55 | pool.flowLock.Lock() // Write lock 56 | defer pool.flowLock.Unlock() 57 | 58 | if _, ok := pool.flowRouter[name]; !ok { 59 | pool.flowRouter[name] = flow 60 | } else { 61 | errString := fmt.Sprintf("Pool AddFlow Repeat FlowName=%s\n", name) 62 | panic(errString) 63 | } 64 | 65 | log.Logger().InfoF("Add FlowRouter FlowName=%s", name) 66 | } 67 | 68 | func (pool *KisPool) GetFlow(name string) Flow { 69 | pool.flowLock.RLock() // Read lock 70 | defer pool.flowLock.RUnlock() 71 | 72 | if flow, ok := pool.flowRouter[name]; ok { 73 | return flow 74 | } else { 75 | return nil 76 | } 77 | } 78 | 79 | // FaaS registers Function computation business logic, indexed and registered by Function Name 80 | func (pool *KisPool) FaaS(fnName string, f FaaS) { 81 | 82 | // When registering the FaaS computation logic callback, create a FaaSDesc description object 83 | faaSDesc, err := NewFaaSDesc(fnName, f) 84 | if err != nil { 85 | panic(err) 86 | } 87 | 88 | pool.fnLock.Lock() // Write lock 89 | defer pool.fnLock.Unlock() 90 | 91 | if _, ok := pool.fnRouter[fnName]; !ok { 92 | // Register the FaaSDesc description object to fnRouter 93 | pool.fnRouter[fnName] = faaSDesc 94 | } else { 95 | errString := fmt.Sprintf("KisPoll FaaS Repeat FuncName=%s", fnName) 96 | panic(errString) 97 | } 98 | 99 | log.Logger().InfoF("Add KisPool FuncName=%s", fnName) 100 | } 101 | 102 | // CallFunction schedules Function 103 | func (pool *KisPool) CallFunction(ctx context.Context, fnName string, flow Flow) error { 104 | pool.fnLock.RLock() // Read lock 105 | defer pool.fnLock.RUnlock() 106 | if funcDesc, ok := pool.fnRouter[fnName]; ok { 107 | 108 | // Parameters list for the scheduled Function 109 | params := make([]reflect.Value, 0, funcDesc.ArgNum) 110 | 111 | for _, argType := range funcDesc.ArgsType { 112 | 113 | // If it is a Flow type parameter, pass in the value of flow 114 | if isFlowType(argType) { 115 | params = append(params, reflect.ValueOf(flow)) 116 | continue 117 | } 118 | 119 | // If it is a Context type parameter, pass in the value of ctx 120 | if isContextType(argType) { 121 | params = append(params, reflect.ValueOf(ctx)) 122 | continue 123 | } 124 | 125 | // If it is a Slice type parameter, pass in the value of flow.Input() 126 | if isSliceType(argType) { 127 | 128 | // Deserialize the raw data in flow.Input() to data of type argType 129 | value, err := funcDesc.Serialize.UnMarshal(flow.Input(), argType) 130 | if err != nil { 131 | log.Logger().ErrorFX(ctx, "funcDesc.Serialize.DecodeParam err=%v", err) 132 | } else { 133 | params = append(params, value) 134 | continue 135 | } 136 | 137 | } 138 | 139 | // If the passed parameter is neither a Flow type, nor a Context type, nor a Slice type, it defaults to zero value 140 | params = append(params, reflect.Zero(argType)) 141 | } 142 | 143 | // Call the computation logic of the current Function 144 | retValues := funcDesc.FuncValue.Call(params) 145 | 146 | // Extract the first return value, if it is nil, return nil 147 | ret := retValues[0].Interface() 148 | if ret == nil { 149 | return nil 150 | } 151 | 152 | // If the return value is of type error, return error 153 | return retValues[0].Interface().(error) 154 | 155 | } 156 | 157 | log.Logger().ErrorFX(ctx, "FuncName: %s Can not find in KisPool, Not Added.\n", fnName) 158 | 159 | return errors.New("FuncName: " + fnName + " Can not find in NsPool, Not Added.") 160 | } 161 | 162 | // CaaSInit registers Connector initialization business 163 | func (pool *KisPool) CaaSInit(cname string, c ConnInit) { 164 | pool.ciLock.Lock() // Write lock 165 | defer pool.ciLock.Unlock() 166 | 167 | if _, ok := pool.cInitRouter[cname]; !ok { 168 | pool.cInitRouter[cname] = c 169 | } else { 170 | errString := fmt.Sprintf("KisPool Reg CaaSInit Repeat CName=%s\n", cname) 171 | panic(errString) 172 | } 173 | 174 | log.Logger().InfoF("Add KisPool CaaSInit CName=%s", cname) 175 | } 176 | 177 | // CallConnInit schedules ConnInit 178 | func (pool *KisPool) CallConnInit(conn Connector) error { 179 | pool.ciLock.RLock() // Read lock 180 | defer pool.ciLock.RUnlock() 181 | 182 | init, ok := pool.cInitRouter[conn.GetName()] 183 | 184 | if !ok { 185 | panic(errors.New(fmt.Sprintf("init connector cname = %s not reg..", conn.GetName()))) 186 | } 187 | 188 | return init(conn) 189 | } 190 | 191 | // CaaS registers Connector Call business 192 | func (pool *KisPool) CaaS(cname string, fname string, mode common.KisMode, c CaaS) { 193 | pool.cLock.Lock() // Write lock 194 | defer pool.cLock.Unlock() 195 | 196 | if _, ok := pool.cTree[cname]; !ok { 197 | //cid First registration, does not exist, create a second-level tree NsConnSL 198 | pool.cTree[cname] = make(connSL) 199 | 200 | // Initialize various FunctionMode 201 | pool.cTree[cname][common.S] = make(connFuncRouter) 202 | pool.cTree[cname][common.L] = make(connFuncRouter) 203 | } 204 | 205 | if _, ok := pool.cTree[cname][mode][fname]; !ok { 206 | pool.cTree[cname][mode][fname] = c 207 | } else { 208 | errString := fmt.Sprintf("CaaS Repeat CName=%s, FName=%s, Mode =%s\n", cname, fname, mode) 209 | panic(errString) 210 | } 211 | 212 | log.Logger().InfoF("Add KisPool CaaS CName=%s, FName=%s, Mode =%s", cname, fname, mode) 213 | } 214 | 215 | // CallConnector schedules Connector 216 | func (pool *KisPool) CallConnector(ctx context.Context, flow Flow, conn Connector, args interface{}) (interface{}, error) { 217 | pool.cLock.RLock() // Read lock 218 | defer pool.cLock.RUnlock() 219 | fn := flow.GetThisFunction() 220 | fnConf := fn.GetConfig() 221 | mode := common.KisMode(fnConf.FMode) 222 | 223 | if callback, ok := pool.cTree[conn.GetName()][mode][fnConf.FName]; ok { 224 | return callback(ctx, conn, fn, flow, args) 225 | } 226 | 227 | log.Logger().ErrorFX(ctx, "CName:%s FName:%s mode:%s Can not find in KisPool, Not Added.\n", conn.GetName(), fnConf.FName, mode) 228 | 229 | return nil, errors.New(fmt.Sprintf("CName:%s FName:%s mode:%s Can not find in KisPool, Not Added.", conn.GetName(), fnConf.FName, mode)) 230 | } 231 | 232 | // GetFlows retrieves all Flows 233 | func (pool *KisPool) GetFlows() []Flow { 234 | pool.flowLock.RLock() // Read lock 235 | defer pool.flowLock.RUnlock() 236 | 237 | var flows []Flow 238 | 239 | for _, flow := range pool.flowRouter { 240 | flows = append(flows, flow) 241 | } 242 | 243 | return flows 244 | } 245 | -------------------------------------------------------------------------------- /flow/kis_flow_data.go: -------------------------------------------------------------------------------- 1 | package flow 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "reflect" 8 | "time" 9 | 10 | "github.com/aceld/kis-flow/common" 11 | "github.com/aceld/kis-flow/config" 12 | "github.com/aceld/kis-flow/log" 13 | "github.com/aceld/kis-flow/metrics" 14 | ) 15 | 16 | // CommitRow submits a single row of data to the Flow; multiple rows can be submitted multiple times 17 | func (flow *KisFlow) CommitRow(row interface{}) error { 18 | 19 | flow.buffer = append(flow.buffer, row) 20 | 21 | return nil 22 | } 23 | 24 | // CommitRowBatch submits a batch of data to the Flow 25 | func (flow *KisFlow) CommitRowBatch(rows interface{}) error { 26 | v := reflect.ValueOf(rows) 27 | if v.Kind() != reflect.Slice { 28 | return fmt.Errorf("Commit Data is not a slice") 29 | } 30 | 31 | for i := 0; i < v.Len(); i++ { 32 | row := v.Index(i).Interface().(common.KisRow) 33 | flow.buffer = append(flow.buffer, row) 34 | } 35 | 36 | return nil 37 | } 38 | 39 | // Input gets the input data for the currently executing Function in the Flow 40 | func (flow *KisFlow) Input() common.KisRowArr { 41 | return flow.inPut 42 | } 43 | 44 | // commitSrcData submits the data source data for the current Flow, indicating the first submission of the original data source for the current Flow 45 | // The flow's temporary data buffer is submitted to the flow's data (data is the source data backup for each Function level) 46 | // All previous flow data will be cleared 47 | func (flow *KisFlow) commitSrcData(ctx context.Context) error { 48 | 49 | // Create a batch of data 50 | dataCnt := len(flow.buffer) 51 | batch := make(common.KisRowArr, 0, dataCnt) 52 | 53 | for _, row := range flow.buffer { 54 | batch = append(batch, row) 55 | } 56 | 57 | // Clear all previous data 58 | flow.clearData(flow.data) 59 | 60 | // Record the original data for the flow for the first time 61 | // Because it is the first submission, PrevFunctionId is FirstVirtual because there is no upper Function 62 | flow.data[common.FunctionIDFirstVirtual] = batch 63 | 64 | // Clear the buffer 65 | flow.buffer = flow.buffer[0:0] 66 | 67 | // The first submission of data source data, for statistical total data 68 | if config.GlobalConfig.EnableProm == true { 69 | // Statistics for total data Metrics.DataTotal accumulates by 1 70 | metrics.Metrics.DataTotal.Add(float64(dataCnt)) 71 | 72 | // Statistics for current Flow quantity index 73 | metrics.Metrics.FlowDataTotal.WithLabelValues(flow.Name).Add(float64(dataCnt)) 74 | } 75 | 76 | log.Logger().DebugFX(ctx, "====> After CommitSrcData, flow_name = %s, flow_id = %s\nAll Level Data =\n %+v\n", flow.Name, flow.Id, flow.data) 77 | 78 | return nil 79 | } 80 | 81 | // getCurData gets the input data for the current Function level of the flow 82 | func (flow *KisFlow) getCurData() (common.KisRowArr, error) { 83 | if flow.PrevFunctionId == "" { 84 | return nil, errors.New(fmt.Sprintf("flow.PrevFunctionId is not set")) 85 | } 86 | 87 | if _, ok := flow.data[flow.PrevFunctionId]; !ok { 88 | return nil, errors.New(fmt.Sprintf("[%s] is not in flow.data", flow.PrevFunctionId)) 89 | } 90 | 91 | return flow.data[flow.PrevFunctionId], nil 92 | } 93 | 94 | // commitReuseData 95 | func (flow *KisFlow) commitReuseData(ctx context.Context) error { 96 | 97 | // Check if there are result data from the upper layer; if not, exit the current Flow Run loop 98 | if len(flow.data[flow.PrevFunctionId]) == 0 { 99 | flow.abort = true 100 | return nil 101 | } 102 | 103 | // This layer's result data is equal to the upper layer's result data (reuse the upper layer's result data to this layer) 104 | flow.data[flow.ThisFunctionId] = flow.data[flow.PrevFunctionId] 105 | 106 | // Clear the buffer (If it is a ReuseData option, all the submitted data will not be carried to the next layer) 107 | flow.buffer = flow.buffer[0:0] 108 | 109 | log.Logger().DebugFX(ctx, " ====> After commitReuseData, flow_name = %s, flow_id = %s\nAll Level Data =\n %+v\n", flow.Name, flow.Id, flow.data) 110 | 111 | return nil 112 | } 113 | 114 | func (flow *KisFlow) commitVoidData(ctx context.Context) error { 115 | if len(flow.buffer) != 0 { 116 | return nil 117 | } 118 | 119 | // Create empty data 120 | batch := make(common.KisRowArr, 0) 121 | 122 | // Submit the calculated buffer data of this layer to the result data of this layer 123 | flow.data[flow.ThisFunctionId] = batch 124 | 125 | log.Logger().DebugFX(ctx, " ====> After commitVoidData, flow_name = %s, flow_id = %s\nAll Level Data =\n %+v\n", flow.Name, flow.Id, flow.data) 126 | 127 | return nil 128 | } 129 | 130 | // commitCurData submits the result data of the currently executing Function in the Flow 131 | func (flow *KisFlow) commitCurData(ctx context.Context) error { 132 | 133 | // Check if this layer's calculation has result data; if not, exit the current Flow Run loop 134 | if len(flow.buffer) == 0 { 135 | flow.abort = true 136 | return nil 137 | } 138 | 139 | // Create a batch of data 140 | batch := make(common.KisRowArr, 0, len(flow.buffer)) 141 | 142 | // If strBuf is empty, no data has been added 143 | for _, row := range flow.buffer { 144 | batch = append(batch, row) 145 | } 146 | 147 | // Submit the calculated buffer data of this layer to the result data of this layer 148 | flow.data[flow.ThisFunctionId] = batch 149 | 150 | // Clear the buffer 151 | flow.buffer = flow.buffer[0:0] 152 | 153 | log.Logger().DebugFX(ctx, " ====> After commitCurData, flow_name = %s, flow_id = %s\nAll Level Data =\n %+v\n", flow.Name, flow.Id, flow.data) 154 | 155 | return nil 156 | } 157 | 158 | // clearData clears all flow data 159 | func (flow *KisFlow) clearData(data common.KisDataMap) { 160 | for k := range data { 161 | delete(data, k) 162 | } 163 | } 164 | 165 | func (flow *KisFlow) GetCacheData(key string) interface{} { 166 | 167 | if data, found := flow.cache.Get(key); found { 168 | return data 169 | } 170 | 171 | return nil 172 | } 173 | 174 | func (flow *KisFlow) SetCacheData(key string, value interface{}, Exp time.Duration) { 175 | if Exp == common.DefaultExpiration { 176 | flow.cache.Set(key, value, 0) 177 | } else { 178 | flow.cache.Set(key, value, Exp) 179 | } 180 | } 181 | 182 | // GetMetaData gets the temporary data of the current Flow object 183 | func (flow *KisFlow) GetMetaData(key string) interface{} { 184 | flow.mLock.RLock() 185 | defer flow.mLock.RUnlock() 186 | 187 | data, ok := flow.metaData[key] 188 | if !ok { 189 | return nil 190 | } 191 | 192 | return data 193 | } 194 | 195 | // SetMetaData sets the temporary data of the current Flow object 196 | func (flow *KisFlow) SetMetaData(key string, value interface{}) { 197 | flow.mLock.Lock() 198 | defer flow.mLock.Unlock() 199 | 200 | flow.metaData[key] = value 201 | } 202 | 203 | // GetFuncParam gets the default configuration parameters of the currently executing Function in the Flow, retrieves a key-value pair 204 | func (flow *KisFlow) GetFuncParam(key string) string { 205 | flow.fplock.RLock() 206 | defer flow.fplock.RUnlock() 207 | 208 | if param, ok := flow.funcParams[flow.ThisFunctionId]; ok { 209 | if value, vok := param[key]; vok { 210 | return value 211 | } 212 | } 213 | 214 | return "" 215 | } 216 | 217 | // GetFuncParamAll gets the default configuration parameters of the currently executing Function in the Flow, retrieves all Key-Value pairs 218 | func (flow *KisFlow) GetFuncParamAll() config.FParam { 219 | flow.fplock.RLock() 220 | defer flow.fplock.RUnlock() 221 | 222 | param, ok := flow.funcParams[flow.ThisFunctionId] 223 | if !ok { 224 | return nil 225 | } 226 | 227 | return param 228 | } 229 | 230 | // GetFuncParamsAllFuncs gets the FuncParams of all Functions in the Flow, retrieves all Key-Value pairs 231 | func (flow *KisFlow) GetFuncParamsAllFuncs() map[string]config.FParam { 232 | flow.fplock.RLock() 233 | defer flow.fplock.RUnlock() 234 | 235 | return flow.funcParams 236 | } 237 | -------------------------------------------------------------------------------- /flow/kis_flow.go: -------------------------------------------------------------------------------- 1 | package flow 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "sync" 7 | "time" 8 | 9 | "github.com/aceld/kis-flow/common" 10 | "github.com/aceld/kis-flow/config" 11 | "github.com/aceld/kis-flow/conn" 12 | "github.com/aceld/kis-flow/function" 13 | "github.com/aceld/kis-flow/id" 14 | "github.com/aceld/kis-flow/kis" 15 | "github.com/aceld/kis-flow/log" 16 | "github.com/aceld/kis-flow/metrics" 17 | "github.com/prometheus/client_golang/prometheus" 18 | 19 | cache "github.com/patrickmn/go-cache" 20 | ) 21 | 22 | // KisFlow is used to manage the context environment of the entire streaming computation. 23 | type KisFlow struct { 24 | // Basic information 25 | Id string // Distributed instance ID of the Flow (used internally by KisFlow to distinguish different instances) 26 | Name string // Readable name of the Flow 27 | Conf *config.KisFlowConfig // Flow configuration policy 28 | 29 | // List of Functions 30 | Funcs map[string]kis.Function // All managed Function objects of the current flow, key: FunctionName 31 | FlowHead kis.Function // Head of the Function list owned by the current Flow 32 | FlowTail kis.Function // Tail of the Function list owned by the current Flow 33 | flock sync.RWMutex // Lock for managing linked list insertion and reading 34 | ThisFunction kis.Function // KisFunction object currently being executed in the Flow 35 | ThisFunctionId string // ID of the Function currently being executed 36 | PrevFunctionId string // ID of the previous layer Function 37 | 38 | // Function list parameters 39 | funcParams map[string]config.FParam // Custom fixed configuration parameters of the Flow in the current Function, Key: KisID of the function instance, value: FParam 40 | fplock sync.RWMutex // Lock for managing funcParams read and write 41 | 42 | // Data 43 | buffer common.KisRowArr // Internal buffer used to temporarily store input byte data, one data is interface{}, multiple data is []interface{} i.e. KisBatch 44 | data common.KisDataMap // Data sources at various levels of the streaming computation 45 | inPut common.KisRowArr // Input data for the current Function computation 46 | abort bool // Whether to abort the Flow 47 | action kis.Action // Action carried by the current Flow 48 | 49 | // Local cache of the flow 50 | cache *cache.Cache // Temporary cache context environment of the Flow 51 | 52 | // metaData of the flow 53 | metaData map[string]interface{} // Custom temporary data of the Flow 54 | mLock sync.RWMutex // Lock for managing metaData read and write 55 | } 56 | 57 | // NewKisFlow creates a KisFlow. 58 | func NewKisFlow(conf *config.KisFlowConfig) kis.Flow { 59 | flow := new(KisFlow) 60 | // Instance Id 61 | flow.Id = id.KisID(common.KisIDTypeFlow) 62 | 63 | // Basic information 64 | flow.Name = conf.FlowName 65 | flow.Conf = conf 66 | 67 | // List of Functions 68 | flow.Funcs = make(map[string]kis.Function) 69 | flow.funcParams = make(map[string]config.FParam) 70 | 71 | // Data 72 | flow.data = make(common.KisDataMap) 73 | 74 | // Initialize local cache 75 | flow.cache = cache.New(cache.NoExpiration, common.DeFaultFlowCacheCleanUp*time.Minute) 76 | 77 | // Initialize temporary data 78 | flow.metaData = make(map[string]interface{}) 79 | 80 | return flow 81 | } 82 | 83 | // Fork gets a copy (deep copy) of the Flow. 84 | func (flow *KisFlow) Fork(ctx context.Context) kis.Flow { 85 | 86 | cfg := flow.Conf 87 | 88 | // Generate a new Flow based on the previous configuration 89 | newFlow := NewKisFlow(cfg) 90 | 91 | for _, fp := range flow.Conf.Flows { 92 | if _, ok := flow.funcParams[flow.Funcs[fp.FuncName].GetID()]; !ok { 93 | // The current function has no Params configured 94 | _ = newFlow.AppendNewFunction(flow.Funcs[fp.FuncName].GetConfig(), nil) 95 | } else { 96 | // The current function has configured Params 97 | _ = newFlow.AppendNewFunction(flow.Funcs[fp.FuncName].GetConfig(), fp.Params) 98 | } 99 | } 100 | 101 | log.Logger().DebugFX(ctx, "=====>Flow Fork, oldFlow.funcParams = %+v\n", flow.funcParams) 102 | log.Logger().DebugFX(ctx, "=====>Flow Fork, newFlow.funcParams = %+v\n", newFlow.GetFuncParamsAllFuncs()) 103 | 104 | return newFlow 105 | } 106 | 107 | // Link links the Function to the Flow, and also adds the Function's configuration parameters to the Flow's configuration. 108 | // fConf: Current Function strategy 109 | // fParams: Dynamic parameters carried by the current Flow's Function 110 | func (flow *KisFlow) Link(fConf *config.KisFuncConfig, fParams config.FParam) error { 111 | 112 | // Add Function to Flow 113 | _ = flow.AppendNewFunction(fConf, fParams) 114 | 115 | // Add Function to FlowConfig 116 | flowFuncParam := config.KisFlowFunctionParam{ 117 | FuncName: fConf.FName, 118 | Params: fParams, 119 | } 120 | flow.Conf.AppendFunctionConfig(flowFuncParam) 121 | 122 | return nil 123 | } 124 | 125 | // AppendNewFunction appends a new Function to the Flow. 126 | func (flow *KisFlow) AppendNewFunction(fConf *config.KisFuncConfig, fParams config.FParam) error { 127 | // Create Function instance 128 | f := function.NewKisFunction(flow, fConf) 129 | 130 | if fConf.Option.CName != "" { 131 | // The current Function has a Connector association and needs to initialize the Connector instance 132 | 133 | // Get Connector configuration 134 | connConfig, err := fConf.GetConnConfig() 135 | if err != nil { 136 | panic(err) 137 | } 138 | 139 | // Create Connector object 140 | connector := conn.NewKisConnector(connConfig) 141 | 142 | // Initialize Connector, execute the Connector Init method 143 | if err = connector.Init(); err != nil { 144 | panic(err) 145 | } 146 | 147 | // Associate the Function instance with the Connector instance 148 | _ = f.AddConnector(connector) 149 | } 150 | 151 | // Add Function to Flow 152 | if err := flow.appendFunc(f, fParams); err != nil { 153 | return err 154 | } 155 | 156 | return nil 157 | } 158 | 159 | // appendFunc adds the Function to the Flow, linked list operation 160 | func (flow *KisFlow) appendFunc(function kis.Function, fParam config.FParam) error { 161 | 162 | if function == nil { 163 | return errors.New("AppendFunc append nil to List") 164 | } 165 | 166 | flow.flock.Lock() 167 | defer flow.flock.Unlock() 168 | 169 | if flow.FlowHead == nil { 170 | // First time adding a node 171 | flow.FlowHead = function 172 | flow.FlowTail = function 173 | 174 | function.SetN(nil) 175 | function.SetP(nil) 176 | 177 | } else { 178 | // Insert the function at the end of the linked list 179 | function.SetP(flow.FlowTail) 180 | function.SetN(nil) 181 | 182 | flow.FlowTail.SetN(function) 183 | flow.FlowTail = function 184 | } 185 | 186 | // Add the detailed Function Name-Hash correspondence to the flow object 187 | flow.Funcs[function.GetConfig().FName] = function 188 | 189 | // First add the Params parameters carried by the function by default 190 | params := make(config.FParam) 191 | for key, value := range function.GetConfig().Option.Params { 192 | params[key] = value 193 | } 194 | 195 | // Then add the function definition parameters carried by the flow (overwriting duplicates) 196 | for key, value := range fParam { 197 | params[key] = value 198 | } 199 | 200 | // Store the obtained FParams in the flow structure for direct access by the function 201 | // The key is the KisId of the current Function, not using Fid to prevent adding two Functions with the same strategy Id to a Flow 202 | flow.funcParams[function.GetID()] = params 203 | 204 | return nil 205 | } 206 | 207 | // Run starts the streaming computation of KisFlow, starting from the initial Function. 208 | func (flow *KisFlow) Run(ctx context.Context) error { 209 | 210 | var fn kis.Function 211 | 212 | fn = flow.FlowHead 213 | flow.abort = false 214 | 215 | if flow.Conf.Status == int(common.FlowDisable) { 216 | // Flow is configured to be disabled 217 | return nil 218 | } 219 | 220 | // Metrics 221 | var funcStart time.Time 222 | var flowStart time.Time 223 | 224 | // Since no Function has been executed at this time, PrevFunctionId is FirstVirtual because there is no previous layer Function 225 | flow.PrevFunctionId = common.FunctionIDFirstVirtual 226 | 227 | // Commit the original data stream 228 | if err := flow.commitSrcData(ctx); err != nil { 229 | return err 230 | } 231 | 232 | // Metrics 233 | if config.GlobalConfig.EnableProm == true { 234 | // Count the number of Flow schedules 235 | metrics.Metrics.FlowScheduleCntsToTal.WithLabelValues(flow.Name).Inc() 236 | // Count the execution time of Flow 237 | flowStart = time.Now() 238 | } 239 | 240 | // Streaming chain call 241 | for fn != nil && flow.abort == false { 242 | 243 | // Record the current Function being executed by the flow 244 | fid := fn.GetID() 245 | flow.ThisFunction = fn 246 | flow.ThisFunctionId = fid 247 | 248 | fName := fn.GetConfig().FName 249 | fMode := fn.GetConfig().FMode 250 | 251 | if config.GlobalConfig.EnableProm == true { 252 | // Count the number of Function schedules 253 | metrics.Metrics.FuncScheduleCntsTotal.WithLabelValues(fName, fMode).Inc() 254 | 255 | // Count the time consumed by Function, record the start time 256 | funcStart = time.Now() 257 | } 258 | 259 | // Get the source data that the current Function needs to process 260 | if inputData, err := flow.getCurData(); err != nil { 261 | log.Logger().ErrorFX(ctx, "flow.Run(): getCurData err = %s\n", err.Error()) 262 | return err 263 | } else { 264 | flow.inPut = inputData 265 | } 266 | 267 | if err := fn.Call(ctx, flow); err != nil { 268 | // Error 269 | return err 270 | } else { 271 | // Success 272 | fn, err = flow.dealAction(ctx, fn) 273 | if err != nil { 274 | return err 275 | } 276 | 277 | // Count the time consumed by Function 278 | if config.GlobalConfig.EnableProm == true { 279 | // Function consumption time 280 | duration := time.Since(funcStart) 281 | 282 | // Count the current Function metrics, do time statistics 283 | metrics.Metrics.FunctionDuration.With( 284 | prometheus.Labels{ 285 | common.LabelFunctionName: fName, 286 | common.LabelFunctionMode: fMode}).Observe(duration.Seconds() * 1000) 287 | } 288 | 289 | } 290 | } 291 | 292 | // Metrics 293 | if config.GlobalConfig.EnableProm == true { 294 | // Count the execution time of Flow 295 | duration := time.Since(flowStart) 296 | metrics.Metrics.FlowDuration.WithLabelValues(flow.Name).Observe(duration.Seconds() * 1000) 297 | } 298 | 299 | return nil 300 | } 301 | 302 | // Next the current Flow enters the Action action carried by the next layer Function. 303 | func (flow *KisFlow) Next(acts ...kis.ActionFunc) error { 304 | 305 | // Load the Action actions carried by Function FaaS 306 | flow.action = kis.LoadActions(acts) 307 | 308 | return nil 309 | } 310 | 311 | func (flow *KisFlow) GetName() string { 312 | return flow.Name 313 | } 314 | 315 | func (flow *KisFlow) GetID() string { 316 | return flow.Id 317 | } 318 | 319 | func (flow *KisFlow) GetThisFunction() kis.Function { 320 | return flow.ThisFunction 321 | } 322 | 323 | func (flow *KisFlow) GetThisFuncConf() *config.KisFuncConfig { 324 | return flow.ThisFunction.GetConfig() 325 | } 326 | 327 | // GetConnector gets the Connector of the Function currently being executed by the Flow. 328 | func (flow *KisFlow) GetConnector() (kis.Connector, error) { 329 | if connector := flow.ThisFunction.GetConnector(); connector != nil { 330 | return connector, nil 331 | } else { 332 | return nil, errors.New("GetConnector(): Connector is nil") 333 | } 334 | } 335 | 336 | // GetConnConf gets the Connector configuration of the Function currently being executed by the Flow. 337 | func (flow *KisFlow) GetConnConf() (*config.KisConnConfig, error) { 338 | if connector := flow.ThisFunction.GetConnector(); connector != nil { 339 | return connector.GetConfig(), nil 340 | } else { 341 | return nil, errors.New("GetConnConf(): Connector is nil") 342 | } 343 | } 344 | 345 | func (flow *KisFlow) GetConfig() *config.KisFlowConfig { 346 | return flow.Conf 347 | } 348 | 349 | // GetFuncConfigByName gets the configuration of the current Flow by Function name. 350 | func (flow *KisFlow) GetFuncConfigByName(funcName string) *config.KisFuncConfig { 351 | if f, ok := flow.Funcs[funcName]; ok { 352 | return f.GetConfig() 353 | } else { 354 | log.Logger().ErrorF("GetFuncConfigByName(): Function %s not found", funcName) 355 | return nil 356 | } 357 | } 358 | -------------------------------------------------------------------------------- /README-CN.md: -------------------------------------------------------------------------------- 1 | # 2 | [English](README.md) | 简体中文 3 | 4 | [![License](https://img.shields.io/badge/License-MIT-black.svg)](LICENSE) 5 | [![Discord](https://img.shields.io/badge/KisFlow-Discord-blue.svg)](https://discord.gg/xQ8Xxfyfcz) 6 | [![KisFlow-tutorial](https://img.shields.io/badge/KisFlowTutorial-YuQue-red.svg)](https://www.yuque.com/aceld/kis-flow) 7 | [![KisFlow-Doc](https://img.shields.io/badge/KisFlow-Doc-green.svg)](https://www.yuque.com/aceld/kis-flow-doc) 8 | 9 | 10 | #### KisFlow(Keep It Simple Flowing) 11 | 12 | 基于Golang的流式计算框架. 为保持简单的流动，强调在进行各种活动或工作时保持简洁、清晰、流畅的过程。 13 | 14 | 15 | 16 | ## KisFlow源代码 17 | 18 | Github 19 | Git: https://github.com/aceld/kis-flow 20 | 21 | GitCode 22 | Git: https://gitcode.com/aceld/kis-flow 23 | 24 | Gitee 25 | Git: https://gitee.com/Aceld/kis-flow 26 | 27 | ## 《KisFlow开发者文档》 28 | 29 | [ < KisFlow Wiki : English > ](https://github.com/aceld/kis-flow/wiki) 30 | 31 | [ < KisFlow 文档 : 简体中文> ](https://www.yuque.com/aceld/kis-flow-doc) 32 | 33 | 34 | ## 在线开发教程 35 | 36 | 37 | | platform | Entry | 38 | | ---- |----------------------------------------------------------------------------------------------------------------------------------------------------| 39 | | | [Practical Tutorial for a Streaming Computation Framework Based on Golang](https://dev.to/aceld/part-1-golang-framework-hands-on-kisflow-streaming-computing-framework-overview-8fh) | 40 | || [《基于Golang的流式计算框架实战教程》](https://www.yuque.com/aceld/hsa94o) | 41 | 42 | 43 | ## KisFlow系统定位 44 | 45 | KisFlow为业务上游计算层，上层接数仓/其他业务方ODS层、下游接本业务存储数据中心。
46 | 47 | 48 | 49 | 50 | ## KisFlow整体架构图 51 | 52 | | 层级 | 层级说明 | 包括子模块 | 53 | |-------|------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 54 | | 流式计算层 | 为KisFlow上游计算层，直接对接业务存储及数仓ODS层，如上游可以为Mysql Binlog、日志、接口数据等，为被动消费模式，提供KisFlow实时计算能力。 | **KisFlow**：分布式批量消费者,一个KisFlow是由多个KisFunction组合。

**KisConnectors**：计算数据流流中间状态持久存储及连接器。

**KisFunctions**：支持算子表达式拼接，Connectors集成、策略配置、Stateful Function模式、Slink流式拼接等。

**KisConfig：** KisFunction的绑定的流处理策略，可以绑定ReSource让Function具有固定的独立流处理能力。

**KisSource：** 对接ODS的数据源 | 55 | | 任务调度层 | 定时任务调度及执行器业务逻辑，包括任务调度平台、执行器管理、调度日志及用户管理等。提供KisFlow的定时任务、统计、聚合运算等调度计算能力。 | **任务调度平台可视化**：包括任务的运行报表、调度报表、成功比例、任务管理、配置管理、GLUE IDE等可视化管理平台。

执行器管理**KisJobs**：Golang SDK及计算自定义业务逻辑、执行器的自动注册、任务触发、终止及摘除等。

**执行器场景KisScenes：** 根据业务划分的逻辑任务集合。

**调度日志及用户管理**：任务调度日志收集、调度详细、调度流程痕迹等。 | 56 | 57 | ![KisFlow架构图drawio](https://github.com/aceld/kis-flow/assets/7778936/3b829bdb-600d-4ab9-9e62-e14f90737cc3) 58 | 59 | ![KisFlow架构设计-KisFlow整体结构 drawio](https://github.com/aceld/kis-flow/assets/7778936/efc1b29d-9dd4-4945-a35a-fb9a618002d7) 60 | 61 | KisFlow是一种流式概念形态，具体表现的特征如下：
62 | 63 | 1、一个KisFlow可以由任意KisFunction组成，且KisFlow可以动态的调整长度。
64 | 65 | 2、一个KisFunction可以随时动态的加入到某个KisFlow中，且KisFlow和KisFlow之间的关系可以通过KisFunction的Load和Save节点的加入，进行动态的并流和分流动作。
66 | 67 | 3、KisFlow在编程行为上，从面向流进行数据业务编程，变成了面向KisFunction的函数单计算逻辑的开发，接近FaaS(Function as a 68 | service)体系。 69 | 70 | ## Example 71 | 72 | 下面是简单的应用场景案例，具体应用单元用例请 参考 73 | 74 | https://github.com/aceld/kis-flow-usage 75 | 76 | #### 《KisFlow开发者文档》 77 | 78 | https://www.yuque.com/aceld/kis-flow-doc 79 | 80 | #### 安装KisFlow 81 | 82 | ```bash 83 | $go get github.com/aceld/kis-flow 84 | ``` 85 | 86 |
87 | 1. Quick Start（快速开始） 88 | 89 | ### 案例源代码 90 | 91 | https://github.com/aceld/kis-flow-usage/tree/main/1-quick_start 92 | 93 | ### 项目目录 94 | 95 | ```bash 96 | ├── faas_stu_score_avg.go 97 | ├── faas_stu_score_avg_print.go 98 | └── main.go 99 | ``` 100 | 101 | ### Flow 102 | 103 | image 104 | 105 | ### Main 106 | 107 | > main.go 108 | 109 | ```go 110 | package main 111 | 112 | import ( 113 | "context" 114 | "fmt" 115 | "github.com/aceld/kis-flow/common" 116 | "github.com/aceld/kis-flow/config" 117 | "github.com/aceld/kis-flow/flow" 118 | "github.com/aceld/kis-flow/kis" 119 | ) 120 | 121 | func main() { 122 | ctx := context.Background() 123 | 124 | // Create a new flow configuration 125 | myFlowConfig1 := config.NewFlowConfig("CalStuAvgScore", common.FlowEnable) 126 | 127 | // Create new function configuration 128 | avgStuScoreConfig := config.NewFuncConfig("AvgStuScore", common.C, nil, nil) 129 | printStuScoreConfig := config.NewFuncConfig("PrintStuAvgScore", common.E, nil, nil) 130 | 131 | // Create a new flow 132 | flow1 := flow.NewKisFlow(myFlowConfig1) 133 | 134 | // Link functions to the flow 135 | _ = flow1.Link(avgStuScoreConfig, nil) 136 | _ = flow1.Link(printStuScoreConfig, nil) 137 | 138 | // Submit a string 139 | _ = flow1.CommitRow(`{"stu_id":101, "score_1":100, "score_2":90, "score_3":80}`) 140 | // Submit a string 141 | _ = flow1.CommitRow(`{"stu_id":102, "score_1":100, "score_2":70, "score_3":60}`) 142 | 143 | // Run the flow 144 | if err := flow1.Run(ctx); err != nil { 145 | fmt.Println("err: ", err) 146 | } 147 | 148 | return 149 | } 150 | 151 | func init() { 152 | // Register functions 153 | kis.Pool().FaaS("AvgStuScore", AvgStuScore) 154 | kis.Pool().FaaS("PrintStuAvgScore", PrintStuAvgScore) 155 | } 156 | ``` 157 | 158 | ### Function1 159 | 160 | > faas_stu_score_avg.go 161 | 162 | ```go 163 | package main 164 | 165 | import ( 166 | "context" 167 | "github.com/aceld/kis-flow/kis" 168 | "github.com/aceld/kis-flow/serialize" 169 | ) 170 | 171 | type AvgStuScoreIn struct { 172 | serialize.DefaultSerialize 173 | StuId int `json:"stu_id"` 174 | Score1 int `json:"score_1"` 175 | Score2 int `json:"score_2"` 176 | Score3 int `json:"score_3"` 177 | } 178 | 179 | type AvgStuScoreOut struct { 180 | serialize.DefaultSerialize 181 | StuId int `json:"stu_id"` 182 | AvgScore float64 `json:"avg_score"` 183 | } 184 | 185 | // AvgStuScore(FaaS) 计算学生平均分 186 | func AvgStuScore(ctx context.Context, flow kis.Flow, rows []*AvgStuScoreIn) error { 187 | for _, row := range rows { 188 | 189 | out := AvgStuScoreOut{ 190 | StuId: row.StuId, 191 | AvgScore: float64(row.Score1+row.Score2+row.Score3) / 3, 192 | } 193 | 194 | // 提交结果数据 195 | _ = flow.CommitRow(out) 196 | } 197 | 198 | return nil 199 | } 200 | ``` 201 | 202 | ### Function2 203 | 204 | > faas_stu_score_avg_print.go 205 | 206 | ```go 207 | package main 208 | 209 | import ( 210 | "context" 211 | "fmt" 212 | "github.com/aceld/kis-flow/kis" 213 | "github.com/aceld/kis-flow/serialize" 214 | ) 215 | 216 | type PrintStuAvgScoreIn struct { 217 | serialize.DefaultSerialize 218 | StuId int `json:"stu_id"` 219 | AvgScore float64 `json:"avg_score"` 220 | } 221 | 222 | type PrintStuAvgScoreOut struct { 223 | serialize.DefaultSerialize 224 | } 225 | 226 | func PrintStuAvgScore(ctx context.Context, flow kis.Flow, rows []*PrintStuAvgScoreIn) error { 227 | 228 | for _, row := range rows { 229 | fmt.Printf("stuid: [%+v], avg score: [%+v]\n", row.StuId, row.AvgScore) 230 | } 231 | 232 | return nil 233 | } 234 | ``` 235 | 236 | ### OutPut 237 | 238 | ```bash 239 | Add KisPool FuncName=AvgStuScore 240 | Add KisPool FuncName=PrintStuAvgScore 241 | funcName NewConfig source is nil, funcName = AvgStuScore, use default unNamed Source. 242 | funcName NewConfig source is nil, funcName = PrintStuAvgScore, use default unNamed Source. 243 | stuid: [101], avg score: [90] 244 | stuid: [102], avg score: [76.66666666666667] 245 | ``` 246 | 247 |
248 | 249 | 250 |
251 | 2. Quick Start With Config（快速开始） 252 | 253 | ### 案例源代码 254 | 255 | https://github.com/aceld/kis-flow-usage/tree/main/2-quick_start_with_config 256 | 257 | 项目目录 258 | 259 | ```bash 260 | ├── Makefile 261 | ├── conf 262 | │ ├── flow-CalStuAvgScore.yml 263 | │ ├── func-AvgStuScore.yml 264 | │ └── func-PrintStuAvgScore.yml 265 | ├── faas_stu_score_avg.go 266 | ├── faas_stu_score_avg_print.go 267 | └── main.go 268 | ``` 269 | 270 | ### Flow 271 | 272 | image 273 | 274 | ### Config 275 | 276 | #### (1) Flow Config 277 | 278 | > conf/flow-CalStuAvgScore.yml 279 | 280 | ```yaml 281 | kistype: flow 282 | status: 1 283 | flow_name: CalStuAvgScore 284 | flows: 285 | - fname: AvgStuScore 286 | - fname: PrintStuAvgScore 287 | ``` 288 | 289 | #### (2) Function1 Config 290 | 291 | > conf/func-AvgStuScore.yml 292 | 293 | ```yaml 294 | kistype: func 295 | fname: AvgStuScore 296 | fmode: Calculate 297 | source: 298 | name: 学生学分 299 | must: 300 | - stu_id 301 | ``` 302 | 303 | #### (3) Function2(Slink) Config 304 | 305 | > conf/func-PrintStuAvgScore.yml 306 | 307 | ```yaml 308 | kistype: func 309 | fname: PrintStuAvgScore 310 | fmode: Expand 311 | source: 312 | name: 学生学分 313 | must: 314 | - stu_id 315 | ``` 316 | 317 | ### Main 318 | 319 | > main.go 320 | 321 | ```go 322 | package main 323 | 324 | import ( 325 | "context" 326 | "fmt" 327 | "github.com/aceld/kis-flow/file" 328 | "github.com/aceld/kis-flow/kis" 329 | ) 330 | 331 | func main() { 332 | ctx := context.Background() 333 | 334 | // Load Configuration from file 335 | if err := file.ConfigImportYaml("conf/"); err != nil { 336 | panic(err) 337 | } 338 | 339 | // Get the flow 340 | flow1 := kis.Pool().GetFlow("CalStuAvgScore") 341 | if flow1 == nil { 342 | panic("flow1 is nil") 343 | } 344 | 345 | // Submit a string 346 | _ = flow1.CommitRow(`{"stu_id":101, "score_1":100, "score_2":90, "score_3":80}`) 347 | // Submit a string 348 | _ = flow1.CommitRow(`{"stu_id":102, "score_1":100, "score_2":70, "score_3":60}`) 349 | 350 | // Run the flow 351 | if err := flow1.Run(ctx); err != nil { 352 | fmt.Println("err: ", err) 353 | } 354 | 355 | return 356 | } 357 | 358 | func init() { 359 | // Register functions 360 | kis.Pool().FaaS("AvgStuScore", AvgStuScore) 361 | kis.Pool().FaaS("PrintStuAvgScore", PrintStuAvgScore) 362 | } 363 | ``` 364 | 365 | ### Function1 366 | 367 | > faas_stu_score_avg.go 368 | 369 | ```go 370 | package main 371 | 372 | import ( 373 | "context" 374 | "github.com/aceld/kis-flow/kis" 375 | "github.com/aceld/kis-flow/serialize" 376 | ) 377 | 378 | type AvgStuScoreIn struct { 379 | serialize.DefaultSerialize 380 | StuId int `json:"stu_id"` 381 | Score1 int `json:"score_1"` 382 | Score2 int `json:"score_2"` 383 | Score3 int `json:"score_3"` 384 | } 385 | 386 | type AvgStuScoreOut struct { 387 | serialize.DefaultSerialize 388 | StuId int `json:"stu_id"` 389 | AvgScore float64 `json:"avg_score"` 390 | } 391 | 392 | // AvgStuScore(FaaS) 计算学生平均分 393 | func AvgStuScore(ctx context.Context, flow kis.Flow, rows []*AvgStuScoreIn) error { 394 | for _, row := range rows { 395 | 396 | out := AvgStuScoreOut{ 397 | StuId: row.StuId, 398 | AvgScore: float64(row.Score1+row.Score2+row.Score3) / 3, 399 | } 400 | 401 | // 提交结果数据 402 | _ = flow.CommitRow(out) 403 | } 404 | 405 | return nil 406 | } 407 | ``` 408 | 409 | ### Function2 410 | 411 | > faas_stu_score_avg_print.go 412 | 413 | ```go 414 | package main 415 | 416 | import ( 417 | "context" 418 | "fmt" 419 | "github.com/aceld/kis-flow/kis" 420 | "github.com/aceld/kis-flow/serialize" 421 | ) 422 | 423 | type PrintStuAvgScoreIn struct { 424 | serialize.DefaultSerialize 425 | StuId int `json:"stu_id"` 426 | AvgScore float64 `json:"avg_score"` 427 | } 428 | 429 | type PrintStuAvgScoreOut struct { 430 | serialize.DefaultSerialize 431 | } 432 | 433 | func PrintStuAvgScore(ctx context.Context, flow kis.Flow, rows []*PrintStuAvgScoreIn) error { 434 | 435 | for _, row := range rows { 436 | fmt.Printf("stuid: [%+v], avg score: [%+v]\n", row.StuId, row.AvgScore) 437 | } 438 | 439 | return nil 440 | } 441 | ``` 442 | 443 | ### OutPut 444 | 445 | ```bash 446 | Add KisPool FuncName=AvgStuScore 447 | Add KisPool FuncName=PrintStuAvgScore 448 | Add FlowRouter FlowName=CalStuAvgScore 449 | stuid: [101], avg score: [90] 450 | stuid: [102], avg score: [76.66666666666667] 451 | ``` 452 | 453 |
454 | 455 | 456 | --- 457 | 458 | ### 开发者 459 | 460 | * 刘丹冰([@aceld](https://github.com/aceld)) 461 | * 胡辰豪([@ChenHaoHu](https://github.com/ChenHaoHu)) 462 | 463 | Thanks to all the developers who contributed to KisFlow! 464 | 465 | 466 | 467 | 468 | 469 | ### 加入KisFlow 社区 470 | 471 | | platform | Entry | 472 | |----------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 473 | | | https://discord.gg/xQ8Xxfyfcz | 474 | | | 加微信: `ace_ld` 或扫二维码，备注`flow`即可。
| 475 | | | **WeChat Public Account** | 476 | | | **QQ Group** | 477 | 478 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 2 | English | [简体中文](README-CN.md) 3 | 4 | 5 | [![License](https://img.shields.io/badge/License-MIT-black.svg)](LICENSE) 6 | [![Discord](https://img.shields.io/badge/KisFlow-Discord-blue.svg)](https://discord.gg/xQ8Xxfyfcz) 7 | [![KisFlow-tutorial](https://img.shields.io/badge/KisFlowTutorial-YuQue-red.svg)](https://www.yuque.com/aceld/kis-flow) 8 | [![KisFlow-Doc](https://img.shields.io/badge/KisFlow-Doc-green.svg)](https://www.yuque.com/aceld/kis-flow-doc) 9 | 10 | 11 | #### KisFlow(Keep It Simple Flowing) 12 | 13 | A Streaming Computation Framework Based on Golang. Emphasizes maintaining a simple, clear, and smooth process while performing various activities or tasks. 14 | 15 | ## Source of KisFlow 16 | 17 | Github 18 | Git: https://github.com/aceld/kis-flow 19 | 20 | GitCode 21 | Git: https://gitcode.com/aceld/kis-flow 22 | 23 | Gitee 24 | Git: https://gitee.com/Aceld/kis-flow 25 | 26 | ## Documentation 27 | 28 | [ < KisFlow Wiki : English > ](https://github.com/aceld/kis-flow/wiki) 29 | 30 | [ < KisFlow 文档 : 简体中文> ](https://www.yuque.com/aceld/kis-flow-doc) 31 | 32 | 33 | 34 | ## Online Tutorial 35 | 36 | 37 | 38 | | platform | Entry | 39 | | ---- |----------------------------------------------------------------------------------------------------------------------------------------------------| 40 | | | [Practical Tutorial for a Streaming Computation Framework Based on Golang](https://dev.to/aceld/part-1-golang-framework-hands-on-kisflow-streaming-computing-framework-overview-8fh) | 41 | || [《基于Golang的流式计算框架实战教程》](https://www.yuque.com/aceld/hsa94o) | 42 | 43 | 44 | ## Positioning of the KisFlow System 45 | 46 | KisFlow serves as the upstream computing layer for business, connecting to the ODS layer of data warehouses or other business methods upstream, and connecting to the data center of this business's storage downstream.
47 | 48 | 49 | 50 | 51 | ## KisFlow Overall Architecture Diagram 52 | 53 | 54 | | Levels | Level Explanation | Sub-modules | 55 | |-------|------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 56 | | Flowing Computation Layer | The upstream computing layer for KisFlow, which directly connects to business storage and the ODS (Operational Data Store) layer of data warehouses. The upstream can be MySQL Binlog, logs, interface data, etc., and it supports a passive consumption mode, providing KisFlow with real-time computing capabilities. | **KisFlow**: Distributed batch consumer; a KisFlow is composed of multiple KisFunctions.

**KisConnectors**: Computing data stream intermediate state persistence and connectors.

**KisFunctions**: Supports operator expression splicing, connector integration, strategy configuration, Stateful Function mode, and Slink stream splicing.

**KisConfig**: Binding of flow processing policies for KisFunctions, allowing Functions to have fixed independent processing capabilities.

**KisSource**: Interface for connecting to ODS data sources. | 57 | | Task Scheduling Layer | Timed task scheduling and execution business logic, including task scheduling platform, executor management, scheduling logs, and user management. Provides KisFlow's timed task, statistics, and aggregation calculation capabilities. | **The task scheduling platform has a visual interface.**：ncludes running reports, scheduling reports, success rate, task management, configuration management, and GLUE IDE as visual management platforms.

**Executor management KisJobs**: Golang SDK, custom business logic, executor automatic registration, task triggering, termination, and removal.

**Executor scenarios KisScenes**: Logical task sets divided according to business needs.

**Scheduling logs and user management**: Collection of task scheduling logs, detailed scheduling, and scheduling process traces. | 58 | 59 | ![KisFlow](https://github.com/aceld/kis-flow/assets/7778936/59dceaf3-16e7-41fc-979f-98297638416f) 60 | 61 | 62 | 63 | 64 | 65 | KisFlow is a flow-based conceptual form, and its specific characteristics are as follows:
66 | 67 | 1. A KisFlow can be composed of any KisFunction(s), and the length of a KisFlow can be dynamically adjusted.
68 | 69 | 2. A KisFunction can be dynamically added to a specific KisFlow at any time, and the relationship between KisFlows can be dynamically adjusted through the addition of KisFunction's Load and Save nodes for parallel and branching actions.
70 | 71 | 3. In programming behavior, KisFlow has shifted from data business programming to function-based single computing logic development, approaching the FaaS (Function as a Service) system. 72 | 73 | ## Example 74 | 75 | Below is a simple application scenario case, please refer to the specific application unit cases. 76 | 77 | https://github.com/aceld/kis-flow-usage 78 | 79 | #### [KisFlow Developer Documentation](https://github.com/aceld/kis-flow/wiki) 80 | 81 | 82 | 83 | #### Install KisFlow 84 | 85 | ```bash 86 | $go get github.com/aceld/kis-flow 87 | ``` 88 | 89 |
90 | 1. Quick Start 91 | 92 | ### Source Code 93 | 94 | https://github.com/aceld/kis-flow-usage/tree/main/1-quick_start 95 | 96 | ### Project Directory 97 | 98 | ```bash 99 | ├── faas_stu_score_avg.go 100 | ├── faas_stu_score_avg_print.go 101 | └── main.go 102 | ``` 103 | 104 | ### Flow 105 | 106 | image 107 | 108 | ### Main 109 | 110 | > main.go 111 | 112 | ```go 113 | package main 114 | 115 | import ( 116 | "context" 117 | "fmt" 118 | "github.com/aceld/kis-flow/common" 119 | "github.com/aceld/kis-flow/config" 120 | "github.com/aceld/kis-flow/flow" 121 | "github.com/aceld/kis-flow/kis" 122 | ) 123 | 124 | func main() { 125 | ctx := context.Background() 126 | 127 | // Create a new flow configuration 128 | myFlowConfig1 := config.NewFlowConfig("CalStuAvgScore", common.FlowEnable) 129 | 130 | // Create new function configuration 131 | avgStuScoreConfig := config.NewFuncConfig("AvgStuScore", common.C, nil, nil) 132 | printStuScoreConfig := config.NewFuncConfig("PrintStuAvgScore", common.E, nil, nil) 133 | 134 | // Create a new flow 135 | flow1 := flow.NewKisFlow(myFlowConfig1) 136 | 137 | // Link functions to the flow 138 | _ = flow1.Link(avgStuScoreConfig, nil) 139 | _ = flow1.Link(printStuScoreConfig, nil) 140 | 141 | // Submit a string 142 | _ = flow1.CommitRow(`{"stu_id":101, "score_1":100, "score_2":90, "score_3":80}`) 143 | // Submit a string 144 | _ = flow1.CommitRow(`{"stu_id":102, "score_1":100, "score_2":70, "score_3":60}`) 145 | 146 | // Run the flow 147 | if err := flow1.Run(ctx); err != nil { 148 | fmt.Println("err: ", err) 149 | } 150 | 151 | return 152 | } 153 | 154 | func init() { 155 | // Register functions 156 | kis.Pool().FaaS("AvgStuScore", AvgStuScore) 157 | kis.Pool().FaaS("PrintStuAvgScore", PrintStuAvgScore) 158 | } 159 | ``` 160 | 161 | ### Function1 162 | 163 | > faas_stu_score_avg.go 164 | 165 | ```go 166 | package main 167 | 168 | import ( 169 | "context" 170 | "github.com/aceld/kis-flow/kis" 171 | "github.com/aceld/kis-flow/serialize" 172 | ) 173 | 174 | type AvgStuScoreIn struct { 175 | serialize.DefaultSerialize 176 | StuId int `json:"stu_id"` 177 | Score1 int `json:"score_1"` 178 | Score2 int `json:"score_2"` 179 | Score3 int `json:"score_3"` 180 | } 181 | 182 | type AvgStuScoreOut struct { 183 | serialize.DefaultSerialize 184 | StuId int `json:"stu_id"` 185 | AvgScore float64 `json:"avg_score"` 186 | } 187 | 188 | // AvgStuScore(FaaS) 计算学生平均分 189 | func AvgStuScore(ctx context.Context, flow kis.Flow, rows []*AvgStuScoreIn) error { 190 | for _, row := range rows { 191 | 192 | out := AvgStuScoreOut{ 193 | StuId: row.StuId, 194 | AvgScore: float64(row.Score1+row.Score2+row.Score3) / 3, 195 | } 196 | 197 | // 提交结果数据 198 | _ = flow.CommitRow(out) 199 | } 200 | 201 | return nil 202 | } 203 | ``` 204 | 205 | ### Function2 206 | 207 | > faas_stu_score_avg_print.go 208 | 209 | ```go 210 | package main 211 | 212 | import ( 213 | "context" 214 | "fmt" 215 | "github.com/aceld/kis-flow/kis" 216 | "github.com/aceld/kis-flow/serialize" 217 | ) 218 | 219 | type PrintStuAvgScoreIn struct { 220 | serialize.DefaultSerialize 221 | StuId int `json:"stu_id"` 222 | AvgScore float64 `json:"avg_score"` 223 | } 224 | 225 | type PrintStuAvgScoreOut struct { 226 | serialize.DefaultSerialize 227 | } 228 | 229 | func PrintStuAvgScore(ctx context.Context, flow kis.Flow, rows []*PrintStuAvgScoreIn) error { 230 | 231 | for _, row := range rows { 232 | fmt.Printf("stuid: [%+v], avg score: [%+v]\n", row.StuId, row.AvgScore) 233 | } 234 | 235 | return nil 236 | } 237 | ``` 238 | 239 | ### OutPut 240 | 241 | ```bash 242 | Add KisPool FuncName=AvgStuScore 243 | Add KisPool FuncName=PrintStuAvgScore 244 | funcName NewConfig source is nil, funcName = AvgStuScore, use default unNamed Source. 245 | funcName NewConfig source is nil, funcName = PrintStuAvgScore, use default unNamed Source. 246 | stuid: [101], avg score: [90] 247 | stuid: [102], avg score: [76.66666666666667] 248 | ``` 249 | 250 |
251 | 252 | 253 |
254 | 2. Quick Start With Config 255 | 256 | ### Source Code 257 | 258 | https://github.com/aceld/kis-flow-usage/tree/main/2-quick_start_with_config 259 | 260 | ### Project Directory 261 | 262 | ```bash 263 | ├── Makefile 264 | ├── conf 265 | │ ├── flow-CalStuAvgScore.yml 266 | │ ├── func-AvgStuScore.yml 267 | │ └── func-PrintStuAvgScore.yml 268 | ├── faas_stu_score_avg.go 269 | ├── faas_stu_score_avg_print.go 270 | └── main.go 271 | ``` 272 | 273 | ### Flow 274 | 275 | image 276 | 277 | ### Config 278 | 279 | #### (1) Flow Config 280 | 281 | > conf/flow-CalStuAvgScore.yml 282 | 283 | ```yaml 284 | kistype: flow 285 | status: 1 286 | flow_name: CalStuAvgScore 287 | flows: 288 | - fname: AvgStuScore 289 | - fname: PrintStuAvgScore 290 | ``` 291 | 292 | #### (2) Function1 Config 293 | 294 | > conf/func-AvgStuScore.yml 295 | 296 | ```yaml 297 | kistype: func 298 | fname: AvgStuScore 299 | fmode: Calculate 300 | source: 301 | name: StudentScore 302 | must: 303 | - stu_id 304 | ``` 305 | 306 | #### (3) Function2(Slink) Config 307 | 308 | > conf/func-PrintStuAvgScore.yml 309 | 310 | ```yaml 311 | kistype: func 312 | fname: PrintStuAvgScore 313 | fmode: Expand 314 | source: 315 | name: StudentScore 316 | must: 317 | - stu_id 318 | ``` 319 | 320 | ### Main 321 | 322 | > main.go 323 | 324 | ```go 325 | package main 326 | 327 | import ( 328 | "context" 329 | "fmt" 330 | "github.com/aceld/kis-flow/file" 331 | "github.com/aceld/kis-flow/kis" 332 | ) 333 | 334 | func main() { 335 | ctx := context.Background() 336 | 337 | // Load Configuration from file 338 | if err := file.ConfigImportYaml("conf/"); err != nil { 339 | panic(err) 340 | } 341 | 342 | // Get the flow 343 | flow1 := kis.Pool().GetFlow("CalStuAvgScore") 344 | if flow1 == nil { 345 | panic("flow1 is nil") 346 | } 347 | 348 | // Submit a string 349 | _ = flow1.CommitRow(`{"stu_id":101, "score_1":100, "score_2":90, "score_3":80}`) 350 | // Submit a string 351 | _ = flow1.CommitRow(`{"stu_id":102, "score_1":100, "score_2":70, "score_3":60}`) 352 | 353 | // Run the flow 354 | if err := flow1.Run(ctx); err != nil { 355 | fmt.Println("err: ", err) 356 | } 357 | 358 | return 359 | } 360 | 361 | func init() { 362 | // Register functions 363 | kis.Pool().FaaS("AvgStuScore", AvgStuScore) 364 | kis.Pool().FaaS("PrintStuAvgScore", PrintStuAvgScore) 365 | } 366 | ``` 367 | 368 | ### Function1 369 | 370 | > faas_stu_score_avg.go 371 | 372 | ```go 373 | package main 374 | 375 | import ( 376 | "context" 377 | "github.com/aceld/kis-flow/kis" 378 | "github.com/aceld/kis-flow/serialize" 379 | ) 380 | 381 | type AvgStuScoreIn struct { 382 | serialize.DefaultSerialize 383 | StuId int `json:"stu_id"` 384 | Score1 int `json:"score_1"` 385 | Score2 int `json:"score_2"` 386 | Score3 int `json:"score_3"` 387 | } 388 | 389 | type AvgStuScoreOut struct { 390 | serialize.DefaultSerialize 391 | StuId int `json:"stu_id"` 392 | AvgScore float64 `json:"avg_score"` 393 | } 394 | 395 | // AvgStuScore(FaaS) 计 396 | func AvgStuScore(ctx context.Context, flow kis.Flow, rows []*AvgStuScoreIn) error { 397 | for _, row := range rows { 398 | 399 | out := AvgStuScoreOut{ 400 | StuId: row.StuId, 401 | AvgScore: float64(row.Score1+row.Score2+row.Score3) / 3, 402 | } 403 | 404 | // Commit Result Data 405 | _ = flow.CommitRow(out) 406 | } 407 | 408 | return nil 409 | } 410 | ``` 411 | 412 | ### Function2 413 | 414 | > faas_stu_score_avg_print.go 415 | 416 | ```go 417 | package main 418 | 419 | import ( 420 | "context" 421 | "fmt" 422 | "github.com/aceld/kis-flow/kis" 423 | "github.com/aceld/kis-flow/serialize" 424 | ) 425 | 426 | type PrintStuAvgScoreIn struct { 427 | serialize.DefaultSerialize 428 | StuId int `json:"stu_id"` 429 | AvgScore float64 `json:"avg_score"` 430 | } 431 | 432 | type PrintStuAvgScoreOut struct { 433 | serialize.DefaultSerialize 434 | } 435 | 436 | func PrintStuAvgScore(ctx context.Context, flow kis.Flow, rows []*PrintStuAvgScoreIn) error { 437 | 438 | for _, row := range rows { 439 | fmt.Printf("stuid: [%+v], avg score: [%+v]\n", row.StuId, row.AvgScore) 440 | } 441 | 442 | return nil 443 | } 444 | ``` 445 | 446 | ### OutPut 447 | 448 | ```bash 449 | Add KisPool FuncName=AvgStuScore 450 | Add KisPool FuncName=PrintStuAvgScore 451 | Add FlowRouter FlowName=CalStuAvgScore 452 | stuid: [101], avg score: [90] 453 | stuid: [102], avg score: [76.66666666666667] 454 | ``` 455 | 456 |
457 | 458 | 459 | --- 460 | 461 | ### KisFlow Contributors 462 | 463 | * 刘丹冰([@aceld](https://github.com/aceld)) 464 | * 胡辰豪([@ChenHaoHu](https://github.com/ChenHaoHu)) 465 | 466 | Thanks to all the developers who contributed to KisFlow! 467 | 468 | 469 | 470 | 471 | 472 | ### Join the KisFlow Community 473 | 474 | | platform | Entry | 475 | |----------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 476 | | | https://discord.gg/xQ8Xxfyfcz | 477 | | | Add WeChat: ace_ld or scan the QR code, and note flow to proceed.
| 478 | | | **WeChat Public Account** | 479 | | | **QQ Group** | 480 | 481 | --------------------------------------------------------------------------------