├── .gitignore ├── .vscode └── launch.json ├── LICENSE ├── Makefile ├── README.md ├── api ├── apps │ ├── action │ │ ├── action.pb.go │ │ ├── action.pb_enum_generate.go │ │ ├── action_ext.go │ │ ├── action_grpc.pb.go │ │ ├── action_http.pb.go │ │ ├── app.go │ │ ├── http │ │ │ ├── action.go │ │ │ └── http.go │ │ ├── impl │ │ │ ├── action.go │ │ │ ├── impl.go │ │ │ └── query.go │ │ └── pb │ │ │ └── action.proto │ ├── all │ │ ├── grpc.go │ │ └── http.go │ ├── approval │ │ ├── app.go │ │ ├── approval.pb.go │ │ ├── approval_grpc.pb.go │ │ ├── http │ │ │ └── http.go │ │ ├── impl │ │ │ └── impl.go │ │ ├── pb │ │ │ └── approval.proto │ │ └── provider │ │ │ └── feishu │ │ │ ├── approval.go │ │ │ ├── client.go │ │ │ └── client_test.go │ ├── node │ │ ├── etcd │ │ │ └── etcd.go │ │ ├── node.go │ │ └── prefix.go │ ├── pipeline │ │ ├── app.go │ │ ├── http │ │ │ ├── enum.go │ │ │ ├── http.go │ │ │ ├── pipeline.go │ │ │ ├── proxy.go │ │ │ ├── proxy_test.go │ │ │ └── step.go │ │ ├── impl │ │ │ ├── impl.go │ │ │ ├── pipeline.go │ │ │ └── step.go │ │ ├── pb │ │ │ └── pipeline.proto │ │ ├── pipeline.pb.go │ │ ├── pipeline.pb_enum_generate.go │ │ ├── pipeline_ext.go │ │ ├── pipeline_grpc.pb.go │ │ ├── pipeline_test.go │ │ ├── prefix.go │ │ ├── step_ext.go │ │ ├── step_test.go │ │ └── variable │ │ │ └── getter.go │ ├── step │ │ ├── README.md │ │ └── pb │ │ │ ├── rpc.proto │ │ │ └── step.proto │ └── template │ │ ├── app.go │ │ ├── http │ │ ├── http.go │ │ └── template.go │ │ ├── impl │ │ ├── dao.go │ │ ├── impl.go │ │ ├── query.go │ │ └── template.go │ │ ├── pb │ │ └── template.proto │ │ ├── template.pb.go │ │ ├── template_ext.go │ │ └── template_grpc.pb.go ├── client │ ├── client.go │ └── config.go ├── cmd │ ├── root.go │ └── start.go ├── main.go └── protocol │ ├── grpc.go │ └── http.go ├── common ├── cache │ ├── cache.go │ ├── cache_test.go │ ├── index.go │ ├── store.go │ └── thread_safe_store.go ├── enum │ └── enum.go ├── hooks │ ├── hooks.go │ └── webhook │ │ ├── dingding │ │ ├── card.go │ │ └── message.go │ │ ├── feishu │ │ ├── card.go │ │ ├── color.go │ │ └── message.go │ │ ├── request.go │ │ ├── request_test.go │ │ ├── webhook.go │ │ └── wechat │ │ └── message.go └── informers │ ├── node │ ├── etcd │ │ ├── imformer.go │ │ ├── lister.go │ │ └── watcher.go │ ├── indexer.go │ └── informer.go │ ├── pipeline │ ├── etcd │ │ ├── imformer.go │ │ ├── lister.go │ │ ├── recorder.go │ │ └── watcher.go │ ├── indexer.go │ └── informer.go │ └── step │ ├── etcd │ ├── imformer.go │ ├── lister.go │ ├── recorder.go │ └── watcher.go │ ├── indexer.go │ └── informer.go ├── conf ├── config.go ├── load.go └── log.go ├── docs ├── deploy │ └── install.md ├── etcd │ └── deploy.md ├── keypoint │ ├── docker-in-docker.md │ ├── git-pull.md │ └── web-hook-test.md └── sample │ ├── create_applicaiton.json │ ├── gitlab_hook.json │ ├── gitlab_hook.md │ ├── pipeline.json │ └── pipeline.yaml ├── etc └── workflow_sample.toml ├── go.mod ├── go.sum ├── node ├── cmd │ ├── root.go │ └── start.go ├── controller │ └── step │ │ ├── controller.go │ │ ├── engine │ │ ├── cancel.go │ │ ├── engine.go │ │ └── run.go │ │ ├── handler.go │ │ ├── runner │ │ ├── docker │ │ │ ├── request.go │ │ │ ├── runner.go │ │ │ └── runner_test.go │ │ ├── http │ │ │ └── runner.go │ │ ├── k8s │ │ │ └── runner.go │ │ ├── local │ │ │ └── runner.go │ │ └── runner.go │ │ └── store │ │ ├── file │ │ ├── uploader.go │ │ └── uploader_test.go │ │ └── store.go └── main.go ├── scheduler ├── README.md ├── algorithm │ ├── picker.go │ └── roundrobin │ │ └── roundrobin.go ├── cmd │ ├── root.go │ └── start.go ├── controller │ ├── cronjob │ │ ├── controller.go │ │ └── handler.go │ ├── node │ │ ├── controller.go │ │ └── handler.go │ ├── pipeline │ │ ├── controller.go │ │ └── handler.go │ └── step │ │ ├── controller.go │ │ └── handler.go └── main.go └── version └── version.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | __debug_bin 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | *.idea 15 | *vendor 16 | dist/* 17 | 18 | cover.out 19 | coverage.txt 20 | runner_log 21 | 22 | etc/workflow.toml 23 | etc/workflow.env 24 | etc/ssl 25 | etc/* 26 | workflow 27 | workflow-api 28 | workflow-scheduler 29 | workflow-node 30 | .vscode/* -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // 使用 IntelliSense 了解相关属性。 3 | // 悬停以查看现有属性的描述。 4 | // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "API Server", 9 | "type": "go", 10 | "request": "launch", 11 | "mode": "auto", 12 | "program": "${workspaceFolder}/api/main.go", 13 | "args": ["start"], 14 | "cwd": "${workspaceFolder}" 15 | }, 16 | { 17 | "name": "Scheduler", 18 | "type": "go", 19 | "request": "launch", 20 | "mode": "auto", 21 | "program": "${workspaceFolder}/scheduler/main.go", 22 | "args": ["start"], 23 | "cwd": "${workspaceFolder}" 24 | }, 25 | { 26 | "name": "Node", 27 | "type": "go", 28 | "request": "launch", 29 | "mode": "auto", 30 | "program": "${workspaceFolder}/node/main.go", 31 | "args": ["start"], 32 | "cwd": "${workspaceFolder}" 33 | } 34 | ] 35 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 infraboard 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | API_PROJECT_NAME := "workflow-api" 2 | API_MAIN_FILE_PAHT := "api/main.go" 3 | SCH_PROJECT_NAME := "workflow-scheduler" 4 | SCH_MAIN_FILE_PAHT := "scheduler/main.go" 5 | NODE_PROJECT_NAME := "workflow-node" 6 | NODE_MAIN_FILE_PAHT := "node/main.go" 7 | PKG := "github.com/infraboard/workflow" 8 | IMAGE_PREFIX := "github.com/infraboard/workflow" 9 | 10 | BUILD_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) 11 | BUILD_COMMIT := ${shell git rev-parse HEAD} 12 | BUILD_TIME := ${shell date '+%Y-%m-%d %H:%M:%S'} 13 | BUILD_GO_VERSION := $(shell go version | grep -o 'go[0-9].[0-9].*') 14 | VERSION_PATH := "${PKG}/version" 15 | 16 | PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/ | grep -v redis) 17 | GO_FILES := $(shell find . -name '*.go' | grep -v /vendor/ | grep -v _test.go) 18 | 19 | .PHONY: all dep lint vet test test-coverage build clean 20 | 21 | all: build 22 | 23 | dep: ## Get the dependencies 24 | @go mod tidy 25 | 26 | lint: ## Lint Golang files 27 | @golint -set_exit_status ${PKG_LIST} 28 | 29 | vet: ## Run go vet 30 | @go vet ${PKG_LIST} 31 | 32 | test: ## Run unittests 33 | @go test -short ${PKG_LIST} 34 | 35 | test-coverage: ## Run tests with coverage 36 | @go test -short -coverprofile cover.out -covermode=atomic ${PKG_LIST} 37 | @cat cover.out >> coverage.txt 38 | 39 | build: dep ## Build the binary file 40 | @go build -a -o dist/${API_PROJECT_NAME} -ldflags "-s -w" -ldflags "-X '${VERSION_PATH}.GIT_BRANCH=${BUILD_BRANCH}' -X '${VERSION_PATH}.GIT_COMMIT=${BUILD_COMMIT}' -X '${VERSION_PATH}.BUILD_TIME=${BUILD_TIME}' -X '${VERSION_PATH}.GO_VERSION=${BUILD_GO_VERSION}'" ${API_MAIN_FILE_PAHT} 41 | @go build -a -o dist/${SCH_PROJECT_NAME} -ldflags "-s -w" -ldflags "-X '${VERSION_PATH}.GIT_BRANCH=${BUILD_BRANCH}' -X '${VERSION_PATH}.GIT_COMMIT=${BUILD_COMMIT}' -X '${VERSION_PATH}.BUILD_TIME=${BUILD_TIME}' -X '${VERSION_PATH}.GO_VERSION=${BUILD_GO_VERSION}'" ${SCH_MAIN_FILE_PAHT} 42 | @go build -a -o dist/${NODE_PROJECT_NAME} -ldflags "-s -w" -ldflags "-X '${VERSION_PATH}.GIT_BRANCH=${BUILD_BRANCH}' -X '${VERSION_PATH}.GIT_COMMIT=${BUILD_COMMIT}' -X '${VERSION_PATH}.BUILD_TIME=${BUILD_TIME}' -X '${VERSION_PATH}.GO_VERSION=${BUILD_GO_VERSION}'" ${NODE_MAIN_FILE_PAHT} 43 | 44 | linux: ## Linux build 45 | @GOOS=linux GOARCH=amd64 go build -a -o dist/${API_PROJECT_NAME} -ldflags "-s -w" -ldflags "-X '${VERSION_PATH}.GIT_BRANCH=${BUILD_BRANCH}' -X '${VERSION_PATH}.GIT_COMMIT=${BUILD_COMMIT}' -X '${VERSION_PATH}.BUILD_TIME=${BUILD_TIME}' -X '${VERSION_PATH}.GO_VERSION=${BUILD_GO_VERSION}'" ${API_MAIN_FILE_PAHT} 46 | @GOOS=linux GOARCH=amd64 go build -a -o dist/${SCH_PROJECT_NAME} -ldflags "-s -w" -ldflags "-X '${VERSION_PATH}.GIT_BRANCH=${BUILD_BRANCH}' -X '${VERSION_PATH}.GIT_COMMIT=${BUILD_COMMIT}' -X '${VERSION_PATH}.BUILD_TIME=${BUILD_TIME}' -X '${VERSION_PATH}.GO_VERSION=${BUILD_GO_VERSION}'" ${SCH_MAIN_FILE_PAHT} 47 | @GOOS=linux GOARCH=amd64 go build -a -o dist/${NODE_PROJECT_NAME} -ldflags "-s -w" -ldflags "-X '${VERSION_PATH}.GIT_BRANCH=${BUILD_BRANCH}' -X '${VERSION_PATH}.GIT_COMMIT=${BUILD_COMMIT}' -X '${VERSION_PATH}.BUILD_TIME=${BUILD_TIME}' -X '${VERSION_PATH}.GO_VERSION=${BUILD_GO_VERSION}'" ${NODE_MAIN_FILE_PAHT} 48 | 49 | run-api: dep ## Run Server 50 | @go run ${API_MAIN_FILE_PAHT} start 51 | 52 | build-api: dep ## Build the binary file 53 | @GOOS=linux GOARCH=amd64 go build -a -o dist/${API_PROJECT_NAME} -ldflags "-s -w" -ldflags "-X '${VERSION_PATH}.GIT_BRANCH=${BUILD_BRANCH}' -X '${VERSION_PATH}.GIT_COMMIT=${BUILD_COMMIT}' -X '${VERSION_PATH}.BUILD_TIME=${BUILD_TIME}' -X '${VERSION_PATH}.GO_VERSION=${BUILD_GO_VERSION}'" ${API_MAIN_FILE_PAHT} 54 | 55 | run-sch: dep build-sch ## Run schedule 56 | @go run ${SCH_MAIN_FILE_PAHT} start 57 | 58 | build-sch: dep ## Build the binary file 59 | @GOOS=linux GOARCH=amd64 go build -a -o dist/${SCH_PROJECT_NAME} -ldflags "-s -w" -ldflags "-X '${VERSION_PATH}.GIT_BRANCH=${BUILD_BRANCH}' -X '${VERSION_PATH}.GIT_COMMIT=${BUILD_COMMIT}' -X '${VERSION_PATH}.BUILD_TIME=${BUILD_TIME}' -X '${VERSION_PATH}.GO_VERSION=${BUILD_GO_VERSION}'" ${SCH_MAIN_FILE_PAHT} 60 | 61 | run-node: dep build-node ## Run node 62 | @go run ${NODE_MAIN_FILE_PAHT} start 63 | 64 | build-node: dep ## Build the binary file 65 | @GOOS=linux GOARCH=amd64 go build -a -o dist/${NODE_PROJECT_NAME} -ldflags "-s -w" -ldflags "-X '${VERSION_PATH}.GIT_BRANCH=${BUILD_BRANCH}' -X '${VERSION_PATH}.GIT_COMMIT=${BUILD_COMMIT}' -X '${VERSION_PATH}.BUILD_TIME=${BUILD_TIME}' -X '${VERSION_PATH}.GO_VERSION=${BUILD_GO_VERSION}'" ${NODE_MAIN_FILE_PAHT} 66 | 67 | clean: ## Remove previous build 68 | @go clean . 69 | @rm -f dist/* 70 | 71 | install: ## Install depence go package 72 | @go install google.golang.org/protobuf/cmd/protoc-gen-go@latest 73 | @go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest 74 | 75 | gen: ## Init Service 76 | @protoc -I=. -I=/usr/local/include --go_out=. --go_opt=module=${PKG} --go-grpc_out=. --go-grpc_opt=module=${PKG} api/apps/*/pb/*.proto 77 | @protoc-go-inject-tag -input=api/apps/*/*.pb.go 78 | @go generate ./... 79 | 80 | help: ## Display this help screen 81 | @grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # workflow 2 | 3 | 应用研发交付中心, 以应用为中心的分布式自定义流水线, 支持的常见场景包括: 4 | + 应用研发流, 常见CI CD 5 | + 应用审批流, 比如应用资源审批与自动化流程处理编排 6 | + 应用事件流, 基于应用事件, 为其编排应急预案流水线, 比如扩容预案流水线和缩容预案流水线 7 | 8 | 9 | ## 架构图 10 | 11 | 12 | ## 快速开发 13 | 14 | ## 注意 15 | 16 | 依赖的 mod包: go.etcd.io/etcd v3.3.25+incompatible, 直接替换的v3.5.0-beta.3 17 | 18 | make脚手架 19 | ```sh 20 | ➜ workflow git:(master) ✗ make help 21 | dep Get the dependencies 22 | lint Lint Golang files 23 | vet Run go vet 24 | test Run unittests 25 | test-coverage Run tests with coverage 26 | build Local build 27 | linux Linux build 28 | run Run Server 29 | clean Remove previous build 30 | help Display this help screen 31 | ``` 32 | 33 | 1. 使用go mod下载项目依赖 34 | ```sh 35 | $ make dep 36 | ``` 37 | 38 | 2. 添加配置文件(默认读取位置: etc/workflow.toml) 39 | ```sh 40 | $ 编辑样例配置文件 etc/workflow.toml.example 41 | $ mv etc/workflow.toml.example etc/workflow.toml 42 | ``` 43 | 44 | 3. 启动服务 45 | ```sh 46 | $ make run 47 | ``` 48 | 49 | ## 相关文档 -------------------------------------------------------------------------------- /api/apps/action/action.pb_enum_generate.go: -------------------------------------------------------------------------------- 1 | // Code generated by github.com/infraboard/mcube 2 | // DO NOT EDIT 3 | 4 | package action 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "strings" 10 | ) 11 | 12 | // ParseRUNNER_TYPEFromString Parse RUNNER_TYPE from string 13 | func ParseRUNNER_TYPEFromString(str string) (RUNNER_TYPE, error) { 14 | key := strings.Trim(string(str), `"`) 15 | v, ok := RUNNER_TYPE_value[strings.ToUpper(key)] 16 | if !ok { 17 | return 0, fmt.Errorf("unknown RUNNER_TYPE: %s", str) 18 | } 19 | 20 | return RUNNER_TYPE(v), nil 21 | } 22 | 23 | // Equal type compare 24 | func (t RUNNER_TYPE) Equal(target RUNNER_TYPE) bool { 25 | return t == target 26 | } 27 | 28 | // IsIn todo 29 | func (t RUNNER_TYPE) IsIn(targets ...RUNNER_TYPE) bool { 30 | for _, target := range targets { 31 | if t.Equal(target) { 32 | return true 33 | } 34 | } 35 | 36 | return false 37 | } 38 | 39 | // MarshalJSON todo 40 | func (t RUNNER_TYPE) MarshalJSON() ([]byte, error) { 41 | b := bytes.NewBufferString(`"`) 42 | b.WriteString(strings.ToUpper(t.String())) 43 | b.WriteString(`"`) 44 | return b.Bytes(), nil 45 | } 46 | 47 | // UnmarshalJSON todo 48 | func (t *RUNNER_TYPE) UnmarshalJSON(b []byte) error { 49 | ins, err := ParseRUNNER_TYPEFromString(string(b)) 50 | if err != nil { 51 | return err 52 | } 53 | *t = ins 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /api/apps/action/action_ext.go: -------------------------------------------------------------------------------- 1 | package action 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | "time" 8 | 9 | "github.com/go-playground/validator/v10" 10 | "github.com/infraboard/keyauth/app/token" 11 | "github.com/infraboard/mcube/http/request" 12 | "github.com/rs/xid" 13 | ) 14 | 15 | // use a single instance of Validate, it caches struct info 16 | var ( 17 | validate = validator.New() 18 | ) 19 | 20 | func NewCreateActionRequest() *CreateActionRequest { 21 | return &CreateActionRequest{} 22 | } 23 | 24 | // NewQueryActionRequest 查询book列表 25 | func NewQueryActionRequest(page *request.PageRequest) *QueryActionRequest { 26 | return &QueryActionRequest{ 27 | Page: page, 28 | } 29 | } 30 | 31 | func (req *CreateActionRequest) Validate() error { 32 | return validate.Struct(req) 33 | } 34 | 35 | func (req *CreateActionRequest) UpdateOwner(tk *token.Token) { 36 | req.CreateBy = tk.Account 37 | req.Domain = tk.Domain 38 | req.Namespace = tk.Namespace 39 | } 40 | 41 | func LoadActionFromBytes(payload []byte) (*Action, error) { 42 | ins := NewDefaultAction() 43 | 44 | // 解析Value 45 | if err := json.Unmarshal(payload, ins); err != nil { 46 | return nil, fmt.Errorf("unmarshal step error, vaule(%s) %s", string(payload), err) 47 | } 48 | 49 | // 校验合法性 50 | if err := ins.Validate(); err != nil { 51 | return nil, err 52 | } 53 | 54 | return ins, nil 55 | } 56 | 57 | func NewDefaultAction() *Action { 58 | return &Action{} 59 | } 60 | 61 | func NewAction(req *CreateActionRequest) (*Action, error) { 62 | if err := req.Validate(); err != nil { 63 | return nil, err 64 | } 65 | 66 | p := &Action{ 67 | Id: xid.New().String(), 68 | CreateAt: time.Now().UnixMilli(), 69 | UpdateAt: time.Now().UnixMilli(), 70 | Domain: req.Domain, 71 | Namespace: req.Namespace, 72 | CreateBy: req.CreateBy, 73 | Logo: req.Logo, 74 | DisplayName: req.DisplayName, 75 | IsLatest: true, 76 | Name: req.Name, 77 | Version: req.Version, 78 | VisiableMode: req.VisiableMode, 79 | RunnerType: req.RunnerType, 80 | RunnerParams: req.RunnerParams, 81 | RunParams: req.RunParams, 82 | Tags: req.Tags, 83 | Description: req.Description, 84 | } 85 | 86 | return p, nil 87 | } 88 | 89 | func (a *Action) InitNil() { 90 | if a.RunnerParams == nil { 91 | a.RunnerParams = map[string]string{} 92 | } 93 | if a.RunParams == nil { 94 | a.RunParams = []*RunParamDesc{} 95 | } 96 | if a.Tags == nil { 97 | a.Tags = map[string]string{} 98 | } 99 | } 100 | 101 | func (a *Action) DefaultRunParam() map[string]string { 102 | ret := map[string]string{} 103 | for i := range a.RunParams { 104 | param := a.RunParams[i] 105 | if param.DefaultValue != "" { 106 | ret[param.KeyName] = param.DefaultValue 107 | } 108 | } 109 | return ret 110 | } 111 | 112 | func (a *Action) RunnerParam() map[string]string { 113 | param := map[string]string{} 114 | for k, v := range a.RunnerParams { 115 | if v != "" { 116 | param[k] = v 117 | } 118 | } 119 | return param 120 | } 121 | 122 | // ValidateParam 按照action的定义, 检查必传参数是否传人 123 | func (a *Action) ValidateRunParam(params map[string]string) error { 124 | msg := []string{} 125 | for i := range a.RunParams { 126 | param := a.RunParams[i] 127 | if param.Required { 128 | if pv, ok := params[param.KeyName]; !ok || pv == "" { 129 | msg = append(msg, "required param "+param.KeyName) 130 | } 131 | } 132 | } 133 | 134 | if len(msg) > 0 { 135 | return fmt.Errorf("validate run params error, %s", strings.Join(msg, ",")) 136 | } 137 | 138 | return nil 139 | } 140 | 141 | func (a *Action) Validate() error { 142 | return validate.Struct(a) 143 | } 144 | 145 | func (a *Action) Update(req *UpdateActionRequest) { 146 | a.VisiableMode = req.VisiableMode 147 | a.RunParams = req.RunParams 148 | a.Tags = req.Tags 149 | a.Description = req.Description 150 | } 151 | 152 | func (a *Action) Key() string { 153 | return fmt.Sprintf("%s@%s", a.Name, a.Version) 154 | } 155 | 156 | // NewActionSet todo 157 | func NewActionSet() *ActionSet { 158 | return &ActionSet{ 159 | Items: []*Action{}, 160 | } 161 | } 162 | 163 | func (s *ActionSet) InitNil() { 164 | for i := range s.Items { 165 | s.Items[i].InitNil() 166 | } 167 | } 168 | 169 | func (s *ActionSet) Add(item *Action) { 170 | s.Items = append(s.Items, item) 171 | } 172 | 173 | // NewQueryActionRequest 查询book列表 174 | func NewDescribeActionRequest(name, version string) *DescribeActionRequest { 175 | return &DescribeActionRequest{ 176 | Name: name, 177 | Version: version, 178 | } 179 | } 180 | 181 | // NewDeleteActionRequest 查询book列表 182 | func NewDeleteActionRequest(name, version string) *DeleteActionRequest { 183 | return &DeleteActionRequest{ 184 | Name: name, 185 | Version: version, 186 | } 187 | } 188 | 189 | func (req *DescribeActionRequest) Validate() error { 190 | return validate.Struct(req) 191 | } 192 | 193 | func ParseActionKey(key string) (name, version string) { 194 | parseKey := strings.Split(key, "@") 195 | name = parseKey[0] 196 | if len(parseKey) > 1 { 197 | version = parseKey[1] 198 | } 199 | 200 | return 201 | } 202 | 203 | func (req *DeleteActionRequest) Validate() error { 204 | return validate.Struct(req) 205 | } 206 | 207 | func NewUpdateActionRequest() *UpdateActionRequest { 208 | return &UpdateActionRequest{} 209 | } 210 | 211 | func (req *UpdateActionRequest) Validate() error { 212 | return validate.Struct(req) 213 | } 214 | -------------------------------------------------------------------------------- /api/apps/action/action_http.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-http. DO NOT EDIT. 2 | 3 | package action 4 | 5 | import ( 6 | http "github.com/infraboard/mcube/pb/http" 7 | ) 8 | 9 | // HttpEntry todo 10 | func HttpEntry() *http.EntrySet { 11 | set := &http.EntrySet{ 12 | Items: []*http.Entry{ 13 | { 14 | Path: "/workflow.action.Service/CreateAction", 15 | FunctionName: "CreateAction", 16 | }, 17 | { 18 | Path: "/workflow.action.Service/QueryAction", 19 | FunctionName: "QueryAction", 20 | }, 21 | { 22 | Path: "/workflow.action.Service/DescribeAction", 23 | FunctionName: "DescribeAction", 24 | }, 25 | { 26 | Path: "/workflow.action.Service/UpdateAction", 27 | FunctionName: "UpdateAction", 28 | }, 29 | { 30 | Path: "/workflow.action.Service/DeleteAction", 31 | FunctionName: "DeleteAction", 32 | }, 33 | }, 34 | } 35 | return set 36 | } 37 | -------------------------------------------------------------------------------- /api/apps/action/app.go: -------------------------------------------------------------------------------- 1 | package action 2 | 3 | const ( 4 | AppName = "action" 5 | ) 6 | -------------------------------------------------------------------------------- /api/apps/action/http/action.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/infraboard/keyauth/app/token" 7 | "github.com/infraboard/mcube/http/context" 8 | "github.com/infraboard/mcube/http/request" 9 | "github.com/infraboard/mcube/http/response" 10 | "github.com/infraboard/mcube/pb/resource" 11 | 12 | "github.com/infraboard/workflow/api/apps/action" 13 | "github.com/infraboard/workflow/node/controller/step/runner/docker" 14 | "github.com/infraboard/workflow/node/controller/step/runner/k8s" 15 | "github.com/infraboard/workflow/node/controller/step/runner/local" 16 | ) 17 | 18 | // Action 19 | func (h *handler) CreateAction(w http.ResponseWriter, r *http.Request) { 20 | ctx := context.GetContext(r) 21 | tk := ctx.AuthInfo.(*token.Token) 22 | 23 | req := action.NewCreateActionRequest() 24 | if err := request.GetDataFromRequest(r, req); err != nil { 25 | response.Failed(w, err) 26 | return 27 | } 28 | req.VisiableMode = resource.VisiableMode_NAMESPACE 29 | req.UpdateOwner(tk) 30 | 31 | ins, err := h.service.CreateAction( 32 | r.Context(), 33 | req, 34 | ) 35 | if err != nil { 36 | response.Failed(w, err) 37 | return 38 | } 39 | 40 | response.Success(w, ins) 41 | } 42 | 43 | func (h *handler) UpdateAction(w http.ResponseWriter, r *http.Request) { 44 | ctx := context.GetContext(r) 45 | 46 | name, version := action.ParseActionKey(ctx.PS.ByName("key")) 47 | req := action.NewUpdateActionRequest() 48 | if err := request.GetDataFromRequest(r, req); err != nil { 49 | response.Failed(w, err) 50 | return 51 | } 52 | req.Name = name 53 | req.Version = version 54 | 55 | ins, err := h.service.UpdateAction( 56 | r.Context(), 57 | req, 58 | ) 59 | if err != nil { 60 | response.Failed(w, err) 61 | return 62 | } 63 | 64 | response.Success(w, ins) 65 | } 66 | 67 | func (h *handler) QueryAction(w http.ResponseWriter, r *http.Request) { 68 | ctx := context.GetContext(r) 69 | tk := ctx.AuthInfo.(*token.Token) 70 | 71 | page := request.NewPageRequestFromHTTP(r) 72 | req := action.NewQueryActionRequest(page) 73 | req.Namespace = tk.Namespace 74 | 75 | actions, err := h.service.QueryAction( 76 | r.Context(), 77 | req, 78 | ) 79 | if err != nil { 80 | response.Failed(w, err) 81 | return 82 | } 83 | 84 | // 避免前端处理null 85 | actions.InitNil() 86 | response.Success(w, actions) 87 | } 88 | 89 | func (h *handler) DescribeAction(w http.ResponseWriter, r *http.Request) { 90 | ctx := context.GetContext(r) 91 | 92 | name, version := action.ParseActionKey(ctx.PS.ByName("key")) 93 | req := action.NewDescribeActionRequest(name, version) 94 | 95 | ins, err := h.service.DescribeAction( 96 | r.Context(), 97 | req, 98 | ) 99 | if err != nil { 100 | response.Failed(w, err) 101 | return 102 | } 103 | 104 | response.Success(w, ins) 105 | } 106 | 107 | func (h *handler) DeleteAction(w http.ResponseWriter, r *http.Request) { 108 | ctx := context.GetContext(r) 109 | tk := ctx.AuthInfo.(*token.Token) 110 | 111 | hc := context.GetContext(r) 112 | name, version := action.ParseActionKey(hc.PS.ByName("key")) 113 | req := action.NewDeleteActionRequest(name, version) 114 | 115 | req.Namespace = tk.Namespace 116 | 117 | action, err := h.service.DeleteAction( 118 | r.Context(), 119 | req, 120 | ) 121 | if err != nil { 122 | response.Failed(w, err) 123 | return 124 | } 125 | response.Success(w, action) 126 | } 127 | 128 | func (h *handler) QueryRunner(w http.ResponseWriter, r *http.Request) { 129 | ins := NewRunnerParamDescSet() 130 | ins.Add(action.RUNNER_TYPE_DOCKER, docker.ParamsDesc()) 131 | ins.Add(action.RUNNER_TYPE_K8s, k8s.ParamsDesc()) 132 | ins.Add(action.RUNNER_TYPE_LOCAL, local.ParamsDesc()) 133 | response.Success(w, ins) 134 | } 135 | 136 | func NewRunnerParamDescSet() *RunnerParamDescSet { 137 | return &RunnerParamDescSet{ 138 | Items: []*RunnerParamDesc{}, 139 | } 140 | } 141 | 142 | type RunnerParamDescSet struct { 143 | Items []*RunnerParamDesc `json:"items"` 144 | } 145 | 146 | func (s *RunnerParamDescSet) Add(t action.RUNNER_TYPE, desc []*action.RunParamDesc) { 147 | s.Items = append(s.Items, &RunnerParamDesc{ 148 | Type: t, 149 | ParamDesc: desc, 150 | }) 151 | } 152 | 153 | type RunnerParamDesc struct { 154 | Type action.RUNNER_TYPE `json:"type"` 155 | ParamDesc []*action.RunParamDesc `json:"param_desc"` 156 | } 157 | -------------------------------------------------------------------------------- /api/apps/action/http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "github.com/infraboard/mcube/app" 5 | "github.com/infraboard/mcube/http/label" 6 | "github.com/infraboard/mcube/http/router" 7 | "github.com/infraboard/mcube/logger" 8 | "github.com/infraboard/mcube/logger/zap" 9 | 10 | "github.com/infraboard/workflow/api/apps/action" 11 | ) 12 | 13 | var ( 14 | api = &handler{log: zap.L().Named("Action")} 15 | ) 16 | 17 | type handler struct { 18 | service action.ServiceServer 19 | log logger.Logger 20 | } 21 | 22 | // Registry 注册HTTP服务路由 23 | func (h *handler) Registry(router router.SubRouter) { 24 | r := router.ResourceRouter("actions") 25 | r.Permission(true) 26 | r.BasePath("actions") 27 | r.Handle("POST", "/", h.CreateAction).AddLabel(label.Create) 28 | r.Handle("GET", "/", h.QueryAction).AddLabel(label.List) 29 | r.Handle("GET", "/:key", h.DescribeAction).AddLabel(label.Get) 30 | r.Handle("PUT", "/:key", h.UpdateAction).AddLabel() 31 | r.Handle("DELETE", "/:key", h.DeleteAction).AddLabel(label.Delete) 32 | 33 | r.BasePath("runners") 34 | r.Handle("GET", "/", h.QueryRunner).AddLabel(label.List) 35 | } 36 | 37 | func (h *handler) Config() error { 38 | h.service = app.GetGrpcApp(action.AppName).(action.ServiceServer) 39 | return nil 40 | } 41 | 42 | func (h *handler) Name() string { 43 | return action.AppName 44 | } 45 | 46 | func init() { 47 | app.RegistryHttpApp(api) 48 | } 49 | -------------------------------------------------------------------------------- /api/apps/action/impl/action.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/infraboard/mcube/exception" 8 | "go.mongodb.org/mongo-driver/bson" 9 | "go.mongodb.org/mongo-driver/mongo" 10 | 11 | "github.com/infraboard/workflow/api/apps/action" 12 | ) 13 | 14 | func (i *service) CreateAction(ctx context.Context, req *action.CreateActionRequest) ( 15 | *action.Action, error) { 16 | a, err := action.NewAction(req) 17 | if err != nil { 18 | return nil, err 19 | } 20 | 21 | // 获取之前最新的版本 22 | if _, err := i.col.InsertOne(context.TODO(), a); err != nil { 23 | return nil, exception.NewInternalServerError("inserted a action document error, %s", err) 24 | } 25 | 26 | return a, nil 27 | } 28 | 29 | func (i *service) QueryAction(ctx context.Context, req *action.QueryActionRequest) ( 30 | *action.ActionSet, error) { 31 | query := newQueryActionRequest(req) 32 | resp, err := i.col.Find(context.TODO(), query.FindFilter(), query.FindOptions()) 33 | 34 | if err != nil { 35 | return nil, exception.NewInternalServerError("find action error, error is %s", err) 36 | } 37 | 38 | set := action.NewActionSet() 39 | // 循环 40 | for resp.Next(context.TODO()) { 41 | a := action.NewDefaultAction() 42 | if err := resp.Decode(a); err != nil { 43 | return nil, exception.NewInternalServerError("decode action error, error is %s", err) 44 | } 45 | 46 | set.Add(a) 47 | } 48 | 49 | // count 50 | count, err := i.col.CountDocuments(context.TODO(), query.FindFilter()) 51 | if err != nil { 52 | return nil, exception.NewInternalServerError("get action count error, error is %s", err) 53 | } 54 | set.Total = count 55 | return set, nil 56 | } 57 | 58 | func (i *service) DescribeAction(ctx context.Context, req *action.DescribeActionRequest) ( 59 | *action.Action, error) { 60 | if err := req.Validate(); err != nil { 61 | return nil, exception.NewBadRequest("validate DescribeActionRequest error, %s", err) 62 | } 63 | 64 | desc, err := newDescActionRequest(req) 65 | if err != nil { 66 | return nil, exception.NewBadRequest(err.Error()) 67 | } 68 | 69 | ins := action.NewDefaultAction() 70 | if err := i.col.FindOne(context.TODO(), desc.FindFilter()).Decode(ins); err != nil { 71 | if err == mongo.ErrNoDocuments { 72 | return nil, exception.NewNotFound("action %s not found", req) 73 | } 74 | 75 | return nil, exception.NewInternalServerError("find action %s error, %s", req.Name, err) 76 | } 77 | 78 | return ins, nil 79 | } 80 | 81 | func (i *service) UpdateAction(ctx context.Context, req *action.UpdateActionRequest) (*action.Action, error) { 82 | if err := req.Validate(); err != nil { 83 | return nil, exception.NewBadRequest(err.Error()) 84 | } 85 | 86 | ins, err := i.DescribeAction(ctx, action.NewDescribeActionRequest(req.Name, req.Version)) 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | ins.Update(req) 92 | ins.UpdateAt = time.Now().UnixMilli() 93 | _, err = i.col.UpdateOne(context.TODO(), bson.M{"name": req.Name, "version": req.Version}, bson.M{"$set": ins}) 94 | if err != nil { 95 | return nil, exception.NewInternalServerError("update action(%s) error, %s", ins.Key(), err) 96 | } 97 | 98 | return ins, nil 99 | } 100 | 101 | func (i *service) DeleteAction(ctx context.Context, req *action.DeleteActionRequest) ( 102 | *action.Action, error) { 103 | ins, err := i.DescribeAction(ctx, action.NewDescribeActionRequest(req.Name, req.Version)) 104 | if err != nil { 105 | return nil, err 106 | } 107 | 108 | if ins.Namespace != req.Namespace { 109 | return nil, exception.NewBadRequest("target action namespace not match your namespace") 110 | } 111 | 112 | delReq, err := newDeleteActionRequest(req) 113 | if err != nil { 114 | return nil, exception.NewBadRequest(err.Error()) 115 | } 116 | 117 | if _, err := i.col.DeleteOne(context.TODO(), delReq.DeleteFilter()); err != nil { 118 | return nil, err 119 | } 120 | 121 | return ins, nil 122 | } 123 | -------------------------------------------------------------------------------- /api/apps/action/impl/impl.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/infraboard/mcube/app" 7 | "github.com/infraboard/mcube/logger" 8 | "github.com/infraboard/mcube/logger/zap" 9 | 10 | "go.mongodb.org/mongo-driver/mongo" 11 | "go.mongodb.org/mongo-driver/mongo/options" 12 | "go.mongodb.org/mongo-driver/x/bsonx" 13 | "google.golang.org/grpc" 14 | 15 | "github.com/infraboard/workflow/api/apps/action" 16 | "github.com/infraboard/workflow/conf" 17 | ) 18 | 19 | var ( 20 | // Service 服务实例 21 | svr = &service{} 22 | ) 23 | 24 | type service struct { 25 | col *mongo.Collection 26 | action.UnimplementedServiceServer 27 | 28 | log logger.Logger 29 | } 30 | 31 | func (s *service) Config() error { 32 | db := conf.C().Mongo.GetDB() 33 | dc := db.Collection("actions") 34 | 35 | indexs := []mongo.IndexModel{ 36 | { 37 | Keys: bsonx.Doc{ 38 | {Key: "name", Value: bsonx.Int32(-1)}, 39 | {Key: "version", Value: bsonx.Int32(-1)}, 40 | }, 41 | Options: options.Index().SetUnique(true), 42 | }, 43 | { 44 | Keys: bsonx.Doc{{Key: "create_at", Value: bsonx.Int32(-1)}}, 45 | }, 46 | } 47 | 48 | _, err := dc.Indexes().CreateMany(context.Background(), indexs) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | s.col = dc 54 | s.log = zap.L().Named("Action") 55 | 56 | return nil 57 | } 58 | 59 | func (s *service) Name() string { 60 | return action.AppName 61 | } 62 | 63 | func (s *service) Registry(server *grpc.Server) { 64 | action.RegisterServiceServer(server, svr) 65 | } 66 | 67 | func init() { 68 | app.RegistryGrpcApp(svr) 69 | } 70 | -------------------------------------------------------------------------------- /api/apps/action/impl/query.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "go.mongodb.org/mongo-driver/bson" 5 | "go.mongodb.org/mongo-driver/mongo/options" 6 | 7 | "github.com/infraboard/mcube/pb/resource" 8 | "github.com/infraboard/workflow/api/apps/action" 9 | ) 10 | 11 | func newQueryActionRequest(req *action.QueryActionRequest) *queryRequest { 12 | return &queryRequest{ 13 | QueryActionRequest: req, 14 | } 15 | } 16 | 17 | type queryRequest struct { 18 | *action.QueryActionRequest 19 | } 20 | 21 | func (r *queryRequest) FindOptions() *options.FindOptions { 22 | pageSize := int64(r.Page.PageSize) 23 | skip := int64(r.Page.PageSize) * int64(r.Page.PageNumber-1) 24 | 25 | opt := &options.FindOptions{ 26 | Sort: bson.D{{Key: "create_at", Value: -1}}, 27 | Limit: &pageSize, 28 | Skip: &skip, 29 | } 30 | 31 | return opt 32 | } 33 | 34 | func (r *queryRequest) FindFilter() bson.M { 35 | filter := bson.M{} 36 | 37 | cond1 := bson.M{} 38 | if r.Namespace != "" { 39 | cond1["namespace"] = r.Namespace 40 | } 41 | if r.Name != "" { 42 | cond1["name"] = r.Name 43 | } 44 | 45 | filter["$or"] = bson.A{ 46 | cond1, 47 | bson.M{"visiable_mode": resource.VisiableMode_GLOBAL}, 48 | } 49 | return filter 50 | } 51 | 52 | func newDescActionRequest(req *action.DescribeActionRequest) (*describeRequest, error) { 53 | if err := req.Validate(); err != nil { 54 | return nil, err 55 | } 56 | return &describeRequest{req}, nil 57 | } 58 | 59 | type describeRequest struct { 60 | *action.DescribeActionRequest 61 | } 62 | 63 | func (req *describeRequest) FindFilter() bson.M { 64 | filter := bson.M{} 65 | 66 | filter["name"] = req.Name 67 | filter["version"] = req.Version 68 | 69 | return filter 70 | } 71 | 72 | func newDeleteActionRequest(req *action.DeleteActionRequest) (*deleteRequest, error) { 73 | if err := req.Validate(); err != nil { 74 | return nil, err 75 | } 76 | return &deleteRequest{req}, nil 77 | } 78 | 79 | type deleteRequest struct { 80 | *action.DeleteActionRequest 81 | } 82 | 83 | func (req *deleteRequest) DeleteFilter() bson.M { 84 | filter := bson.M{} 85 | 86 | filter["name"] = req.Name 87 | filter["version"] = req.Version 88 | filter["namespace"] = req.Namespace 89 | 90 | return filter 91 | } 92 | -------------------------------------------------------------------------------- /api/apps/all/grpc.go: -------------------------------------------------------------------------------- 1 | package all 2 | 3 | import ( 4 | _ "github.com/infraboard/workflow/api/apps/action/impl" 5 | _ "github.com/infraboard/workflow/api/apps/approval/impl" 6 | _ "github.com/infraboard/workflow/api/apps/pipeline/impl" 7 | _ "github.com/infraboard/workflow/api/apps/template/impl" 8 | ) 9 | -------------------------------------------------------------------------------- /api/apps/all/http.go: -------------------------------------------------------------------------------- 1 | package all 2 | 3 | import ( 4 | // 加载服务模块 5 | _ "github.com/infraboard/workflow/api/apps/action/http" 6 | _ "github.com/infraboard/workflow/api/apps/pipeline/http" 7 | _ "github.com/infraboard/workflow/api/apps/template/http" 8 | ) 9 | -------------------------------------------------------------------------------- /api/apps/approval/app.go: -------------------------------------------------------------------------------- 1 | package approval 2 | 3 | const ( 4 | AppName = "approval" 5 | ) 6 | -------------------------------------------------------------------------------- /api/apps/approval/http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "github.com/infraboard/mcube/app" 5 | "github.com/infraboard/mcube/http/router" 6 | "github.com/infraboard/mcube/logger" 7 | "github.com/infraboard/mcube/logger/zap" 8 | 9 | "github.com/infraboard/workflow/api/apps/approval" 10 | ) 11 | 12 | var ( 13 | api = &handler{} 14 | ) 15 | 16 | type handler struct { 17 | service approval.ServiceServer 18 | 19 | log logger.Logger 20 | } 21 | 22 | // Registry 注册HTTP服务路由 23 | func (h *handler) Registry(router router.SubRouter) { 24 | r := router.ResourceRouter("approval") 25 | r.Auth(true) 26 | } 27 | 28 | func (h *handler) Config() error { 29 | h.log = zap.L().Named(h.Name()) 30 | h.service = app.GetGrpcApp(approval.AppName).(approval.ServiceServer) 31 | return nil 32 | } 33 | 34 | func (h *handler) Name() string { 35 | return approval.AppName 36 | } 37 | 38 | func init() { 39 | app.RegistryHttpApp(api) 40 | } 41 | -------------------------------------------------------------------------------- /api/apps/approval/impl/impl.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "github.com/infraboard/mcube/app" 5 | "github.com/infraboard/mcube/logger" 6 | "github.com/infraboard/mcube/logger/zap" 7 | "go.mongodb.org/mongo-driver/mongo" 8 | "google.golang.org/grpc" 9 | 10 | "github.com/infraboard/workflow/api/apps/approval" 11 | "github.com/infraboard/workflow/conf" 12 | ) 13 | 14 | var ( 15 | // Service 服务实例 16 | svr = &service{} 17 | ) 18 | 19 | type service struct { 20 | col *mongo.Collection 21 | log logger.Logger 22 | 23 | approval.UnimplementedServiceServer 24 | } 25 | 26 | func (s *service) Config() error { 27 | db := conf.C().Mongo.GetDB() 28 | dc := db.Collection("approval") 29 | 30 | s.col = dc 31 | s.log = zap.L().Named(s.Name()) 32 | return nil 33 | } 34 | 35 | func (s *service) Name() string { 36 | return approval.AppName 37 | } 38 | 39 | func (s *service) Registry(server *grpc.Server) { 40 | approval.RegisterServiceServer(server, svr) 41 | } 42 | 43 | func init() { 44 | app.RegistryGrpcApp(svr) 45 | } 46 | -------------------------------------------------------------------------------- /api/apps/approval/pb/approval.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package infraboard.workorder.approval; 4 | option go_package = "github.com/infraboard/workflow/api/apps/approval"; 5 | 6 | import "github.com/infraboard/mcube/pb/page/page.proto"; 7 | import "github.com/infraboard/mcube/pb/request/request.proto"; 8 | 9 | service Service { 10 | rpc CreateApproval(CreateApprovalRequest) returns(Approval); 11 | rpc QueryApproval(QueryApprovalRequest) returns(ApprovalSet); 12 | rpc DescribeApproval(DescribeApprovalRequest) returns(Approval); 13 | rpc UpdateApproval(UpdateApprovalRequest) returns(Approval); 14 | rpc DeleteApproval(DeleteApprovalRequest) returns(Approval); 15 | } 16 | 17 | message Approval { 18 | 19 | } 20 | 21 | message ApprovalSet { 22 | // 分页时，返回总数量 23 | // @gotags: json:"total" 24 | int64 total = 1; 25 | // 一页的数据 26 | // @gotags: json:"items" 27 | repeated Approval items = 2; 28 | } 29 | 30 | enum Provider { 31 | DEVCLOUD = 0; 32 | FEISHU = 1; 33 | } 34 | 35 | message CreateApprovalRequest { 36 | // 工单对接的第三方系统 37 | // @gotags: json:"provider" bson:"provider" 38 | Provider provider = 1; 39 | // 工单模版编号, 用于对接 40 | // @gotags: json:"approval_code" bson:"approval_code" 41 | string approval_code = 2; 42 | // 工单状态 43 | // @gotags: json:"status" bson:"status" 44 | string status = 3; 45 | } 46 | 47 | message QueryApprovalRequest { 48 | // 分页参数 49 | // @gotags: json:"page" 50 | infraboard.mcube.page.PageRequest page = 1; 51 | } 52 | 53 | message DescribeApprovalRequest { 54 | 55 | } 56 | 57 | message UpdateApprovalRequest { 58 | // 更新模式 59 | // @gotags: json:"update_mode" 60 | mcube.request.UpdateMode update_mode = 1; 61 | } 62 | 63 | message DeleteApprovalRequest { 64 | 65 | } -------------------------------------------------------------------------------- /api/apps/approval/provider/feishu/approval.go: -------------------------------------------------------------------------------- 1 | // 对接开放平台审批API使用手册: https://open.feishu.cn/document/ukTMukTMukTM/uIjN4UjLyYDO14iM2gTN 2 | 3 | package feishu 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/chyroc/lark" 9 | ) 10 | 11 | func (c *Client) GetApproval(ctx context.Context, req *lark.GetApprovalReq) (*lark.GetApprovalResp, error) { 12 | resp, _, err := c.client.Approval.GetApproval(ctx, req) 13 | return resp, err 14 | } 15 | 16 | func (c *Client) CreateApprovalInstance(ctx context.Context, req *lark.CreateApprovalInstanceReq) (*lark.CreateApprovalInstanceResp, error) { 17 | resp, _, err := c.client.Approval.CreateApprovalInstance(ctx, req) 18 | return resp, err 19 | } 20 | -------------------------------------------------------------------------------- /api/apps/approval/provider/feishu/client.go: -------------------------------------------------------------------------------- 1 | package feishu 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/caarlos0/env/v6" 8 | "github.com/chyroc/lark" 9 | ) 10 | 11 | func NewClient(appId, appSecret string) *Client { 12 | c := &Client{ 13 | AppId: appId, 14 | AppSecret: appSecret, 15 | } 16 | c.init() 17 | return c 18 | } 19 | 20 | func LoadClientFromEnv() (*Client, error) { 21 | c := &Client{} 22 | if err := env.Parse(c); err != nil { 23 | return nil, err 24 | } 25 | c.init() 26 | return c, nil 27 | } 28 | 29 | // 获取应用身份访问凭证 https://open.feishu.cn/document/ukTMukTMukTM/ukDNz4SO0MjL5QzM/g#top_anchor 30 | type Client struct { 31 | // 开发流程与工具介绍 https://open.feishu.cn/document/home/introduction-to-custom-app-development/self-built-application-development-process?lang=zh-CN 32 | AppId string `env:"FEISHU_APP_ID"` 33 | AppSecret string `env:"FEISHU_APP_SECRET"` 34 | 35 | // 事件订阅概述 https://open.feishu.cn/document/ukTMukTMukTM/uUTNz4SN1MjL1UzM?lang=zh-CN 36 | // Encrypt Key 配置示例与解密: https://open.feishu.cn/document/ukTMukTMukTM/uYDNxYjL2QTM24iN0EjN/event-subscription-configure-/encrypt-key-encryption-configuration-case 37 | EncryptKey string `env:"FEISHU_ENCRYPT_KEY"` 38 | // 订阅事件时, 该自动会放置于token字段中, 用于校验发送方身份 39 | VerificationToken string `env:"FEISHU_VERIFICATION_TOKEN"` 40 | 41 | client *lark.Lark 42 | } 43 | 44 | func (c *Client) init() { 45 | c.client = lark.New(lark.WithAppCredential(c.AppId, c.AppSecret)) 46 | } 47 | 48 | func (c *Client) Notify() { 49 | // us, resp, err := c.client.Contact.GetUser(context.Background(), &lark.GetUserReq{UserID: "ou_86791cfdf42c886435bdc65547d3ad8d"}) 50 | // fmt.Println(*us.User, resp, err) 51 | mresp, resp, err := c.client.Message.Send().ToUserID("2fbc2b39").SendText(context.Background(), "测试个人通知") 52 | fmt.Println(mresp) 53 | fmt.Println(resp) 54 | fmt.Println(err) 55 | } 56 | -------------------------------------------------------------------------------- /api/apps/approval/provider/feishu/client_test.go: -------------------------------------------------------------------------------- 1 | package feishu_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/infraboard/workflow/api/apps/approval/provider/feishu" 7 | ) 8 | 9 | var ( 10 | client *feishu.Client 11 | ) 12 | 13 | func TestAuth(t *testing.T) { 14 | client.Notify() 15 | } 16 | 17 | func init() { 18 | c, err := feishu.LoadClientFromEnv() 19 | if err != nil { 20 | panic(err) 21 | } 22 | client = c 23 | } 24 | -------------------------------------------------------------------------------- /api/apps/node/etcd/etcd.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "strings" 9 | "time" 10 | 11 | clientv3 "go.etcd.io/etcd/client/v3" 12 | 13 | "github.com/infraboard/mcube/logger" 14 | "github.com/infraboard/workflow/api/apps/node" 15 | "github.com/infraboard/workflow/conf" 16 | ) 17 | 18 | type etcd struct { 19 | leaseID clientv3.LeaseID 20 | client *clientv3.Client 21 | requestTimeout time.Duration 22 | isStopped bool 23 | instanceKey string 24 | instanceValue string 25 | stopInstance chan struct{} 26 | keepAliveStop context.CancelFunc 27 | node *node.Node 28 | logger.Logger 29 | } 30 | 31 | // NewEtcdRegister 初始化一个基于etcd的实例注册器 32 | func NewEtcdRegister(node *node.Node) (node.Register, error) { 33 | if err := node.Validate(); err != nil { 34 | return nil, err 35 | } 36 | 37 | etcdR := new(etcd) 38 | etcdR.client = conf.C().Etcd.GetClient() 39 | etcdR.stopInstance = make(chan struct{}, 1) 40 | etcdR.requestTimeout = time.Duration(5) * time.Second 41 | etcdR.node = node 42 | 43 | // 注册服务的key 44 | sjson, err := json.Marshal(node) 45 | if err != nil { 46 | return nil, err 47 | } 48 | etcdR.instanceValue = string(sjson) 49 | etcdR.instanceKey = node.MakeObjectKey() 50 | return etcdR, nil 51 | } 52 | 53 | // node use to registe serice endpoint to etcd. when etcd is down, 54 | // node can retry to registe util the etcd up. 55 | // 56 | // name is service name, use to discovery service address, eg. keyauth 57 | // host and port is service endpoint, eg. 127.0.0.0:50000 58 | // target is etcd addr, eg. 127.0.0.0:2379 59 | // interval is service refresh interval, eg. 10s 60 | // ttl is service ttl, eg. 15 61 | // TODO: 判断服务是否已经被其他人注册了, 如果注册了 则需要更换名称才能注册 62 | func (e *etcd) Registe() error { 63 | // 后台续约 64 | // 并没有直接使用KeepAlive, 因为存在偶然端口, 就不续约的情况 65 | ctx, cancel := context.WithCancel(context.Background()) 66 | e.keepAliveStop = cancel 67 | 68 | // 初始化注册 69 | if err := e.addOnce(); err != nil { 70 | e.Errorf("registry error, %s", err) 71 | return err 72 | } 73 | e.Infof("服务实例(%s)注册成功", e.instanceKey) 74 | 75 | // keep alive 76 | go e.keepAlive(ctx) 77 | return nil 78 | } 79 | 80 | func (e *etcd) Debug(log logger.Logger) { 81 | e.Logger = log 82 | } 83 | 84 | func (e *etcd) addOnce() error { 85 | // 获取leaseID 86 | resp, err := e.client.Lease.Grant(context.TODO(), e.node.TTL) 87 | if err != nil { 88 | return fmt.Errorf("get etcd lease id error, %s", err) 89 | } 90 | e.leaseID = resp.ID 91 | 92 | // 写入key 93 | if _, err := e.client.Put(context.Background(), e.instanceKey, e.instanceValue, clientv3.WithLease(e.leaseID)); err != nil { 94 | return fmt.Errorf("registe service '%s' with ttl to etcd3 failed: %s", e.instanceKey, err.Error()) 95 | } 96 | e.instanceKey = e.instanceKey 97 | return nil 98 | } 99 | 100 | func (e *etcd) keepAlive(ctx context.Context) { 101 | // 不停的续约 102 | interval := e.node.TTL / 5 103 | e.Infof("keep alive lease interval is %d second", interval) 104 | tk := time.NewTicker(time.Duration(interval) * time.Second) 105 | defer tk.Stop() 106 | for { 107 | select { 108 | case <-ctx.Done(): 109 | e.Infof("keepalive goroutine exit") 110 | return 111 | case <-tk.C: 112 | Opctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 113 | defer cancel() 114 | 115 | _, err := e.client.Lease.KeepAliveOnce(Opctx, e.leaseID) 116 | if err != nil { 117 | if strings.Contains(err.Error(), "requested lease not found") { 118 | // 避免程序卡顿造成leaseID失效(比如mac 电脑休眠)) 119 | if err := e.addOnce(); err != nil { 120 | e.Errorf("refresh registry error, %s", err) 121 | } else { 122 | e.Warn("refresh registry success") 123 | } 124 | } 125 | e.Errorf("lease keep alive error, %s", err) 126 | } else { 127 | e.Debugf("lease keep alive key: %s", e.instanceKey) 128 | } 129 | } 130 | } 131 | } 132 | 133 | // UnRegiste delete nodeed service from etcd, if etcd is down 134 | // unnode while timeout. 135 | func (e *etcd) UnRegiste() error { 136 | if e.isStopped { 137 | return errors.New("the instance has unregisted") 138 | } 139 | // delete instance key 140 | e.stopInstance <- struct{}{} 141 | ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) 142 | defer cancel() 143 | if resp, err := e.client.Delete(ctx, e.instanceKey); err != nil { 144 | e.Warnf("unregiste '%s' failed: connect to etcd server timeout, %s", e.instanceKey, err.Error()) 145 | } else { 146 | if resp.Deleted == 0 { 147 | e.Infof("unregiste '%s' failed, the key not exist", e.instanceKey) 148 | } else { 149 | e.Infof("服务实例(%s)注销成功", e.instanceKey) 150 | } 151 | } 152 | // revoke lease 153 | _, err := e.client.Lease.Revoke(context.TODO(), e.leaseID) 154 | if err != nil { 155 | e.Warnf("revoke lease error, %s", err) 156 | return err 157 | } 158 | e.isStopped = true 159 | // 停止续约心态 160 | e.keepAliveStop() 161 | return nil 162 | } 163 | -------------------------------------------------------------------------------- /api/apps/node/node.go: -------------------------------------------------------------------------------- 1 | package node 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/infraboard/mcube/logger" 10 | ) 11 | 12 | const ( 13 | // API 提供API访问的服务 14 | APIType = Type("api") 15 | // Worker 后台作业服务 16 | NodeType = Type("node") 17 | // Scheduler 调度器 18 | SchedulerType = Type("scheduler") 19 | ) 20 | 21 | type Type string 22 | 23 | // ParseEtcdNode tdo 24 | func LoadNodeFromBytes(value []byte) (*Node, error) { 25 | n := new(Node) 26 | n.Tag = map[string]string{} 27 | // 解析Value 28 | if len(value) > 0 { 29 | if err := json.Unmarshal(value, n); err != nil { 30 | return nil, fmt.Errorf("unmarshal node error, vaule(%s) %s", string(value), err) 31 | } 32 | } 33 | 34 | // 校验合法性 35 | if string(value) == "" { 36 | return nil, nil 37 | } 38 | 39 | if err := n.Validate(); err != nil { 40 | return nil, err 41 | } 42 | return n, nil 43 | } 44 | 45 | // Node todo 46 | type Node struct { 47 | Region string `json:"region,omitempty"` 48 | ResourceVersion int64 `json:"resourceVersion,omitempty"` 49 | InstanceName string `json:"instance_name,omitempty"` 50 | ServiceName string `json:"service_name,omitempty"` 51 | Type Type `json:"type,omitempty"` 52 | Address string `json:"address,omitempty"` 53 | Version string `json:"version,omitempty"` 54 | GitBranch string `json:"git_branch,omitempty"` 55 | GitCommit string `json:"git_commit,omitempty"` 56 | BuildEnv string `json:"build_env,omitempty"` 57 | BuildAt string `json:"build_at,omitempty"` 58 | Online int64 `json:"online,omitempty"` 59 | Tag map[string]string `json:"tag,omitempty"` 60 | 61 | Prefix string `json:"-"` 62 | Interval time.Duration `json:"-"` 63 | TTL int64 `json:"-"` 64 | } 65 | 66 | func (n *Node) Name() string { 67 | return fmt.Sprintf("%s.%s", n.ServiceName, n.InstanceName) 68 | } 69 | 70 | func (n *Node) ShortDescribe() string { 71 | return n.Name() 72 | } 73 | 74 | func (n *Node) Validate() error { 75 | if n.InstanceName == "" && n.ServiceName == "" || n.Type == "" { 76 | return errors.New("service instance name or service_name or type not config") 77 | } 78 | return nil 79 | } 80 | 81 | func (n *Node) IsMatch(nodeName string) bool { 82 | return n.InstanceName == nodeName 83 | } 84 | 85 | // MakeObjectKey 构建etcd对应的key 86 | // 例如: inforboard/workflow/service/node/node-01 87 | func (n *Node) MakeObjectKey() string { 88 | return fmt.Sprintf("%s/%s/%s", EtcdNodePrefix(), n.Type, n.InstanceName) 89 | } 90 | 91 | // Register 服务注册接口 92 | type Register interface { 93 | Debug(logger.Logger) 94 | Registe() error 95 | UnRegiste() error 96 | } 97 | -------------------------------------------------------------------------------- /api/apps/node/prefix.go: -------------------------------------------------------------------------------- 1 | package node 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/infraboard/workflow/conf" 9 | "github.com/infraboard/workflow/version" 10 | ) 11 | 12 | // ParseInstanceKey 解析key中的服务信息 13 | func ParseInstanceKey(key string) (serviceName, instanceName string, serviceType Type, err error) { 14 | kl := strings.Split(key, "/") 15 | if len(kl) != 5 { 16 | err = errors.New("key format error, must like inforboard/workflow/service/node/node-dev") 17 | return 18 | } 19 | serviceName, serviceType, instanceName = kl[1], Type(kl[3]), kl[4] 20 | return 21 | } 22 | 23 | func NodeObjectKey(key string) string { 24 | return fmt.Sprintf("%s/%s", EtcdNodePrefix(), key) 25 | } 26 | 27 | func EtcdNodePrefix() string { 28 | return fmt.Sprintf("%s/%s/service", conf.C().Etcd.Prefix, version.ServiceName) 29 | } 30 | 31 | func EtcdNodePrefixWithType(t Type) string { 32 | return fmt.Sprintf("%s/%s/service/%s", conf.C().Etcd.Prefix, version.ServiceName, t) 33 | } 34 | -------------------------------------------------------------------------------- /api/apps/pipeline/app.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | const ( 4 | AppName = "pipeline" 5 | ) 6 | -------------------------------------------------------------------------------- /api/apps/pipeline/http/enum.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/infraboard/mcube/http/response" 7 | 8 | "github.com/infraboard/workflow/api/apps/pipeline" 9 | "github.com/infraboard/workflow/common/enum" 10 | ) 11 | 12 | func (h *handler) QueryStepStatusEnum(w http.ResponseWriter, r *http.Request) { 13 | response.Success(w, stepStatusEnums) 14 | } 15 | 16 | var ( 17 | stepStatusEnums = []*enum.EnumDesc{ 18 | {Value: pipeline.STEP_STATUS_PENDDING.String(), Name: "调度中", Desc: "当任务被创建后触发"}, 19 | {Value: pipeline.STEP_STATUS_RUNNING.String(), Name: "运行中", Desc: "当任务开始运行时触发"}, 20 | {Value: pipeline.STEP_STATUS_SUCCEEDED.String(), Name: "运行成功", Desc: "当任务运行成功后触发"}, 21 | {Value: pipeline.STEP_STATUS_FAILED.String(), Name: "运行失败", Desc: "当任务运行失败时触发"}, 22 | {Value: pipeline.STEP_STATUS_CANCELED.String(), Name: "任务取消", Desc: "当任务被成功取消时触发"}, 23 | {Value: pipeline.STEP_STATUS_SKIP.String(), Name: "任务跳过", Desc: "当前任务忽略执行时触发"}, 24 | {Value: pipeline.STEP_STATUS_AUDITING.String(), Name: "审批中", Desc: "当前任务需要审批时触发"}, 25 | {Value: pipeline.STEP_STATUS_REFUSE.String(), Name: "审批拒绝", Desc: "当审批的任务被拒绝时触发"}, 26 | {Value: pipeline.STEP_STATUS_SCHEDULE_FAILED.String(), Name: "调度失败", Desc: "当任务由于调度失败无法执行时触发"}, 27 | } 28 | ) 29 | 30 | func (h *handler) QueryVariableTemplate(w http.ResponseWriter, r *http.Request) { 31 | if !tempateIsInit { 32 | for k, v := range pipeline.VALUE_TYPE_ID_MAP { 33 | for i := range valueTempate { 34 | if valueTempate[i].Type == v { 35 | valueTempate[i].Prefix = k 36 | } 37 | } 38 | } 39 | tempateIsInit = true 40 | } 41 | 42 | response.Success(w, valueTempate) 43 | } 44 | 45 | var ( 46 | tempateIsInit = false 47 | valueTempate = []*ValueTypeDesc{ 48 | { 49 | Type: pipeline.PARAM_VALUE_TYPE_PLAIN, 50 | Prefix: "", 51 | Name: "明文", 52 | Desc: "明文文本,敏感信息请不要使用这个类型", 53 | IsEdit: true, 54 | }, 55 | { 56 | Type: pipeline.PARAM_VALUE_TYPE_PASSWORD, 57 | Prefix: "", 58 | Name: "秘文", 59 | Desc: "敏感信息,由系统加密存储,运行时解密注入", 60 | IsEdit: true, 61 | }, 62 | { 63 | Type: pipeline.PARAM_VALUE_TYPE_CRYPTO, 64 | Prefix: "", 65 | Name: "解密", 66 | Desc: "敏感信息加密后的密文,无法修改", 67 | }, 68 | { 69 | Type: pipeline.PARAM_VALUE_TYPE_APP_VAR, 70 | Prefix: "", 71 | Name: "应用变量", 72 | Desc: "应用属性,也包含自定义变量,运行时由系统动态注入", 73 | IsEdit: true, 74 | }, 75 | { 76 | Type: pipeline.PARAM_VALUE_TYPE_SECRET_REF, 77 | Prefix: "", 78 | Name: "Secret引用", 79 | Desc: "运行时由系统查询Secret系统后动态注入", 80 | IsEdit: true, 81 | }, 82 | } 83 | ) 84 | 85 | type ValueTypeDesc struct { 86 | Type pipeline.PARAM_VALUE_TYPE `json:"type"` 87 | Prefix string `json:"prefix"` 88 | Name string `json:"name"` 89 | Desc string `json:"desc"` 90 | Value string `json:"value"` 91 | IsEdit bool `json:"is_edit"` 92 | } 93 | -------------------------------------------------------------------------------- /api/apps/pipeline/http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "github.com/infraboard/mcube/app" 5 | "github.com/infraboard/mcube/http/label" 6 | "github.com/infraboard/mcube/http/router" 7 | "github.com/infraboard/mcube/logger" 8 | "github.com/infraboard/mcube/logger/zap" 9 | 10 | "github.com/infraboard/workflow/api/apps/pipeline" 11 | ) 12 | 13 | var ( 14 | api = &handler{} 15 | ) 16 | 17 | type handler struct { 18 | service pipeline.ServiceServer 19 | log logger.Logger 20 | proxy *Proxy 21 | } 22 | 23 | // Registry 注册HTTP服务路由 24 | func (h *handler) Registry(router router.SubRouter) { 25 | r := router.ResourceRouter("pipeline") 26 | r.Permission(true) 27 | r.BasePath("pipelines") 28 | r.Handle("POST", "/", h.CreatePipeline).AddLabel(label.Create) 29 | r.Handle("GET", "/", h.QueryPipeline).AddLabel(label.List) 30 | r.Handle("GET", "/:id", h.DescribePipeline).AddLabel(label.Get) 31 | r.Handle("DELETE", "/:id", h.DeletePipeline).AddLabel(label.Delete) 32 | r.Handle("GET", "/:id/watch_check", h.WatchPipelineCheck).AddLabel(label.Get) 33 | r.BasePath("websocket") 34 | r.Handle("GET", "pipelines/:id/watch", h.WatchPipeline).AddLabel(label.Get) 35 | 36 | r.BasePath("steps") 37 | r.Handle("GET", "/", h.QueryStep).AddLabel(label.List) 38 | r.Handle("POST", "/", h.CreateStep).AddLabel(label.Create) 39 | r.Handle("GET", "/:id", h.DescribeStep).AddLabel(label.Get) 40 | r.Handle("DELETE", "/:id", h.DeleteStep).AddLabel(label.Delete) 41 | r.Handle("POST", "/:id/audit", h.AuditStep).AddLabel(label.Update) 42 | r.Handle("POST", "/:id/cancel", h.CancelStep).AddLabel(label.Update) 43 | 44 | r.BasePath("variable_templates") 45 | r.Handle("GET", "/", h.QueryVariableTemplate).AddLabel(label.List) 46 | r.BasePath("enums") 47 | r.Handle("GET", "/step_status", h.QueryStepStatusEnum).AddLabel(label.List) 48 | } 49 | 50 | func (h *handler) Config() error { 51 | h.service = app.GetGrpcApp(pipeline.AppName).(pipeline.ServiceServer) 52 | h.proxy = NewProxy() 53 | 54 | h.log = zap.L().Named("Pipeline") 55 | return nil 56 | } 57 | 58 | func (h *handler) Name() string { 59 | return pipeline.AppName 60 | } 61 | 62 | func init() { 63 | app.RegistryHttpApp(api) 64 | } 65 | -------------------------------------------------------------------------------- /api/apps/pipeline/http/proxy.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "io" 7 | "time" 8 | 9 | "github.com/gorilla/websocket" 10 | "github.com/infraboard/mcube/logger" 11 | "github.com/infraboard/mcube/logger/zap" 12 | ) 13 | 14 | func NewProxy() *Proxy { 15 | return &Proxy{ 16 | maxRespBodyBufferBytes: 64 * 1024, 17 | log: zap.L().Named("Websocket Proxy"), 18 | pingInterval: 5 * time.Second, 19 | pingWait: 3 * time.Second, 20 | pongWait: 15 * time.Second, 21 | } 22 | } 23 | 24 | type Proxy struct { 25 | maxRespBodyBufferBytes int 26 | log logger.Logger 27 | pingInterval time.Duration 28 | pingWait time.Duration 29 | pongWait time.Duration 30 | cancel context.CancelFunc 31 | } 32 | 33 | func (p *Proxy) Proxy(ctx context.Context, conn *websocket.Conn, stream io.ReadWriter) { 34 | go p.handleRead(ctx, conn, stream) 35 | go p.handleWrite(ctx, conn, stream) 36 | 37 | ctx, p.cancel = context.WithCancel(ctx) 38 | p.handlePing(ctx, conn) 39 | p.log.Debug("proxy down") 40 | } 41 | 42 | // read loop -- take messages from websocket and write to http request 43 | func (p *Proxy) handleWrite(ctx context.Context, conn *websocket.Conn, writer io.Writer) { 44 | p.log.Debugf("start write to: websocket connection --> writer ...") 45 | defer func() { 46 | p.log.Debugf("[read] read from websocket and write to writer down") 47 | p.cancel() 48 | }() 49 | 50 | if p.pingInterval > 0 && p.pingWait > 0 && p.pongWait > 0 { 51 | conn.SetReadDeadline(time.Now().Add(p.pongWait)) 52 | conn.SetPongHandler(func(string) error { conn.SetReadDeadline(time.Now().Add(p.pongWait)); return nil }) 53 | } 54 | 55 | for { 56 | select { 57 | case <-ctx.Done(): 58 | p.log.Debug("read loop done") 59 | return 60 | default: 61 | } 62 | p.log.Debug("[read] reading from socket.") 63 | _, payload, err := conn.ReadMessage() 64 | if err != nil { 65 | if websocket.IsCloseError(err) { 66 | p.log.Debugf("[read] websocket closed: %s", err) 67 | return 68 | } 69 | p.log.Warnf("error reading websocket message: %s", err) 70 | return 71 | } 72 | p.log.Debugf("[read] read payload: %s", string(payload)) 73 | p.log.Debug("[read] writing to requestBody:") 74 | n, err := writer.Write(payload) 75 | writer.Write([]byte("\n")) 76 | p.log.Debugf("[read] write to requestBody", n) 77 | if err != nil { 78 | p.log.Warnf("[read] error writing message to http server:", err) 79 | return 80 | } 81 | } 82 | } 83 | 84 | func (p *Proxy) handleRead(ctx context.Context, conn *websocket.Conn, reader io.Reader) { 85 | p.log.Debugf("start dumpping read : reader --> websocket connection ...") 86 | defer func() { 87 | p.log.Debugf("[write] read from reader and write to websocket down") 88 | p.cancel() 89 | }() 90 | 91 | scanner := bufio.NewScanner(reader) 92 | var scannerBuf []byte 93 | if p.maxRespBodyBufferBytes > 0 { 94 | scannerBuf = make([]byte, 0, 64*1024) 95 | scanner.Buffer(scannerBuf, p.maxRespBodyBufferBytes) 96 | } 97 | 98 | p.log.Debug("[write] writing to socket.") 99 | for scanner.Scan() { 100 | if len(scanner.Bytes()) == 0 { 101 | p.log.Warnf("[write] empty scan", scanner.Err()) 102 | continue 103 | } 104 | p.log.Debugf("[write] scanned: %s", scanner.Text()) 105 | if err := conn.WriteMessage(websocket.TextMessage, scanner.Bytes()); err != nil { 106 | p.log.Errorf("[write] error writing websocket message:", err) 107 | return 108 | } 109 | } 110 | 111 | if err := scanner.Err(); err != nil { 112 | p.log.Errorf("scanner err: %s", err) 113 | } 114 | } 115 | 116 | func (p *Proxy) handlePing(ctx context.Context, conn *websocket.Conn) { 117 | if !(p.pingInterval > 0 && p.pingWait > 0 && p.pongWait > 0) { 118 | return 119 | } 120 | 121 | ticker := time.NewTicker(p.pingInterval) 122 | defer func() { 123 | ticker.Stop() 124 | conn.Close() 125 | }() 126 | 127 | p.log.Debug("start websocket ping") 128 | for { 129 | select { 130 | case <-ctx.Done(): 131 | p.log.Debug("ping loop done") 132 | return 133 | case <-ticker.C: 134 | conn.SetWriteDeadline(time.Now().Add(p.pingWait)) 135 | if err := conn.WriteMessage(websocket.PingMessage, nil); err != nil { 136 | p.log.Debugf("write ping message error, %s", err) 137 | return 138 | } 139 | } 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /api/apps/pipeline/http/proxy_test.go: -------------------------------------------------------------------------------- 1 | package http_test 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "testing" 7 | 8 | "github.com/gorilla/websocket" 9 | "github.com/infraboard/mcube/logger/zap" 10 | ) 11 | 12 | func TestWebsocket(t *testing.T) { 13 | // URL 14 | target := "ws://127.0.0.1:9948/workflow/api/v1/websocket/pipelines/c4s8iiea0brugin5hl30/watch" 15 | t.Logf("connnect to: %s", target) 16 | 17 | // Connect to the server 18 | h := http.Header{} 19 | h.Add("Authorization", "R9bRjrYpAtunMM9VDUPhCIgL") 20 | 21 | ws, _, err := websocket.DefaultDialer.Dial(target, h) 22 | if err != nil { 23 | t.Fatalf("%v", err) 24 | } 25 | defer ws.Close() 26 | 27 | _, p, err := ws.ReadMessage() 28 | if err != nil { 29 | t.Fatalf("%v", err) 30 | } 31 | 32 | fmt.Println(string(p)) 33 | } 34 | 35 | func init() { 36 | zap.DevelopmentSetup() 37 | } 38 | -------------------------------------------------------------------------------- /api/apps/pipeline/http/step.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | "github.com/infraboard/keyauth/app/token" 8 | "github.com/infraboard/mcube/grpc/gcontext" 9 | "github.com/infraboard/mcube/http/context" 10 | "github.com/infraboard/mcube/http/request" 11 | "github.com/infraboard/mcube/http/response" 12 | 13 | "github.com/infraboard/workflow/api/apps/pipeline" 14 | ) 15 | 16 | // Action 17 | func (h *handler) CreateStep(w http.ResponseWriter, r *http.Request) { 18 | ctx, err := gcontext.NewGrpcOutCtxFromHTTPRequest(r) 19 | if err != nil { 20 | response.Failed(w, err) 21 | return 22 | } 23 | 24 | hc := context.GetContext(r) 25 | tk, ok := hc.AuthInfo.(*token.Token) 26 | if !ok { 27 | response.Failed(w, fmt.Errorf("auth info is not an *token.Token")) 28 | return 29 | } 30 | 31 | req := pipeline.NewCreateStepRequest() 32 | if err := request.GetDataFromRequest(r, req); err != nil { 33 | response.Failed(w, err) 34 | return 35 | } 36 | req.Namespace = tk.Namespace 37 | 38 | ins, err := h.service.CreateStep( 39 | ctx.Context(), 40 | req, 41 | ) 42 | if err != nil { 43 | response.Failed(w, err) 44 | return 45 | } 46 | response.Success(w, ins) 47 | } 48 | 49 | func (h *handler) QueryStep(w http.ResponseWriter, r *http.Request) { 50 | ctx, err := gcontext.NewGrpcOutCtxFromHTTPRequest(r) 51 | if err != nil { 52 | response.Failed(w, err) 53 | return 54 | } 55 | 56 | page := request.NewPageRequestFromHTTP(r) 57 | req := pipeline.NewQueryStepRequest() 58 | req.Page = page 59 | 60 | dommains, err := h.service.QueryStep( 61 | ctx.Context(), 62 | req, 63 | ) 64 | if err != nil { 65 | response.Failed(w, err) 66 | return 67 | } 68 | response.Success(w, dommains) 69 | } 70 | 71 | func (h *handler) DescribeStep(w http.ResponseWriter, r *http.Request) { 72 | ctx, err := gcontext.NewGrpcOutCtxFromHTTPRequest(r) 73 | if err != nil { 74 | response.Failed(w, err) 75 | return 76 | } 77 | 78 | hc := context.GetContext(r) 79 | tk, ok := hc.AuthInfo.(*token.Token) 80 | if !ok { 81 | response.Failed(w, fmt.Errorf("auth info is not an *token.Token")) 82 | return 83 | } 84 | 85 | req := pipeline.NewDescribeStepRequestWithKey(hc.PS.ByName("id")) 86 | req.Namespace = tk.Namespace 87 | 88 | dommains, err := h.service.DescribeStep( 89 | ctx.Context(), 90 | req, 91 | ) 92 | if err != nil { 93 | response.Failed(w, err) 94 | return 95 | } 96 | response.Success(w, dommains) 97 | } 98 | 99 | func (h *handler) DeleteStep(w http.ResponseWriter, r *http.Request) { 100 | ctx, err := gcontext.NewGrpcOutCtxFromHTTPRequest(r) 101 | if err != nil { 102 | response.Failed(w, err) 103 | return 104 | } 105 | 106 | hc := context.GetContext(r) 107 | req := pipeline.NewDeleteStepRequestWithKey(hc.PS.ByName("id")) 108 | 109 | dommains, err := h.service.DeleteStep( 110 | ctx.Context(), 111 | req, 112 | ) 113 | if err != nil { 114 | response.Failed(w, err) 115 | return 116 | } 117 | response.Success(w, dommains) 118 | } 119 | 120 | func (h *handler) AuditStep(w http.ResponseWriter, r *http.Request) { 121 | ctx, err := gcontext.NewGrpcOutCtxFromHTTPRequest(r) 122 | if err != nil { 123 | response.Failed(w, err) 124 | return 125 | } 126 | 127 | hc := context.GetContext(r) 128 | req := pipeline.NewAuditStepRequest() 129 | if err := request.GetDataFromRequest(r, req); err != nil { 130 | response.Failed(w, err) 131 | return 132 | } 133 | req.Key = hc.PS.ByName("id") 134 | 135 | dommains, err := h.service.AuditStep( 136 | ctx.Context(), 137 | req, 138 | ) 139 | if err != nil { 140 | response.Failed(w, err) 141 | return 142 | } 143 | response.Success(w, dommains) 144 | } 145 | 146 | func (h *handler) CancelStep(w http.ResponseWriter, r *http.Request) { 147 | ctx, err := gcontext.NewGrpcOutCtxFromHTTPRequest(r) 148 | if err != nil { 149 | response.Failed(w, err) 150 | return 151 | } 152 | 153 | hc := context.GetContext(r) 154 | req := pipeline.NewCancelStepRequestWithKey(hc.PS.ByName("id")) 155 | 156 | dommains, err := h.service.CancelStep( 157 | ctx.Context(), 158 | req, 159 | ) 160 | if err != nil { 161 | response.Failed(w, err) 162 | return 163 | } 164 | response.Success(w, dommains) 165 | } 166 | -------------------------------------------------------------------------------- /api/apps/pipeline/impl/impl.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/infraboard/mcube/app" 8 | "github.com/infraboard/mcube/logger" 9 | "github.com/infraboard/mcube/logger/zap" 10 | clientv3 "go.etcd.io/etcd/client/v3" 11 | "google.golang.org/grpc" 12 | 13 | "github.com/infraboard/workflow/api/apps/action" 14 | "github.com/infraboard/workflow/api/apps/pipeline" 15 | "github.com/infraboard/workflow/conf" 16 | ) 17 | 18 | var ( 19 | // Service 服务实例 20 | svr = &impl{ 21 | watchCancel: make(map[int64]context.CancelFunc), 22 | } 23 | ) 24 | 25 | type impl struct { 26 | client *clientv3.Client 27 | log logger.Logger 28 | action action.ServiceServer 29 | 30 | watchCancel map[int64]context.CancelFunc 31 | currentNumber int64 32 | l sync.Mutex 33 | 34 | pipeline.UnimplementedServiceServer 35 | } 36 | 37 | func (i *impl) SetWatcherCancelFn(fn context.CancelFunc) int64 { 38 | i.l.Lock() 39 | defer i.l.Unlock() 40 | 41 | i.currentNumber++ 42 | i.watchCancel[i.currentNumber] = fn 43 | return i.currentNumber 44 | } 45 | 46 | func (s *impl) Config() error { 47 | s.log = zap.L().Named("Pipeline") 48 | s.client = conf.C().Etcd.GetClient() 49 | s.action = app.GetGrpcApp(action.AppName).(action.ServiceServer) 50 | return nil 51 | } 52 | 53 | func (e *impl) Debug(log logger.Logger) { 54 | e.log = log 55 | } 56 | 57 | func (s *impl) Name() string { 58 | return pipeline.AppName 59 | } 60 | 61 | func (s *impl) Registry(server *grpc.Server) { 62 | pipeline.RegisterServiceServer(server, svr) 63 | } 64 | 65 | func init() { 66 | app.RegistryGrpcApp(svr) 67 | } 68 | -------------------------------------------------------------------------------- /api/apps/pipeline/prefix.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/infraboard/workflow/conf" 7 | "github.com/infraboard/workflow/version" 8 | ) 9 | 10 | func PipeLineObjectKey(namespace, id string) string { 11 | return fmt.Sprintf("%s/%s/%s", EtcdPipelinePrefix(), namespace, id) 12 | } 13 | 14 | func StepObjectKey(key string) string { 15 | return fmt.Sprintf("%s/%s", EtcdStepPrefix(), key) 16 | } 17 | 18 | func ActionObjectKey(namespace, name, version string) string { 19 | return fmt.Sprintf("%s/%s/%s@%s", EtcdActionPrefix(), namespace, name, version) 20 | } 21 | 22 | func EtcdPipelinePrefix() string { 23 | return fmt.Sprintf("%s/%s/pipelines", conf.C().Etcd.Prefix, version.ServiceName) 24 | } 25 | 26 | func EtcdStepPrefix() string { 27 | return fmt.Sprintf("%s/%s/steps", conf.C().Etcd.Prefix, version.ServiceName) 28 | } 29 | 30 | func EtcdActionPrefix() string { 31 | return fmt.Sprintf("%s/%s/actions", conf.C().Etcd.Prefix, version.ServiceName) 32 | } 33 | -------------------------------------------------------------------------------- /api/apps/pipeline/step_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/infraboard/workflow/api/apps/pipeline" 9 | ) 10 | 11 | func TestStepNamespace(t *testing.T) { 12 | should := assert.New(t) 13 | 14 | s := pipeline.NewDefaultStep() 15 | s.Key = "c16mhsddrei91m4ri0jg.c3iqcama0brimaq08e40.2.1" 16 | should.Equal(s.GetNamespace(), "c16mhsddrei91m4ri0jg") 17 | } 18 | -------------------------------------------------------------------------------- /api/apps/pipeline/variable/getter.go: -------------------------------------------------------------------------------- 1 | package variable 2 | 3 | // 变量替换 4 | type ValueGetter interface { 5 | // 传人变量, 获取值, 如果不是变量，这返回本身 6 | Get(v string) string 7 | } 8 | -------------------------------------------------------------------------------- /api/apps/step/README.md: -------------------------------------------------------------------------------- 1 | # Step管理 2 | 3 | -------------------------------------------------------------------------------- /api/apps/step/pb/rpc.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package infraboard.workflow.step; 4 | option go_package = "github.com/infraboard/workflow/api/apps/step"; 5 | 6 | import "api/apps/step/pb/step.proto"; 7 | 8 | service RPC { 9 | rpc CreateStep(CreateStepRequest) returns(Step); 10 | rpc QueryStep(QueryStepRequest) returns(StepSet); 11 | rpc DescribeStep(DescribeStepRequest) returns(Step); 12 | rpc DeleteStep(DeleteStepRequest) returns(Step); 13 | rpc CancelStep(CancelStepRequest) returns(Step); 14 | rpc AuditStep(AuditStepRequest) returns(Step); 15 | } 16 | 17 | message CreateStepRequest { 18 | // 名称 19 | // @gotags: json:"name" validate:"required" 20 | string name = 1; 21 | // 具体动作 22 | // @gotags: json:"action" validate:"required" 23 | string action = 2; 24 | // 是否需要审批, 审批通过后才能执行 25 | // @gotags: json:"with_audit" 26 | bool with_audit =3; 27 | // 审批参数, 有审批模块做具体实现 28 | // @gotags: json:"audit_params" 29 | map audit_params = 4; 30 | // 参数 31 | // @gotags: json:"with" 32 | map with = 5; 33 | // step执行完成后, 是否需要通知 34 | // @gotags: json:"with_notify" 35 | bool with_notify = 6; 36 | // 通知参数, 由通知模块做具体实现 37 | // @gotags: json:"notify_params" 38 | map notify_params = 7; 39 | // WebHook配置, 用于和其他系统联动, 比如各种机器人 40 | // @gotags: json:"webhooks" 41 | repeated WebHook webhooks = 8; 42 | // 调度标签 43 | // @gotags: json:"node_selector" 44 | map node_selector = 9; 45 | // 空间 46 | // @gotags: json:"namespace" 47 | string namespace = 10; 48 | } 49 | 50 | // QueryStepRequest 查询Book请求 51 | message QueryStepRequest { 52 | // @gotags: json:"key" 53 | string key = 2; 54 | } 55 | 56 | // DescribeStepRequest todo 57 | message DescribeStepRequest { 58 | // 唯一ID 59 | // @gotags: json:"key" 60 | string key = 1; 61 | // 唯一name 62 | // @gotags: json:"namespace" 63 | string namespace = 2; 64 | } 65 | 66 | message DeleteStepRequest { 67 | // 唯一ID 68 | // @gotags: json:"key" validate:"required" 69 | string key = 1; 70 | } 71 | 72 | 73 | message CancelStepRequest { 74 | // 取消step对应的key 75 | // @gotags: json:"id" 76 | string key = 1; 77 | } 78 | 79 | message AuditStepRequest { 80 | // 取消step对应的key 81 | // @gotags: json:"key" 82 | string key = 1; 83 | // 审核的结果 84 | // @gotags: json:"audit_reponse" 85 | AUDIT_RESPONSE audit_reponse = 2; 86 | // 审批时的反馈信息 87 | // @gotags: json:"audit_message" 88 | string audit_message = 3; 89 | } -------------------------------------------------------------------------------- /api/apps/template/app.go: -------------------------------------------------------------------------------- 1 | package template 2 | 3 | const ( 4 | AppName = "template" 5 | ) 6 | -------------------------------------------------------------------------------- /api/apps/template/http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "github.com/infraboard/mcube/app" 5 | "github.com/infraboard/mcube/http/router" 6 | "github.com/infraboard/mcube/logger" 7 | "github.com/infraboard/mcube/logger/zap" 8 | 9 | "github.com/infraboard/workflow/api/apps/template" 10 | ) 11 | 12 | var ( 13 | api = &handler{} 14 | ) 15 | 16 | type handler struct { 17 | service template.ServiceServer 18 | log logger.Logger 19 | } 20 | 21 | // Registry 注册HTTP服务路由 22 | func (h *handler) Registry(router router.SubRouter) { 23 | r := router.ResourceRouter("templates") 24 | r.Permission(true) 25 | r.BasePath("templates") 26 | r.Handle("POST", "/", h.CreateTemplate) 27 | r.Handle("GET", "/", h.QueryTemplate) 28 | r.Handle("GET", "/:id", h.DescribeTemplate) 29 | r.Handle("PUT", "/:id", h.PutTemplate) 30 | r.Handle("PATCH", "/:id", h.PatchTemplate) 31 | r.Handle("DELETE", "/:id", h.DeleteTemplate) 32 | } 33 | 34 | func (h *handler) Config() error { 35 | h.service = nil 36 | 37 | h.log = zap.L().Named(template.AppName) 38 | return nil 39 | } 40 | 41 | func (h *handler) Name() string { 42 | return template.AppName 43 | } 44 | 45 | func init() { 46 | app.RegistryHttpApp(api) 47 | } 48 | -------------------------------------------------------------------------------- /api/apps/template/http/template.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/infraboard/keyauth/app/token" 7 | "github.com/infraboard/mcube/http/context" 8 | "github.com/infraboard/mcube/http/request" 9 | "github.com/infraboard/mcube/http/response" 10 | pb "github.com/infraboard/mcube/pb/request" 11 | 12 | "github.com/infraboard/workflow/api/apps/template" 13 | ) 14 | 15 | // Action 16 | func (h *handler) CreateTemplate(w http.ResponseWriter, r *http.Request) { 17 | ctx := context.GetContext(r) 18 | tk := ctx.AuthInfo.(*token.Token) 19 | 20 | req := template.NewCreateTemplateRequest() 21 | if err := request.GetDataFromRequest(r, req); err != nil { 22 | response.Failed(w, err) 23 | return 24 | } 25 | req.UpdateOwner(tk) 26 | 27 | ins, err := h.service.CreateTemplate( 28 | r.Context(), 29 | req, 30 | ) 31 | if err != nil { 32 | response.Failed(w, err) 33 | return 34 | } 35 | response.Success(w, ins) 36 | } 37 | 38 | func (h *handler) QueryTemplate(w http.ResponseWriter, r *http.Request) { 39 | ctx := context.GetContext(r) 40 | tk := ctx.AuthInfo.(*token.Token) 41 | 42 | page := request.NewPageRequestFromHTTP(r) 43 | req := template.NewQueryTemplateRequest(page) 44 | req.Namespace = tk.Namespace 45 | 46 | actions, err := h.service.QueryTemplate( 47 | r.Context(), 48 | req, 49 | ) 50 | if err != nil { 51 | response.Failed(w, err) 52 | return 53 | } 54 | response.Success(w, actions) 55 | } 56 | 57 | func (h *handler) DescribeTemplate(w http.ResponseWriter, r *http.Request) { 58 | ctx := context.GetContext(r) 59 | req := template.NewDescribeTemplateRequestWithID(ctx.PS.ByName("id")) 60 | 61 | ins, err := h.service.DescribeTemplate( 62 | r.Context(), 63 | req, 64 | ) 65 | if err != nil { 66 | response.Failed(w, err) 67 | return 68 | } 69 | response.Success(w, ins) 70 | } 71 | 72 | func (h *handler) DeleteTemplate(w http.ResponseWriter, r *http.Request) { 73 | ctx := context.GetContext(r) 74 | req := template.NewDeleteTemplateRequestWithID(ctx.PS.ByName("id")) 75 | 76 | action, err := h.service.DeleteTemplate( 77 | r.Context(), 78 | req, 79 | ) 80 | if err != nil { 81 | response.Failed(w, err) 82 | return 83 | } 84 | response.Success(w, action) 85 | } 86 | 87 | func (h *handler) PutTemplate(w http.ResponseWriter, r *http.Request) { 88 | ctx := context.GetContext(r) 89 | tk := ctx.AuthInfo.(*token.Token) 90 | 91 | req := template.NewUpdateTemplateRequest(ctx.PS.ByName("id")) 92 | req.UpdateBy = tk.Account 93 | if err := request.GetDataFromRequest(r, req.Data); err != nil { 94 | response.Failed(w, err) 95 | return 96 | } 97 | 98 | ins, err := h.service.UpdateTemplate( 99 | r.Context(), 100 | req, 101 | ) 102 | if err != nil { 103 | response.Failed(w, err) 104 | return 105 | } 106 | 107 | response.Success(w, ins) 108 | } 109 | 110 | func (h *handler) PatchTemplate(w http.ResponseWriter, r *http.Request) { 111 | ctx := context.GetContext(r) 112 | tk := ctx.AuthInfo.(*token.Token) 113 | 114 | req := template.NewUpdateTemplateRequest(ctx.PS.ByName("id")) 115 | req.UpdateMode = pb.UpdateMode_PATCH 116 | req.UpdateBy = tk.Account 117 | if err := request.GetDataFromRequest(r, req.Data); err != nil { 118 | response.Failed(w, err) 119 | return 120 | } 121 | 122 | ins, err := h.service.UpdateTemplate( 123 | r.Context(), 124 | req, 125 | ) 126 | if err != nil { 127 | response.Failed(w, err) 128 | return 129 | } 130 | 131 | response.Success(w, ins) 132 | } 133 | -------------------------------------------------------------------------------- /api/apps/template/impl/dao.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | 6 | "go.mongodb.org/mongo-driver/bson" 7 | 8 | "github.com/infraboard/mcube/exception" 9 | 10 | "github.com/infraboard/workflow/api/apps/template" 11 | ) 12 | 13 | func (s *impl) update(ctx context.Context, app *template.Template) error { 14 | _, err := s.col.UpdateOne(ctx, bson.M{"_id": app.Id}, bson.M{"$set": app}) 15 | if err != nil { 16 | return exception.NewInternalServerError("update template(%s) error, %s", app.Id, err) 17 | } 18 | 19 | return nil 20 | } 21 | -------------------------------------------------------------------------------- /api/apps/template/impl/impl.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/infraboard/mcube/app" 7 | "github.com/infraboard/mcube/logger" 8 | "github.com/infraboard/mcube/logger/zap" 9 | "go.mongodb.org/mongo-driver/mongo" 10 | "go.mongodb.org/mongo-driver/mongo/options" 11 | "go.mongodb.org/mongo-driver/x/bsonx" 12 | "google.golang.org/grpc" 13 | 14 | "github.com/infraboard/workflow/api/apps/template" 15 | "github.com/infraboard/workflow/conf" 16 | ) 17 | 18 | var ( 19 | // Service 服务实例 20 | svr = &impl{} 21 | ) 22 | 23 | type impl struct { 24 | col *mongo.Collection 25 | log logger.Logger 26 | 27 | template.UnimplementedServiceServer 28 | } 29 | 30 | func (s *impl) Config() error { 31 | db := conf.C().Mongo.GetDB() 32 | dc := db.Collection("actions") 33 | 34 | indexs := []mongo.IndexModel{ 35 | { 36 | Keys: bsonx.Doc{ 37 | {Key: "name", Value: bsonx.Int32(-1)}, 38 | {Key: "version", Value: bsonx.Int32(-1)}, 39 | }, 40 | Options: options.Index().SetUnique(true), 41 | }, 42 | { 43 | Keys: bsonx.Doc{{Key: "create_at", Value: bsonx.Int32(-1)}}, 44 | }, 45 | } 46 | 47 | _, err := dc.Indexes().CreateMany(context.Background(), indexs) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | s.col = dc 53 | s.log = zap.L().Named("Action") 54 | 55 | return nil 56 | } 57 | 58 | func (s *impl) Name() string { 59 | return template.AppName 60 | } 61 | 62 | func (s *impl) Registry(server *grpc.Server) { 63 | template.RegisterServiceServer(server, svr) 64 | } 65 | 66 | func init() { 67 | app.RegistryGrpcApp(svr) 68 | } 69 | -------------------------------------------------------------------------------- /api/apps/template/impl/query.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "go.mongodb.org/mongo-driver/bson" 5 | "go.mongodb.org/mongo-driver/mongo/options" 6 | 7 | "github.com/infraboard/mcube/pb/resource" 8 | "github.com/infraboard/workflow/api/apps/template" 9 | ) 10 | 11 | func newQueryActionRequest(req *template.QueryTemplateRequest) *queryRequest { 12 | return &queryRequest{ 13 | QueryTemplateRequest: req, 14 | } 15 | } 16 | 17 | type queryRequest struct { 18 | *template.QueryTemplateRequest 19 | } 20 | 21 | func (r *queryRequest) FindOptions() *options.FindOptions { 22 | pageSize := int64(r.Page.PageSize) 23 | skip := int64(r.Page.PageSize) * int64(r.Page.PageNumber-1) 24 | 25 | opt := &options.FindOptions{ 26 | Sort: bson.D{{Key: "create_at", Value: -1}}, 27 | Limit: &pageSize, 28 | Skip: &skip, 29 | } 30 | 31 | return opt 32 | } 33 | 34 | func (r *queryRequest) FindFilter() bson.M { 35 | filter := bson.M{} 36 | 37 | cond1 := bson.M{} 38 | if r.Namespace != "" { 39 | cond1["namespace"] = r.Namespace 40 | } 41 | if r.Name != "" { 42 | cond1["name"] = r.Name 43 | } 44 | 45 | filter["$or"] = bson.A{ 46 | cond1, 47 | bson.M{"visiable_mode": resource.VisiableMode_GLOBAL}, 48 | } 49 | 50 | return filter 51 | } 52 | 53 | func newDescTemplateRequest(req *template.DescribeTemplateRequest) (*describeRequest, error) { 54 | if err := req.Validate(); err != nil { 55 | return nil, err 56 | } 57 | return &describeRequest{req}, nil 58 | } 59 | 60 | type describeRequest struct { 61 | *template.DescribeTemplateRequest 62 | } 63 | 64 | func (req *describeRequest) FindFilter() bson.M { 65 | filter := bson.M{} 66 | 67 | filter["_id"] = req.Id 68 | 69 | return filter 70 | } 71 | -------------------------------------------------------------------------------- /api/apps/template/impl/template.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/infraboard/mcube/exception" 8 | "github.com/infraboard/mcube/pb/request" 9 | "go.mongodb.org/mongo-driver/bson" 10 | "go.mongodb.org/mongo-driver/mongo" 11 | 12 | "github.com/infraboard/workflow/api/apps/template" 13 | ) 14 | 15 | func (i *impl) CreateTemplate(ctx context.Context, req *template.CreateTemplateRequest) ( 16 | *template.Template, error) { 17 | a, err := template.NewTemplate(req) 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | if _, err := i.col.InsertOne(context.TODO(), a); err != nil { 23 | return nil, exception.NewInternalServerError("inserted a template document error, %s", err) 24 | } 25 | return a, nil 26 | } 27 | 28 | func (i *impl) QueryTemplate(ctx context.Context, req *template.QueryTemplateRequest) ( 29 | *template.TemplateSet, error) { 30 | query := newQueryActionRequest(req) 31 | resp, err := i.col.Find(context.TODO(), query.FindFilter(), query.FindOptions()) 32 | 33 | if err != nil { 34 | return nil, exception.NewInternalServerError("find template error, error is %s", err) 35 | } 36 | 37 | set := template.NewTemplateSet() 38 | // 循环 39 | for resp.Next(context.TODO()) { 40 | a := template.NewDefaultTemplate() 41 | if err := resp.Decode(a); err != nil { 42 | return nil, exception.NewInternalServerError("decode template error, error is %s", err) 43 | } 44 | 45 | set.Add(a) 46 | } 47 | 48 | // count 49 | count, err := i.col.CountDocuments(context.TODO(), query.FindFilter()) 50 | if err != nil { 51 | return nil, exception.NewInternalServerError("get template count error, error is %s", err) 52 | } 53 | set.Total = count 54 | return set, nil 55 | } 56 | 57 | func (i *impl) DescribeTemplate(ctx context.Context, req *template.DescribeTemplateRequest) ( 58 | *template.Template, error) { 59 | if err := req.Validate(); err != nil { 60 | return nil, exception.NewBadRequest("validate DescribeTemplateRequest error, %s", err) 61 | } 62 | 63 | desc, err := newDescTemplateRequest(req) 64 | if err != nil { 65 | return nil, exception.NewBadRequest(err.Error()) 66 | } 67 | 68 | ins := template.NewDefaultTemplate() 69 | if err := i.col.FindOne(context.TODO(), desc.FindFilter()).Decode(ins); err != nil { 70 | if err == mongo.ErrNoDocuments { 71 | return nil, exception.NewNotFound("template %s not found", req) 72 | } 73 | 74 | return nil, exception.NewInternalServerError("find template %s error, %s", req.Id, err) 75 | } 76 | 77 | return ins, nil 78 | } 79 | 80 | func (i *impl) UpdateTemplate(ctx context.Context, req *template.UpdateTemplateRequest) ( 81 | *template.Template, error) { 82 | if err := req.Validate(); err != nil { 83 | return nil, exception.NewBadRequest("validate update template error, %s", err) 84 | } 85 | 86 | temp, err := i.DescribeTemplate(ctx, template.NewDescribeTemplateRequestWithID(req.Id)) 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | switch req.UpdateMode { 92 | case request.UpdateMode_PUT: 93 | temp.Update(req.UpdateBy, req.Data) 94 | case request.UpdateMode_PATCH: 95 | temp.Patch(req.UpdateBy, req.Data) 96 | default: 97 | return nil, fmt.Errorf("unknown update mode: %s", req.UpdateMode) 98 | } 99 | 100 | if err := i.update(ctx, temp); err != nil { 101 | return nil, err 102 | } 103 | return temp, nil 104 | } 105 | 106 | func (i *impl) DeleteTemplate(ctx context.Context, req *template.DeleteTemplateRequest) ( 107 | *template.Template, error) { 108 | ins, err := i.DescribeTemplate(ctx, template.NewDescribeTemplateRequestWithID(req.Id)) 109 | if err != nil { 110 | return nil, err 111 | } 112 | 113 | if _, err := i.col.DeleteOne(context.TODO(), bson.M{"_id": req.Id}); err != nil { 114 | return nil, err 115 | } 116 | 117 | return ins, nil 118 | } 119 | -------------------------------------------------------------------------------- /api/apps/template/pb/template.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package infraboard.workflow.template; 4 | option go_package = "github.com/infraboard/workflow/api/apps/template"; 5 | 6 | import "api/apps/pipeline/pb/pipeline.proto"; 7 | import "github.com/infraboard/mcube/pb/page/page.proto"; 8 | import "github.com/infraboard/mcube/pb/resource/base.proto"; 9 | import "github.com/infraboard/mcube/pb/request/request.proto"; 10 | 11 | 12 | service Service { 13 | rpc CreateTemplate(CreateTemplateRequest) returns(Template); 14 | rpc QueryTemplate(QueryTemplateRequest) returns(TemplateSet); 15 | rpc DescribeTemplate(DescribeTemplateRequest) returns(Template); 16 | rpc UpdateTemplate(UpdateTemplateRequest) returns(Template); 17 | rpc DeleteTemplate(DeleteTemplateRequest) returns(Template); 18 | } 19 | 20 | // Template Pipeline参数模版 21 | message Template { 22 | // 唯一ID 23 | // @gotags: bson:"_id" json:"id" 24 | string id = 1; 25 | // 所属域 26 | // @gotags: bson:"domain" json:"domain" 27 | string domain = 2; 28 | // 所属空间 29 | // @gotags: bson:"namespace" json:"namespace" 30 | string namespace = 3; 31 | // 创建时间 32 | // @gotags: bson:"create_at" json:"create_at" 33 | int64 create_at = 4; 34 | // 创建人 35 | // @gotags: bson:"create_by" json:"create_by" 36 | string create_by = 5; 37 | // 创建时间 38 | // @gotags: bson:"update_at" json:"update_at" 39 | int64 update_at = 6; 40 | // 创建人 41 | // @gotags: bson:"update_by" json:"update_by" 42 | string update_by = 7; 43 | // 用于创建pipeline的请求参数 44 | // @gotags: bson:"pipelines" json:"pipelines" 45 | repeated infraboard.workflow.pipeline.CreatePipelineRequest pipelines = 8; 46 | // 可见模式 47 | // @gotags: bson:"visiable_mode" json:"visiable_mode" 48 | infraboard.mcube.resource.VisiableMode visiable_mode = 9; 49 | // 模版的名字 50 | // @gotags: bson:"name" json:"name" 51 | string name = 10; 52 | // 标签 53 | // @gotags: bson:"tags" json:"tags" 54 | map tags = 11; 55 | // 描述 56 | // @gotags: bson:"description" json:"description" 57 | string description = 12; 58 | } 59 | 60 | 61 | // TemplateSet todo 62 | message TemplateSet { 63 | // @gotags: json:"total" 64 | int64 total = 1; 65 | // @gotags: json:"items" 66 | repeated Template items = 2; 67 | } 68 | 69 | // CreateTemplateRequest todo 70 | message CreateTemplateRequest { 71 | // 所属域 72 | // @gotags: json:"domain" validate:"required" 73 | string domain = 1; 74 | // 所属空间 75 | // @gotags: json:"namespace" validate:"required" 76 | string namespace = 2; 77 | // 创建人 78 | // @gotags: json:"create_by" validate:"required" 79 | string create_by = 3; 80 | // 用于创建pipeline的请求参数 81 | // @gotags: json:"pipelines" 82 | repeated workflow.pipeline.CreatePipelineRequest pipelines = 4; 83 | // 可见模式 84 | // @gotags: json:"visiable_mode" 85 | infraboard.mcube.resource.VisiableMode visiable_mode = 5; 86 | // 模版的名字 87 | // @gotags: json:"name" validate:"required" 88 | string name = 6; 89 | // 标签 90 | // @gotags: json:"tags" 91 | map tags = 7; 92 | // 描述 93 | // @gotags: json:"description" 94 | string description = 8; 95 | } 96 | 97 | // UpdateTemplateRequest todo 98 | message UpdateTemplateRequest { 99 | // 更新模式 100 | // @gotags: json:"update_mode" 101 | infraboard.mcube.request.UpdateMode update_mode = 1; 102 | // 更新人 103 | // @gotags: json:"update_by" validate:"required" 104 | string update_by = 2; 105 | // 模版id 106 | // @gotags: json:"id" validate:"required" 107 | string id = 3; 108 | // 具体需要更新的数据 109 | // @gotags: json:"data" 110 | UpdateTemplateData data = 4; 111 | } 112 | 113 | message UpdateTemplateData { 114 | // 用于创建pipeline的请求参数 115 | // @gotags: json:"pipelines" 116 | repeated workflow.pipeline.CreatePipelineRequest pipelines = 1; 117 | // 可见模式 118 | // @gotags: json:"visiable_mode" 119 | infraboard.mcube.resource.VisiableMode visiable_mode = 2; 120 | // 模版的名字 121 | // @gotags: json:"name" 122 | string name = 3; 123 | // 标签 124 | // @gotags: json:"tags" 125 | map tags = 4; 126 | // 描述 127 | // @gotags: json:"description" 128 | string description = 5; 129 | } 130 | 131 | // QueryTemplateRequest 查询Book请求 132 | message QueryTemplateRequest { 133 | infraboard.mcube.page.PageRequest page = 1; 134 | string namespace = 4; 135 | string name = 2; 136 | string version = 3; 137 | } 138 | 139 | // DescribeTemplateRequest todo 140 | message DescribeTemplateRequest { 141 | // id 142 | // @gotags: json:"id" validate:"required" 143 | string id = 1; 144 | } 145 | 146 | // DeleteTemplateRequest todo 147 | message DeleteTemplateRequest { 148 | // id 149 | // @gotags: json:"id" validate:"required" 150 | string id = 1; 151 | } -------------------------------------------------------------------------------- /api/apps/template/template_ext.go: -------------------------------------------------------------------------------- 1 | package template 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/go-playground/validator/v10" 8 | "github.com/infraboard/keyauth/app/token" 9 | "github.com/infraboard/mcube/http/request" 10 | "github.com/rs/xid" 11 | ) 12 | 13 | // use a single instance of Validate, it caches struct info 14 | var ( 15 | validate = validator.New() 16 | ) 17 | 18 | func NewTemplate(req *CreateTemplateRequest) (*Template, error) { 19 | if err := req.Validate(); err != nil { 20 | return nil, err 21 | } 22 | 23 | p := &Template{ 24 | Domain: req.Domain, 25 | Namespace: req.Namespace, 26 | CreateBy: req.CreateBy, 27 | Id: xid.New().String(), 28 | CreateAt: time.Now().UnixMilli(), 29 | UpdateAt: time.Now().UnixMilli(), 30 | Name: req.Name, 31 | Tags: req.Tags, 32 | VisiableMode: req.VisiableMode, 33 | Pipelines: req.Pipelines, 34 | Description: req.Description, 35 | } 36 | 37 | return p, nil 38 | } 39 | 40 | func (req *CreateTemplateRequest) Validate() error { 41 | // 判断同一个模版里面,Pipeline名字是否相同 42 | nameMap := map[string]struct{}{} 43 | for i := range req.Pipelines { 44 | _, ok := nameMap[req.Pipelines[i].Name] 45 | if ok { 46 | return fmt.Errorf("in this template name is ready exist") 47 | } 48 | nameMap[req.Pipelines[i].Name] = struct{}{} 49 | } 50 | 51 | return validate.Struct(req) 52 | } 53 | 54 | func NewCreateTemplateRequest() *CreateTemplateRequest { 55 | return &CreateTemplateRequest{} 56 | } 57 | 58 | func (req *CreateTemplateRequest) UpdateOwner(tk *token.Token) { 59 | req.Domain = tk.Domain 60 | req.Namespace = tk.Namespace 61 | req.CreateBy = tk.Account 62 | } 63 | 64 | // NewTemplateSet todo 65 | func NewTemplateSet() *TemplateSet { 66 | return &TemplateSet{ 67 | Items: []*Template{}, 68 | } 69 | } 70 | 71 | func (s *TemplateSet) Add(item *Template) { 72 | s.Items = append(s.Items, item) 73 | } 74 | 75 | func NewDefaultTemplate() *Template { 76 | return &Template{} 77 | } 78 | 79 | func (t *Template) Update(updater string, req *UpdateTemplateData) { 80 | t.UpdateAt = time.Now().UnixMilli() 81 | t.UpdateBy = updater 82 | t.Name = req.Name 83 | t.Tags = req.Tags 84 | t.Description = req.Description 85 | t.Pipelines = req.Pipelines 86 | } 87 | 88 | func (t *Template) Patch(updater string, req *UpdateTemplateData) { 89 | t.UpdateAt = time.Now().UnixMilli() 90 | t.UpdateBy = updater 91 | 92 | if req.Name != "" { 93 | t.Name = req.Name 94 | } 95 | if req.Description != "" { 96 | t.Description = req.Description 97 | } 98 | if len(req.Tags) > 0 { 99 | t.Tags = req.Tags 100 | } 101 | if len(req.Pipelines) > 0 { 102 | t.Pipelines = req.Pipelines 103 | } 104 | } 105 | 106 | func (req *DescribeTemplateRequest) Validate() error { 107 | return validate.Struct(req) 108 | } 109 | 110 | // NewQueryTemplateRequest 查询book列表 111 | func NewQueryTemplateRequest(page *request.PageRequest) *QueryTemplateRequest { 112 | return &QueryTemplateRequest{ 113 | Page: page, 114 | } 115 | } 116 | 117 | // NewDescribeTemplateRequestWithID 查询book列表 118 | func NewDescribeTemplateRequestWithID(id string) *DescribeTemplateRequest { 119 | return &DescribeTemplateRequest{ 120 | Id: id, 121 | } 122 | } 123 | 124 | // NewDeleteTemplateRequestWithID 查询book列表 125 | func NewDeleteTemplateRequestWithID(id string) *DeleteTemplateRequest { 126 | return &DeleteTemplateRequest{ 127 | Id: id, 128 | } 129 | } 130 | 131 | func (req *UpdateTemplateRequest) Validate() error { 132 | return validate.Struct(req) 133 | } 134 | 135 | func NewUpdateTemplateRequest(id string) *UpdateTemplateRequest { 136 | return &UpdateTemplateRequest{ 137 | Id: id, 138 | Data: &UpdateTemplateData{}, 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /api/client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | kc "github.com/infraboard/keyauth/client" 5 | "github.com/infraboard/mcube/logger" 6 | "github.com/infraboard/mcube/logger/zap" 7 | "google.golang.org/grpc" 8 | 9 | "github.com/infraboard/workflow/api/apps/action" 10 | "github.com/infraboard/workflow/api/apps/pipeline" 11 | "github.com/infraboard/workflow/api/apps/template" 12 | ) 13 | 14 | var ( 15 | client *ClientSet 16 | ) 17 | 18 | // SetGlobal todo 19 | func SetGlobal(cli *ClientSet) { 20 | client = cli 21 | } 22 | 23 | // C Global 24 | func C() *ClientSet { 25 | return client 26 | } 27 | 28 | // NewClient todo 29 | func NewClientSet(conf *kc.Config) (*ClientSet, error) { 30 | zap.DevelopmentSetup() 31 | log := zap.L() 32 | 33 | conn, err := grpc.Dial(conf.Address(), grpc.WithInsecure(), grpc.WithPerRPCCredentials(conf.Authentication)) 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | return &ClientSet{ 39 | conn: conn, 40 | log: log, 41 | }, nil 42 | } 43 | 44 | // Client 客户端 45 | type ClientSet struct { 46 | conn *grpc.ClientConn 47 | log logger.Logger 48 | } 49 | 50 | // Example todo 51 | func (c *ClientSet) Pipeline() pipeline.ServiceClient { 52 | return pipeline.NewServiceClient(c.conn) 53 | } 54 | 55 | // Example todo 56 | func (c *ClientSet) Action() action.ServiceClient { 57 | return action.NewServiceClient(c.conn) 58 | } 59 | 60 | // Example todo 61 | func (c *ClientSet) Template() template.ServiceClient { 62 | return template.NewServiceClient(c.conn) 63 | } 64 | -------------------------------------------------------------------------------- /api/client/config.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | kc "github.com/infraboard/keyauth/client" 5 | ) 6 | 7 | // NewDefaultConfig todo 8 | func NewDefaultConfig() *kc.Config { 9 | return kc.NewDefaultConfig() 10 | } 11 | -------------------------------------------------------------------------------- /api/cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/spf13/cobra" 9 | 10 | "github.com/infraboard/workflow/version" 11 | ) 12 | 13 | var vers bool 14 | 15 | // RootCmd represents the base command when called without any subcommands 16 | var RootCmd = &cobra.Command{ 17 | Use: "workflow", 18 | Short: "应用研发交付中心", 19 | Long: "应用研发交付中心", 20 | RunE: func(cmd *cobra.Command, args []string) error { 21 | if vers { 22 | fmt.Println(version.FullVersion()) 23 | return nil 24 | } 25 | return errors.New("no flags find") 26 | }, 27 | } 28 | 29 | // Execute adds all child commands to the root command sets flags appropriately. 30 | // This is called by main.main(). It only needs to happen once to the rootCmd. 31 | func Execute() { 32 | if err := RootCmd.Execute(); err != nil { 33 | fmt.Println(err) 34 | os.Exit(-1) 35 | } 36 | } 37 | 38 | func init() { 39 | RootCmd.PersistentFlags().BoolVarP(&vers, "version", "v", false, "the workflow version") 40 | } 41 | -------------------------------------------------------------------------------- /api/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/infraboard/workflow/api/cmd" 5 | ) 6 | 7 | func main() { 8 | cmd.Execute() 9 | } 10 | -------------------------------------------------------------------------------- /api/protocol/grpc.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net" 7 | 8 | "github.com/infraboard/mcube/app" 9 | "github.com/infraboard/mcube/logger" 10 | "github.com/infraboard/mcube/logger/zap" 11 | "google.golang.org/grpc" 12 | 13 | "github.com/infraboard/workflow/conf" 14 | ) 15 | 16 | // NewGRPCService todo 17 | func NewGRPCService(interceptors ...grpc.UnaryServerInterceptor) *GRPCService { 18 | grpcServer := grpc.NewServer(grpc.ChainUnaryInterceptor( 19 | interceptors..., 20 | )) 21 | 22 | return &GRPCService{ 23 | svr: grpcServer, 24 | l: zap.L().Named("GRPC Service"), 25 | c: conf.C(), 26 | } 27 | } 28 | 29 | // GRPCService grpc服务 30 | type GRPCService struct { 31 | svr *grpc.Server 32 | l logger.Logger 33 | c *conf.Config 34 | } 35 | 36 | // Start 启动GRPC服务 37 | func (s *GRPCService) Start() error { 38 | // 装载所有GRPC服务 39 | app.LoadGrpcApp(s.svr) 40 | 41 | // 启动HTTP服务 42 | s.l.Infof("GRPC 开始启动, 监听地址: %s", s.c.GRPC.Addr()) 43 | lis, err := net.Listen("tcp", s.c.GRPC.Addr()) 44 | if err != nil { 45 | log.Fatal(err) 46 | } 47 | if err := s.svr.Serve(lis); err != nil { 48 | if err == grpc.ErrServerStopped { 49 | s.l.Info("service is stopped") 50 | } 51 | 52 | return fmt.Errorf("start service error, %s", err.Error()) 53 | } 54 | 55 | return nil 56 | } 57 | 58 | // Stop 停止GRPC服务 59 | func (s *GRPCService) Stop() error { 60 | s.l.Info("start graceful shutdown") 61 | 62 | // 优雅关闭HTTP服务 63 | s.svr.GracefulStop() 64 | 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /api/protocol/http.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "fmt" 7 | "net/http" 8 | "time" 9 | 10 | "github.com/infraboard/keyauth/app/endpoint" 11 | "github.com/infraboard/keyauth/client/interceptor" 12 | "github.com/infraboard/mcube/app" 13 | "github.com/infraboard/mcube/http/middleware/accesslog" 14 | "github.com/infraboard/mcube/http/middleware/cors" 15 | "github.com/infraboard/mcube/http/middleware/recovery" 16 | "github.com/infraboard/mcube/http/router" 17 | "github.com/infraboard/mcube/http/router/httprouter" 18 | "github.com/infraboard/mcube/logger" 19 | "github.com/infraboard/mcube/logger/zap" 20 | 21 | "github.com/infraboard/workflow/conf" 22 | "github.com/infraboard/workflow/version" 23 | ) 24 | 25 | // NewHTTPService 构建函数 26 | func NewHTTPService() *HTTPService { 27 | c, err := conf.C().Keyauth.Client() 28 | if err != nil { 29 | panic(err) 30 | } 31 | auther := interceptor.NewHTTPAuther(c) 32 | 33 | r := httprouter.New() 34 | r.Use(recovery.NewWithLogger(zap.L().Named("Recovery"))) 35 | r.Use(accesslog.NewWithLogger(zap.L().Named("AccessLog"))) 36 | r.Use(cors.AllowAll()) 37 | r.EnableAPIRoot() 38 | r.SetAuther(auther) 39 | r.Auth(true) 40 | r.Permission(true) 41 | r.AuditLog(true) 42 | r.RequiredNamespace(true) 43 | 44 | server := &http.Server{ 45 | ReadHeaderTimeout: 20 * time.Second, 46 | ReadTimeout: 20 * time.Second, 47 | WriteTimeout: 25 * time.Second, 48 | IdleTimeout: 120 * time.Second, 49 | MaxHeaderBytes: 1 << 20, 50 | Addr: conf.C().HTTP.Addr(), 51 | Handler: r, 52 | } 53 | return &HTTPService{ 54 | r: r, 55 | server: server, 56 | l: zap.L().Named("HTTP Service"), 57 | c: conf.C(), 58 | endpoint: c.Endpoint(), 59 | } 60 | } 61 | 62 | // HTTPService http服务 63 | type HTTPService struct { 64 | r router.Router 65 | l logger.Logger 66 | c *conf.Config 67 | server *http.Server 68 | 69 | endpoint endpoint.ServiceClient 70 | } 71 | 72 | func (s *HTTPService) PathPrefix() string { 73 | return fmt.Sprintf("/%s/api/v1", s.c.App.Name) 74 | } 75 | 76 | // Start 启动服务 77 | func (s *HTTPService) Start() error { 78 | hc := s.c.HTTP 79 | 80 | // 装置子服务路由 81 | app.LoadHttpApp(s.PathPrefix(), s.r) 82 | 83 | // 注册路由 84 | s.RegistryEndpoint() 85 | 86 | // 启动HTTPS服务 87 | if hc.EnableSSL { 88 | // 安全的算法挑选标准依赖: https://wiki.mozilla.org/Security/Server_Side_TLS 89 | cfg := &tls.Config{ 90 | MinVersion: tls.VersionTLS12, 91 | CurvePreferences: []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256, tls.X25519}, 92 | PreferServerCipherSuites: true, 93 | CipherSuites: []uint16{ 94 | // tls 1.2 95 | tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, 96 | tls.TLS_RSA_WITH_AES_256_GCM_SHA384, 97 | tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, 98 | tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, 99 | tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, 100 | tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, 101 | // tls 1.3 102 | tls.TLS_AES_128_GCM_SHA256, 103 | tls.TLS_AES_256_GCM_SHA384, 104 | tls.TLS_CHACHA20_POLY1305_SHA256, 105 | }, 106 | } 107 | s.server.TLSConfig = cfg 108 | s.server.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0) 109 | s.l.Infof("HTTPS服务启动成功, 监听地址: %s", s.server.Addr) 110 | if err := s.server.ListenAndServeTLS(hc.CertFile, hc.KeyFile); err != nil { 111 | if err == http.ErrServerClosed { 112 | s.l.Info("service is stopped") 113 | } 114 | return fmt.Errorf("start service error, %s", err.Error()) 115 | } 116 | } 117 | // 启动 HTTP服务 118 | s.l.Infof("HTTP服务启动成功, 监听地址: %s", s.server.Addr) 119 | if err := s.server.ListenAndServe(); err != nil { 120 | if err == http.ErrServerClosed { 121 | s.l.Info("service is stopped") 122 | } 123 | return fmt.Errorf("start service error, %s", err.Error()) 124 | } 125 | return nil 126 | } 127 | 128 | // Stop 停止server 129 | func (s *HTTPService) Stop() error { 130 | s.l.Info("start graceful shutdown") 131 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 132 | defer cancel() 133 | // 优雅关闭HTTP服务 134 | if err := s.server.Shutdown(ctx); err != nil { 135 | s.l.Errorf("graceful shutdown timeout, force exit") 136 | } 137 | return nil 138 | } 139 | 140 | // registryEndpoints 注册条目 141 | func (s *HTTPService) RegistryEndpoint() { 142 | // 注册服务权限条目 143 | s.l.Info("start registry endpoints ...") 144 | 145 | req := endpoint.NewRegistryRequest(version.Short(), s.r.GetEndpoints().UniquePathEntry()) 146 | _, err := s.endpoint.RegistryEndpoint(context.Background(), req) 147 | if err != nil { 148 | s.l.Warnf("registry endpoints error, %s", err) 149 | } else { 150 | s.l.Debug("service endpoints registry success") 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /common/cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // cache responsibilities are limited to: 4 | // 1. Computing keys for objects via keyFunc 5 | // 2. Invoking methods of a ThreadSafeStorage interface 6 | type cache struct { 7 | // cacheStorage bears the burden of thread safety for the cache 8 | cacheStorage ThreadSafeStore 9 | // keyFunc is used to make the key for objects stored in and retrieved from items, and 10 | // should be deterministic. 11 | keyFunc KeyFunc 12 | } 13 | 14 | var _ Store = &cache{} 15 | 16 | // Add inserts an item into the cache. 17 | func (c *cache) Add(obj interface{}) error { 18 | key, err := c.keyFunc(obj) 19 | if err != nil { 20 | return KeyError{obj, err} 21 | } 22 | c.cacheStorage.Add(key, obj) 23 | return nil 24 | } 25 | 26 | // Update sets an item in the cache to its updated state. 27 | func (c *cache) Update(obj interface{}) error { 28 | key, err := c.keyFunc(obj) 29 | if err != nil { 30 | return KeyError{obj, err} 31 | } 32 | c.cacheStorage.Update(key, obj) 33 | return nil 34 | } 35 | 36 | // Delete removes an item from the cache. 37 | func (c *cache) Delete(obj interface{}) error { 38 | key, err := c.keyFunc(obj) 39 | if err != nil { 40 | return KeyError{obj, err} 41 | } 42 | c.cacheStorage.Delete(key) 43 | return nil 44 | } 45 | 46 | // List returns a list of all the items. 47 | // List is completely threadsafe as long as you treat all items as immutable. 48 | func (c *cache) List() []interface{} { 49 | return c.cacheStorage.List() 50 | } 51 | 52 | // ListKeys returns a list of all the keys of the objects currently 53 | // in the cache. 54 | func (c *cache) ListKeys() []string { 55 | return c.cacheStorage.ListKeys() 56 | } 57 | 58 | // ListKeys returns a list of all the keys of the objects currently 59 | // in the cache. 60 | func (c *cache) Len() int { 61 | return c.cacheStorage.Len() 62 | } 63 | 64 | // GetIndexers returns the indexers of cache 65 | func (c *cache) GetIndexers() Indexers { 66 | return c.cacheStorage.GetIndexers() 67 | } 68 | 69 | // Index returns a list of items that match on the index function 70 | // Index is thread-safe so long as you treat all items as immutable 71 | func (c *cache) Index(indexName string, obj interface{}) ([]interface{}, error) { 72 | return c.cacheStorage.Index(indexName, obj) 73 | } 74 | func (c *cache) IndexKeys(indexName, indexKey string) ([]string, error) { 75 | return c.cacheStorage.IndexKeys(indexName, indexKey) 76 | } 77 | 78 | // ListIndexFuncValues returns the list of generated values of an Index func 79 | func (c *cache) ListIndexFuncValues(indexName string) []string { 80 | return c.cacheStorage.ListIndexFuncValues(indexName) 81 | } 82 | func (c *cache) ByIndex(indexName, indexKey string) ([]interface{}, error) { 83 | return c.cacheStorage.ByIndex(indexName, indexKey) 84 | } 85 | func (c *cache) AddIndexers(newIndexers Indexers) error { 86 | return c.cacheStorage.AddIndexers(newIndexers) 87 | } 88 | 89 | // Get returns the requested item, or sets exists=false. 90 | // Get is completely threadsafe as long as you treat all items as immutable. 91 | func (c *cache) Get(obj interface{}) (item interface{}, exists bool, err error) { 92 | key, err := c.keyFunc(obj) 93 | if err != nil { 94 | return nil, false, KeyError{obj, err} 95 | } 96 | return c.GetByKey(key) 97 | } 98 | 99 | // GetByKey returns the request item, or exists=false. 100 | // GetByKey is completely threadsafe as long as you treat all items as immutable. 101 | func (c *cache) GetByKey(key string) (item interface{}, exists bool, err error) { 102 | item, exists = c.cacheStorage.Get(key) 103 | return item, exists, nil 104 | } 105 | 106 | // Replace will delete the contents of 'c', using instead the given list. 107 | // 'c' takes ownership of the list, you should not reference the list again 108 | // after calling this function. 109 | func (c *cache) Replace(list []interface{}, resourceVersion string) error { 110 | items := make(map[string]interface{}, len(list)) 111 | for _, item := range list { 112 | key, err := c.keyFunc(item) 113 | if err != nil { 114 | return KeyError{item, err} 115 | } 116 | items[key] = item 117 | } 118 | c.cacheStorage.Replace(items, resourceVersion) 119 | return nil 120 | } 121 | 122 | // Resync touches all items in the store to force processing 123 | func (c *cache) Resync() error { 124 | return c.cacheStorage.Resync() 125 | } 126 | 127 | // NewStore returns a Store implemented simply with a map and a lock. 128 | func NewStore(keyFunc KeyFunc) Store { 129 | return &cache{ 130 | cacheStorage: NewThreadSafeStore(Indexers{}, Indices{}), 131 | keyFunc: keyFunc, 132 | } 133 | } 134 | 135 | // NewIndexer returns an Indexer implemented simply with a map and a lock. 136 | func NewIndexer(keyFunc KeyFunc, indexers Indexers) Indexer { 137 | return &cache{ 138 | cacheStorage: NewThreadSafeStore(indexers, Indices{}), 139 | keyFunc: keyFunc, 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /common/cache/cache_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "testing" 5 | 6 | "k8s.io/apimachinery/pkg/util/sets" 7 | ) 8 | 9 | // Test public interface 10 | func doTestStore(t *testing.T, store Store) { 11 | mkObj := func(id string, val string) testStoreObject { 12 | return testStoreObject{id: id, val: val} 13 | } 14 | store.Add(mkObj("foo", "bar")) 15 | if item, ok, _ := store.Get(mkObj("foo", "")); !ok { 16 | t.Errorf("didn't find inserted item") 17 | } else { 18 | if e, a := "bar", item.(testStoreObject).val; e != a { 19 | t.Errorf("expected %v, got %v", e, a) 20 | } 21 | } 22 | store.Update(mkObj("foo", "baz")) 23 | if item, ok, _ := store.Get(mkObj("foo", "")); !ok { 24 | t.Errorf("didn't find inserted item") 25 | } else { 26 | if e, a := "baz", item.(testStoreObject).val; e != a { 27 | t.Errorf("expected %v, got %v", e, a) 28 | } 29 | } 30 | store.Delete(mkObj("foo", "")) 31 | if _, ok, _ := store.Get(mkObj("foo", "")); ok { 32 | t.Errorf("found deleted item??") 33 | } 34 | // Test List. 35 | store.Add(mkObj("a", "b")) 36 | store.Add(mkObj("c", "d")) 37 | store.Add(mkObj("e", "e")) 38 | { 39 | found := sets.String{} 40 | for _, item := range store.List() { 41 | found.Insert(item.(testStoreObject).val) 42 | } 43 | if !found.HasAll("b", "d", "e") { 44 | t.Errorf("missing items, found: %v", found) 45 | } 46 | if len(found) != 3 { 47 | t.Errorf("extra items") 48 | } 49 | } 50 | // Test Replace. 51 | store.Replace([]interface{}{ 52 | mkObj("foo", "foo"), 53 | mkObj("bar", "bar"), 54 | }, "0") 55 | { 56 | found := sets.String{} 57 | for _, item := range store.List() { 58 | found.Insert(item.(testStoreObject).val) 59 | } 60 | if !found.HasAll("foo", "bar") { 61 | t.Errorf("missing items") 62 | } 63 | if len(found) != 2 { 64 | t.Errorf("extra items") 65 | } 66 | } 67 | } 68 | 69 | // Test public interface 70 | func doTestIndex(t *testing.T, indexer Indexer) { 71 | mkObj := func(id string, val string) testStoreObject { 72 | return testStoreObject{id: id, val: val} 73 | } 74 | // Test Index 75 | expected := map[string]sets.String{} 76 | expected["b"] = sets.NewString("a", "c") 77 | expected["f"] = sets.NewString("e") 78 | expected["h"] = sets.NewString("g") 79 | indexer.Add(mkObj("a", "b")) 80 | indexer.Add(mkObj("c", "b")) 81 | indexer.Add(mkObj("e", "f")) 82 | indexer.Add(mkObj("g", "h")) 83 | { 84 | for k, v := range expected { 85 | found := sets.String{} 86 | indexResults, err := indexer.Index("by_val", mkObj("", k)) 87 | if err != nil { 88 | t.Errorf("Unexpected error %v", err) 89 | } 90 | for _, item := range indexResults { 91 | found.Insert(item.(testStoreObject).id) 92 | } 93 | items := v.List() 94 | if !found.HasAll(items...) { 95 | t.Errorf("missing items, index %s, expected %v but found %v", k, items, found.List()) 96 | } 97 | } 98 | } 99 | } 100 | func testStoreKeyFunc(obj interface{}) (string, error) { 101 | return obj.(testStoreObject).id, nil 102 | } 103 | func testStoreIndexFunc(obj interface{}) ([]string, error) { 104 | return []string{obj.(testStoreObject).val}, nil 105 | } 106 | func testStoreIndexers() Indexers { 107 | indexers := Indexers{} 108 | indexers["by_val"] = testStoreIndexFunc 109 | return indexers 110 | } 111 | 112 | type testStoreObject struct { 113 | id string 114 | val string 115 | } 116 | 117 | func TestCache(t *testing.T) { 118 | doTestStore(t, NewStore(testStoreKeyFunc)) 119 | } 120 | func TestIndex(t *testing.T) { 121 | doTestIndex(t, NewIndexer(testStoreKeyFunc, testStoreIndexers())) 122 | } 123 | -------------------------------------------------------------------------------- /common/cache/index.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "fmt" 5 | 6 | "k8s.io/apimachinery/pkg/util/sets" 7 | ) 8 | 9 | // Indexer is a storage interface that lets you list objects using multiple indexing functions. 10 | // There are three kinds of strings here. 11 | // One is a storage key, as defined in the Store interface. 12 | // Another kind is a name of an index. 13 | // The third kind of string is an "indexed value", which is produced by an 14 | // IndexFunc and can be a field value or any other string computed from the object. 15 | type Indexer interface { 16 | Store 17 | // Index returns the stored objects whose set of indexed values 18 | // intersects the set of indexed values of the given object, for 19 | // the named index 20 | Index(indexName string, obj interface{}) ([]interface{}, error) 21 | // IndexKeys returns the storage keys of the stored objects whose 22 | // set of indexed values for the named index includes the given 23 | // indexed value 24 | IndexKeys(indexName, indexedValue string) ([]string, error) 25 | // ListIndexFuncValues returns all the indexed values of the given index 26 | ListIndexFuncValues(indexName string) []string 27 | // ByIndex returns the stored objects whose set of indexed values 28 | // for the named index includes the given indexed value 29 | ByIndex(indexName, indexedValue string) ([]interface{}, error) 30 | // GetIndexer return the indexers 31 | GetIndexers() Indexers 32 | // AddIndexers adds more indexers to this store. If you call this after you already have data 33 | // in the store, the results are undefined. 34 | AddIndexers(newIndexers Indexers) error 35 | } 36 | 37 | // IndexFunc knows how to compute the set of indexed values for an object. 38 | type IndexFunc func(obj interface{}) ([]string, error) 39 | 40 | // IndexFuncToKeyFuncAdapter adapts an indexFunc to a keyFunc. This is only useful if your index function returns 41 | // unique values for every object. This is conversion can create errors when more than one key is found. You 42 | // should prefer to make proper key and index functions. 43 | func IndexFuncToKeyFuncAdapter(indexFunc IndexFunc) KeyFunc { 44 | return func(obj interface{}) (string, error) { 45 | indexKeys, err := indexFunc(obj) 46 | if err != nil { 47 | return "", err 48 | } 49 | if len(indexKeys) > 1 { 50 | return "", fmt.Errorf("too many keys: %v", indexKeys) 51 | } 52 | if len(indexKeys) == 0 { 53 | return "", fmt.Errorf("unexpected empty indexKeys") 54 | } 55 | return indexKeys[0], nil 56 | } 57 | } 58 | 59 | const ( 60 | // NamespaceIndex is the lookup name for the most comment index function, which is to index by the namespace field. 61 | NamespaceIndex string = "namespace" 62 | ) 63 | 64 | // Index maps the indexed value to a set of keys in the store that match on that value 65 | type Index map[string]sets.String 66 | 67 | // Indexers maps a name to a IndexFunc 68 | type Indexers map[string]IndexFunc 69 | 70 | // Indices maps a name to an Index 71 | type Indices map[string]Index 72 | -------------------------------------------------------------------------------- /common/cache/store.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // Store is a generic object storage interface. Reflector knows how to watch a server 8 | // and update a store. A generic store is provided, which allows Reflector to be used 9 | // as a local caching system, and an LRU store, which allows Reflector to work like a 10 | // queue of items yet to be processed. 11 | // 12 | // Store makes no assumptions about stored object identity; it is the responsibility 13 | // of a Store implementation to provide a mechanism to correctly key objects and to 14 | // define the contract for obtaining objects by some arbitrary key type. 15 | type Store interface { 16 | Reader 17 | Writer 18 | Manager 19 | } 20 | 21 | // Reader 读取数据 22 | type Reader interface { 23 | Len() int 24 | List() []interface{} 25 | ListKeys() []string 26 | Get(obj interface{}) (item interface{}, exists bool, err error) 27 | GetByKey(key string) (item interface{}, exists bool, err error) 28 | } 29 | 30 | // Writer 写数据 31 | type Writer interface { 32 | Add(obj interface{}) error 33 | Update(obj interface{}) error 34 | Delete(obj interface{}) error 35 | } 36 | 37 | // Manager 管理Store 38 | type Manager interface { 39 | // Replace will delete the contents of the store, using instead the 40 | // given list. Store takes ownership of the list, you should not reference 41 | // it after calling this function. 42 | Replace([]interface{}, string) error 43 | Resync() error 44 | } 45 | 46 | // KeyFunc knows how to make a key from an object. Implementations should be deterministic. 47 | type KeyFunc func(obj interface{}) (string, error) 48 | 49 | // KeyError will be returned any time a KeyFunc gives an error; it includes the object 50 | // at fault. 51 | type KeyError struct { 52 | Obj interface{} 53 | Err error 54 | } 55 | 56 | // Error gives a human-readable description of the error. 57 | func (k KeyError) Error() string { 58 | return fmt.Sprintf("couldn't create key for object %+v: %v", k.Obj, k.Err) 59 | } 60 | 61 | // ExplicitKey can be passed to MetaNamespaceKeyFunc if you have the key for 62 | // the object but not the object itself. 63 | type ExplicitKey string 64 | -------------------------------------------------------------------------------- /common/enum/enum.go: -------------------------------------------------------------------------------- 1 | package enum 2 | 3 | type EnumDesc struct { 4 | Name string `json:"name"` 5 | Icon string `json:"icon"` 6 | Value string `json:"value"` 7 | Desc string `json:"desc"` 8 | } 9 | -------------------------------------------------------------------------------- /common/hooks/hooks.go: -------------------------------------------------------------------------------- 1 | package hooks 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/infraboard/workflow/api/apps/pipeline" 7 | "github.com/infraboard/workflow/common/hooks/webhook" 8 | ) 9 | 10 | type StepNotifyEvent struct { 11 | StepKey string `json:"step_key"` 12 | NotifyParams map[string]string `json:"notify_params"` 13 | *pipeline.StepStatus 14 | } 15 | 16 | // StepWebHooker step状态变化时，通知其他系统 17 | type StepWebHookPusher interface { 18 | Send(context.Context, []*pipeline.WebHook, *pipeline.Step) error 19 | } 20 | 21 | func NewDefaultStepWebHookPusher() StepWebHookPusher { 22 | return webhook.NewWebHook() 23 | } 24 | -------------------------------------------------------------------------------- /common/hooks/webhook/dingding/card.go: -------------------------------------------------------------------------------- 1 | package dingding 2 | 3 | type ActionCard struct { 4 | Title string `json:"title"` 5 | Text string `json:"text"` 6 | Buttons []*Button `json:"btns"` 7 | ButtonsOrientation string `json:"btnOrientation"` 8 | SingleTitle string `json:"singleTitle"` 9 | SingleURL string `json:"singleURL"` 10 | } 11 | 12 | type Button struct { 13 | Title string `json:"title"` 14 | ActionURL string `json:"actionURL"` 15 | } 16 | -------------------------------------------------------------------------------- /common/hooks/webhook/dingding/message.go: -------------------------------------------------------------------------------- 1 | package dingding 2 | 3 | import "github.com/infraboard/workflow/api/apps/pipeline" 4 | 5 | const ( 6 | URL_PREFIX = "https://oapi.dingtalk.com/robot/send" 7 | ) 8 | 9 | const ( 10 | CardMessage = "actionCard" 11 | ) 12 | 13 | type MessageType string 14 | 15 | func NewStepCardMessage(s *pipeline.Step) *Message { 16 | return &Message{ 17 | MsgType: CardMessage, 18 | ActionCard: newActionCard(s), 19 | } 20 | } 21 | 22 | // 自定义机器人接入: https://developers.dingtalk.com/document/app/custom-robot-access 23 | // 默认使用钉钉的actionCard数据模式 24 | type Message struct { 25 | MsgType MessageType `json:"msgtype"` 26 | ActionCard *ActionCard `json:"actionCard"` 27 | } 28 | 29 | func newActionCard(s *pipeline.Step) *ActionCard { 30 | return &ActionCard{ 31 | Title: s.ShowTitle(), 32 | Text: s.Status.String(), 33 | ButtonsOrientation: "0", 34 | SingleTitle: "详情", 35 | SingleURL: "https://www.dingtalk.com/", 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /common/hooks/webhook/feishu/card.go: -------------------------------------------------------------------------------- 1 | package feishu 2 | 3 | // card说明: https://open.feishu.cn/document/ukTMukTMukTM/ugTNwUjL4UDM14CO1ATN 4 | // card可视化工具: https://open.feishu.cn/tool/cardbuilder?from=custom_bot_doc 5 | type Card struct { 6 | Config *CardConfig `json:"config"` 7 | Header *CardHeader `json:"header"` 8 | Elements []interface{} `json:"elements"` 9 | } 10 | 11 | // card config说明: https://open.feishu.cn/document/ukTMukTMukTM/uAjNwUjLwYDM14CM2ATN 12 | type CardConfig struct { 13 | WideScreenMode bool `json:"wide_screen_mode"` 14 | EnableForward bool `json:"enable_forward"` 15 | } 16 | 17 | // card header说明: https://open.feishu.cn/document/ukTMukTMukTM/ukTNwUjL5UDM14SO1ATN 18 | type CardHeader struct { 19 | Title map[string]string `json:"title"` 20 | Template string `json:"template"` 21 | } 22 | 23 | type ElementType string 24 | 25 | const ( 26 | ElementType_Content = "content" 27 | ElementType_Hr = "hr" 28 | ElementType_Image = "image" 29 | ElementType_Action = "action" 30 | ElementType_Note = "note" 31 | ) 32 | 33 | func NewMarkdownContent(content string) *ContentElement { 34 | return &ContentElement{ 35 | Tag: "div", 36 | Text: &Text{ 37 | Content: content, 38 | Tag: "lark_md", 39 | }, 40 | } 41 | } 42 | func NewFiledMarkdownContent(fileds []*NotifyFiled) *ContentElement { 43 | element := &ContentElement{ 44 | Tag: "div", 45 | Fields: []*Field{}, 46 | } 47 | for i := range fileds { 48 | element.Fields = append(element.Fields, NewField(fileds[i].IsShort, fileds[i].FiledFormat())) 49 | } 50 | return element 51 | } 52 | 53 | // car element说明: https://open.feishu.cn/document/ukTMukTMukTM/uEjNwUjLxYDM14SM2ATN 54 | type ContentElement struct { 55 | Tag string `json:"tag"` 56 | Text *Text `json:"text"` 57 | Fields []*Field `json:"fields"` 58 | } 59 | 60 | func NewNoteContent(fileds []string) *NoteElement { 61 | element := &NoteElement{ 62 | Tag: "note", 63 | Elements: []*Text{}, 64 | } 65 | for i := range fileds { 66 | element.Elements = append(element.Elements, &Text{ 67 | Content: fileds[i], 68 | Tag: "plain_text", 69 | }) 70 | } 71 | return element 72 | } 73 | 74 | // https://open.feishu.cn/document/ukTMukTMukTM/ucjNwUjL3YDM14yN2ATN 75 | type NoteElement struct { 76 | Tag string `json:"tag"` 77 | Elements []*Text `json:"elements"` 78 | } 79 | 80 | // 说明文档: https://open.feishu.cn/document/ukTMukTMukTM/uUzNwUjL1cDM14SN3ATN 81 | type Text struct { 82 | Tag string `json:"tag"` 83 | Content string `json:"content"` 84 | Lines int `json:"lines"` 85 | } 86 | 87 | func NewField(isShort bool, content string) *Field { 88 | return &Field{ 89 | IsShort: isShort, 90 | Text: Text{ 91 | Content: content, 92 | Tag: "lark_md", 93 | }, 94 | } 95 | } 96 | 97 | // 说明文档 https://open.feishu.cn/document/ukTMukTMukTM/uYzNwUjL2cDM14iN3ATN 98 | type Field struct { 99 | IsShort bool `json:"is_short"` 100 | Text Text `json:"text"` 101 | } 102 | 103 | func NewHrElement() *HrElement { 104 | return &HrElement{ 105 | Tag: "hr", 106 | } 107 | } 108 | 109 | type HrElement struct { 110 | Tag string `json:"tag"` 111 | } 112 | -------------------------------------------------------------------------------- /common/hooks/webhook/feishu/color.go: -------------------------------------------------------------------------------- 1 | package feishu 2 | 3 | type Color string 4 | 5 | func (c Color) String() string { 6 | return string(c) 7 | } 8 | 9 | const ( 10 | COLOR_BLUE Color = "blue" 11 | COLOR_WATHET Color = "wathet" 12 | COLOR_TURQUOISE Color = "turquoise" 13 | COLOR_GREEN Color = "green" 14 | COLOR_YELLOW Color = "yellow" 15 | COLOR_ORANGE Color = "orange" 16 | COLOR_RED Color = "red" 17 | COLOR_CARMINE Color = "carmine" 18 | COLOR_VIOLET Color = "violet" 19 | COLOR_PURPLE Color = "purple" 20 | COLOR_INDIGO Color = "indigo" 21 | COLOR_GREY Color = "grey" 22 | ) 23 | -------------------------------------------------------------------------------- /common/hooks/webhook/feishu/message.go: -------------------------------------------------------------------------------- 1 | package feishu 2 | 3 | import "fmt" 4 | 5 | const ( 6 | URL_PREFIX = "https://open.feishu.cn/open-apis/bot" 7 | ) 8 | const ( 9 | CardMessage = "interactive" 10 | ) 11 | 12 | type MessageType string 13 | 14 | func NewMarkdownNotifyMessage(robotURL, title, content string) *NotifyMessage { 15 | return &NotifyMessage{ 16 | Title: title, 17 | Content: content, 18 | RobotURL: robotURL, 19 | Note: []string{}, 20 | } 21 | } 22 | func NewFiledMarkdownMessage(robotURL, title string, color Color, groups ...*FiledGroup) *NotifyMessage { 23 | return &NotifyMessage{ 24 | Title: title, 25 | FiledGroup: groups, 26 | RobotURL: robotURL, 27 | Color: color, 28 | Note: []string{}, 29 | } 30 | } 31 | 32 | type NotifyMessage struct { 33 | Title string 34 | Content string 35 | RobotURL string 36 | FiledGroup []*FiledGroup 37 | Note []string 38 | Color Color 39 | } 40 | 41 | func (m *NotifyMessage) HasFiledGroup() bool { 42 | return len(m.FiledGroup) > 0 43 | } 44 | 45 | func (m *NotifyMessage) HasNote() bool { 46 | return len(m.Note) > 0 47 | } 48 | 49 | func (m *NotifyMessage) AddFiledGroup(group *FiledGroup) { 50 | m.FiledGroup = append(m.FiledGroup, group) 51 | } 52 | 53 | func (m *NotifyMessage) AddNote(n string) { 54 | m.Note = append(m.Note, n) 55 | } 56 | 57 | type FiledGroupEndType string 58 | 59 | const ( 60 | FiledGroupEndType_None FiledGroupEndType = "none" 61 | FiledGroupEndType_Hr FiledGroupEndType = "hr" 62 | FiledGroupEndType_Line FiledGroupEndType = "line" 63 | ) 64 | 65 | func NewEndHrGroup(fileds []*NotifyFiled) *FiledGroup { 66 | return &FiledGroup{ 67 | EndType: FiledGroupEndType_Hr, 68 | Items: fileds, 69 | } 70 | } 71 | 72 | func NewEndNoneGroup() *FiledGroup { 73 | return &FiledGroup{ 74 | EndType: FiledGroupEndType_None, 75 | Items: []*NotifyFiled{}, 76 | } 77 | } 78 | 79 | type FiledGroup struct { 80 | EndType FiledGroupEndType 81 | Items []*NotifyFiled 82 | } 83 | 84 | func (g *FiledGroup) Add(f *NotifyFiled) { 85 | g.Items = append(g.Items, f) 86 | } 87 | 88 | func NewNotifyFiled(key, value string, short bool) *NotifyFiled { 89 | return &NotifyFiled{ 90 | Key: key, 91 | Value: value, 92 | IsShort: short, 93 | } 94 | } 95 | 96 | type NotifyFiled struct { 97 | IsShort bool 98 | Key string 99 | Value string 100 | } 101 | 102 | func (f *NotifyFiled) FiledFormat() string { 103 | return fmt.Sprintf("**%s**\n%s", f.Key, f.Value) 104 | } 105 | 106 | // 如何寻找emoji字符: https://emojipedia.org/light-bulb/ 107 | func NewCardMessage(m *NotifyMessage) *Message { 108 | return &Message{ 109 | MsgType: CardMessage, 110 | Card: Card{ 111 | Config: messageConfig(), 112 | Header: messageHeader(m), 113 | Elements: messageContent(m), 114 | }, 115 | } 116 | } 117 | 118 | // https://www.feishu.cn/hc/zh-CN/articles/360024984973 119 | // 默认使用飞书的card数据模式 120 | type Message struct { 121 | MsgType MessageType `json:"msg_type"` 122 | Card Card `json:"card"` 123 | } 124 | 125 | // https://open.feishu.cn/document/ukTMukTMukTM/uEjNwUjLxYDM14SM2ATN 126 | func messageContent(m *NotifyMessage) (elements []interface{}) { 127 | // 内容模块 128 | if m.HasFiledGroup() { 129 | for i := range m.FiledGroup { 130 | group := m.FiledGroup[i] 131 | content := NewFiledMarkdownContent(group.Items) 132 | elements = append(elements, content) 133 | 134 | switch group.EndType { 135 | case FiledGroupEndType_Hr: 136 | elements = append(elements, NewHrElement()) 137 | case FiledGroupEndType_Line: 138 | content.Fields = append(content.Fields, NewField(false, "")) 139 | } 140 | } 141 | 142 | } else { 143 | content := NewMarkdownContent(m.Content) 144 | elements = append(elements, content) 145 | } 146 | // 备注模块 147 | if m.HasNote() { 148 | note := NewNoteContent(m.Note) 149 | elements = append(elements, NewHrElement(), note) 150 | } 151 | return 152 | } 153 | 154 | func messageHeader(m *NotifyMessage) *CardHeader { 155 | return &CardHeader{ 156 | Title: map[string]string{ 157 | "tag": "plain_text", 158 | "content": m.Title, 159 | }, 160 | Template: m.Color.String(), 161 | } 162 | } 163 | 164 | func messageConfig() *CardConfig { 165 | return &CardConfig{ 166 | WideScreenMode: true, 167 | EnableForward: true, 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /common/hooks/webhook/request.go: -------------------------------------------------------------------------------- 1 | package webhook 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io" 7 | "net/http" 8 | "strings" 9 | "time" 10 | 11 | "github.com/infraboard/workflow/api/apps/pipeline" 12 | "github.com/infraboard/workflow/common/hooks/webhook/dingding" 13 | "github.com/infraboard/workflow/common/hooks/webhook/feishu" 14 | "github.com/infraboard/workflow/common/hooks/webhook/wechat" 15 | ) 16 | 17 | const ( 18 | MAX_WEBHOOKS_PER_SEND = 12 19 | ) 20 | 21 | const ( 22 | feishuBot = "feishu" 23 | dingdingBot = "dingding" 24 | wechatBot = "wechat" 25 | ) 26 | 27 | var ( 28 | client = &http.Client{ 29 | Timeout: 3 * time.Second, 30 | } 31 | ) 32 | 33 | func newRequest(hook *pipeline.WebHook, step *pipeline.Step) *request { 34 | return &request{ 35 | hook: hook, 36 | step: step, 37 | } 38 | } 39 | 40 | type request struct { 41 | hook *pipeline.WebHook 42 | step *pipeline.Step 43 | matchRes string 44 | } 45 | 46 | func (r *request) Push() { 47 | r.hook.StartSend() 48 | 49 | // 准备请求,适配主流机器人 50 | var messageObj interface{} 51 | switch r.BotType() { 52 | case feishuBot: 53 | messageObj = r.NewFeishuMessage() 54 | r.matchRes = `"StatusCode":0,` 55 | case dingdingBot: 56 | messageObj = dingding.NewStepCardMessage(r.step) 57 | r.matchRes = `"errcode":0,` 58 | case wechatBot: 59 | messageObj = wechat.NewStepMarkdownMessage(r.step) 60 | r.matchRes = `"errcode":0,` 61 | default: 62 | messageObj = r.step 63 | } 64 | 65 | body, err := json.Marshal(messageObj) 66 | if err != nil { 67 | r.hook.SendFailed("marshal step to json error, %s", err) 68 | return 69 | } 70 | 71 | req, err := http.NewRequest("POST", r.hook.Url, bytes.NewReader(body)) 72 | if err != nil { 73 | r.hook.SendFailed("new post request error, %s", err) 74 | return 75 | } 76 | 77 | req.Header.Set("Content-Type", "application/json") 78 | for k, v := range r.hook.Header { 79 | req.Header.Add(k, v) 80 | } 81 | 82 | // 发起请求 83 | resp, err := client.Do(req) 84 | if err != nil { 85 | r.hook.SendFailed("send request error, %s", err) 86 | return 87 | } 88 | defer resp.Body.Close() 89 | 90 | // 读取body 91 | bytesB, err := io.ReadAll(resp.Body) 92 | if err != nil { 93 | r.hook.SendFailed("read response error, %s", err) 94 | return 95 | } 96 | respString := string(bytesB) 97 | 98 | if (resp.StatusCode / 100) != 2 { 99 | r.hook.SendFailed("status code[%d] is not 200, response %s", resp.StatusCode, respString) 100 | return 101 | } 102 | 103 | // 通过返回匹配字符串来判断通知是否成功 104 | if r.matchRes != "" { 105 | if !strings.Contains(respString, r.matchRes) { 106 | r.hook.SendFailed("reponse not match string %s, response: %s", 107 | r.matchRes, respString) 108 | return 109 | } 110 | } 111 | 112 | r.hook.Success(respString) 113 | } 114 | 115 | func (r *request) BotType() string { 116 | if strings.HasPrefix(r.hook.Url, feishu.URL_PREFIX) { 117 | return feishuBot 118 | } 119 | if strings.HasPrefix(r.hook.Url, dingding.URL_PREFIX) { 120 | return dingdingBot 121 | } 122 | if strings.HasPrefix(r.hook.Url, wechat.URL_PREFIX) { 123 | return wechatBot 124 | } 125 | 126 | return "" 127 | } 128 | 129 | func (r *request) NewFeishuMessage() *feishu.Message { 130 | s := r.step 131 | msg := &feishu.NotifyMessage{ 132 | Title: s.ShowTitle(), 133 | Content: s.String(), 134 | RobotURL: r.hook.Url, 135 | Note: []string{"💡 该消息由极乐研发云[研发交付系统]提供"}, 136 | Color: feishu.COLOR_PURPLE, 137 | } 138 | return feishu.NewCardMessage(msg) 139 | } 140 | -------------------------------------------------------------------------------- /common/hooks/webhook/request_test.go: -------------------------------------------------------------------------------- 1 | package webhook_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | 9 | "github.com/infraboard/mcube/logger/zap" 10 | "github.com/infraboard/workflow/api/apps/pipeline" 11 | "github.com/infraboard/workflow/common/hooks/webhook" 12 | ) 13 | 14 | var ( 15 | feishuBotURL = "https://open.feishu.cn/open-apis/bot/v2/hook/461ead7b-d856-472c-babc-2d3d0ec9fabb" 16 | dingBotURL = "https://oapi.dingtalk.com/robot/send?access_token=xxxx" 17 | wechatBotURL = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=693axxx6-7aoc-4bc4-97a0-0ec2sifa5aaa" 18 | ) 19 | 20 | func TestFeishuWebHook(t *testing.T) { 21 | should := assert.New(t) 22 | 23 | hooks := testPipelineWebHook(feishuBotURL) 24 | sender := webhook.NewWebHook() 25 | err := sender.Send( 26 | context.Background(), 27 | hooks, 28 | testPipelineStep(), 29 | ) 30 | should.NoError(err) 31 | t.Log(hooks[0]) 32 | } 33 | 34 | func TestDingDingWebHook(t *testing.T) { 35 | should := assert.New(t) 36 | 37 | hooks := testPipelineWebHook(dingBotURL) 38 | sender := webhook.NewWebHook() 39 | err := sender.Send( 40 | context.Background(), 41 | hooks, 42 | testPipelineStep(), 43 | ) 44 | should.NoError(err) 45 | 46 | t.Log(hooks[0]) 47 | } 48 | 49 | func TestWechatWebHook(t *testing.T) { 50 | should := assert.New(t) 51 | 52 | hooks := testPipelineWebHook(wechatBotURL) 53 | sender := webhook.NewWebHook() 54 | err := sender.Send( 55 | context.Background(), 56 | hooks, 57 | testPipelineStep(), 58 | ) 59 | should.NoError(err) 60 | t.Log(hooks[0]) 61 | } 62 | 63 | func testPipelineWebHook(url string) []*pipeline.WebHook { 64 | h1 := &pipeline.WebHook{ 65 | Url: url, 66 | Events: []pipeline.STEP_STATUS{pipeline.STEP_STATUS_SUCCEEDED}, 67 | Description: "测试", 68 | } 69 | return []*pipeline.WebHook{h1} 70 | } 71 | 72 | func testPipelineStep() *pipeline.Step { 73 | return &pipeline.Step{ 74 | Name: "only for test", 75 | Status: &pipeline.StepStatus{ 76 | Status: pipeline.STEP_STATUS_SUCCEEDED, 77 | }, 78 | } 79 | } 80 | 81 | func init() { 82 | zap.DevelopmentSetup() 83 | } 84 | -------------------------------------------------------------------------------- /common/hooks/webhook/webhook.go: -------------------------------------------------------------------------------- 1 | package webhook 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/infraboard/mcube/logger" 8 | "github.com/infraboard/mcube/logger/zap" 9 | 10 | "github.com/infraboard/workflow/api/apps/pipeline" 11 | ) 12 | 13 | func NewWebHook() *WebHook { 14 | return &WebHook{ 15 | log: zap.L().Named("WebHook"), 16 | } 17 | } 18 | 19 | type WebHook struct { 20 | log logger.Logger 21 | } 22 | 23 | func (h *WebHook) Send(ctx context.Context, hooks []*pipeline.WebHook, step *pipeline.Step) error { 24 | if step == nil { 25 | return fmt.Errorf("step is nil") 26 | } 27 | 28 | if err := h.validate(hooks); err != nil { 29 | return err 30 | } 31 | 32 | h.log.Debugf("start send step[%s] webhook, total %d", step.Key, len(hooks)) 33 | for i := range hooks { 34 | req := newRequest(hooks[i], step) 35 | req.Push() 36 | } 37 | 38 | return nil 39 | } 40 | 41 | func (h *WebHook) validate(hooks []*pipeline.WebHook) error { 42 | if len(hooks) == 0 { 43 | return nil 44 | } 45 | 46 | if len(hooks) > MAX_WEBHOOKS_PER_SEND { 47 | return fmt.Errorf("too many webhooks configs current: %d, max: %d", len(hooks), MAX_WEBHOOKS_PER_SEND) 48 | } 49 | 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /common/hooks/webhook/wechat/message.go: -------------------------------------------------------------------------------- 1 | package wechat 2 | 3 | import "github.com/infraboard/workflow/api/apps/pipeline" 4 | 5 | const ( 6 | URL_PREFIX = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send" 7 | ) 8 | 9 | const ( 10 | MarkdownMessage = "markdown" 11 | ) 12 | 13 | type MessageType string 14 | 15 | func NewStepMarkdownMessage(s *pipeline.Step) *Message { 16 | return &Message{ 17 | MsgType: MarkdownMessage, 18 | Markdown: &MarkDownContent{ 19 | Content: s.ShowTitle(), 20 | }, 21 | } 22 | } 23 | 24 | // 群机器人配置说明: https://work.weixin.qq.com/api/doc/90000/90136/91770 25 | type Message struct { 26 | MsgType MessageType `json:"msgtype"` 27 | Markdown *MarkDownContent `json:"markdown"` 28 | } 29 | 30 | type MarkDownContent struct { 31 | Content string `json:"content"` 32 | } 33 | -------------------------------------------------------------------------------- /common/informers/node/etcd/imformer.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "github.com/infraboard/mcube/logger" 5 | "github.com/infraboard/mcube/logger/zap" 6 | clientv3 "go.etcd.io/etcd/client/v3" 7 | 8 | "github.com/infraboard/workflow/common/cache" 9 | "github.com/infraboard/workflow/common/informers/node" 10 | ) 11 | 12 | // NewNodeInformer todo 13 | func NewInformer(client *clientv3.Client, filter node.NodeFilterHandler) node.Informer { 14 | return &Informer{ 15 | log: zap.L().Named("Node"), 16 | client: client, 17 | filter: filter, 18 | indexer: cache.NewIndexer(node.MetaNamespaceKeyFunc, node.DefaultStoreIndexers()), 19 | } 20 | } 21 | 22 | // Informer todo 23 | type Informer struct { 24 | log logger.Logger 25 | client *clientv3.Client 26 | shared *shared 27 | lister *lister 28 | indexer cache.Indexer 29 | filter node.NodeFilterHandler 30 | } 31 | 32 | func (i *Informer) GetStore() cache.Store { 33 | return i.indexer 34 | } 35 | 36 | func (i *Informer) Debug(l logger.Logger) { 37 | i.log = l 38 | i.shared.log = l 39 | i.lister.log = l 40 | } 41 | 42 | func (i *Informer) Watcher() node.Watcher { 43 | if i.shared != nil { 44 | return i.shared 45 | } 46 | i.shared = &shared{ 47 | log: i.log.Named("Watcher"), 48 | client: clientv3.NewWatcher(i.client), 49 | indexer: i.indexer, 50 | filter: i.filter, 51 | } 52 | return i.shared 53 | } 54 | 55 | func (i *Informer) Lister() node.Lister { 56 | if i.lister != nil { 57 | return i.lister 58 | } 59 | i.lister = &lister{ 60 | log: i.log.Named("Lister"), 61 | client: clientv3.NewKV(i.client), 62 | } 63 | return i.lister 64 | } 65 | -------------------------------------------------------------------------------- /common/informers/node/etcd/lister.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/infraboard/mcube/logger" 7 | clientv3 "go.etcd.io/etcd/client/v3" 8 | 9 | "github.com/infraboard/workflow/api/apps/node" 10 | ) 11 | 12 | type lister struct { 13 | log logger.Logger 14 | client clientv3.KV 15 | } 16 | 17 | // List 获取所有Node对象 18 | func (l *lister) List(ctx context.Context, t node.Type) (ret []*node.Node, err error) { 19 | listKey := node.EtcdNodePrefixWithType(t) 20 | return l.list(ctx, listKey) 21 | } 22 | 23 | // List 获取所有Node对象 24 | func (l *lister) ListAll(ctx context.Context) (ret []*node.Node, err error) { 25 | listKey := node.EtcdNodePrefix() 26 | return l.list(ctx, listKey) 27 | } 28 | 29 | func (l *lister) list(ctx context.Context, key string) (ret []*node.Node, err error) { 30 | l.log.Infof("list etcd node resource key: %s", key) 31 | resp, err := l.client.Get(ctx, key, clientv3.WithPrefix()) 32 | if err != nil { 33 | return nil, err 34 | } 35 | nodes := []*node.Node{} 36 | for i := range resp.Kvs { 37 | // 解析对象 38 | node, err := node.LoadNodeFromBytes(resp.Kvs[i].Value) 39 | if err != nil { 40 | l.log.Error(err) 41 | continue 42 | } 43 | node.ResourceVersion = resp.Header.Revision 44 | nodes = append(nodes, node) 45 | } 46 | 47 | l.log.Infof("total nodes: %d", len(nodes)) 48 | return nodes, nil 49 | } 50 | -------------------------------------------------------------------------------- /common/informers/node/etcd/watcher.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/infraboard/mcube/logger" 9 | "go.etcd.io/etcd/api/v3/mvccpb" 10 | clientv3 "go.etcd.io/etcd/client/v3" 11 | 12 | "github.com/infraboard/workflow/api/apps/node" 13 | "github.com/infraboard/workflow/common/cache" 14 | informer "github.com/infraboard/workflow/common/informers/node" 15 | ) 16 | 17 | type shared struct { 18 | log logger.Logger 19 | client clientv3.Watcher 20 | handler informer.NodeEventHandler 21 | filter informer.NodeFilterHandler 22 | watchChan clientv3.WatchChan 23 | indexer cache.Indexer 24 | } 25 | 26 | // AddEventHandler 添加事件处理回调 27 | func (i *shared) AddNodeEventHandler(h informer.NodeEventHandler) { 28 | i.handler = h 29 | } 30 | 31 | // Run 启动 Watch 32 | func (i *shared) Run(ctx context.Context) error { 33 | // 是否准备完成 34 | if err := i.isReady(); err != nil { 35 | return err 36 | } 37 | 38 | // 监听事件 39 | i.watch(ctx) 40 | 41 | // 后台处理事件 42 | go i.dealEvents() 43 | return nil 44 | } 45 | 46 | func (i *shared) dealEvents() { 47 | // 处理所有事件 48 | for { 49 | select { 50 | case nodeResp := <-i.watchChan: 51 | for _, event := range nodeResp.Events { 52 | switch event.Type { 53 | case mvccpb.PUT: 54 | if err := i.handlePut(event, nodeResp.Header.GetRevision()); err != nil { 55 | i.log.Error(err) 56 | } 57 | case mvccpb.DELETE: 58 | if err := i.handleDelete(event); err != nil { 59 | i.log.Error(err) 60 | } 61 | default: 62 | } 63 | } 64 | } 65 | } 66 | } 67 | 68 | func (i *shared) isReady() error { 69 | if i.handler == nil { 70 | return errors.New("NodeEventHandler not add") 71 | } 72 | return nil 73 | } 74 | 75 | func (s *shared) filt(node *node.Node) bool { 76 | if s.filter == nil { 77 | return false 78 | } 79 | 80 | err, ok := s.filter(node) 81 | if err == nil { 82 | s.log.Errorf("filt node error, %s", err) 83 | return false 84 | } 85 | 86 | return ok 87 | } 88 | 89 | func (i *shared) watch(ctx context.Context) { 90 | nodeWatchKey := node.EtcdNodePrefixWithType(node.NodeType) 91 | i.watchChan = i.client.Watch(ctx, nodeWatchKey, clientv3.WithPrefix()) 92 | i.log.Infof("watch etcd node resource key: %s", nodeWatchKey) 93 | } 94 | 95 | func (i *shared) handlePut(event *clientv3.Event, eventVersion int64) error { 96 | i.log.Debugf("receive node put event, %s", event.Kv.Key) 97 | 98 | // 解析对象 99 | new, err := node.LoadNodeFromBytes(event.Kv.Value) 100 | if err != nil { 101 | return err 102 | } 103 | 104 | if new == nil { 105 | return fmt.Errorf("load node from bytes but get node object is nil") 106 | } 107 | 108 | old, hasOld, err := i.indexer.GetByKey(new.MakeObjectKey()) 109 | if err != nil { 110 | return err 111 | } 112 | 113 | // 过滤掉不需要的 114 | if i.filt(new) { 115 | return nil 116 | } 117 | 118 | // 区分Update 119 | if hasOld { 120 | // 更新缓存 121 | i.log.Debugf("update node: %s", new.ShortDescribe()) 122 | if err := i.indexer.Update(new); err != nil { 123 | i.log.Errorf("update indexer cache error, %s", err) 124 | } 125 | i.handler.OnUpdate(old.(*node.Node), new) 126 | } else { 127 | // 添加缓存 128 | i.log.Debugf("add node: %s", new.ShortDescribe()) 129 | if err := i.indexer.Add(new); err != nil { 130 | i.log.Errorf("add indexer cache error, %s", err) 131 | } 132 | i.handler.OnAdd(new) 133 | } 134 | 135 | return nil 136 | } 137 | 138 | func (i *shared) handleDelete(event *clientv3.Event) error { 139 | key := event.Kv.Key 140 | i.log.Debugf("receive node delete event, %s", key) 141 | 142 | obj, ok, err := i.indexer.GetByKey(string(key)) 143 | if err != nil { 144 | i.log.Errorf("get key %s from store error, %s", key) 145 | } 146 | if !ok { 147 | i.log.Warnf("key %s found in store", key) 148 | } 149 | 150 | // 清除缓存 151 | if err := i.indexer.Delete(obj); err != nil { 152 | i.log.Errorf("delete indexer cache error, %s", err) 153 | } 154 | 155 | i.handler.OnDelete(obj.(*node.Node)) 156 | return nil 157 | } 158 | -------------------------------------------------------------------------------- /common/informers/node/indexer.go: -------------------------------------------------------------------------------- 1 | package node 2 | 3 | import ( 4 | "github.com/infraboard/workflow/api/apps/node" 5 | "github.com/infraboard/workflow/common/cache" 6 | ) 7 | 8 | // DefaultStoreIndexFunc todo 9 | func DefaultStoreIndexFunc(obj interface{}) ([]string, error) { 10 | return []string{obj.(*node.Node).MakeObjectKey()}, nil 11 | } 12 | 13 | // DefaultStoreIndexers todo 14 | func DefaultStoreIndexers() cache.Indexers { 15 | indexers := cache.Indexers{} 16 | indexers["by_val"] = DefaultStoreIndexFunc 17 | return indexers 18 | } 19 | 20 | // ExplicitKey can be passed to MetaNamespaceKeyFunc if you have the key for 21 | // the object but not the object itself. 22 | type ExplicitKey string 23 | 24 | // MetaNamespaceKeyFunc is a convenient default KeyFunc which knows how to make 25 | // keys for API objects which implement meta.Interface. 26 | // The key uses the format / unless is empty, then 27 | // it's just . 28 | // 29 | // TODO: replace key-as-string with a key-as-struct so that this 30 | // packing/unpacking won't be necessary. 31 | func MetaNamespaceKeyFunc(obj interface{}) (string, error) { 32 | if key, ok := obj.(ExplicitKey); ok { 33 | return string(key), nil 34 | } 35 | return obj.(*node.Node).MakeObjectKey(), nil 36 | } 37 | -------------------------------------------------------------------------------- /common/informers/node/informer.go: -------------------------------------------------------------------------------- 1 | package node 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/infraboard/workflow/api/apps/node" 7 | "github.com/infraboard/workflow/common/cache" 8 | ) 9 | 10 | // CronJobInformer provides access to a shared informer and lister for 11 | // CronJobs. 12 | type Informer interface { 13 | // Watcher Event 14 | Watcher() Watcher 15 | // List All Node 16 | Lister() Lister 17 | // node节点 18 | GetStore() cache.Store 19 | } 20 | 21 | // Lister 获取所有执行节点 22 | type Lister interface { 23 | // List lists all Node 24 | List(context.Context, node.Type) ([]*node.Node, error) 25 | // List all 26 | ListAll(context.Context) ([]*node.Node, error) 27 | } 28 | 29 | type Watcher interface { 30 | // Run starts and runs the shared informer, returning after it stops. 31 | // The informer will be stopped when stopCh is closed. 32 | Run(ctx context.Context) error 33 | // AddEventHandler adds an event handler to the shared informer using the shared informer's resync 34 | // period. Events to a single handler are delivered sequentially, but there is no coordination 35 | // between different handlers. 36 | AddNodeEventHandler(handler NodeEventHandler) 37 | } 38 | 39 | // NodeEventHandler can handle notifications for events that happen to a 40 | // resource. The events are informational only, so you can't return an 41 | // error. 42 | // * OnAdd is called when an object is added. 43 | // * OnUpdate is called when an object is modified. Note that oldObj is the 44 | // last known state of the object-- it is possible that several changes 45 | // were combined together, so you can't use this to see every single 46 | // change. OnUpdate is also called when a re-list happens, and it will 47 | // get called even if nothing changed. This is useful for periodically 48 | // evaluating or syncing something. 49 | // * OnDelete will get the final state of the item if it is known, otherwise 50 | // it will get an object of type DeletedFinalStateUnknown. This can 51 | // happen if the watch is closed and misses the delete event and we don't 52 | // notice the deletion until the subsequent re-list. 53 | type NodeEventHandler interface { 54 | OnAdd(node *node.Node) 55 | OnUpdate(oldNode, newNode *node.Node) 56 | OnDelete(node *node.Node) 57 | } 58 | 59 | // NodeEventHandlerFuncs is an adaptor to let you easily specify as many or 60 | // as few of the notification functions as you want while still implementing 61 | // ResourceEventHandler. 62 | type NodeEventHandlerFuncs struct { 63 | AddFunc func(obj *node.Node) 64 | UpdateFunc func(oldObj, newObj *node.Node) 65 | DeleteFunc func(obj *node.Node) 66 | } 67 | 68 | // OnAdd calls AddFunc if it's not nil. 69 | func (r NodeEventHandlerFuncs) OnAdd(obj *node.Node) { 70 | if r.AddFunc != nil { 71 | r.AddFunc(obj) 72 | } 73 | } 74 | 75 | // OnUpdate calls UpdateFunc if it's not nil. 76 | func (r NodeEventHandlerFuncs) OnUpdate(oldObj, newObj *node.Node) { 77 | if r.UpdateFunc != nil { 78 | r.UpdateFunc(oldObj, newObj) 79 | } 80 | } 81 | 82 | // OnDelete calls DeleteFunc if it's not nil. 83 | func (r NodeEventHandlerFuncs) OnDelete(obj *node.Node) { 84 | if r.DeleteFunc != nil { 85 | r.DeleteFunc(obj) 86 | } 87 | } 88 | 89 | type NodeFilterHandler func(obj *node.Node) (error, bool) 90 | -------------------------------------------------------------------------------- /common/informers/pipeline/etcd/imformer.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "github.com/infraboard/mcube/logger" 5 | "github.com/infraboard/mcube/logger/zap" 6 | clientv3 "go.etcd.io/etcd/client/v3" 7 | 8 | "github.com/infraboard/workflow/common/cache" 9 | informer "github.com/infraboard/workflow/common/informers/pipeline" 10 | ) 11 | 12 | // NewScheduleInformer todo 13 | func NewInformerr(client *clientv3.Client, filter informer.PipelineFilterHandler) informer.Informer { 14 | return &Informer{ 15 | log: zap.L().Named("Pipeline"), 16 | client: client, 17 | filter: filter, 18 | indexer: cache.NewIndexer(informer.MetaNamespaceKeyFunc, informer.DefaultStoreIndexers()), 19 | } 20 | } 21 | 22 | // Informer todo 23 | type Informer struct { 24 | log logger.Logger 25 | client *clientv3.Client 26 | shared *shared 27 | lister *lister 28 | recorder *recorder 29 | indexer cache.Indexer 30 | filter informer.PipelineFilterHandler 31 | } 32 | 33 | func (i *Informer) GetStore() cache.Store { 34 | return i.indexer 35 | } 36 | 37 | func (i *Informer) Debug(l logger.Logger) { 38 | i.log = l 39 | i.shared.log = l 40 | i.lister.log = l 41 | } 42 | 43 | func (i *Informer) Watcher() informer.Watcher { 44 | if i.shared != nil { 45 | return i.shared 46 | } 47 | i.shared = &shared{ 48 | log: i.log.Named("Watcher"), 49 | client: clientv3.NewWatcher(i.client), 50 | indexer: i.indexer, 51 | filter: i.filter, 52 | } 53 | return i.shared 54 | } 55 | 56 | func (i *Informer) Lister() informer.Lister { 57 | if i.lister != nil { 58 | return i.lister 59 | } 60 | i.lister = &lister{ 61 | log: i.log.Named("Lister"), 62 | client: clientv3.NewKV(i.client), 63 | } 64 | return i.lister 65 | } 66 | 67 | func (i *Informer) Recorder() informer.Recorder { 68 | if i.recorder != nil { 69 | return i.recorder 70 | } 71 | i.recorder = &recorder{ 72 | log: i.log.Named("Recorder"), 73 | client: clientv3.NewKV(i.client), 74 | } 75 | return i.recorder 76 | } 77 | -------------------------------------------------------------------------------- /common/informers/pipeline/etcd/lister.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/infraboard/mcube/logger" 7 | clientv3 "go.etcd.io/etcd/client/v3" 8 | 9 | "github.com/infraboard/workflow/api/apps/pipeline" 10 | ) 11 | 12 | type lister struct { 13 | log logger.Logger 14 | client clientv3.KV 15 | } 16 | 17 | func (l *lister) List(ctx context.Context, opts *pipeline.QueryPipelineOptions) (*pipeline.PipelineSet, error) { 18 | listKey := pipeline.EtcdPipelinePrefix() 19 | l.log.Infof("list etcd pipeline resource key: %s", listKey) 20 | resp, err := l.client.Get(ctx, listKey, clientv3.WithPrefix()) 21 | if err != nil { 22 | return nil, err 23 | } 24 | ps := pipeline.NewPipelineSet() 25 | for i := range resp.Kvs { 26 | // 解析对象 27 | pt, err := pipeline.LoadPipelineFromBytes(resp.Kvs[i].Value) 28 | if err != nil { 29 | l.log.Errorf("load pipeline [key: %s, value: %s] error, %s", resp.Kvs[i].Key, string(resp.Kvs[i].Value), err) 30 | continue 31 | } 32 | 33 | pt.ResourceVersion = resp.Header.Revision 34 | ps.Add(pt) 35 | } 36 | return ps, nil 37 | } 38 | -------------------------------------------------------------------------------- /common/informers/pipeline/etcd/recorder.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "github.com/infraboard/mcube/logger" 9 | clientv3 "go.etcd.io/etcd/client/v3" 10 | 11 | "github.com/infraboard/workflow/api/apps/pipeline" 12 | ) 13 | 14 | type recorder struct { 15 | log logger.Logger 16 | client clientv3.KV 17 | } 18 | 19 | func (l *recorder) Update(t *pipeline.Pipeline) error { 20 | if t == nil { 21 | return fmt.Errorf("update nil pipeline") 22 | } 23 | 24 | if l.client == nil { 25 | return fmt.Errorf("etcd client is nil") 26 | } 27 | 28 | objKey := t.EtcdObjectKey() 29 | objValue, err := json.Marshal(t) 30 | if err != nil { 31 | return err 32 | } 33 | if _, err := l.client.Put(context.Background(), objKey, string(objValue)); err != nil { 34 | return fmt.Errorf("update pipeline task '%s' to etcd3 failed: %s", objKey, err.Error()) 35 | } 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /common/informers/pipeline/etcd/watcher.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/infraboard/mcube/logger" 8 | "go.etcd.io/etcd/api/v3/mvccpb" 9 | clientv3 "go.etcd.io/etcd/client/v3" 10 | 11 | "github.com/infraboard/workflow/api/apps/pipeline" 12 | "github.com/infraboard/workflow/common/cache" 13 | informer "github.com/infraboard/workflow/common/informers/pipeline" 14 | ) 15 | 16 | type shared struct { 17 | log logger.Logger 18 | client clientv3.Watcher 19 | indexer cache.Indexer 20 | handler informer.PipelineEventHandler 21 | filter informer.PipelineFilterHandler 22 | watchChan clientv3.WatchChan 23 | } 24 | 25 | // AddPipelineEventHandler 添加事件处理回调 26 | func (i *shared) AddPipelineTaskEventHandler(h informer.PipelineEventHandler) { 27 | i.handler = h 28 | } 29 | 30 | // Run 启动 Watch 31 | func (i *shared) Run(ctx context.Context) error { 32 | // 是否准备完成 33 | if err := i.isReady(); err != nil { 34 | return err 35 | } 36 | 37 | // 监听事件 38 | i.watch(ctx) 39 | 40 | // 后台处理事件 41 | go i.dealEvents() 42 | return nil 43 | } 44 | 45 | func (i *shared) dealEvents() { 46 | // 处理所有事件 47 | for { 48 | select { 49 | case nodeResp := <-i.watchChan: 50 | for _, event := range nodeResp.Events { 51 | switch event.Type { 52 | case mvccpb.PUT: 53 | if err := i.handlePut(event, nodeResp.Header.GetRevision()); err != nil { 54 | i.log.Error(err) 55 | } 56 | case mvccpb.DELETE: 57 | if err := i.handleDelete(event); err != nil { 58 | i.log.Error(err) 59 | } 60 | default: 61 | } 62 | } 63 | } 64 | } 65 | } 66 | 67 | func (i *shared) isReady() error { 68 | if i.handler == nil { 69 | return errors.New("PipelineEventHandler not add") 70 | } 71 | return nil 72 | } 73 | 74 | func (i *shared) watch(ctx context.Context) { 75 | ppWatchKey := pipeline.EtcdPipelinePrefix() 76 | i.watchChan = i.client.Watch(ctx, ppWatchKey, clientv3.WithPrefix()) 77 | i.log.Infof("watch etcd pipeline resource key: %s", ppWatchKey) 78 | } 79 | 80 | func (i *shared) handlePut(event *clientv3.Event, eventVersion int64) error { 81 | i.log.Debugf("receive pipeline put event, %s", event.Kv.Key) 82 | 83 | // 解析对象 84 | new, err := pipeline.LoadPipelineFromBytes(event.Kv.Value) 85 | if err != nil { 86 | return err 87 | } 88 | new.ResourceVersion = eventVersion 89 | 90 | old, hasOld, err := i.indexer.GetByKey(new.MakeObjectKey()) 91 | if err != nil { 92 | return err 93 | } 94 | 95 | if i.filter != nil { 96 | if err := i.filter(new); err != nil { 97 | return err 98 | } 99 | } 100 | 101 | // 区分Update 102 | if hasOld { 103 | // 更新缓存 104 | i.log.Debugf("update pipeline: %s", new.ShortDescribe()) 105 | if err := i.indexer.Update(new); err != nil { 106 | i.log.Errorf("update indexer cache error, %s", err) 107 | } 108 | i.handler.OnUpdate(old.(*pipeline.Pipeline), new) 109 | } else { 110 | // 添加缓存 111 | i.log.Debugf("add pipeline: %s", new.ShortDescribe()) 112 | if err := i.indexer.Add(new); err != nil { 113 | i.log.Errorf("add indexer cache error, %s", err) 114 | } 115 | i.handler.OnAdd(new) 116 | } 117 | 118 | return nil 119 | } 120 | 121 | func (i *shared) handleDelete(event *clientv3.Event) error { 122 | key := event.Kv.Key 123 | i.log.Debugf("receive pipeline delete event, %s", key) 124 | 125 | obj, ok, err := i.indexer.GetByKey(string(key)) 126 | if err != nil { 127 | i.log.Errorf("get key %s from store error, %s", key) 128 | } 129 | if !ok { 130 | i.log.Warnf("key %s found in store", key) 131 | } 132 | 133 | // 清除缓存 134 | if err := i.indexer.Delete(obj); err != nil { 135 | i.log.Errorf("delete indexer cache error, %s", err) 136 | } 137 | 138 | pl, ok := obj.(*pipeline.Pipeline) 139 | if ok { 140 | i.handler.OnDelete(pl) 141 | } 142 | 143 | return nil 144 | } 145 | -------------------------------------------------------------------------------- /common/informers/pipeline/indexer.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "github.com/infraboard/workflow/api/apps/pipeline" 5 | "github.com/infraboard/workflow/common/cache" 6 | ) 7 | 8 | // DefaultStoreIndexFunc todo 9 | func DefaultStoreIndexFunc(obj interface{}) ([]string, error) { 10 | return []string{obj.(*pipeline.Pipeline).MakeObjectKey()}, nil 11 | } 12 | 13 | // DefaultStoreIndexers todo 14 | func DefaultStoreIndexers() cache.Indexers { 15 | indexers := cache.Indexers{} 16 | indexers["by_val"] = DefaultStoreIndexFunc 17 | return indexers 18 | } 19 | 20 | // ExplicitKey can be passed to MetaNamespaceKeyFunc if you have the key for 21 | // the object but not the object itself. 22 | type ExplicitKey string 23 | 24 | // MetaNamespaceKeyFunc is a convenient default KeyFunc which knows how to make 25 | // keys for API objects which implement meta.Interface. 26 | // The key uses the format / unless is empty, then 27 | // it's just . 28 | // 29 | // TODO: replace key-as-string with a key-as-struct so that this 30 | // packing/unpacking won't be necessary. 31 | func MetaNamespaceKeyFunc(obj interface{}) (string, error) { 32 | if key, ok := obj.(ExplicitKey); ok { 33 | return string(key), nil 34 | } 35 | 36 | pl, ok := obj.(*pipeline.Pipeline) 37 | if ok { 38 | return pl.MakeObjectKey(), nil 39 | } 40 | 41 | return "", nil 42 | } 43 | -------------------------------------------------------------------------------- /common/informers/pipeline/informer.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/infraboard/workflow/api/apps/pipeline" 7 | "github.com/infraboard/workflow/common/cache" 8 | ) 9 | 10 | // Informer 负责事件通知 11 | type Informer interface { 12 | Watcher() Watcher 13 | Lister() Lister 14 | Recorder() Recorder 15 | GetStore() cache.Store 16 | } 17 | 18 | type Lister interface { 19 | List(ctx context.Context, opts *pipeline.QueryPipelineOptions) (*pipeline.PipelineSet, error) 20 | } 21 | 22 | type Recorder interface { 23 | Update(*pipeline.Pipeline) error 24 | } 25 | 26 | // Watcher 负责事件通知 27 | type Watcher interface { 28 | // Run starts and runs the shared informer, returning after it stops. 29 | // The informer will be stopped when stopCh is closed. 30 | Run(ctx context.Context) error 31 | // AddEventHandler adds an event handler to the shared informer using the shared informer's resync 32 | // period. Events to a single handler are delivered sequentially, but there is no coordination 33 | // between different handlers. 34 | AddPipelineTaskEventHandler(handler PipelineEventHandler) 35 | } 36 | 37 | // PipelineEventHandler can handle notifications for events that happen to a 38 | // resource. The events are informational only, so you can't return an 39 | // error. 40 | // * OnAdd is called when an object is added. 41 | // * OnUpdate is called when an object is modified. Note that oldObj is the 42 | // last known state of the object-- it is possible that several changes 43 | // were combined together, so you can't use this to see every single 44 | // change. OnUpdate is also called when a re-list happens, and it will 45 | // get called even if nothing changed. This is useful for periodically 46 | // evaluating or syncing something. 47 | // * OnDelete will get the final state of the item if it is known, otherwise 48 | // it will get an object of type DeletedFinalStateUnknown. This can 49 | // happen if the watch is closed and misses the delete event and we don't 50 | // notice the deletion until the subsequent re-list. 51 | type PipelineEventHandler interface { 52 | OnAdd(obj *pipeline.Pipeline) 53 | OnUpdate(old, new *pipeline.Pipeline) 54 | OnDelete(obj *pipeline.Pipeline) 55 | } 56 | 57 | // PipelineEventHandlerFuncs is an adaptor to let you easily specify as many or 58 | // as few of the notification functions as you want while still implementing 59 | // ResourceEventHandler. 60 | type PipelineTaskEventHandlerFuncs struct { 61 | AddFunc func(obj *pipeline.Pipeline) 62 | UpdateFunc func(oldObj, newObj *pipeline.Pipeline) 63 | DeleteFunc func(obj *pipeline.Pipeline) 64 | } 65 | 66 | // OnAdd calls AddFunc if it's not nil. 67 | func (r PipelineTaskEventHandlerFuncs) OnAdd(obj *pipeline.Pipeline) { 68 | if r.AddFunc != nil { 69 | r.AddFunc(obj) 70 | } 71 | } 72 | 73 | // OnUpdate calls UpdateFunc if it's not nil. 74 | func (r PipelineTaskEventHandlerFuncs) OnUpdate(oldObj, newObj *pipeline.Pipeline) { 75 | if r.UpdateFunc != nil { 76 | r.UpdateFunc(oldObj, newObj) 77 | } 78 | } 79 | 80 | // OnDelete calls DeleteFunc if it's not nil. 81 | func (r PipelineTaskEventHandlerFuncs) OnDelete(obj *pipeline.Pipeline) { 82 | if r.DeleteFunc != nil { 83 | r.DeleteFunc(obj) 84 | } 85 | } 86 | 87 | type PipelineFilterHandler func(obj *pipeline.Pipeline) error 88 | -------------------------------------------------------------------------------- /common/informers/step/etcd/imformer.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "github.com/infraboard/mcube/logger" 5 | "github.com/infraboard/mcube/logger/zap" 6 | clientv3 "go.etcd.io/etcd/client/v3" 7 | 8 | "github.com/infraboard/workflow/common/cache" 9 | informer "github.com/infraboard/workflow/common/informers/step" 10 | ) 11 | 12 | func NewFilterInformer(client *clientv3.Client, filter informer.StepFilterHandler) informer.Informer { 13 | return &Informer{ 14 | log: zap.L().Named("Step"), 15 | client: client, 16 | filter: filter, 17 | indexer: cache.NewIndexer(informer.MetaNamespaceKeyFunc, informer.DefaultStoreIndexers()), 18 | } 19 | } 20 | 21 | // NewSInformer todo 22 | func NewInformer(client *clientv3.Client) informer.Informer { 23 | return NewFilterInformer(client, nil) 24 | } 25 | 26 | // Informer todo 27 | type Informer struct { 28 | log logger.Logger 29 | client *clientv3.Client 30 | shared *shared 31 | lister *lister 32 | recorder *recorder 33 | indexer cache.Indexer 34 | filter informer.StepFilterHandler 35 | } 36 | 37 | func (i *Informer) GetStore() cache.Store { 38 | return i.indexer 39 | } 40 | 41 | func (i *Informer) Debug(l logger.Logger) { 42 | i.log = l 43 | i.shared.log = l 44 | i.lister.log = l 45 | } 46 | 47 | func (i *Informer) Watcher() informer.Watcher { 48 | if i.shared != nil { 49 | return i.shared 50 | } 51 | i.shared = &shared{ 52 | log: i.log.Named("Watcher"), 53 | client: clientv3.NewWatcher(i.client), 54 | indexer: i.indexer, 55 | filter: i.filter, 56 | } 57 | return i.shared 58 | } 59 | 60 | func (i *Informer) Lister() informer.Lister { 61 | if i.lister != nil { 62 | return i.lister 63 | } 64 | i.lister = &lister{ 65 | log: i.log.Named("Lister"), 66 | client: clientv3.NewKV(i.client), 67 | filter: i.filter, 68 | } 69 | return i.lister 70 | } 71 | 72 | func (i *Informer) Recorder() informer.Recorder { 73 | if i.recorder != nil { 74 | return i.recorder 75 | } 76 | i.recorder = &recorder{ 77 | log: i.log.Named("Recorder"), 78 | client: clientv3.NewKV(i.client), 79 | } 80 | return i.recorder 81 | } 82 | -------------------------------------------------------------------------------- /common/informers/step/etcd/lister.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/infraboard/mcube/exception" 7 | "github.com/infraboard/mcube/logger" 8 | clientv3 "go.etcd.io/etcd/client/v3" 9 | 10 | "github.com/infraboard/workflow/api/apps/pipeline" 11 | "github.com/infraboard/workflow/common/informers/step" 12 | ) 13 | 14 | type lister struct { 15 | log logger.Logger 16 | client clientv3.KV 17 | filter step.StepFilterHandler 18 | } 19 | 20 | func (l *lister) List(ctx context.Context) (ret []*pipeline.Step, err error) { 21 | listKey := pipeline.EtcdStepPrefix() 22 | 23 | l.log.Infof("list etcd step resource key: %s", listKey) 24 | resp, err := l.client.Get(ctx, listKey, clientv3.WithPrefix()) 25 | if err != nil { 26 | return nil, err 27 | } 28 | set := pipeline.NewStepSet() 29 | for i := range resp.Kvs { 30 | // 解析对象 31 | ins, err := pipeline.LoadStepFromBytes(resp.Kvs[i].Value) 32 | if err != nil { 33 | l.log.Error(err) 34 | continue 35 | } 36 | 37 | if l.filter != nil { 38 | if err := l.filter(ins); err != nil { 39 | l.log.Error(err) 40 | continue 41 | } 42 | } 43 | 44 | ins.ResourceVersion = resp.Header.Revision 45 | set.Add(ins) 46 | } 47 | 48 | return set.Items, nil 49 | } 50 | 51 | func (l *lister) Get(ctx context.Context, key string) (*pipeline.Step, error) { 52 | descKey := pipeline.StepObjectKey(key) 53 | l.log.Infof("describe etcd step resource key: %s", descKey) 54 | resp, err := l.client.Get(ctx, descKey) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | if resp.Count == 0 { 60 | return nil, nil 61 | } 62 | 63 | if resp.Count > 1 { 64 | return nil, exception.NewInternalServerError("step find more than one: %d", resp.Count) 65 | } 66 | 67 | ins := pipeline.NewDefaultStep() 68 | for index := range resp.Kvs { 69 | // 解析对象 70 | ins, err = pipeline.LoadStepFromBytes(resp.Kvs[index].Value) 71 | if err != nil { 72 | l.log.Error(err) 73 | continue 74 | } 75 | } 76 | return ins, nil 77 | } 78 | -------------------------------------------------------------------------------- /common/informers/step/etcd/recorder.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/infraboard/mcube/logger" 10 | clientv3 "go.etcd.io/etcd/client/v3" 11 | 12 | "github.com/infraboard/workflow/api/apps/pipeline" 13 | ) 14 | 15 | type recorder struct { 16 | log logger.Logger 17 | client clientv3.KV 18 | } 19 | 20 | func (l *recorder) Update(step *pipeline.Step) error { 21 | step.UpdateAt = time.Now().UnixMilli() 22 | objKey := pipeline.StepObjectKey(step.Key) 23 | objValue, err := json.Marshal(step) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | l.log.Debugf("update step %s status %s %s ...", objKey, step.Status, string(objValue)) 29 | if _, err := l.client.Put(context.Background(), objKey, string(objValue)); err != nil { 30 | return fmt.Errorf("update pipeline step '%s' to etcd3 failed: %s", objKey, err.Error()) 31 | } 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /common/informers/step/etcd/watcher.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/infraboard/mcube/logger" 8 | "go.etcd.io/etcd/api/v3/mvccpb" 9 | clientv3 "go.etcd.io/etcd/client/v3" 10 | 11 | "github.com/infraboard/workflow/api/apps/pipeline" 12 | "github.com/infraboard/workflow/common/cache" 13 | informer "github.com/infraboard/workflow/common/informers/step" 14 | ) 15 | 16 | type shared struct { 17 | log logger.Logger 18 | client clientv3.Watcher 19 | indexer cache.Indexer 20 | handler informer.StepEventHandler 21 | filter informer.StepFilterHandler 22 | watchChan clientv3.WatchChan 23 | } 24 | 25 | func (i *shared) AddStepEventHandler(h informer.StepEventHandler) { 26 | i.handler = h 27 | } 28 | 29 | // Run 启动 Watch 30 | func (i *shared) Run(ctx context.Context) error { 31 | // 是否准备完成 32 | if err := i.isReady(); err != nil { 33 | return err 34 | } 35 | 36 | // 监听事件 37 | i.watch(ctx) 38 | 39 | go i.dealEvent() 40 | return nil 41 | } 42 | 43 | func (i *shared) dealEvent() { 44 | // 处理所有事件 45 | for { 46 | select { 47 | case nodeResp := <-i.watchChan: 48 | for _, event := range nodeResp.Events { 49 | switch event.Type { 50 | case mvccpb.PUT: 51 | if err := i.handlePut(event, nodeResp.Header.GetRevision()); err != nil { 52 | i.log.Error(err) 53 | } 54 | case mvccpb.DELETE: 55 | if err := i.handleDelete(event); err != nil { 56 | i.log.Error(err) 57 | } 58 | default: 59 | } 60 | } 61 | } 62 | } 63 | } 64 | 65 | func (i *shared) isReady() error { 66 | if i.handler == nil { 67 | return errors.New("StepEventHandler not add") 68 | } 69 | return nil 70 | } 71 | 72 | func (i *shared) watch(ctx context.Context) { 73 | // 监听事件 74 | stepWatchKey := pipeline.EtcdStepPrefix() 75 | i.watchChan = i.client.Watch(ctx, stepWatchKey, clientv3.WithPrefix()) 76 | i.log.Infof("watch etcd step resource key: %s", stepWatchKey) 77 | } 78 | 79 | func (i *shared) handlePut(event *clientv3.Event, eventVersion int64) error { 80 | i.log.Debugf("receive step put event, %s", event.Kv.Key) 81 | 82 | // 解析对象 83 | new, err := pipeline.LoadStepFromBytes(event.Kv.Value) 84 | if err != nil { 85 | return err 86 | } 87 | new.ResourceVersion = eventVersion 88 | 89 | old, hasOld, err := i.indexer.GetByKey(new.MakeObjectKey()) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | if i.filter != nil { 95 | if err := i.filter(new); err != nil { 96 | return err 97 | } 98 | } 99 | 100 | // 区分Update 101 | if hasOld { 102 | // 更新缓存 103 | i.log.Debugf("update step store key: %s, status %s", new.Key, new.Status) 104 | if err := i.indexer.Update(new); err != nil { 105 | i.log.Errorf("update indexer cache error, %s", err) 106 | } 107 | i.handler.OnUpdate(old.(*pipeline.Step), new) 108 | } else { 109 | // 添加缓存 110 | i.log.Debugf("add step store key: %s, status %s", new.Key, new.Status) 111 | if err := i.indexer.Add(new); err != nil { 112 | i.log.Errorf("add indexer cache error, %s", err) 113 | } 114 | i.handler.OnAdd(new) 115 | } 116 | 117 | return nil 118 | } 119 | 120 | func (i *shared) handleDelete(event *clientv3.Event) error { 121 | key := event.Kv.Key 122 | i.log.Debugf("receive step delete event, %s", key) 123 | 124 | obj, ok, err := i.indexer.GetByKey(string(key)) 125 | if err != nil { 126 | i.log.Errorf("get key %s from store error, %s", key) 127 | } 128 | if !ok { 129 | i.log.Warnf("key %s found in store", key) 130 | } 131 | if obj == nil { 132 | return nil 133 | } 134 | 135 | // 清除缓存 136 | if err := i.indexer.Delete(obj); err != nil { 137 | i.log.Errorf("delete indexer cache error, %s", err) 138 | } 139 | 140 | i.handler.OnDelete(obj.(*pipeline.Step)) 141 | return nil 142 | } 143 | -------------------------------------------------------------------------------- /common/informers/step/indexer.go: -------------------------------------------------------------------------------- 1 | package step 2 | 3 | import ( 4 | "github.com/infraboard/workflow/api/apps/pipeline" 5 | "github.com/infraboard/workflow/common/cache" 6 | ) 7 | 8 | // DefaultStoreIndexFunc todo 9 | func DefaultStoreIndexFunc(obj interface{}) ([]string, error) { 10 | return []string{obj.(*pipeline.Step).MakeObjectKey()}, nil 11 | } 12 | 13 | // DefaultStoreIndexers todo 14 | func DefaultStoreIndexers() cache.Indexers { 15 | indexers := cache.Indexers{} 16 | indexers["by_val"] = DefaultStoreIndexFunc 17 | return indexers 18 | } 19 | 20 | // ExplicitKey can be passed to MetaNamespaceKeyFunc if you have the key for 21 | // the object but not the object itself. 22 | type ExplicitKey string 23 | 24 | // MetaNamespaceKeyFunc is a convenient default KeyFunc which knows how to make 25 | // keys for API objects which implement meta.Interface. 26 | // The key uses the format / unless is empty, then 27 | // it's just . 28 | // 29 | // TODO: replace key-as-string with a key-as-struct so that this 30 | // packing/unpacking won't be necessary. 31 | func MetaNamespaceKeyFunc(obj interface{}) (string, error) { 32 | if key, ok := obj.(ExplicitKey); ok { 33 | return string(key), nil 34 | } 35 | 36 | if obj, ok := obj.(*pipeline.Step); ok { 37 | return obj.MakeObjectKey(), nil 38 | } 39 | 40 | return "", nil 41 | } 42 | -------------------------------------------------------------------------------- /common/informers/step/informer.go: -------------------------------------------------------------------------------- 1 | package step 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/infraboard/workflow/api/apps/node" 8 | "github.com/infraboard/workflow/api/apps/pipeline" 9 | "github.com/infraboard/workflow/common/cache" 10 | ) 11 | 12 | // Informer 负责事件通知 13 | type Informer interface { 14 | Watcher() Watcher 15 | Lister() Lister 16 | Recorder() Recorder 17 | GetStore() cache.Store 18 | } 19 | 20 | type Recorder interface { 21 | Update(*pipeline.Step) error 22 | } 23 | 24 | func NewListOptions() *ListOptions { 25 | return &ListOptions{} 26 | } 27 | 28 | type ListOptions struct { 29 | Key string 30 | } 31 | 32 | func (o *ListOptions) With(key string) *ListOptions { 33 | o.Key = key 34 | return o 35 | } 36 | 37 | type Lister interface { 38 | Get(ctx context.Context, key string) (*pipeline.Step, error) 39 | List(ctx context.Context) ([]*pipeline.Step, error) 40 | } 41 | 42 | // Watcher 负责事件通知 43 | type Watcher interface { 44 | // Run starts and runs the shared informer, returning after it stops. 45 | // The informer will be stopped when stopCh is closed. 46 | Run(ctx context.Context) error 47 | // AddEventHandler adds an event handler to the shared informer using the shared informer's resync 48 | // period. Events to a single handler are delivered sequentially, but there is no coordination 49 | // between different handlers. 50 | AddStepEventHandler(handler StepEventHandler) 51 | } 52 | 53 | // StepEventHandler can handle notifications for events that happen to a 54 | // resource. The events are informational only, so you can't return an 55 | // error. 56 | // * OnAdd is called when an object is added. 57 | // * OnUpdate is called when an object is modified. Note that oldObj is the 58 | // last known state of the object-- it is possible that several changes 59 | // were combined together, so you can't use this to see every single 60 | // change. OnUpdate is also called when a re-list happens, and it will 61 | // get called even if nothing changed. This is useful for periodically 62 | // evaluating or syncing something. 63 | // * OnDelete will get the final state of the item if it is known, otherwise 64 | // it will get an object of type DeletedFinalStateUnknown. This can 65 | // happen if the watch is closed and misses the delete event and we don't 66 | // notice the deletion until the subsequent re-list. 67 | type StepEventHandler interface { 68 | OnAdd(obj *pipeline.Step) 69 | OnUpdate(old, new *pipeline.Step) 70 | OnDelete(obj *pipeline.Step) 71 | } 72 | 73 | // StepEventHandlerFuncs is an adaptor to let you easily specify as many or 74 | // as few of the notification functions as you want while still implementing 75 | // ResourceEventHandler. 76 | type StepEventHandlerFuncs struct { 77 | AddFunc func(obj *pipeline.Step) 78 | UpdateFunc func(oldObj, newObj *pipeline.Step) 79 | DeleteFunc func(obj *pipeline.Step) 80 | } 81 | 82 | type UpdateStepCallback func(old, new *pipeline.Step) 83 | 84 | // OnAdd calls AddFunc if it's not nil. 85 | func (r StepEventHandlerFuncs) OnAdd(obj *pipeline.Step) { 86 | if r.AddFunc != nil { 87 | r.AddFunc(obj) 88 | } 89 | } 90 | 91 | // OnUpdate calls UpdateFunc if it's not nil. 92 | func (r StepEventHandlerFuncs) OnUpdate(oldObj, newObj *pipeline.Step) { 93 | if r.UpdateFunc != nil { 94 | r.UpdateFunc(oldObj, newObj) 95 | } 96 | } 97 | 98 | // OnDelete calls DeleteFunc if it's not nil. 99 | func (r StepEventHandlerFuncs) OnDelete(obj *pipeline.Step) { 100 | if r.DeleteFunc != nil { 101 | r.DeleteFunc(obj) 102 | } 103 | } 104 | 105 | type StepFilterHandler func(obj *pipeline.Step) error 106 | 107 | func NewNodeFilter(node *node.Node) StepFilterHandler { 108 | return func(obj *pipeline.Step) error { 109 | if !node.IsMatch(obj.ScheduledNodeName()) { 110 | return fmt.Errorf("step %s not match this node [%s], expect [%s]", obj.Key, node.Name(), obj.ScheduledNodeName()) 111 | } 112 | return nil 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /conf/load.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "github.com/BurntSushi/toml" 5 | "github.com/caarlos0/env/v6" 6 | ) 7 | 8 | var ( 9 | global *Config 10 | ) 11 | 12 | // C 全局配置对象 13 | func C() *Config { 14 | if global == nil { 15 | panic("Load Config first") 16 | } 17 | return global 18 | } 19 | 20 | func initGloabalInstance(cfg *Config) error { 21 | c, err := cfg.Etcd.getClient() 22 | if err != nil { 23 | return err 24 | } 25 | etcdClient = c 26 | 27 | mgo, err := cfg.Mongo.getClient() 28 | if err != nil { 29 | return err 30 | } 31 | mgoClient = mgo 32 | return nil 33 | } 34 | 35 | // LoadConfigFromToml 从toml中添加配置文件, 并初始化全局对象 36 | func LoadConfigFromToml(filePath string) error { 37 | cfg := newConfig() 38 | if _, err := toml.DecodeFile(filePath, cfg); err != nil { 39 | return err 40 | } 41 | 42 | if err := initGloabalInstance(cfg); err != nil { 43 | return err 44 | } 45 | 46 | // 加载全局配置单例 47 | global = cfg 48 | return nil 49 | } 50 | 51 | // LoadConfigFromEnv 从环境变量中加载配置 52 | func LoadConfigFromEnv() error { 53 | cfg := newConfig() 54 | if err := env.Parse(cfg); err != nil { 55 | return err 56 | } 57 | 58 | if err := initGloabalInstance(cfg); err != nil { 59 | return err 60 | } 61 | 62 | // 加载全局配置单例 63 | global = cfg 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /conf/log.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | // LogFormat 日志格式 4 | type LogFormat string 5 | 6 | const ( 7 | // TextFormat 文本格式 8 | TextFormat = LogFormat("text") 9 | // JSONFormat json格式 10 | JSONFormat = LogFormat("json") 11 | ) 12 | 13 | // LogTo 日志记录到哪儿 14 | type LogTo string 15 | 16 | const ( 17 | // ToFile 保存到文件 18 | ToFile = LogTo("file") 19 | // ToStdout 打印到标准输出 20 | ToStdout = LogTo("stdout") 21 | ) 22 | -------------------------------------------------------------------------------- /docs/deploy/install.md: -------------------------------------------------------------------------------- 1 | # 部署 2 | 3 | ## 简单部署 4 | ``` 5 | nohup ./workflow-api start -f /etc/workflow/workflow.toml &> workflow-api.log & 6 | nohup ./workflow-scheduler start -f /etc/workflow/workflow.toml &> workflow-scheduler.log & 7 | nohup ./workflow-node start -f /etc/workflow/workflow.toml &> workflow-node.log & 8 | ``` -------------------------------------------------------------------------------- /docs/etcd/deploy.md: -------------------------------------------------------------------------------- 1 | # 部署 2 | 3 | 4 | ## 单机部署 5 | 6 | 不带数据卷 7 | ``` 8 | docker run -d \ 9 | -p 2379:2379 \ 10 | -p 2380:2380 \ 11 | --name exam-etcd \ 12 | quay.io/coreos/etcd:latest \ 13 | /usr/local/bin/etcd \ 14 | --name s1 \ 15 | --listen-client-urls http://0.0.0.0:2379 \ 16 | --advertise-client-urls http://0.0.0.0:2379 \ 17 | --listen-peer-urls http://0.0.0.0:2380 \ 18 | --initial-advertise-peer-urls http://0.0.0.0:2380 \ 19 | --initial-cluster s1=http://0.0.0.0:2380 \ 20 | --initial-cluster-token tkn \ 21 | --initial-cluster-state new 22 | ``` 23 | 24 | 带数据卷 25 | ```sh 26 | mkdir /data/etcd -p 27 | docker run -d \ 28 | -p 32379:2379 \ 29 | -p 32380:2380 \ 30 | -v /data/etcd:/etcd-data/member \ 31 | --name exam-etcd \ 32 | quay.io/coreos/etcd:latest \ 33 | /usr/local/bin/etcd \ 34 | --name s1 \ 35 | --data-dir /etcd-data \ 36 | --listen-client-urls http://0.0.0.0:2379 \ 37 | --advertise-client-urls http://0.0.0.0:2379 \ 38 | --listen-peer-urls http://0.0.0.0:2380 \ 39 | --initial-advertise-peer-urls http://0.0.0.0:2380 \ 40 | --initial-cluster s1=http://0.0.0.0:2380 \ 41 | --initial-cluster-token tkn \ 42 | --initial-cluster-state new 43 | ``` 44 | 45 | ## 常用操作 46 | ```sh 47 | # 添加 48 | docker exec -e ETCDCTL_API=3 exam-etcd etcdctl --endpoints=http://127.0.0.1:2379 put foo bar 49 | # 查看 50 | docker exec -e ETCDCTL_API=3 exam-etcd etcdctl --endpoints=http://127.0.0.1:2379 get --prefix foo 51 | # 删除 52 | docker exec -e ETCDCTL_API=3 exam-etcd etcdctl --endpoints=http://127.0.0.1:2379 del foo 53 | ``` -------------------------------------------------------------------------------- /docs/keypoint/docker-in-docker.md: -------------------------------------------------------------------------------- 1 | # 如何为容器提供Docker工具 2 | 3 | 4 | 简单而言: 把宿主机的Docker工具挂载到容器里面, 扩展而言，我们可以在宿主机层面做一个工具箱, 上层容器按需挂载 5 | ``` 6 | # docker run -it -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker dockerindocker:1.0 /bin/bash 7 | ``` 8 | 9 | 参考: [docker 嵌套技术](https://blog.csdn.net/shida_csdn/article/details/79812817) -------------------------------------------------------------------------------- /docs/keypoint/git-pull.md: -------------------------------------------------------------------------------- 1 | 2 | # Git下载指定文件/文件夹 3 | 4 | 参考: [Sparse checkout解决pull远程库特定文件失败问题](https://blog.csdn.net/zzh920625/article/details/77073816) -------------------------------------------------------------------------------- /docs/keypoint/web-hook-test.md: -------------------------------------------------------------------------------- 1 | # webhook 如何调试 2 | 3 | 使用内网穿透工具, 比如[网云穿](https://i.xiaomy.net/#/tunnel) -------------------------------------------------------------------------------- /docs/sample/create_applicaiton.json: -------------------------------------------------------------------------------- 1 | { 2 | "object_kind": "push", 3 | "event_name": "push", 4 | "before": "f8a831144634f5810e17014582b5ba21267bb257", 5 | "after": "f8a831144634f5810e17014582b5ba21267bb257", 6 | "ref": "refs/heads/master", 7 | "checkout_sha": "f8a831144634f5810e17014582b5ba21267bb257", 8 | "message": null, 9 | "user_id": 9556442, 10 | "user_name": "紫川秀", 11 | "user_username": "yumaojun03", 12 | "user_email": "", 13 | "user_avatar": "https://secure.gravatar.com/avatar/1c8f622795d244227b2982871bc925d6?s=80&d=identicon", 14 | "project_id": 29032549, 15 | "project": { 16 | "id": 29032549, 17 | "name": "sample-devcloud", 18 | "description": "测试使用", 19 | "web_url": "https://gitlab.com/yumaojun03/sample-devcloud", 20 | "avatar_url": null, 21 | "git_ssh_url": "git@gitlab.com:yumaojun03/sample-devcloud.git", 22 | "git_http_url": "https://gitlab.com/yumaojun03/sample-devcloud.git", 23 | "namespace": "紫川秀", 24 | "visibility_level": 0, 25 | "path_with_namespace": "yumaojun03/sample-devcloud", 26 | "default_branch": "main", 27 | "ci_config_path": "", 28 | "homepage": "https://gitlab.com/yumaojun03/sample-devcloud", 29 | "url": "git@gitlab.com:yumaojun03/sample-devcloud.git", 30 | "ssh_url": "git@gitlab.com:yumaojun03/sample-devcloud.git", 31 | "http_url": "https://gitlab.com/yumaojun03/sample-devcloud.git" 32 | }, 33 | "commits": [ 34 | { 35 | "id": "f8a831144634f5810e17014582b5ba21267bb257", 36 | "message": "Initial commit", 37 | "title": "Initial commit", 38 | "timestamp": "2021-08-22T03:44:35+00:00", 39 | "url": "https://gitlab.com/yumaojun03/sample-devcloud/-/commit/f8a831144634f5810e17014582b5ba21267bb257", 40 | "author": { 41 | "name": "紫川秀", 42 | "email": "9556442-yumaojun03@users.noreply.gitlab.com" 43 | }, 44 | "added": [ 45 | "README.md" 46 | ], 47 | "modified": [], 48 | "removed": [] 49 | } 50 | ], 51 | "total_commits_count": 1, 52 | "push_options": {}, 53 | "repository": { 54 | "name": "sample-devcloud", 55 | "url": "git@gitlab.com:yumaojun03/sample-devcloud.git", 56 | "description": "测试使用", 57 | "homepage": "https://gitlab.com/yumaojun03/sample-devcloud", 58 | "git_http_url": "https://gitlab.com/yumaojun03/sample-devcloud.git", 59 | "git_ssh_url": "git@gitlab.com:yumaojun03/sample-devcloud.git", 60 | "visibility_level": 0 61 | } 62 | } -------------------------------------------------------------------------------- /docs/sample/gitlab_hook.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | header 4 | ``` 5 | Content-Type: application/json 6 | User-Agent: GitLab/14.3.0-pre 7 | X-Gitlab-Event: Push Hook 8 | X-Gitlab-Token: c4i3qfma0brjg21o7aag 9 | ``` 10 | 11 | body 12 | ```json 13 | { 14 | "object_kind": "push", 15 | "event_name": "push", 16 | "before": "f8a831144634f5810e17014582b5ba21267bb257", 17 | "after": "f8a831144634f5810e17014582b5ba21267bb257", 18 | "ref": "refs/heads/main", 19 | "checkout_sha": "f8a831144634f5810e17014582b5ba21267bb257", 20 | "message": null, 21 | "user_id": 9556442, 22 | "user_name": "紫川秀", 23 | "user_username": "yumaojun03", 24 | "user_email": "", 25 | "user_avatar": "https://secure.gravatar.com/avatar/1c8f622795d244227b2982871bc925d6?s=80&d=identicon", 26 | "project_id": 29032549, 27 | "project": { 28 | "id": 29032549, 29 | "name": "sample-devcloud", 30 | "description": "测试使用", 31 | "web_url": "https://gitlab.com/yumaojun03/sample-devcloud", 32 | "avatar_url": null, 33 | "git_ssh_url": "git@gitlab.com:yumaojun03/sample-devcloud.git", 34 | "git_http_url": "https://gitlab.com/yumaojun03/sample-devcloud.git", 35 | "namespace": "紫川秀", 36 | "visibility_level": 0, 37 | "path_with_namespace": "yumaojun03/sample-devcloud", 38 | "default_branch": "main", 39 | "ci_config_path": "", 40 | "homepage": "https://gitlab.com/yumaojun03/sample-devcloud", 41 | "url": "git@gitlab.com:yumaojun03/sample-devcloud.git", 42 | "ssh_url": "git@gitlab.com:yumaojun03/sample-devcloud.git", 43 | "http_url": "https://gitlab.com/yumaojun03/sample-devcloud.git" 44 | }, 45 | "commits": [ 46 | { 47 | "id": "f8a831144634f5810e17014582b5ba21267bb257", 48 | "message": "Initial commit", 49 | "title": "Initial commit", 50 | "timestamp": "2021-08-22T03:44:35+00:00", 51 | "url": "https://gitlab.com/yumaojun03/sample-devcloud/-/commit/f8a831144634f5810e17014582b5ba21267bb257", 52 | "author": { 53 | "name": "紫川秀", 54 | "email": "9556442-yumaojun03@users.noreply.gitlab.com" 55 | }, 56 | "added": [ 57 | "README.md" 58 | ], 59 | "modified": [ 60 | 61 | ], 62 | "removed": [ 63 | 64 | ] 65 | } 66 | ], 67 | "total_commits_count": 1, 68 | "push_options": { 69 | }, 70 | "repository": { 71 | "name": "sample-devcloud", 72 | "url": "git@gitlab.com:yumaojun03/sample-devcloud.git", 73 | "description": "测试使用", 74 | "homepage": "https://gitlab.com/yumaojun03/sample-devcloud", 75 | "git_http_url": "https://gitlab.com/yumaojun03/sample-devcloud.git", 76 | "git_ssh_url": "git@gitlab.com:yumaojun03/sample-devcloud.git", 77 | "visibility_level": 0 78 | } 79 | } 80 | ```` -------------------------------------------------------------------------------- /docs/sample/pipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"test44", 3 | "stages":[ 4 | { 5 | "name":"stage1", 6 | "steps":[ 7 | { 8 | "name":"step1.1", 9 | "action":"go_build@v2", 10 | "with":{ 11 | "ENV1":"env1", 12 | "ENV2":"env2" 13 | }, 14 | "with_audit":false, 15 | "is_parallel":false 16 | }, 17 | { 18 | "name":"step1.2", 19 | "action":"go_build@v2", 20 | "with":{ 21 | "ENV1":"env1", 22 | "ENV2":"env2" 23 | }, 24 | "with_audit":false, 25 | "is_parallel":false 26 | } 27 | ] 28 | }, 29 | { 30 | "name":"stage2", 31 | "steps":[ 32 | { 33 | "name":"step2.1", 34 | "action":"go_build@v2", 35 | "with":{ 36 | "ENV1":"env1", 37 | "ENV2":"env2" 38 | }, 39 | "with_audit":false, 40 | "is_parallel":false 41 | }, 42 | { 43 | "name":"step2.2", 44 | "action":"go_build@v2", 45 | "with":{ 46 | "ENV1":"env1", 47 | "ENV2":"env2" 48 | }, 49 | "with_audit":false, 50 | "is_parallel":false 51 | } 52 | ] 53 | } 54 | ] 55 | } -------------------------------------------------------------------------------- /docs/sample/pipeline.yaml: -------------------------------------------------------------------------------- 1 | name: test44 2 | stages: 3 | - name: stage1 4 | steps: 5 | - name: step1.1 6 | action: action01@v1 7 | with: 8 | ENV1: env1 9 | ENV2: env2 10 | with_audit: false 11 | is_parallel: true 12 | webhooks: 13 | - url: >- 14 | https://open.feishu.cn/open-apis/bot/v2/hook/83bde95c-00b2-4df1-91e4-705f66102479 15 | events: 16 | - SUCCEEDED 17 | - name: step1.2 18 | action: action01@v1 19 | with_audit: false 20 | is_parallel: true 21 | with: 22 | ENV1: env1 23 | ENV2: env2 24 | webhooks: 25 | - url: >- 26 | https://open.feishu.cn/open-apis/bot/v2/hook/83bde95c-00b2-4df1-91e4-705f66102479 27 | events: 28 | - SUCCEEDED 29 | - name: step1.3 30 | action: action01@v1 31 | with_audit: true 32 | with: 33 | ENV1: env1 34 | ENV2: env2 35 | webhooks: 36 | - url: >- 37 | https://open.feishu.cn/open-apis/bot/v2/hook/83bde95c-00b2-4df1-91e4-705f66102479 38 | events: 39 | - SUCCEEDED 40 | - name: stage2 41 | steps: 42 | - name: step2.1 43 | action: action01@v1 44 | with: 45 | ENV1: env3 46 | ENV2: env4 47 | webhooks: 48 | - url: >- 49 | https://open.feishu.cn/open-apis/bot/v2/hook/83bde95c-00b2-4df1-91e4-705f66102479 50 | events: 51 | - SUCCEEDED 52 | - name: step2.2 53 | action: action01@v1 54 | with: 55 | ENV1: env1 56 | ENV2: env2 57 | webhooks: 58 | - url: >- 59 | https://open.feishu.cn/open-apis/bot/v2/hook/83bde95c-00b2-4df1-91e4-705f66102479 60 | events: 61 | - SUCCEEDED -------------------------------------------------------------------------------- /etc/workflow_sample.toml: -------------------------------------------------------------------------------- 1 | [app] 2 | name = "workflow" 3 | key = "workflow_key" 4 | 5 | [http] 6 | host = "127.0.0.1" 7 | port = "8848" 8 | enable_ssl = false 9 | cert_file = "" 10 | key_file = "" 11 | 12 | [grpc] 13 | host = "127.0.0.1" 14 | port = "18848" 15 | enable_ssl = false 16 | cert_file = "" 17 | key_file = "" 18 | 19 | [mongodb] 20 | endpoints = ["127.0.0.1:27017"] 21 | username = "workflow" 22 | password = "workflow" 23 | database = "workflow" 24 | 25 | [etcd] 26 | endpoints = ["127.0.0.1:2379"] 27 | username = "workflow" 28 | password = "workflow" 29 | prefix = "inforboard" 30 | instance_ttl = 300 31 | 32 | 33 | [log] 34 | level = "debug" 35 | path = "logs" 36 | format = "text" 37 | to = "stdout" 38 | 39 | [keyauth] 40 | host = "10.40.44.101" 41 | port = "8050" 42 | client_id = "" 43 | client_secret = "" 44 | 45 | [cache] 46 | 47 | [nats] 48 | 49 | [bus] 50 | type = "nats" 51 | 52 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/infraboard/workflow 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/BurntSushi/toml v0.3.1 7 | github.com/caarlos0/env/v6 v6.6.0 8 | github.com/chyroc/lark v0.0.92 9 | github.com/containerd/containerd v1.5.2 // indirect 10 | github.com/docker/docker v20.10.7+incompatible 11 | github.com/docker/go-connections v0.4.0 // indirect 12 | github.com/go-playground/validator/v10 v10.9.0 13 | github.com/gorilla/websocket v1.4.2 14 | github.com/infraboard/keyauth v0.6.4 15 | github.com/infraboard/mcube v1.5.4 16 | github.com/morikuni/aec v1.0.0 // indirect 17 | github.com/rs/xid v1.3.0 18 | github.com/spf13/cobra v1.2.1 19 | github.com/stretchr/testify v1.7.0 20 | go.etcd.io/etcd/api/v3 v3.5.1 21 | go.etcd.io/etcd/client/v3 v3.5.1 22 | go.mongodb.org/mongo-driver v1.7.1 23 | google.golang.org/grpc v1.38.0 24 | google.golang.org/protobuf v1.27.1 25 | k8s.io/apimachinery v0.20.6 26 | k8s.io/client-go v0.20.6 27 | ) 28 | -------------------------------------------------------------------------------- /node/cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/spf13/cobra" 9 | 10 | "github.com/infraboard/workflow/version" 11 | ) 12 | 13 | var vers bool 14 | 15 | // RootCmd represents the base command when called without any subcommands 16 | var RootCmd = &cobra.Command{ 17 | Use: "workflow-node", 18 | Short: "workflow-node 流水线node", 19 | Long: `workflow-node 流水线node`, 20 | RunE: func(cmd *cobra.Command, args []string) error { 21 | if vers { 22 | fmt.Println(version.FullVersion()) 23 | return nil 24 | } 25 | return errors.New("no flags find") 26 | }, 27 | } 28 | 29 | // Execute adds all child commands to the root command sets flags appropriately. 30 | // This is called by main.main(). It only needs to happen once to the rootCmd. 31 | func Execute() { 32 | if err := RootCmd.Execute(); err != nil { 33 | fmt.Println(err) 34 | os.Exit(-1) 35 | } 36 | } 37 | func init() { 38 | RootCmd.PersistentFlags().BoolVarP(&vers, "version", "v", false, "the workflow version") 39 | } 40 | -------------------------------------------------------------------------------- /node/controller/step/engine/cancel.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/infraboard/mcube/grpc/gcontext" 7 | 8 | "github.com/infraboard/workflow/api/apps/action" 9 | "github.com/infraboard/workflow/api/apps/pipeline" 10 | "github.com/infraboard/workflow/node/controller/step/runner" 11 | ) 12 | 13 | func (e *Engine) CancelStep(s *pipeline.Step) { 14 | if !e.init { 15 | s.Failed("engine not init") 16 | return 17 | } 18 | 19 | e.log.Debugf("start cancel step: %s", s.Key) 20 | // 构造运行请求 21 | req := runner.NewCancelRequest(s) 22 | 23 | // 1.查询step对应的action定义 24 | descA := action.NewDescribeActionRequest(s.ActionName(), s.ActionVersion()) 25 | ctx := gcontext.NewGrpcOutCtx() 26 | actionIns, err := e.wc.Action().DescribeAction(ctx.Context(), descA) 27 | if err != nil { 28 | s.Failed("describe step action error, %s", err) 29 | return 30 | } 31 | 32 | // 3.根据action定义的runner_type, 调用具体的runner 33 | switch actionIns.RunnerType { 34 | case action.RUNNER_TYPE_DOCKER: 35 | go e.docker.Cancel(context.Background(), req) 36 | case action.RUNNER_TYPE_K8s: 37 | go e.k8s.Cancel(context.Background(), req) 38 | case action.RUNNER_TYPE_LOCAL: 39 | go e.local.Cancel(context.Background(), req) 40 | default: 41 | s.Failed("unknown runner type: %s", actionIns.RunnerType) 42 | return 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /node/controller/step/engine/engine.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/infraboard/mcube/logger" 8 | "github.com/infraboard/mcube/logger/zap" 9 | 10 | "github.com/infraboard/workflow/api/apps/pipeline" 11 | "github.com/infraboard/workflow/api/client" 12 | "github.com/infraboard/workflow/common/informers/step" 13 | "github.com/infraboard/workflow/node/controller/step/runner" 14 | "github.com/infraboard/workflow/node/controller/step/runner/docker" 15 | "github.com/infraboard/workflow/node/controller/step/runner/k8s" 16 | "github.com/infraboard/workflow/node/controller/step/runner/local" 17 | ) 18 | 19 | var ( 20 | engine = &Engine{} 21 | ) 22 | 23 | func RunStep(ctx context.Context, s *pipeline.Step) { 24 | // 开始执行, 更新状态 25 | s.Run() 26 | engine.updateStep(s) 27 | 28 | // 执行step 29 | go engine.Run(ctx, s) 30 | } 31 | 32 | func CancelStep(s *pipeline.Step) { 33 | engine.CancelStep(s) 34 | } 35 | 36 | func Init(wc *client.ClientSet, recorder step.Recorder) (err error) { 37 | if wc == nil { 38 | return fmt.Errorf("init runner error, workflow client is nil") 39 | } 40 | 41 | engine.log = zap.L().Named("Runner.Engine") 42 | engine.recorder = recorder 43 | engine.wc = wc 44 | engine.docker, err = docker.NewRunner() 45 | engine.k8s = k8s.NewRunner() 46 | engine.local = local.NewRunner() 47 | 48 | if err != nil { 49 | return err 50 | } 51 | 52 | engine.init = true 53 | return nil 54 | } 55 | 56 | type Engine struct { 57 | recorder step.Recorder 58 | wc *client.ClientSet 59 | docker runner.Runner 60 | k8s runner.Runner 61 | local runner.Runner 62 | init bool 63 | log logger.Logger 64 | } 65 | -------------------------------------------------------------------------------- /node/controller/step/engine/run.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/infraboard/workflow/api/apps/action" 7 | "github.com/infraboard/workflow/api/apps/pipeline" 8 | "github.com/infraboard/workflow/node/controller/step/runner" 9 | ) 10 | 11 | func (e *Engine) Run(ctx context.Context, s *pipeline.Step) { 12 | req := runner.NewRunRequest(s) 13 | resp := runner.NewRunReponse(e.updateStep) 14 | 15 | e.run(ctx, req, resp) 16 | 17 | if resp.HasError() { 18 | s.Failed(resp.ErrorMessage()) 19 | } else { 20 | s.Success("") 21 | } 22 | 23 | e.updateStep(s) 24 | } 25 | 26 | // Run 运行Step 27 | // step的参数加载优先级: 28 | // 1. step 本身传人的 29 | // 2. pipeline 运行中产生的 30 | // 3. pipeline 全局传人 31 | // 4. action 默认默认值 32 | func (e *Engine) run(ctx context.Context, req *runner.RunRequest, resp *runner.RunResponse) { 33 | if !e.init { 34 | resp.Failed("engine not init") 35 | return 36 | } 37 | 38 | s := req.Step 39 | 40 | e.log.Debugf("start run step: %s status %s", s.Key, s.Status) 41 | 42 | // 1.查询step对应的action定义 43 | descA := action.NewDescribeActionRequest(s.ActionName(), s.ActionVersion()) 44 | actionIns, err := e.wc.Action().DescribeAction(ctx, descA) 45 | if err != nil { 46 | resp.Failed("describe step action error, %s", err) 47 | return 48 | } 49 | 50 | // 2.加载Action默认参数 51 | req.LoadRunParams(actionIns.DefaultRunParam()) 52 | 53 | // 3.查询Pipeline, 加载全局参数 54 | if s.IsCreateByPipeline() { 55 | descP := pipeline.NewDescribePipelineRequestWithID(s.GetPipelineId()) 56 | descP.Namespace = s.GetNamespace() 57 | pl, err := e.wc.Pipeline().DescribePipeline(ctx, descP) 58 | if err != nil { 59 | resp.Failed("describe step pipeline error, %s", err) 60 | return 61 | } 62 | req.LoadRunParams(pl.With) 63 | req.LoadMount(pl.Mount) 64 | } 65 | 66 | // 4. 加载step传递的参数 67 | req.LoadRunParams(s.With) 68 | 69 | // 校验run参数合法性 70 | if err := actionIns.ValidateRunParam(req.RunParams); err != nil { 71 | resp.Failed(err.Error()) 72 | return 73 | } 74 | 75 | // 加载Runner运行需要的参数 76 | req.LoadRunnerParams(actionIns.RunnerParam()) 77 | 78 | e.log.Debugf("choice %s runner to run step", actionIns.RunnerType) 79 | // 3.根据action定义的runner_type, 调用具体的runner 80 | switch actionIns.RunnerType { 81 | case action.RUNNER_TYPE_DOCKER: 82 | e.docker.Run(context.Background(), req, resp) 83 | case action.RUNNER_TYPE_K8s: 84 | e.k8s.Run(context.Background(), req, resp) 85 | case action.RUNNER_TYPE_LOCAL: 86 | e.local.Run(context.Background(), req, resp) 87 | default: 88 | resp.Failed("unknown runner type: %s", actionIns.RunnerType) 89 | return 90 | } 91 | } 92 | 93 | // 如果step执行完成 94 | func (e *Engine) updateStep(s *pipeline.Step) { 95 | e.log.Debugf("receive step %s update, status %s", s.Key, s.Status) 96 | if err := e.recorder.Update(s.Clone()); err != nil { 97 | e.log.Errorf("update step status error, %s", err) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /node/controller/step/handler.go: -------------------------------------------------------------------------------- 1 | package step 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/infraboard/workflow/api/apps/pipeline" 9 | "github.com/infraboard/workflow/node/controller/step/engine" 10 | ) 11 | 12 | // syncHandler compares the actual state with the desired, and attempts to 13 | // converge the two. It then updates the Status block of the Network resource 14 | // with the current status of the resource. 15 | func (c *Controller) syncHandler(key string) error { 16 | obj, ok, err := c.informer.GetStore().GetByKey(key) 17 | if err != nil { 18 | return err 19 | } 20 | 21 | // 如果不存在, 这期望行为为删除 (DEL) 22 | if !ok { 23 | c.log.Debugf("remove step: %s, skip", key) 24 | } 25 | 26 | st, isOK := obj.(*pipeline.Step) 27 | if !isOK { 28 | return errors.New("invalidate *pipeline.Step obj") 29 | } 30 | 31 | // 添加step 32 | if err := c.addStep(st); err != nil { 33 | return err 34 | } 35 | return nil 36 | } 37 | 38 | func (c *Controller) addStep(s *pipeline.Step) error { 39 | status := s.Status.Status 40 | switch status { 41 | case pipeline.STEP_STATUS_PENDDING: 42 | engine.RunStep(context.Background(), s) 43 | return nil 44 | case pipeline.STEP_STATUS_RUNNING: 45 | // TODO: 判断引擎中该step状态是否一致 46 | // 如果不一致则同步状态, 但是不作再次运行 47 | c.log.Debugf("step is running, no thing todo") 48 | case pipeline.STEP_STATUS_CANCELING: 49 | return c.cancelStep(s) 50 | case pipeline.STEP_STATUS_SUCCEEDED, 51 | pipeline.STEP_STATUS_FAILED, 52 | pipeline.STEP_STATUS_CANCELED, 53 | pipeline.STEP_STATUS_SKIP, 54 | pipeline.STEP_STATUS_REFUSE: 55 | return fmt.Errorf("step %s status is %s has complete", s.Key, status) 56 | case pipeline.STEP_STATUS_AUDITING: 57 | return fmt.Errorf("step %s is %s, is auditing", s.Key, status) 58 | } 59 | 60 | return nil 61 | } 62 | 63 | func (c *Controller) cancelStep(s *pipeline.Step) error { 64 | c.log.Infof("receive cancel object: %s", s) 65 | if err := s.Validate(); err != nil { 66 | c.log.Errorf("invalidate node error, %s", err) 67 | return nil 68 | } 69 | 70 | // 已经完成的step不作处理 71 | if s.IsComplete() { 72 | c.log.Debugf("step [%s] is complete, skip cancel", s.Key) 73 | } 74 | 75 | engine.CancelStep(s) 76 | return nil 77 | } 78 | 79 | // 当step删除时, 如果任务还在运行, 直接kill掉该任务 80 | func (c *Controller) deleteStep(key string) error { 81 | // 取消任务 82 | 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /node/controller/step/runner/docker/request.go: -------------------------------------------------------------------------------- 1 | package docker 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/infraboard/workflow/api/apps/pipeline" 8 | "github.com/infraboard/workflow/node/controller/step/runner" 9 | ) 10 | 11 | func newDockerRunRequest(r *runner.RunRequest) *dockerRunRequest { 12 | if r.Step.Status == nil { 13 | r.Step.Status = pipeline.NewDefaultStepStatus() 14 | } 15 | return &dockerRunRequest{r} 16 | } 17 | 18 | type dockerRunRequest struct { 19 | *runner.RunRequest 20 | } 21 | 22 | func (r *dockerRunRequest) Image() string { 23 | if r.ImageVersion() == "" { 24 | return r.ImageURL() 25 | } 26 | return fmt.Sprintf("%s:%s", r.ImageURL(), r.ImageVersion()) 27 | } 28 | 29 | func (r *dockerRunRequest) ImageURL() string { 30 | return r.RunnerParams[IMAGE_URL_KEY] 31 | } 32 | 33 | func (r *dockerRunRequest) ImageVersion() string { 34 | return r.RunnerParams[IMAGE_VERSION_KEY] 35 | } 36 | 37 | func (r *dockerRunRequest) ContainerName() string { 38 | return r.Step.Key 39 | } 40 | 41 | func (r *dockerRunRequest) ContainerEnv() []string { 42 | envs := []string{} 43 | for k, v := range r.mergeParams() { 44 | envs = append(envs, fmt.Sprintf("%s=%s", k, v)) 45 | } 46 | return envs 47 | } 48 | 49 | func (r *dockerRunRequest) ContainerCMD() []string { 50 | return strings.Split(r.RunnerParams[IMAGE_CMD_KEY], ",") 51 | } 52 | 53 | func (r *dockerRunRequest) mergeParams() map[string]string { 54 | m := r.RunParams 55 | for k, v := range r.Step.With { 56 | m[k] = v 57 | } 58 | return m 59 | } 60 | 61 | func (r *dockerRunRequest) Validate() error { 62 | if r.Step == nil || r.Step.Key == "" { 63 | return fmt.Errorf("step is nil or step key is \"\"") 64 | } 65 | 66 | if r.ImageURL() == "" { 67 | return fmt.Errorf("%s missed", IMAGE_URL_KEY) 68 | } 69 | 70 | return nil 71 | } 72 | 73 | func newDockerCancelRequest(r *runner.CancelRequest) *dockerCancelRequest { 74 | if r.Step.Status == nil { 75 | r.Step.Status = pipeline.NewDefaultStepStatus() 76 | } 77 | return &dockerCancelRequest{r} 78 | } 79 | 80 | type dockerCancelRequest struct { 81 | *runner.CancelRequest 82 | } 83 | 84 | func (r *dockerCancelRequest) ContainerID() string { 85 | if r.Step == nil || r.Step.Status == nil || r.Step.Status.Response == nil { 86 | return "" 87 | } 88 | 89 | return r.Step.Status.Response[CONTAINER_ID_KEY] 90 | } 91 | 92 | func (r *dockerCancelRequest) Validate() error { 93 | if r.Step == nil || r.Step.Key == "" { 94 | return fmt.Errorf("step is nil or step key is \"\"") 95 | } 96 | 97 | if r.ContainerID() == "" { 98 | return fmt.Errorf("%s missed", CONTAINER_ID_KEY) 99 | } 100 | 101 | return nil 102 | } 103 | -------------------------------------------------------------------------------- /node/controller/step/runner/docker/runner_test.go: -------------------------------------------------------------------------------- 1 | package docker_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | "github.com/infraboard/mcube/logger/zap" 10 | "github.com/infraboard/workflow/api/apps/pipeline" 11 | "github.com/infraboard/workflow/node/controller/step/runner" 12 | "github.com/infraboard/workflow/node/controller/step/runner/docker" 13 | ) 14 | 15 | var ( 16 | dr *docker.Runner 17 | ) 18 | 19 | var ( 20 | smapleStep = &pipeline.Step{Key: "test"} 21 | runnerParams = map[string]string{ 22 | "IMAGE_URL": "busybox", 23 | "IMAGE_CMD": "date", 24 | } 25 | resp = runner.NewRunReponse(testUpdater) 26 | ) 27 | 28 | func testUpdater(s *pipeline.Step) { 29 | fmt.Println(s) 30 | } 31 | 32 | func TestRunNilStep(t *testing.T) { 33 | req := runner.NewRunRequest(nil) 34 | 35 | dr.Run(context.Background(), req, resp) 36 | } 37 | 38 | func TestRunNULLStep(t *testing.T) { 39 | req := runner.NewRunRequest(&pipeline.Step{}) 40 | dr.Run(context.Background(), req, resp) 41 | t.Log(req.Step) 42 | } 43 | 44 | func TestDockerRunSampleStep(t *testing.T) { 45 | req := runner.NewRunRequest(smapleStep) 46 | dr.Run(context.Background(), req, resp) 47 | t.Log(smapleStep) 48 | } 49 | 50 | func TestRunStepWithRunnerParams(t *testing.T) { 51 | req := runner.NewRunRequest(smapleStep) 52 | req.LoadRunnerParams(runnerParams) 53 | dr.Run(context.Background(), req, resp) 54 | t.Log(smapleStep) 55 | } 56 | 57 | func TestCancelStep(t *testing.T) { 58 | req := runner.NewRunRequest(smapleStep) 59 | req.LoadRunnerParams(cmdRunnerParams("busybox", "/bin/sleep,10")) 60 | go dr.Run(context.Background(), req, resp) 61 | 62 | time.Sleep(3 * time.Second) 63 | dr.Cancel(context.Background(), runner.NewCancelRequest(req.Step)) 64 | // 等待容器退出 65 | time.Sleep(3 * time.Second) 66 | t.Log(resp) 67 | 68 | } 69 | 70 | func cmdRunnerParams(image, cmd string) map[string]string { 71 | return map[string]string{ 72 | "IMAGE_URL": image, 73 | "IMAGE_CMD": cmd, 74 | } 75 | } 76 | 77 | func init() { 78 | if err := zap.DevelopmentSetup(); err != nil { 79 | panic(err) 80 | } 81 | r, err := docker.NewRunner() 82 | if err != nil { 83 | panic(err) 84 | } 85 | dr = r 86 | 87 | } 88 | -------------------------------------------------------------------------------- /node/controller/step/runner/http/runner.go: -------------------------------------------------------------------------------- 1 | package http 2 | -------------------------------------------------------------------------------- /node/controller/step/runner/k8s/runner.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/infraboard/workflow/api/apps/action" 8 | "github.com/infraboard/workflow/node/controller/step/runner" 9 | ) 10 | 11 | func ParamsDesc() []*action.RunParamDesc { 12 | return []*action.RunParamDesc{} 13 | } 14 | 15 | func NewRunner() *Runner { 16 | return &Runner{} 17 | } 18 | 19 | type Runner struct { 20 | } 21 | 22 | func (r *Runner) Run(ctx context.Context, in *runner.RunRequest, out *runner.RunResponse) { 23 | } 24 | 25 | func (r *Runner) Log(context.Context, *runner.LogRequest) (io.ReadCloser, error) { 26 | return nil, nil 27 | } 28 | 29 | func (r *Runner) Connect(context.Context, *runner.ConnectRequest) error { 30 | return nil 31 | } 32 | 33 | func (r *Runner) Cancel(context.Context, *runner.CancelRequest) { 34 | return 35 | } 36 | -------------------------------------------------------------------------------- /node/controller/step/runner/local/runner.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/infraboard/workflow/api/apps/action" 8 | "github.com/infraboard/workflow/node/controller/step/runner" 9 | ) 10 | 11 | func ParamsDesc() []*action.RunParamDesc { 12 | return []*action.RunParamDesc{} 13 | } 14 | 15 | func NewRunner() *Runner { 16 | return &Runner{} 17 | } 18 | 19 | type Runner struct { 20 | } 21 | 22 | func (r *Runner) Run(ctx context.Context, in *runner.RunRequest, out *runner.RunResponse) { 23 | } 24 | 25 | func (r *Runner) Log(context.Context, *runner.LogRequest) (io.ReadCloser, error) { 26 | return nil, nil 27 | } 28 | 29 | func (r *Runner) Connect(context.Context, *runner.ConnectRequest) error { 30 | return nil 31 | } 32 | 33 | func (r *Runner) Cancel(context.Context, *runner.CancelRequest) { 34 | return 35 | } 36 | -------------------------------------------------------------------------------- /node/controller/step/runner/runner.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "strings" 8 | 9 | "github.com/infraboard/workflow/api/apps/pipeline" 10 | ) 11 | 12 | type Runner interface { 13 | // 执行Step, 执行过后的关联信息保存在Status的Response里面 14 | Run(context.Context, *RunRequest, *RunResponse) 15 | // 连接到该执行环境 16 | Connect(context.Context, *ConnectRequest) error 17 | // 取消Step的执行 18 | Cancel(context.Context, *CancelRequest) 19 | } 20 | 21 | func NewRunRequest(s *pipeline.Step) *RunRequest { 22 | return &RunRequest{ 23 | Step: s, 24 | RunnerParams: map[string]string{}, 25 | RunParams: map[string]string{}, 26 | } 27 | } 28 | 29 | type RunRequest struct { 30 | RunnerParams map[string]string // runner 运行需要的参数 31 | RunParams map[string]string // step 运行需要的参数 32 | Mount *pipeline.MountData // 挂载文件 33 | Step *pipeline.Step // 具体step 34 | } 35 | 36 | func (r *RunRequest) LoadMount(m *pipeline.MountData) { 37 | r.Mount = m 38 | } 39 | 40 | func (r *RunRequest) LoadRunParams(params map[string]string) { 41 | for k, v := range params { 42 | r.RunParams[k] = v 43 | } 44 | } 45 | 46 | func (r *RunRequest) LoadRunnerParams(params map[string]string) { 47 | for k, v := range params { 48 | r.RunnerParams[k] = v 49 | } 50 | } 51 | 52 | func NewRunReponse(updater UpdateStepCallback) *RunResponse { 53 | return &RunResponse{ 54 | updater: updater, 55 | resp: map[string]string{}, 56 | ctx: map[string]string{}, 57 | } 58 | } 59 | 60 | type UpdateStepCallback func(*pipeline.Step) 61 | 62 | type RunResponse struct { 63 | updater UpdateStepCallback // 更新状态的回调 64 | errs []string 65 | resp map[string]string 66 | ctx map[string]string 67 | } 68 | 69 | func (r *RunResponse) UpdateReponseMap(k, v string) { 70 | r.resp[k] = v 71 | } 72 | 73 | func (r *RunResponse) UpdateCtxMap(k, v string) { 74 | r.ctx[k] = v 75 | } 76 | 77 | func (r *RunResponse) UpdateResponse(s *pipeline.Step) { 78 | s.UpdateResponse(r.resp) 79 | s.UpdateCtx(r.ctx) 80 | r.updater(s) 81 | } 82 | 83 | func (r *RunResponse) Failed(format string, a ...interface{}) { 84 | r.errs = append(r.errs, fmt.Sprintf(format, a...)) 85 | } 86 | 87 | func (r *RunResponse) HasError() bool { 88 | return len(r.errs) > 0 89 | } 90 | 91 | func (r *RunResponse) ErrorMessage() string { 92 | return strings.Join(r.errs, ",") 93 | } 94 | 95 | type LogRequest struct { 96 | Step *pipeline.Step 97 | } 98 | 99 | func NewCancelRequest(s *pipeline.Step) *CancelRequest { 100 | return &CancelRequest{ 101 | Step: s, 102 | } 103 | } 104 | 105 | type CancelRequest struct { 106 | Step *pipeline.Step 107 | } 108 | 109 | // // ConnectRequest holds information pertaining to the current streaming session: 110 | // // input/output streams, if the client is requesting a TTY, and a terminal size queue to 111 | // // support terminal resizing. 112 | type ConnectRequest struct { 113 | Step *pipeline.Step 114 | Stdin io.Reader 115 | Stdout io.Writer 116 | Stderr io.Writer 117 | Tty bool 118 | TerminalSizeQueue TerminalSizeQueue 119 | } 120 | 121 | // TerminalSize and TerminalSizeQueue was a part of k8s.io/kubernetes/pkg/util/term 122 | // and were moved in order to decouple client from other term dependencies 123 | 124 | // TerminalSize represents the width and height of a terminal. 125 | type TerminalSize struct { 126 | Width uint16 127 | Height uint16 128 | } 129 | 130 | // TerminalSizeQueue is capable of returning terminal resize events as they occur. 131 | type TerminalSizeQueue interface { 132 | // Next returns the new terminal size after the terminal has been resized. It returns nil when 133 | // monitoring has been stopped. 134 | Next() *TerminalSize 135 | } 136 | -------------------------------------------------------------------------------- /node/controller/step/store/file/uploader.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | "os" 9 | "path" 10 | "time" 11 | ) 12 | 13 | func NewUploader(id string) *Uploader { 14 | return &Uploader{ 15 | id: id, 16 | root: "runner_log", 17 | parent: dateDir(), 18 | } 19 | } 20 | 21 | type Uploader struct { 22 | id string 23 | root string 24 | parent string 25 | } 26 | 27 | func (u *Uploader) DriverName() string { 28 | return "local_file" 29 | } 30 | func (u *Uploader) ObjectID() string { 31 | return path.Join(u.root, u.parent, u.id) 32 | } 33 | func (u *Uploader) Upload(ctx context.Context, stream io.ReadCloser) error { 34 | defer stream.Close() 35 | 36 | f, err := u.createFile(ctx) 37 | if err != nil { 38 | return fmt.Errorf("create file error, %s", err) 39 | } 40 | defer f.Close() 41 | 42 | w := bufio.NewWriter(f) 43 | _, err = w.ReadFrom(stream) 44 | if err != nil { 45 | return err 46 | } 47 | if err := w.Flush(); err != nil { 48 | return fmt.Errorf("flush file error, %s", err) 49 | } 50 | return nil 51 | } 52 | 53 | func (u *Uploader) createFile(ctx context.Context) (*os.File, error) { 54 | fp := u.ObjectID() 55 | if checkFileIsExist(fp) { 56 | return os.OpenFile(fp, os.O_TRUNC|os.O_WRONLY, os.ModePerm) 57 | } 58 | 59 | if err := os.MkdirAll(path.Dir(fp), os.ModePerm); err != nil { 60 | return nil, err 61 | } 62 | 63 | return os.Create(fp) 64 | } 65 | 66 | func dateDir() string { 67 | year, month, day := time.Now().Date() 68 | return fmt.Sprintf("%d/%d/%d", year, int(month), day) 69 | } 70 | 71 | // 判断文件是否存在 存在返回 true 不存在返回false 72 | func checkFileIsExist(filepath string) bool { 73 | var exist = true 74 | if _, err := os.Stat(filepath); os.IsNotExist(err) { 75 | exist = false 76 | } 77 | return exist 78 | } 79 | -------------------------------------------------------------------------------- /node/controller/step/store/file/uploader_test.go: -------------------------------------------------------------------------------- 1 | package file_test 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | 11 | "github.com/infraboard/workflow/node/controller/step/store/file" 12 | ) 13 | 14 | func TestUpload(t *testing.T) { 15 | should := assert.New(t) 16 | 17 | buffer := io.NopCloser(strings.NewReader("hello world")) 18 | store := file.NewUploader("c16mhsddrei91m4ri0jg.c3iqcama0brimaq08e40.2.1") 19 | err := store.Upload(context.Background(), buffer) 20 | should.NoError(err) 21 | } 22 | -------------------------------------------------------------------------------- /node/controller/step/store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/infraboard/workflow/node/controller/step/store/file" 8 | ) 9 | 10 | // 保存Runner运行中的日志 11 | type StoreFactory interface { 12 | NewFileUploader(key string) Uploader 13 | } 14 | 15 | // 用于上传日志 16 | type Uploader interface { 17 | DriverName() string 18 | ObjectID() string 19 | Upload(ctx context.Context, steam io.ReadCloser) error 20 | } 21 | 22 | func NewStore() *Store { 23 | return &Store{} 24 | } 25 | 26 | type Store struct{} 27 | 28 | func (s *Store) NewFileUploader(key string) Uploader { 29 | return file.NewUploader(key) 30 | } 31 | -------------------------------------------------------------------------------- /node/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/infraboard/workflow/node/cmd" 5 | ) 6 | 7 | func main() { 8 | cmd.Execute() 9 | } 10 | -------------------------------------------------------------------------------- /scheduler/README.md: -------------------------------------------------------------------------------- 1 | # 主要流程 2 | + 加载 pipeline 3 | + 加载 node -------------------------------------------------------------------------------- /scheduler/algorithm/picker.go: -------------------------------------------------------------------------------- 1 | package algorithm 2 | 3 | import ( 4 | "github.com/infraboard/workflow/api/apps/node" 5 | "github.com/infraboard/workflow/api/apps/pipeline" 6 | ) 7 | 8 | // Picker 挑选一个合适的node 运行Step 9 | type StepPicker interface { 10 | Pick(*pipeline.Step) (*node.Node, error) 11 | } 12 | 13 | type PipelinePicker interface { 14 | Pick(*pipeline.Pipeline) (*node.Node, error) 15 | } 16 | -------------------------------------------------------------------------------- /scheduler/algorithm/roundrobin/roundrobin.go: -------------------------------------------------------------------------------- 1 | package roundrobin 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/infraboard/workflow/api/apps/node" 8 | "github.com/infraboard/workflow/api/apps/pipeline" 9 | "github.com/infraboard/workflow/common/cache" 10 | "github.com/infraboard/workflow/scheduler/algorithm" 11 | ) 12 | 13 | type roundrobinPicker struct { 14 | mu *sync.Mutex 15 | next int 16 | store cache.Store 17 | } 18 | 19 | // NewStepPicker 实现分调度 20 | func NewStepPicker(nodestore cache.Store) (algorithm.StepPicker, error) { 21 | base := &roundrobinPicker{ 22 | store: nodestore, 23 | mu: new(sync.Mutex), 24 | next: 0, 25 | } 26 | return &stepPicker{base}, nil 27 | } 28 | 29 | type stepPicker struct { 30 | *roundrobinPicker 31 | } 32 | 33 | func (p *stepPicker) Pick(step *pipeline.Step) (*node.Node, error) { 34 | p.mu.Lock() 35 | defer p.mu.Unlock() 36 | 37 | nodes := p.store.List() 38 | if len(nodes) == 0 { 39 | return nil, fmt.Errorf("has no available nodes") 40 | } 41 | 42 | ns := []*node.Node{} 43 | for i := range nodes { 44 | n := nodes[i].(*node.Node) 45 | if n.Type == node.NodeType { 46 | ns = append(ns, n) 47 | } 48 | } 49 | 50 | if len(ns) == 0 { 51 | return nil, fmt.Errorf("has no available node nodes") 52 | } 53 | 54 | n := ns[p.next] 55 | // 修改状态 56 | p.next = (p.next + 1) % len(ns) 57 | 58 | return n, nil 59 | } 60 | 61 | // NewPipelinePicker 实现分调度 62 | func NewPipelinePicker(nodestore cache.Store) (algorithm.PipelinePicker, error) { 63 | base := &roundrobinPicker{ 64 | store: nodestore, 65 | mu: new(sync.Mutex), 66 | next: 0, 67 | } 68 | return &pipelinePicker{base}, nil 69 | } 70 | 71 | type pipelinePicker struct { 72 | *roundrobinPicker 73 | } 74 | 75 | func (p *pipelinePicker) Pick(step *pipeline.Pipeline) (*node.Node, error) { 76 | p.mu.Lock() 77 | defer p.mu.Unlock() 78 | 79 | nodes := p.store.List() 80 | if len(nodes) == 0 { 81 | return nil, fmt.Errorf("has no available nodes") 82 | } 83 | 84 | schs := []*node.Node{} 85 | for i := range nodes { 86 | n := nodes[i].(*node.Node) 87 | if n.Type == node.SchedulerType { 88 | schs = append(schs, n) 89 | } 90 | } 91 | 92 | if len(schs) == 0 { 93 | return nil, fmt.Errorf("has no available sch nodes") 94 | } 95 | 96 | n := schs[p.next] 97 | // 修改状态 98 | p.next = (p.next + 1) % len(schs) 99 | 100 | return n, nil 101 | } 102 | -------------------------------------------------------------------------------- /scheduler/cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/spf13/cobra" 9 | 10 | "github.com/infraboard/workflow/version" 11 | ) 12 | 13 | var vers bool 14 | 15 | // RootCmd represents the base command when called without any subcommands 16 | var RootCmd = &cobra.Command{ 17 | Use: "workflow-scheduler", 18 | Short: "workflow-scheduler 流水线调度器", 19 | Long: `workflow-scheduler 流水线调度器`, 20 | RunE: func(cmd *cobra.Command, args []string) error { 21 | if vers { 22 | fmt.Println(version.FullVersion()) 23 | return nil 24 | } 25 | return errors.New("no flags find") 26 | }, 27 | } 28 | 29 | // Execute adds all child commands to the root command sets flags appropriately. 30 | // This is called by main.main(). It only needs to happen once to the rootCmd. 31 | func Execute() { 32 | if err := RootCmd.Execute(); err != nil { 33 | fmt.Println(err) 34 | os.Exit(-1) 35 | } 36 | } 37 | func init() { 38 | RootCmd.PersistentFlags().BoolVarP(&vers, "version", "v", false, "the workflow version") 39 | } 40 | -------------------------------------------------------------------------------- /scheduler/controller/cronjob/controller.go: -------------------------------------------------------------------------------- 1 | package cronjob 2 | -------------------------------------------------------------------------------- /scheduler/controller/cronjob/handler.go: -------------------------------------------------------------------------------- 1 | package cronjob 2 | -------------------------------------------------------------------------------- /scheduler/controller/node/handler.go: -------------------------------------------------------------------------------- 1 | package node 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/infraboard/workflow/api/apps/node" 8 | ) 9 | 10 | // syncHandler compares the actual state with the desired, and attempts to 11 | // converge the two. It then updates the Status block of the Network resource 12 | // with the current status of the resource. 13 | func (c *Controller) syncHandler(key string) error { 14 | c.log.Debugf("sync key: %s", key) 15 | 16 | obj, ok, err := c.informer.GetStore().GetByKey(key) 17 | if err != nil { 18 | return err 19 | } 20 | 21 | // 如果不存在, 这期望行为为删除 (DEL) 22 | if !ok { 23 | c.log.Debugf("remove node: %s, skip", key) 24 | return nil 25 | } 26 | 27 | n, isOK := obj.(*node.Node) 28 | if !isOK { 29 | return fmt.Errorf("object %T invalidate, is not *node.Node obj, ", obj) 30 | } 31 | 32 | return c.HandleAdd(n) 33 | } 34 | 35 | // 当有新的节点加入时, 那些调度失败的节点需要重新调度 36 | func (c *Controller) HandleAdd(n *node.Node) error { 37 | // 补充重新调度的逻辑 38 | steps, err := c.stepLister.List(context.Background()) 39 | if err != nil { 40 | c.log.Errorf("list steps error, %s", err) 41 | return nil 42 | } 43 | 44 | // 该删除节点上运行中的step进行重新调度 45 | for i := range steps { 46 | s := steps[i] 47 | if s.IsScheduledFailed() { 48 | c.log.Infof("step %s schedule failed, need reschedule ...", s.Key) 49 | s.SetScheduleNode("") 50 | err := c.stepRecorder.Update(s) 51 | if err != nil { 52 | c.log.Errorf("update step for reschedule error, %s", err) 53 | continue 54 | } 55 | c.log.Infof("reset step %s schedule node to \"\", waiting for reschedule", s.Key) 56 | } 57 | } 58 | 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /scheduler/controller/step/handler.go: -------------------------------------------------------------------------------- 1 | package step 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/infraboard/workflow/api/apps/pipeline" 7 | ) 8 | 9 | // syncHandler compares the actual state with the desired, and attempts to 10 | // converge the two. It then updates the Status block of the Network resource 11 | // with the current status of the resource. 12 | func (c *Controller) syncHandler(key string) error { 13 | obj, ok, err := c.informer.GetStore().GetByKey(key) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | // 如果不存在, 这期望行为为删除 (DEL) 19 | if !ok { 20 | c.log.Debugf("removed step: %s, skip", key) 21 | return nil 22 | } 23 | 24 | st, isOK := obj.(*pipeline.Step) 25 | if !isOK { 26 | return fmt.Errorf("object %T invalidate, is not *pipeline.Step obj, ", obj) 27 | } 28 | 29 | // 添加 30 | if err := c.addStep(st); err != nil { 31 | return err 32 | } 33 | 34 | return nil 35 | } 36 | 37 | func (c *Controller) addStep(s *pipeline.Step) error { 38 | c.log.Infof("receive add step: %s", s) 39 | if err := s.Validate(); err != nil { 40 | return fmt.Errorf("invalidate node error, %s", err) 41 | } 42 | 43 | // 已经调度的任务不处理 44 | if s.IsScheduled() { 45 | return fmt.Errorf("step %s has schedule to node %s, skip add", s.Key, s.ScheduledNodeName()) 46 | } 47 | 48 | // 已经调度的任务不处理, 为了防止不断重复调度形成死循环 49 | // 有用户自己通过API 进行Step的重新调度(重置) 50 | if s.IsScheduledFailed() { 51 | return fmt.Errorf("step %s schedule failed, skip add", s.Key) 52 | } 53 | 54 | // 如果开启审核，需要通过后，才能调度执行 55 | if !c.isAllow(s) { 56 | return fmt.Errorf("step not allow") 57 | } 58 | 59 | if err := c.scheduleStep(s); err != nil { 60 | return err 61 | } 62 | 63 | return nil 64 | } 65 | 66 | func (c *Controller) isAllow(s *pipeline.Step) bool { 67 | if !s.WithAudit { 68 | return true 69 | } 70 | 71 | // 判断审批状态是否同步 72 | 73 | // TODO: 如果未处理, 发送通知 74 | if !s.HasSendAuditNotify() { 75 | // TODO: 76 | c.log.Errorf("send notify ...") 77 | s.MarkSendAuditNotify() 78 | // 更新step 79 | if err := c.informer.Recorder().Update(s.Clone()); err != nil { 80 | c.log.Errorf("update scheduled step to auditing error, %s", err) 81 | } 82 | } 83 | 84 | // 审核通过 允许执行 85 | if s.AuditPass() { 86 | c.log.Debugf("step %s waiting for audit", s.Key) 87 | return true 88 | } 89 | 90 | return false 91 | } 92 | 93 | // Step任务调度 94 | func (c *Controller) scheduleStep(step *pipeline.Step) error { 95 | node, err := c.picker.Pick(step) 96 | if err != nil || node == nil { 97 | c.log.Warnf("step %s pick node error, %s", step.Name, err) 98 | step.ScheduleFailed(err.Error()) 99 | // 清除一下其他数据 100 | if err := c.informer.Recorder().Update(step.Clone()); err != nil { 101 | c.log.Errorf("update scheduled step error, %s", err) 102 | } 103 | return err 104 | } 105 | 106 | c.log.Debugf("choice [%s] %s for step %s", node.Type, node.InstanceName, step.Key) 107 | step.SetScheduleNode(node.InstanceName) 108 | // 清除一下其他数据 109 | if err := c.informer.Recorder().Update(step.Clone()); err != nil { 110 | c.log.Errorf("update scheduled step error, %s", err) 111 | } 112 | return nil 113 | } 114 | -------------------------------------------------------------------------------- /scheduler/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/infraboard/workflow/scheduler/cmd" 5 | ) 6 | 7 | func main() { 8 | cmd.Execute() 9 | } 10 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | var ( 8 | // ServiceName 服务名称 9 | ServiceName = "workflow" 10 | ) 11 | 12 | var ( 13 | GIT_COMMIT string 14 | GIT_BRANCH string 15 | BUILD_TIME string 16 | GO_VERSION string 17 | ) 18 | 19 | // FullVersion show the version info 20 | func FullVersion() string { 21 | version := fmt.Sprintf("Build Time: %s\nGit Branch: %s\nGit Commit: %s\nGo Version: %s\n", BUILD_TIME, GIT_BRANCH, GIT_COMMIT, GO_VERSION) 22 | return version 23 | } 24 | 25 | // Short 版本缩写 26 | func Short() string { 27 | commit := "" 28 | if len(GIT_COMMIT) > 8 { 29 | commit = GIT_COMMIT[:8] 30 | } 31 | return fmt.Sprintf("%s[%s]", GIT_BRANCH, commit) 32 | } 33 | --------------------------------------------------------------------------------