├── .gitignore
├── .run
└── flow-agent-01.run.xml
├── Dockerfile
├── LICENSE
├── Makefile
├── README.md
├── _testdata
├── agent_connect_response.json
└── agent_settings.json
├── api
├── client.go
├── client_test.go
├── const.go
├── counter_write.go
├── factory.go
├── util.go
└── util_test.go
├── app.go
├── build.sh
├── config
├── errors.go
├── factory.go
├── manager.go
└── manager_test.go
├── controller
├── cmd_controller.go
├── cmd_controller_test.go
├── health_controller.go
└── root_controller.go
├── dao
├── base_test.go
├── builder.go
├── builder_test.go
├── client.go
├── client_test.go
├── entity.go
├── entity_test.go
├── errors.go
├── helper.go
└── helper_test.go
├── debug.yaml
├── domain
├── agent.go
├── cmd.go
├── cmd_shell.go
├── cmd_tty.go
├── config.go
├── config_smtp.go
├── config_text.go
├── docker.go
├── docker_config.go
├── docker_option.go
├── docker_test.go
├── events.go
├── job_cache.go
├── k8s_config.go
├── response.go
├── secret.go
├── secret_auth.go
├── secret_rsa.go
├── secret_test.go
├── secret_token.go
├── variables.go
└── variables_test.go
├── executor
├── bin_file.go
├── bindata.go
├── docker.go
├── docker_test.go
├── docker_util.go
├── docker_util_test.go
├── executor.go
├── executor_test.go
├── shell.go
├── shell_bash.go
├── shell_bash_test.go
├── shell_powershell.go
├── shell_powershell_test.go
├── util.go
├── util_unix.go
└── util_win.go
├── go.mod
├── go.sum
├── service
├── cache_manager.go
├── cmd_service.go
├── errors.go
├── factory.go
└── plugin_manager.go
├── util
├── common.go
├── common_test.go
├── files.go
├── http.go
├── logger.go
├── obj.go
├── sync.go
├── sync_test.go
├── zk.go
└── zk_test.go
└── wait-for-it.sh
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | *.exe
3 | flow-agent-x
4 | vendor/
5 | .idea/
6 | .vscode
7 | *.log
8 | bin/
9 | *.pb.go
10 | .cache/
11 | .vender/
12 | mocks/
--------------------------------------------------------------------------------
/.run/flow-agent-01.run.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM docker:20.10-cli as docker
2 |
3 | FROM python:3.10.6-alpine3.16
4 |
5 | RUN apk update
6 | RUN apk add bash git curl wget
7 |
8 | ## docker v20.10.18 ##
9 | COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker
10 |
11 | ## docker compose v2.11.1 ##
12 | COPY --from=docker /usr/libexec/docker/cli-plugins/docker-compose /usr/local/bin/docker-compose
13 |
14 | ## ssh config
15 | RUN mkdir -p $HOME/.ssh
16 | RUN echo "StrictHostKeyChecking=no" >> $HOME/.ssh/config
17 |
18 | ## upgrade pip
19 | RUN pip install --upgrade pip
20 |
21 | ## install required pip packages
22 | RUN python3 -m pip install requests==2.22.0 python-lib-flow.ci==1.21.6
23 |
24 | ## default work dir
25 | ENV FLOWCI_AGENT_WORKSPACE=/ws
26 | RUN mkdir -p $FLOWCI_AGENT_WORKSPACE
27 |
28 | WORKDIR $FLOWCI_AGENT_WORKSPACE
29 | COPY ./flow-agent-x-linux /usr/bin
30 |
31 | ENV FLOWCI_DOCKER_AGENT=true
32 |
33 | ## start docker ##
34 | CMD flow-agent-x-linux
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | PROJECT=flow-agent-x
2 | CURRENT_DIR := $(shell pwd)
3 |
4 | LINUX_AMD64 := GOOS=linux GOARCH=amd64
5 | MAC_AMD64 := GOOS=darwin GOARCH=amd64
6 | MAC_ARM64 := GOOS=darwin GOARCH=arm64
7 | WIN_AMD64 := GOOS=windows GOARCH=amd64
8 |
9 | GO := go
10 | GOGEN := $(GO) generate ./...
11 | GOBUILD_LINUX := $(LINUX_AMD64) $(GO) build -o bin/$(PROJECT)-linux -v
12 | GOBUILD_MAC := $(MAC_AMD64) $(GO) build -o bin/$(PROJECT)-mac -v
13 | GOBUILD_MAC_ARM := $(MAC_ARM64) $(GO) build -o bin/$(PROJECT)-mac-arm -v
14 | GOBUILD_WIN := $(WIN_AMD64) $(GO) build -o bin/$(PROJECT)-win -v
15 |
16 | GOTEST_MOCK_GEN := docker run --rm -v "$(CURRENT_DIR)":/src -w /src vektra/mockery --all
17 | GOTEST := $(GO) test ./... -v -timeout 10s
18 | GOENV := -e GOCACHE=/ws/.cache -e GOPATH=/ws/.vender
19 |
20 | DOCKER_IMG := golang:1.17
21 | DOCKER_RUN := docker run -it --rm -v $(CURRENT_DIR):/ws $(GOENV) -w /ws --network host $(DOCKER_IMG) /bin/bash -c
22 |
23 | DOCKER_BUILD := ./build.sh
24 |
25 | .PHONY: build protogen test image clean cleanall
26 |
27 | build:
28 | $(DOCKER_RUN) "$(GOGEN) && $(GOBUILD_LINUX) && $(GOBUILD_MAC) && $(GOBUILD_MAC_ARM) && $(GOBUILD_WIN)"
29 |
30 | test:
31 | $(GOTEST_MOCK_GEN)
32 | $(DOCKER_RUN) "$(GOTEST)"
33 |
34 | image: build
35 | $(DOCKER_BUILD) $(tag)
36 |
37 | clean:
38 | $(GO) clean -i ./...
39 | rm -rf bin
40 |
41 | cleanall: clean
42 | rm -rf .cache
43 | rm -rf .vender
44 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | flow-agent-x
2 | ============
3 |
4 | 
5 | 
6 | 
7 |
8 | The new version agent for flow.ci
9 |
10 | ## How to start
11 |
12 | - [Start from docker](https://github.com/FlowCI/docker)
13 |
14 | - For more detail, please refer [doc](https://github.com/flowci/docs)
15 |
16 | ## Build binary
17 |
18 | ```bash
19 | make build
20 |
21 | # binary will be created at ./bin/flow-agent-x-mac
22 | # binary will be created at ./bin/flow-agent-x-linux
23 | ```
24 |
25 | ## Run Unit Test
26 |
27 | ```bash
28 | make test
29 | ```
30 |
31 | ## Build docker image
32 | ```bash
33 | make docker
34 |
35 | # docker image with name 'flowci/agent'
36 | ```
--------------------------------------------------------------------------------
/_testdata/agent_connect_response.json:
--------------------------------------------------------------------------------
1 | {
2 | "code": 200,
3 | "message": "mock",
4 | "data": {
5 | "agent": {
6 | "id": "1",
7 | "name": "local",
8 | "token": "xxx-xxx",
9 | "host": "test",
10 | "tags": [
11 | "ios",
12 | "mac"
13 | ],
14 | "status": "OFFLINE",
15 | "jobid": "job-id"
16 | },
17 | "queue": {
18 | "uri": "amqp://guest:guest@127.0.0.1:5672",
19 | "callback": "callback-q",
20 | "shellLogEx": "shelllog-exchange-ut",
21 | "ttyLogEx": "ttylog-exchange-ut"
22 | },
23 | "zookeeper": {
24 | "host": "127.0.0.1:2181",
25 | "root": "/flow-x"
26 | }
27 | }
28 | }
--------------------------------------------------------------------------------
/_testdata/agent_settings.json:
--------------------------------------------------------------------------------
1 | {
2 | "agent": {
3 | "id": "1",
4 | "name": "local",
5 | "token": "xxx-xxx",
6 | "host": "test",
7 | "tags": [
8 | "ios",
9 | "mac"
10 | ],
11 | "status": "OFFLINE",
12 | "jobid": "job-id"
13 | },
14 | "queue": {
15 | "uri": "amqp://guest:guest@127.0.0.1:5672",
16 | "callback": "callback-q",
17 | "shellLog": "shelllog",
18 | "ttyLog": "ttylog"
19 | },
20 | "zookeeper": {
21 | "host": "127.0.0.1:2181",
22 | "root": "/flow-x"
23 | }
24 | }
--------------------------------------------------------------------------------
/api/client.go:
--------------------------------------------------------------------------------
1 | package api
2 |
3 | import (
4 | "bytes"
5 | "encoding/json"
6 | "fmt"
7 | "github.com/gorilla/websocket"
8 | "io"
9 | "io/ioutil"
10 | "mime/multipart"
11 | "net/http"
12 | "net/url"
13 | "os"
14 | "path/filepath"
15 | "strings"
16 | "sync/atomic"
17 | "time"
18 |
19 | "github.com/flowci/flow-agent-x/domain"
20 | "github.com/flowci/flow-agent-x/util"
21 | )
22 |
23 | const (
24 | timeout = 30 * time.Second
25 | connStateConnected = 1
26 | connStateReconnecting = 2
27 | bufferSize = 64 * 1024
28 | )
29 |
30 | var (
31 | newline = []byte{'\n'}
32 | space = []byte{' '}
33 | disconnected = &message{
34 | event: "disconnected",
35 | }
36 | )
37 |
38 | type (
39 | Client interface {
40 | Connect(*domain.AgentInit) (*domain.AgentConfig, error)
41 | ReConn() <-chan struct{}
42 |
43 | UploadLog(filePath string) error
44 | ReportProfile(profile *domain.AgentProfile) error
45 |
46 | GetCmdIn() <-chan []byte
47 | SendCmdOut(out domain.CmdOut) error
48 | SendShellLog(jobId, stepId, b64Log string)
49 | SendTtyLog(ttyId, b64Log string)
50 |
51 | CachePut(jobId, name, workspace string, paths []string) error
52 | CacheGet(jobId, name string) *domain.JobCache
53 | CacheDownload(cacheId, workspace, file string, progress io.Writer)
54 |
55 | GetSecret(name string) (domain.Secret, error)
56 | GetConfig(name string) (domain.Config, error)
57 |
58 | Close()
59 | }
60 |
61 | client struct {
62 | token string
63 | server string
64 | client *http.Client
65 | cmdInbound chan []byte
66 | pending chan *message
67 |
68 | reConn chan struct{}
69 | connState int32
70 | conn *websocket.Conn
71 | }
72 |
73 | message struct {
74 | event string
75 | body []byte
76 | }
77 |
78 | // part: multipart data
79 | part struct {
80 | key string
81 | file string
82 | }
83 | )
84 |
85 | func (c *client) Connect(init *domain.AgentInit) (*domain.AgentConfig, error) {
86 | u, err := url.Parse(c.server)
87 | if err != nil {
88 | return nil, err
89 | }
90 |
91 | u.Scheme = "ws"
92 | u.Path = "agent"
93 |
94 | header := http.Header{}
95 | header.Add(headerToken, c.token)
96 |
97 | dialer := websocket.Dialer{
98 | Proxy: http.ProxyFromEnvironment,
99 | HandshakeTimeout: timeout,
100 | ReadBufferSize: bufferSize,
101 | WriteBufferSize: bufferSize,
102 | }
103 |
104 | // build connection
105 | c.conn, _, err = dialer.Dial(u.String(), header)
106 | if err != nil {
107 | return nil, err
108 | }
109 |
110 | c.conn.SetReadLimit(bufferSize)
111 | c.setConnState(connStateConnected)
112 |
113 | // send init connect event
114 | resp := &domain.AgentConfigResponse{}
115 | err = c.sendMessageWithResp(eventConnect, init, resp)
116 | if err != nil {
117 | return nil, err
118 | }
119 |
120 | // start to read message
121 | go c.readMessage()
122 | go c.consumePendingMessage()
123 |
124 | util.LogInfo("Agent is connected to server %s", c.server)
125 | return resp.Data, nil
126 | }
127 |
128 | func (c *client) ReConn() <-chan struct{} {
129 | return c.reConn
130 | }
131 |
132 | func (c *client) ReportProfile(r *domain.AgentProfile) (err error) {
133 | _ = c.sendMessageWithJson(eventProfile, r)
134 | return
135 | }
136 |
137 | func (c *client) UploadLog(filePath string) (err error) {
138 | defer util.RecoverPanic(func(e error) {
139 | err = e
140 | })
141 |
142 | buffer, contentType := c.buildMultipartContent([]*part{
143 | {
144 | key: "file",
145 | file: filePath,
146 | },
147 | })
148 |
149 | // send request
150 | raw, err := c.send("POST", "logs/upload", contentType, buffer)
151 | util.PanicIfErr(err)
152 |
153 | _, err = c.parseResponse(raw, &domain.Response{})
154 | util.PanicIfErr(err)
155 |
156 | util.LogInfo("[Uploaded]: %s", filePath)
157 | return
158 | }
159 |
160 | func (c *client) GetCmdIn() <-chan []byte {
161 | return c.cmdInbound
162 | }
163 |
164 | func (c *client) SendCmdOut(out domain.CmdOut) error {
165 | _ = c.sendMessageWithBytes(eventCmdOut, out.ToBytes())
166 | util.LogDebug("Result of cmd been pushed")
167 | return nil
168 | }
169 |
170 | func (c *client) SendShellLog(jobId, stepId, b64Log string) {
171 | body := &domain.ShellLog{
172 | JobId: jobId,
173 | StepId: stepId,
174 | Log: b64Log,
175 | }
176 |
177 | _ = c.sendMessageWithJson(eventShellLog, body)
178 | }
179 |
180 | func (c *client) SendTtyLog(ttyId, b64Log string) {
181 | body := &domain.TtyLog{
182 | ID: ttyId,
183 | Log: b64Log,
184 | }
185 | _ = c.sendMessageWithJson(eventTtyLog, body)
186 | }
187 |
188 | func (c *client) CachePut(jobId, key, workspace string, paths []string) (out error) {
189 | defer util.RecoverPanic(func(e error) {
190 | out = e
191 | })
192 |
193 | tempDir, err := ioutil.TempDir("", "agent_cache_")
194 | util.PanicIfErr(err)
195 | defer os.RemoveAll(tempDir)
196 |
197 | var parts []*part
198 | for _, path := range paths {
199 | if !util.IsFileExists(path) {
200 | util.LogWarn("the file %s not exist", path)
201 | continue
202 | }
203 |
204 | if !strings.HasPrefix(path, workspace) {
205 | util.LogWarn("the cache path must be under workspace")
206 | continue
207 | }
208 |
209 | cacheZipName := encodeCacheName(workspace, path)
210 | zipPath := tempDir + util.UnixPathSeparator + cacheZipName
211 | err = util.Zip(path, zipPath, util.UnixPathSeparator)
212 |
213 | if err != nil {
214 | util.LogWarn(err.Error())
215 | continue
216 | }
217 |
218 | parts = append(parts, &part{
219 | key: "files",
220 | file: zipPath,
221 | })
222 | }
223 |
224 | buffer, contentType := c.buildMultipartContent(parts)
225 |
226 | path := fmt.Sprintf("cache/%s/%s/%s", jobId, key, util.OS())
227 | raw, err := c.send("POST", path, contentType, buffer)
228 | util.PanicIfErr(err)
229 |
230 | _, err = c.parseResponse(raw, &domain.Response{})
231 | util.PanicIfErr(err)
232 |
233 | util.LogInfo("[CachePut] %d/%d files cached in %s", len(parts), len(paths), key)
234 | return
235 | }
236 |
237 | func (c *client) CacheGet(jobId, key string) *domain.JobCache {
238 | raw, err := c.send("GET", fmt.Sprintf("cache/%s/%s", jobId, key), "", nil)
239 | util.PanicIfErr(err)
240 |
241 | resp, err := c.parseResponse(raw, &domain.JobCacheResponse{})
242 | util.PanicIfErr(err)
243 |
244 | jobCache := resp.(*domain.JobCacheResponse)
245 | return jobCache.Data
246 | }
247 |
248 | func (c *client) CacheDownload(cacheId, workspace, file string, progress io.Writer) {
249 | defer util.RecoverPanic(func(e error) {
250 | util.LogWarn(e.Error())
251 | })
252 |
253 | tmpPath := fmt.Sprintf("%s/%s.tmp", workspace, file)
254 | tmpFile, err := os.Create(tmpPath)
255 | util.PanicIfErr(err)
256 |
257 | err = c.download(fmt.Sprintf("cache/%s?file=%s", cacheId, file), tmpFile, progress)
258 | util.PanicIfErr(err)
259 |
260 | zippedFile := workspace + util.UnixPathSeparator + file
261 | err = os.Rename(tmpPath, zippedFile)
262 | util.PanicIfErr(err)
263 |
264 | defer os.RemoveAll(zippedFile)
265 |
266 | cacheFileName := decodeCacheName(file)
267 | dest := workspace + util.UnixPathSeparator + cacheFileName
268 |
269 | err = util.Unzip(zippedFile, dest)
270 | util.PanicIfErr(err)
271 | }
272 |
273 | func (c *client) GetSecret(name string) (secret domain.Secret, err error) {
274 | defer util.RecoverPanic(func(e error) {
275 | err = e
276 | })
277 |
278 | req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/secret/%s", c.server, name), nil)
279 | req.Header.Set(util.HttpHeaderAgentToken, c.token)
280 |
281 | resp, err := c.client.Do(req)
282 | util.PanicIfErr(err)
283 |
284 | defer resp.Body.Close()
285 |
286 | out, err := ioutil.ReadAll(resp.Body)
287 | util.PanicIfErr(err)
288 |
289 | secretResp, err := c.parseResponse(out, &domain.SecretResponse{})
290 | util.PanicIfErr(err)
291 |
292 | body := secretResp.(*domain.SecretResponse)
293 | util.PanicIfNil(body.Data, "secret data")
294 |
295 | base := body.Data
296 | baseRaw := &domain.ResponseRaw{}
297 | err = json.Unmarshal(out, baseRaw)
298 | util.PanicIfErr(err)
299 |
300 | if base.Category == domain.SecretCategoryAuth {
301 | auth := &domain.AuthSecret{}
302 | err = json.Unmarshal(baseRaw.Raw, auth)
303 | util.PanicIfErr(err)
304 | return auth, nil
305 | }
306 |
307 | if base.Category == domain.SecretCategorySshRsa {
308 | rsa := &domain.RSASecret{}
309 | err = json.Unmarshal(baseRaw.Raw, rsa)
310 | util.PanicIfErr(err)
311 | return rsa, nil
312 | }
313 |
314 | if base.Category == domain.SecretCategoryToken {
315 | token := &domain.TokenSecret{}
316 | err = json.Unmarshal(baseRaw.Raw, token)
317 | util.PanicIfErr(err)
318 | return token, nil
319 | }
320 |
321 | return nil, fmt.Errorf("secret '%s' category '%s' is unsupported", base.GetName(), base.GetCategory())
322 | }
323 |
324 | func (c *client) GetConfig(name string) (config domain.Config, err error) {
325 | defer util.RecoverPanic(func(e error) {
326 | err = e
327 | })
328 |
329 | req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/config/%s", c.server, name), nil)
330 | req.Header.Set(util.HttpHeaderAgentToken, c.token)
331 |
332 | resp, err := c.client.Do(req)
333 | util.PanicIfErr(err)
334 |
335 | defer resp.Body.Close()
336 |
337 | out, err := ioutil.ReadAll(resp.Body)
338 | util.PanicIfErr(err)
339 |
340 | configResp, err := c.parseResponse(out, &domain.ConfigResponse{})
341 | util.PanicIfErr(err)
342 |
343 | body := configResp.(*domain.ConfigResponse)
344 | util.PanicIfNil(body.Data, "config data")
345 |
346 | base := body.Data
347 | baseRaw := &domain.ResponseRaw{}
348 | err = json.Unmarshal(out, baseRaw)
349 | util.PanicIfErr(err)
350 |
351 | if base.Category == domain.ConfigCategorySmtp {
352 | auth := &domain.SmtpConfig{}
353 | err = json.Unmarshal(baseRaw.Raw, auth)
354 | util.PanicIfErr(err)
355 | return auth, nil
356 | }
357 |
358 | if base.Category == domain.ConfigCategoryText {
359 | rsa := &domain.TextConfig{}
360 | err = json.Unmarshal(baseRaw.Raw, rsa)
361 | util.PanicIfErr(err)
362 | return rsa, nil
363 | }
364 |
365 | return nil, fmt.Errorf("config '%s' category '%s' is unsupported", base.GetName(), base.GetCategory())
366 | }
367 |
368 | func (c *client) Close() {
369 | if c.conn != nil {
370 | close(c.pending)
371 | _ = c.conn.Close()
372 | }
373 | }
374 |
375 | func (c *client) buildMultipartContent(parts []*part) (*bytes.Buffer, string) {
376 | buffer := &bytes.Buffer{}
377 | writer := multipart.NewWriter(buffer)
378 | defer writer.Close()
379 |
380 | for _, part := range parts {
381 | file, err := os.Open(part.file)
382 | util.PanicIfErr(err)
383 |
384 | part, err := writer.CreateFormFile(part.key, filepath.Base(part.file))
385 | util.PanicIfErr(err)
386 |
387 | _, err = io.Copy(part, file)
388 | util.PanicIfErr(err)
389 |
390 | _ = file.Close()
391 | }
392 |
393 | return buffer, writer.FormDataContentType()
394 | }
395 |
396 | func (c *client) setConnState(state int32) {
397 | atomic.StoreInt32(&c.connState, state)
398 | }
399 |
400 | func (c *client) isReConnecting() bool {
401 | return atomic.LoadInt32(&c.connState) == connStateReconnecting
402 | }
403 |
404 | func (c *client) isConnected() bool {
405 | return atomic.LoadInt32(&c.connState) == connStateConnected
406 | }
407 |
408 | func (c *client) readMessage() {
409 | // start receive data
410 |
411 | for {
412 | _, message, err := c.conn.ReadMessage()
413 | if err != nil {
414 | util.LogWarn("err on read message: %s", err.Error())
415 | if websocket.IsCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) {
416 | c.setConnState(connStateReconnecting)
417 | c.reConn <- struct{}{}
418 | c.conn = nil
419 | c.pending <- disconnected
420 | break
421 | }
422 |
423 | panic(err)
424 | }
425 | message = bytes.TrimSpace(bytes.Replace(message, newline, space, -1))
426 | c.cmdInbound <- message
427 | }
428 | }
429 |
430 | // method: GET/POST, path: {server}/agents/api/:path
431 | func (c *client) send(method, path, contentType string, body io.Reader) ([]byte, error) {
432 | url := fmt.Sprintf("%s/api/%s", c.server, path)
433 | req, _ := http.NewRequest(method, url, body)
434 |
435 | req.Header.Set(util.HttpHeaderContentType, contentType)
436 | req.Header.Set(util.HttpHeaderAgentToken, c.token)
437 |
438 | resp, err := c.client.Do(req)
439 | if err != nil {
440 | return nil, err
441 | }
442 |
443 | defer resp.Body.Close()
444 |
445 | data, err := ioutil.ReadAll(resp.Body)
446 | if err != nil {
447 | return nil, err
448 | }
449 |
450 | return data, nil
451 | }
452 |
453 | func (c *client) download(path string, dist io.Writer, progress io.Writer) error {
454 | url := fmt.Sprintf("%s/api/%s", c.server, path)
455 |
456 | req, _ := http.NewRequest("GET", url, nil)
457 | req.Header.Set(util.HttpHeaderAgentToken, c.token)
458 |
459 | resp, err := c.client.Do(req)
460 | if err != nil {
461 | return err
462 | }
463 |
464 | defer resp.Body.Close()
465 |
466 | if progress == nil {
467 | progress = &CounterWrite{}
468 | }
469 |
470 | _, err = io.Copy(dist, io.TeeReader(resp.Body, progress))
471 | if err != nil {
472 | return err
473 | }
474 |
475 | return nil
476 | }
477 |
478 | func (c *client) consumePendingMessage() {
479 | for message := range c.pending {
480 | if message == disconnected {
481 | util.LogDebug("exit ws message consumer")
482 | break
483 | }
484 | _ = c.conn.WriteMessage(websocket.BinaryMessage, buildMessage(message.event, message.body))
485 | util.LogDebug("pending message has been sent: %s", message.event)
486 | }
487 | }
488 |
489 | func (c *client) sendMessageWithJson(event string, msg interface{}) error {
490 | body, err := json.Marshal(msg)
491 | if err != nil {
492 | return err
493 | }
494 |
495 | c.pending <- &message{
496 | event: event,
497 | body: body,
498 | }
499 | return nil
500 | }
501 |
502 | func (c *client) sendMessageWithBytes(event string, body []byte) error {
503 | c.pending <- &message{
504 | event: event,
505 | body: body,
506 | }
507 | return nil
508 | }
509 |
510 | func (c *client) sendMessageWithResp(event string, msg interface{}, resp domain.ResponseMessage) (out error) {
511 | defer util.RecoverPanic(func(e error) {
512 | out = e
513 | })
514 |
515 | body, err := json.Marshal(msg)
516 | util.PanicIfErr(err)
517 |
518 | err = c.conn.WriteMessage(websocket.BinaryMessage, buildMessage(event, body))
519 | util.PanicIfErr(err)
520 |
521 | _, data, err := c.conn.ReadMessage()
522 | util.PanicIfErr(err)
523 |
524 | _, err = c.parseResponse(data, resp)
525 | util.PanicIfErr(err)
526 | return
527 | }
528 |
529 | func (c *client) parseResponse(body []byte, resp domain.ResponseMessage) (domain.ResponseMessage, error) {
530 | // get response data
531 | err := json.Unmarshal(body, resp)
532 | if err != nil {
533 | return nil, err
534 | }
535 |
536 | if resp.IsOk() {
537 | return resp, nil
538 | }
539 |
540 | return nil, fmt.Errorf(resp.GetMessage())
541 | }
542 |
--------------------------------------------------------------------------------
/api/client_test.go:
--------------------------------------------------------------------------------
1 | package api
2 |
3 | import (
4 | "github.com/stretchr/testify/assert"
5 | "testing"
6 | )
7 |
8 | func TestShouldCacheFile(t *testing.T) {
9 | t.SkipNow()
10 |
11 | assert := assert.New(t)
12 |
13 | c := NewClient("277a35ad-30d7-47ea-a317-70670fb27306", "http://localhost:8080")
14 |
15 | jobId := "5f9935af5875dd0b92db014b"
16 | workspace := "/ws"
17 | cacheName := "test_cache"
18 |
19 | c.CachePut(jobId, cacheName, workspace, []string{
20 | "/Users/yang/Desktop/cache_1/test",
21 | "/Users/yang/Desktop/cache_2",
22 | })
23 |
24 | jobCache := c.CacheGet(jobId, cacheName)
25 | assert.NotNil(jobCache)
26 |
27 | c.CacheDownload(jobCache.Id, "/ws/out", "Y2FjaGVfMg==", nil)
28 | c.CacheDownload(jobCache.Id, "/ws/out", "Y2FjaGVfMS90ZXN0", nil)
29 | }
30 |
--------------------------------------------------------------------------------
/api/const.go:
--------------------------------------------------------------------------------
1 | package api
2 |
3 | const (
4 | eventConnect = "connect___"
5 | eventProfile = "profile___"
6 | eventCmdOut = "cmd_out___"
7 | eventShellLog = "slog______"
8 | eventTtyLog = "tlog______"
9 |
10 | headerToken = "Token"
11 | )
12 |
--------------------------------------------------------------------------------
/api/counter_write.go:
--------------------------------------------------------------------------------
1 | package api
2 |
3 | import (
4 | "fmt"
5 | "github.com/dustin/go-humanize"
6 | "strings"
7 | )
8 |
9 | type CounterWrite struct {
10 | Total uint64
11 | }
12 |
13 | func (cw *CounterWrite) Write(p []byte) (int, error) {
14 | n := len(p)
15 | cw.Total += uint64(n)
16 | cw.PrintProgress()
17 | return n, nil
18 | }
19 |
20 | func (cw *CounterWrite) PrintProgress() {
21 | fmt.Printf("\r%s", strings.Repeat(" ", 50))
22 | fmt.Printf("\rDownloading... %s complete", humanize.Bytes(cw.Total))
23 | }
24 |
--------------------------------------------------------------------------------
/api/factory.go:
--------------------------------------------------------------------------------
1 | package api
2 |
3 | import (
4 | "net/http"
5 | "time"
6 | )
7 |
8 | func NewClient(token, server string) Client {
9 | transport := &http.Transport{
10 | MaxIdleConns: 5,
11 | IdleConnTimeout: 30 * time.Second,
12 | }
13 |
14 | return &client{
15 | token: token,
16 | server: server,
17 | cmdInbound: make(chan []byte),
18 | reConn: make(chan struct{}),
19 | pending: make(chan *message, 100),
20 | client: &http.Client{
21 | Transport: transport,
22 | Timeout: 10 * time.Second,
23 | },
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/api/util.go:
--------------------------------------------------------------------------------
1 | package api
2 |
3 | import (
4 | "encoding/base64"
5 | "github.com/flowci/flow-agent-x/util"
6 | "strings"
7 | )
8 |
9 | func buildMessage(event string, body []byte) (out []byte) {
10 | out = append([]byte(event), '\n')
11 | out = append(out, body...)
12 | return
13 | }
14 |
15 | func encodeCacheName(workspace, fullPath string) string {
16 | cacheName := util.TrimLeftString(fullPath, workspace)
17 | if strings.HasPrefix(cacheName, util.UnixPathSeparator) {
18 | cacheName = cacheName[1:]
19 | }
20 | return base64.StdEncoding.EncodeToString([]byte(cacheName))
21 | }
22 |
23 | func decodeCacheName(encodedFileName string) string {
24 | cacheName, err := base64.StdEncoding.DecodeString(encodedFileName)
25 | if err != nil {
26 | return ""
27 | }
28 | return string(cacheName)
29 | }
30 |
--------------------------------------------------------------------------------
/api/util_test.go:
--------------------------------------------------------------------------------
1 | package api
2 |
3 | import (
4 | "encoding/base64"
5 | "github.com/stretchr/testify/assert"
6 | "testing"
7 | )
8 |
9 | func TestShouldEncodeCacheName(t *testing.T) {
10 | assert := assert.New(t)
11 |
12 | encoded := encodeCacheName("/ws", "/ws/a/b/c")
13 | assert.Equal(base64.StdEncoding.EncodeToString([]byte("a/b/c")), encoded)
14 |
15 | encoded = encodeCacheName("/ws", "/ws/test.log")
16 | assert.Equal(base64.StdEncoding.EncodeToString([]byte("test.log")), encoded)
17 | }
18 |
19 | func TestShouldDecodeCacheName(t *testing.T) {
20 | assert := assert.New(t)
21 |
22 | decoded := decodeCacheName("YS9iL2M=")
23 | assert.Equal("a/b/c", decoded)
24 |
25 | decoded = decodeCacheName("dGVzdC5sb2c=")
26 | assert.Equal("test.log", decoded)
27 | }
28 |
--------------------------------------------------------------------------------
/app.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "net/http"
6 | "os"
7 | "path/filepath"
8 |
9 | "github.com/flowci/flow-agent-x/config"
10 | "github.com/flowci/flow-agent-x/controller"
11 | "github.com/flowci/flow-agent-x/domain"
12 | "github.com/flowci/flow-agent-x/util"
13 | "github.com/gin-contrib/pprof"
14 | "github.com/gin-gonic/gin"
15 | "github.com/urfave/cli"
16 | )
17 |
18 | const version = "1.23.01"
19 |
20 | func init() {
21 | util.LogInit()
22 | util.EnableDebugLog()
23 | }
24 |
25 | func main() {
26 | app := cli.NewApp()
27 | app.Name = "Agent of flow.ci"
28 | app.Usage = ""
29 | app.Action = start
30 | app.Author = "yang.guo"
31 | app.Version = version
32 |
33 | cm := config.GetInstance()
34 |
35 | app.Flags = []cli.Flag{
36 | cli.BoolFlag{
37 | Name: "debug, d",
38 | Usage: "Enable debug model",
39 | EnvVar: domain.VarAgentDebug,
40 | Destination: &cm.Debug,
41 | },
42 |
43 | cli.StringFlag{
44 | Name: "url, u",
45 | Value: "http://127.0.0.1:8080",
46 | Usage: "flow.ci server url",
47 | EnvVar: domain.VarServerUrl,
48 | Destination: &cm.Server,
49 | },
50 |
51 | cli.StringFlag{
52 | Name: "token, t",
53 | Usage: "Token for agent",
54 | EnvVar: domain.VarAgentToken,
55 | Destination: &cm.Token,
56 | },
57 |
58 | cli.IntFlag{
59 | Name: "port, p",
60 | Value: -1,
61 | Usage: "Port for agent",
62 | EnvVar: domain.VarAgentPort,
63 | Destination: &cm.Port,
64 | },
65 |
66 | cli.StringFlag{
67 | Name: "profile",
68 | Usage: "Enable or disable agent profiling",
69 | EnvVar: domain.VarAgentEnableProfile,
70 | Value: "true",
71 | Destination: &cm.ProfileEnabledStr,
72 | },
73 |
74 | cli.BoolFlag{
75 | Name: "k8sEnabled",
76 | Usage: "Indicate is run from k8s",
77 | EnvVar: domain.VarK8sEnabled,
78 | Destination: &cm.K8sEnabled,
79 | },
80 |
81 | cli.BoolFlag{
82 | Name: "k8sInCluster",
83 | Usage: "Indicate is k8s run in cluster",
84 | EnvVar: domain.VarK8sInCluster,
85 | Destination: &cm.K8sCluster,
86 | },
87 |
88 | cli.StringFlag{
89 | Name: "workspace, w",
90 | Value: filepath.Join(util.HomeDir, ".flow.ci.agent"),
91 | Usage: "Agent working directory",
92 | EnvVar: domain.VarAgentWorkspace,
93 | Destination: &cm.Workspace,
94 | },
95 |
96 | cli.StringFlag{
97 | Name: "volumes, m",
98 | Usage: "List of volume that will mount to docker from step \n" +
99 | "format: name=xxx,dest=xxx,script=xxx;name=xxx,dest=xxx,script=xxx;...",
100 | EnvVar: domain.VarAgentVolumes,
101 | Destination: &cm.VolumesStr,
102 | },
103 | }
104 |
105 | err := app.Run(os.Args)
106 | util.LogIfError(err)
107 | }
108 |
109 | func start(c *cli.Context) error {
110 | util.LogInfo("Staring flow.ci agent (v%s)...", version)
111 | defer func() {
112 | if err := recover(); err != nil {
113 | util.LogIfError(err.(error))
114 | }
115 | util.LogInfo("Agent stopped")
116 | }()
117 |
118 | cm := config.GetInstance()
119 | cm.Init()
120 |
121 | defer cm.Close()
122 |
123 | // connect to ci server
124 | startGin(cm)
125 |
126 | return nil
127 | }
128 |
129 | func startGin(cm *config.Manager) {
130 | router := gin.Default()
131 | controller.NewCmdController(router)
132 | controller.NewHealthController(router)
133 |
134 | if cm.Debug {
135 | pprof.Register(router)
136 | }
137 |
138 | server := &http.Server{
139 | Addr: fmt.Sprintf(":%d", cm.Port),
140 | Handler: router,
141 | }
142 |
143 | go func() {
144 | err := server.ListenAndServe()
145 | if err != nil && err != http.ErrServerClosed {
146 | util.FailOnError(err, "Unable to listen")
147 | }
148 | }()
149 |
150 | // wait
151 | <-cm.AppCtx.Done()
152 |
153 | if err := server.Shutdown(cm.AppCtx); err != nil {
154 | util.FailOnError(err, "Unable to stop the agent")
155 | }
156 | }
157 |
--------------------------------------------------------------------------------
/build.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | version=$1
4 |
5 | if [[ -n $version ]]; then
6 | VersionTag="-t flowci/agent:$version"
7 | fi
8 |
9 | docker run --privileged --rm tonistiigi/binfmt --install all
10 | # docker buildx create --name flowci --use
11 |
12 | docker buildx build -f ./Dockerfile --platform linux/arm64,linux/amd64 --push -t flowci/agent:latest $VersionTag ./bin
13 |
14 | # docker rmi -f $(docker images -f 'dangling=true' -q)
--------------------------------------------------------------------------------
/config/errors.go:
--------------------------------------------------------------------------------
1 | package config
2 |
3 | import "errors"
4 |
5 | var (
6 | ErrSettingsNotBeenLoaded = errors.New("agent: settings has not been initialized")
7 | )
8 |
--------------------------------------------------------------------------------
/config/factory.go:
--------------------------------------------------------------------------------
1 | package config
2 |
3 | import (
4 | "github.com/flowci/flow-agent-x/domain"
5 | "sync"
6 | )
7 |
8 | var (
9 | singleton *Manager
10 | once sync.Once
11 | )
12 |
13 | // GetInstance get singleton of config manager
14 | func GetInstance() *Manager {
15 | once.Do(func() {
16 | singleton = &Manager{
17 | status: domain.AgentIdle,
18 | events: map[domain.AppEvent]func(){},
19 | }
20 | })
21 | return singleton
22 | }
23 |
--------------------------------------------------------------------------------
/config/manager.go:
--------------------------------------------------------------------------------
1 | package config
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "net"
7 | "os"
8 | "path/filepath"
9 | "strconv"
10 | "strings"
11 | "time"
12 |
13 | "github.com/flowci/flow-agent-x/api"
14 | "github.com/flowci/flow-agent-x/domain"
15 | "github.com/flowci/flow-agent-x/util"
16 | "github.com/shirou/gopsutil/v3/cpu"
17 | "github.com/shirou/gopsutil/v3/disk"
18 | "github.com/shirou/gopsutil/v3/mem"
19 | )
20 |
21 | const pluginDir = ".plugins"
22 | const logDir = ".logs"
23 |
24 | type (
25 | // Manager to handle server connection and config
26 | Manager struct {
27 | Zk *util.ZkClient
28 |
29 | Debug bool
30 | Server string
31 | Token string
32 | Port int
33 |
34 | ProfileEnabled bool
35 | ProfileEnabledStr string
36 |
37 | K8sEnabled bool
38 | K8sCluster bool
39 | K8sNodeName string
40 | K8sPodName string
41 | K8sPodIp string
42 | K8sNamespace string
43 |
44 | Workspace string
45 | LoggingDir string
46 | PluginDir string
47 | IsFromDocker bool
48 |
49 | Client api.Client
50 |
51 | VolumesStr string
52 | Volumes []*domain.DockerVolume
53 |
54 | AppCtx context.Context
55 | Cancel context.CancelFunc
56 |
57 | idleTimer *time.Timer
58 | config *domain.AgentConfig
59 | status domain.AgentStatus
60 | events map[domain.AppEvent]func()
61 | }
62 | )
63 |
64 | func (m *Manager) Init() {
65 | // init vars
66 | if m.Port < 0 {
67 | m.Port = m.getDefaultPort()
68 | }
69 | var err error
70 | m.ProfileEnabled, err = strconv.ParseBool(m.ProfileEnabledStr)
71 | util.PanicIfErr(err)
72 |
73 | m.IsFromDocker, err = strconv.ParseBool(util.GetEnv(domain.VarAgentFromDocker, "false"))
74 | util.PanicIfErr(err)
75 |
76 | m.PluginDir = filepath.Join(m.Workspace, pluginDir)
77 | m.LoggingDir = filepath.Join(m.Workspace, logDir)
78 |
79 | m.K8sNodeName = os.Getenv(domain.VarK8sNodeName)
80 | m.K8sPodName = os.Getenv(domain.VarK8sPodName)
81 | m.K8sPodIp = os.Getenv(domain.VarK8sPodIp)
82 | m.K8sNamespace = os.Getenv(domain.VarK8sNamespace)
83 |
84 | // init dir
85 | _ = os.MkdirAll(m.Workspace, os.ModePerm)
86 | _ = os.MkdirAll(m.LoggingDir, os.ModePerm)
87 | _ = os.MkdirAll(m.PluginDir, os.ModePerm)
88 |
89 | ctx, cancel := context.WithCancel(context.Background())
90 | m.AppCtx = ctx
91 | m.Cancel = cancel
92 | m.Client = api.NewClient(m.Token, m.Server)
93 |
94 | // init events
95 | m.events[domain.EventOnIdle] = m.onIdleEvent
96 | m.events[domain.EventOnBusy] = m.onBusyEvent
97 |
98 | m.initVolumes()
99 | util.PanicIfErr(m.connect())
100 |
101 | m.listenReConn()
102 | m.sendAgentProfile()
103 |
104 | m.printInfo()
105 | m.FireEvent(domain.EventOnIdle)
106 | }
107 |
108 | func (m *Manager) FetchProfile() *domain.AgentProfile {
109 | nCpu, _ := cpu.Counts(true)
110 | percent, _ := cpu.Percent(time.Second, false)
111 | vmStat, _ := mem.VirtualMemory()
112 | diskStat, _ := disk.Usage("/")
113 |
114 | cpuUsage := float64(0)
115 | if len(percent) > 0 {
116 | cpuUsage = percent[0]
117 | }
118 |
119 | return &domain.AgentProfile{
120 | CpuNum: nCpu,
121 | CpuUsage: cpuUsage,
122 | TotalMemory: util.ByteToMB(vmStat.Total),
123 | FreeMemory: util.ByteToMB(vmStat.Available),
124 | TotalDisk: util.ByteToMB(diskStat.Total),
125 | FreeDisk: util.ByteToMB(diskStat.Free),
126 | }
127 | }
128 |
129 | func (m *Manager) FireEvent(event domain.AppEvent) {
130 | if f, ok := m.events[event]; ok {
131 | f()
132 | }
133 | }
134 |
135 | // Close release resources and connections
136 | func (m *Manager) Close() {
137 | m.Client.Close()
138 | }
139 |
140 | // --------------------------------
141 | // Events Handler
142 | // --------------------------------
143 |
144 | func (m *Manager) onIdleEvent() {
145 | m.status = domain.AgentIdle
146 | util.LogInfo("[Agent Status] = Idle")
147 |
148 | if m.config.ExitOnIdle <= 0 {
149 | return
150 | }
151 |
152 | if m.idleTimer != nil {
153 | m.idleTimer.Stop()
154 | m.idleTimer = nil
155 | }
156 |
157 | m.idleTimer = time.NewTimer(time.Duration(m.config.ExitOnIdle) * time.Second)
158 | go func() {
159 | t := <-m.idleTimer.C
160 | panic(fmt.Errorf("idle after %d seconds, agent will be exited", t.Second()))
161 | }()
162 | }
163 |
164 | func (m *Manager) onBusyEvent() {
165 | m.status = domain.AgentBusy
166 | util.LogInfo("[Agent Status] = Busy")
167 |
168 | if m.idleTimer != nil {
169 | m.idleTimer.Stop()
170 | m.idleTimer = nil
171 | }
172 | }
173 |
174 | // --------------------------------
175 | // Private Functions
176 | // --------------------------------
177 |
178 | func (m *Manager) printInfo() {
179 | util.LogInfo("--- [Server URL]: %s", m.Server)
180 | util.LogInfo("--- [Token]: %s", m.Token)
181 | util.LogInfo("--- [Port]: %d", m.Port)
182 | util.LogInfo("--- [Workspace]: %s", m.Workspace)
183 | util.LogInfo("--- [Plugin Dir]: %s", m.PluginDir)
184 | util.LogInfo("--- [Log Dir]: %s", m.LoggingDir)
185 | util.LogInfo("--- [Volume Str]: %s", m.VolumesStr)
186 | util.LogInfo("--- [Exit On Idle]: %d (seconds)", m.config.ExitOnIdle)
187 |
188 | if m.K8sEnabled {
189 | util.LogInfo("--- [K8s InCluster]: %d", m.K8sCluster)
190 | util.LogInfo("--- [K8s Node]: %s", m.K8sNodeName)
191 | util.LogInfo("--- [K8s Namespace]: %s", m.K8sNamespace)
192 | util.LogInfo("--- [K8s Pod]: %s", m.K8sPodName)
193 | util.LogInfo("--- [K8s Pod IP]: %s", m.K8sPodIp)
194 | }
195 | }
196 |
197 | func (m *Manager) getDefaultPort() int {
198 | listener, err := net.Listen("tcp", "127.0.0.1:0")
199 | util.FailOnError(err, "Cannot start listen localhost")
200 | defer func() {
201 | _ = listener.Close()
202 | }()
203 |
204 | addressAndPort := listener.Addr().String()
205 |
206 | i, err := strconv.Atoi(addressAndPort[strings.Index(addressAndPort, ":")+1:])
207 | util.FailOnError(err, "Invalid port format")
208 | return i
209 | }
210 |
211 | func (m *Manager) initVolumes() {
212 | if util.IsEmptyString(m.VolumesStr) {
213 | return
214 | }
215 |
216 | m.Volumes = domain.NewVolumesFromString(m.VolumesStr)
217 | }
218 |
219 | func (m *Manager) connect() error {
220 | initData := &domain.AgentInit{
221 | IsK8sCluster: m.K8sCluster,
222 | IsDocker: m.IsFromDocker,
223 | Port: m.Port,
224 | Os: util.OS(),
225 | Status: string(m.status),
226 | }
227 |
228 | config, err := m.Client.Connect(initData)
229 | if err != nil {
230 | return err
231 | }
232 |
233 | m.config = config
234 | return nil
235 | }
236 |
237 | func (m *Manager) listenReConn() {
238 | go func() {
239 | for range m.Client.ReConn() {
240 | util.LogWarn("connection lost from server %s, start reconnecting..", m.Server)
241 | connected := false
242 |
243 | for i := 0; i < 6; i++ {
244 | err := m.connect()
245 | if err == nil {
246 | connected = true
247 | break
248 | }
249 |
250 | util.LogWarn("unable to connect to server %s, retry...", m.Server)
251 | time.Sleep(10 * time.Second)
252 | }
253 |
254 | if !connected {
255 | panic(fmt.Errorf("unable to connect to server %s, exit", m.Server))
256 | }
257 | }
258 | }()
259 | }
260 |
261 | func (m *Manager) sendAgentProfile() {
262 | if !m.ProfileEnabled {
263 | return
264 | }
265 |
266 | go func() {
267 | for {
268 | select {
269 | case <-m.AppCtx.Done():
270 | return
271 | default:
272 | time.Sleep(10 * time.Second)
273 | _ = m.Client.ReportProfile(m.FetchProfile())
274 | }
275 | }
276 | }()
277 | }
278 |
--------------------------------------------------------------------------------
/config/manager_test.go:
--------------------------------------------------------------------------------
1 | package config
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/stretchr/testify/assert"
7 | )
8 |
9 | func TestShouldFetchSystemResource(t *testing.T) {
10 | assert := assert.New(t)
11 |
12 | m := GetInstance()
13 | assert.NotNil(m)
14 |
15 | resource := m.FetchProfile()
16 | assert.NotNil(resource)
17 |
18 | assert.True(resource.Cpu > 0)
19 | assert.True(resource.TotalMemory > 0)
20 | assert.True(resource.FreeMemory > 0)
21 | assert.True(resource.TotalDisk > 0)
22 | assert.True(resource.FreeDisk > 0)
23 | }
24 |
--------------------------------------------------------------------------------
/controller/cmd_controller.go:
--------------------------------------------------------------------------------
1 | package controller
2 |
3 | import (
4 | "net/http"
5 |
6 | "github.com/flowci/flow-agent-x/service"
7 | "github.com/gin-gonic/gin"
8 | )
9 |
10 | type CmdController struct {
11 | RootController `path:"/cmds"`
12 |
13 | GetCmdByID gin.HandlerFunc `path:"/:id"`
14 |
15 | PostExecuteCmd gin.HandlerFunc `path:"/"`
16 |
17 | cmdService *service.CmdService
18 | }
19 |
20 | // NewCmdController create new instance of CmdController
21 | func NewCmdController(router *gin.Engine) *CmdController {
22 | c := new(CmdController)
23 | c.cmdService = service.GetCmdService()
24 |
25 | autoWireController(c, router)
26 | return c
27 | }
28 |
29 | // GetCmdByIDImpl http get to get detail of cmd by id
30 | func (c *CmdController) GetCmdByIDImpl(context *gin.Context) {
31 | id := context.Param("id")
32 | context.String(http.StatusOK, "id : "+id)
33 | }
34 |
35 | // PostExecuteCmdImpl http post request to execute cmd from request body
36 | func (c *CmdController) PostExecuteCmdImpl(context *gin.Context) {
37 | bytes, err := context.GetRawData()
38 | if c.responseIfError(context, err) {
39 | return
40 | }
41 |
42 | err = c.cmdService.Execute(bytes)
43 | if c.responseIfError(context, err) {
44 | return
45 | }
46 |
47 | c.responseOk(context, nil)
48 | }
49 |
--------------------------------------------------------------------------------
/controller/cmd_controller_test.go:
--------------------------------------------------------------------------------
1 | package controller
2 |
3 | import (
4 | "github.com/flowci/flow-agent-x/api"
5 | "github.com/flowci/flow-agent-x/config"
6 | "github.com/flowci/flow-agent-x/mocks"
7 | "github.com/streadway/amqp"
8 | "testing"
9 |
10 | "github.com/gin-gonic/gin"
11 | "github.com/stretchr/testify/assert"
12 | )
13 |
14 | func TestShouldInitCmdController(t *testing.T) {
15 | assert := assert.New(t)
16 |
17 | router := gin.Default()
18 | appConfig := config.GetInstance()
19 | appConfig.Client = mockClient()
20 |
21 | cmdController := NewCmdController(router)
22 | assert.NotNil(cmdController)
23 | }
24 |
25 | func mockClient() api.Client {
26 | mockClient := &mocks.Client{}
27 | deliveries := make(chan amqp.Delivery)
28 | mockClient.On("GetCmdIn").Return(<-deliveries, nil)
29 | return mockClient
30 | }
31 |
--------------------------------------------------------------------------------
/controller/health_controller.go:
--------------------------------------------------------------------------------
1 | package controller
2 |
3 | import (
4 | "net/http"
5 | "runtime"
6 |
7 | "github.com/gin-gonic/gin"
8 | )
9 |
10 | type HealthController struct {
11 | RootController `path:"/health"`
12 |
13 | GetInfo gin.HandlerFunc `path:"/"`
14 | }
15 |
16 | type HealthInfo struct {
17 | CPU int `json:"cpu"`
18 | Memory runtime.MemStats `json:"memory"`
19 | }
20 |
21 | // NewHealthController create new instance of HealthController
22 | func NewHealthController(router *gin.Engine) *HealthController {
23 | c := new(HealthController)
24 | autoWireController(c, router)
25 | return c
26 | }
27 |
28 | func (c *HealthController) GetInfoImpl(context *gin.Context) {
29 | var mem runtime.MemStats
30 | runtime.ReadMemStats(&mem)
31 |
32 | info := HealthInfo{
33 | CPU: runtime.NumCPU(),
34 | Memory: mem,
35 | }
36 |
37 | context.JSON(http.StatusOK, info)
38 | }
39 |
--------------------------------------------------------------------------------
/controller/root_controller.go:
--------------------------------------------------------------------------------
1 | package controller
2 |
3 | import (
4 | "net/http"
5 | "reflect"
6 | "strings"
7 |
8 | "github.com/flowci/flow-agent-x/util"
9 |
10 | "github.com/gin-gonic/gin"
11 | )
12 |
13 | const (
14 | pathTagName = "path"
15 | methodSuffix = "Impl"
16 |
17 | httpGetPrefix = "Get"
18 | httpPostPrefix = "Post"
19 | httpDeletePrefix = "Delete"
20 | )
21 |
22 | // ResponseMessage the response body
23 | type ResponseMessage struct {
24 | Code int `json:"code"`
25 | Message string `json:"message"`
26 | Data interface{} `json:"data"`
27 | }
28 |
29 | // RootController supper controller type
30 | type RootController struct {
31 | }
32 |
33 | func (c *RootController) responseIfError(context *gin.Context, err error) bool {
34 | if err == nil {
35 | return false
36 | }
37 |
38 | context.Abort()
39 |
40 | context.JSON(http.StatusBadRequest, ResponseMessage{
41 | Code: -1,
42 | Message: err.Error(),
43 | })
44 |
45 | return context.IsAborted()
46 | }
47 |
48 | func (c *RootController) responseOk(context *gin.Context, data interface{}) {
49 | context.JSON(http.StatusOK, ResponseMessage{
50 | Code: 0,
51 | Message: "ok",
52 | Data: data,
53 | })
54 | }
55 |
56 | // autoWireController it will regist request mapping automatically
57 | //
58 | // Example:
59 | // type SubController struct {
60 | // RootController 'path:"/roots"'
61 | //
62 | // PostCreateHelloWorld gin.HandlerFunc 'path:"/new"'
63 | // }
64 | //
65 | // The request '/roots/new' for http POST will be registered
66 | //
67 | // first field 'RootController' define the root path by tag 'path'
68 | //
69 | // the field 'PostCreateHelloWorld' define POST request by method prefix
70 | // and sub request mapping by tag 'path'
71 | //
72 | // the method 'PostCreateHelloWorldImpl' has to created to receive the request
73 | func autoWireController(c interface{}, router *gin.Engine) {
74 | t := util.GetType(c)
75 |
76 | rootPath := t.Field(0).Tag.Get(pathTagName)
77 |
78 | for i := 1; i < t.NumField(); i++ {
79 | field := t.Field(i)
80 | subPath := field.Tag.Get(pathTagName)
81 |
82 | if util.IsEmptyString(subPath) {
83 | continue
84 | }
85 |
86 | // get field related method according to the rule
87 | searchMethodName := field.Name + methodSuffix
88 | m := reflect.ValueOf(c).MethodByName(searchMethodName)
89 |
90 | fullPath := rootPath + subPath
91 | handler := toGinHandlerFunc(m)
92 |
93 | if strings.HasPrefix(searchMethodName, httpGetPrefix) {
94 | router.GET(fullPath, handler)
95 | continue
96 | }
97 |
98 | if strings.HasPrefix(searchMethodName, httpPostPrefix) {
99 | router.POST(fullPath, handler)
100 | continue
101 | }
102 |
103 | if strings.HasPrefix(searchMethodName, httpDeletePrefix) {
104 | router.DELETE(fullPath, handler)
105 | continue
106 | }
107 | }
108 | }
109 |
110 | func toGinHandlerFunc(method reflect.Value) gin.HandlerFunc {
111 | return (method.Interface()).(func(*gin.Context))
112 | }
113 |
--------------------------------------------------------------------------------
/dao/base_test.go:
--------------------------------------------------------------------------------
1 | package dao
2 |
3 | import "time"
4 |
5 | type MockSubEntity struct {
6 | ID string `db:"column=id,pk=true,nullable=false"`
7 | Name string `db:"column=name"`
8 | Age int `db:"column=age"`
9 | CreatedAt time.Time
10 | UpdatedAt time.Time
11 | }
12 |
--------------------------------------------------------------------------------
/dao/builder.go:
--------------------------------------------------------------------------------
1 | package dao
2 |
3 | import (
4 | "fmt"
5 | "reflect"
6 | "strings"
7 |
8 | u "github.com/flowci/flow-agent-x/util"
9 | )
10 |
11 | type QueryBuilder struct {
12 | entity interface{}
13 | entityType reflect.Type
14 |
15 | table string
16 | columns []*EntityColumn
17 | key *EntityColumn
18 | }
19 |
20 | // init querybuilder with metadata
21 | func initQueryBuilder(entity interface{}) *QueryBuilder {
22 | t := u.GetType(entity)
23 |
24 | builder := new(QueryBuilder)
25 | builder.entityType = t
26 | builder.entity = entity
27 | builder.table = flatCamelString(t.Name())
28 | builder.columns = make([]*EntityColumn, t.NumField())
29 |
30 | numOfNil := 0
31 |
32 | for i := 0; i < t.NumField(); i++ {
33 | column := parseEntityColumn(t.Field(i))
34 |
35 | if column == nil {
36 | numOfNil++
37 | continue
38 | }
39 |
40 | if column.Pk {
41 | builder.key = column
42 | }
43 |
44 | builder.columns[i-numOfNil] = column
45 | }
46 |
47 | builder.columns = builder.columns[:numOfNil+1]
48 | return builder
49 | }
50 |
51 | // create table by entity
52 | func (builder *QueryBuilder) create() (string, error) {
53 | var sql strings.Builder
54 | sql.WriteString("CREATE TABLE IF NOT EXISTS " + builder.table)
55 | sql.WriteString(" (")
56 |
57 | for i, c := range builder.columns {
58 | if c == nil {
59 | continue
60 | }
61 |
62 | // create sql for field
63 | q, err := c.toQuery()
64 | if u.HasError(err) {
65 | return "", err
66 | }
67 |
68 | if i > 0 {
69 | sql.WriteByte(',')
70 | }
71 |
72 | sql.WriteString(q)
73 | }
74 |
75 | sql.WriteString(");")
76 | return sql.String(), nil
77 | }
78 |
79 | func (builder *QueryBuilder) drop() (string, error) {
80 | return "DROP TABLE IF EXISTS " + builder.table + ";", nil
81 | }
82 |
83 | func (builder *QueryBuilder) insert(data interface{}) (string, error) {
84 | if !isSameType(builder.entityType, data) {
85 | return "", ErrorNotEntity
86 | }
87 |
88 | var sql strings.Builder
89 | sql.WriteString("INSERT INTO ")
90 | sql.WriteString(builder.table)
91 | sql.WriteString(" (")
92 |
93 | for i, c := range builder.columns {
94 | sql.WriteString("'" + c.Column + "'")
95 |
96 | if i < len(builder.columns)-1 {
97 | sql.WriteString(",")
98 | }
99 | }
100 |
101 | sql.WriteString(")")
102 | sql.WriteString(" VALUES ")
103 | sql.WriteString("(")
104 |
105 | value := u.GetValue(data)
106 | for i, c := range builder.columns {
107 | query, err := toString(value.FieldByName(c.Field.Name))
108 | if u.HasError(err) {
109 | return u.EmptyStr, err
110 | }
111 |
112 | sql.WriteString(query)
113 |
114 | if i < len(builder.columns)-1 {
115 | sql.WriteString(",")
116 | }
117 | }
118 |
119 | sql.WriteString(");")
120 |
121 | return sql.String(), nil
122 | }
123 |
124 | func (builder *QueryBuilder) find(id string) (string, error) {
125 | var sql strings.Builder
126 | sql.WriteString("SELECT ")
127 |
128 | for i, c := range builder.columns {
129 | sql.WriteString("'" + c.Column + "'")
130 |
131 | if i < len(builder.columns)-1 {
132 | sql.WriteString(",")
133 | }
134 | }
135 |
136 | sql.WriteString(" FROM " + builder.table)
137 | sql.WriteString(fmt.Sprintf(" WHERE %s='%s';", builder.key.Column, id))
138 |
139 | return sql.String(), nil
140 | }
141 |
142 | func isSameType(source reflect.Type, data interface{}) bool {
143 | t := u.GetType(data)
144 | return t == source
145 | }
146 |
147 | // from value to sql type
148 | func toString(val reflect.Value) (string, error) {
149 | if val.Kind() == reflect.String {
150 | return "'" + val.String() + "'", nil
151 | }
152 |
153 | if val.Kind() == reflect.Bool {
154 | return fmt.Sprintf("%t", val.Bool()), nil
155 | }
156 |
157 | if val.Kind() == reflect.Int {
158 | return fmt.Sprintf("%d", val.Int()), nil
159 | }
160 |
161 | return u.EmptyStr, ErrorUnsupporttedDataType
162 | }
163 |
--------------------------------------------------------------------------------
/dao/builder_test.go:
--------------------------------------------------------------------------------
1 | package dao
2 |
3 | import (
4 | "testing"
5 | "time"
6 |
7 | "github.com/stretchr/testify/assert"
8 | )
9 |
10 | func TestShouldBuildQueryForCreateTable(t *testing.T) {
11 | assert := assert.New(t)
12 |
13 | builder := initQueryBuilder(MockSubEntity{})
14 | query, err := builder.create()
15 | assert.Nil(err)
16 |
17 | expected := "CREATE TABLE IF NOT EXISTS mock_sub_entity (id TEXT NOT NULL PRIMARY KEY,name TEXT,age INTEGER);"
18 | assert.Equal(expected, query)
19 | }
20 |
21 | func TestShouldBuildQueryForDropTable(t *testing.T) {
22 | assert := assert.New(t)
23 |
24 | builder := initQueryBuilder(MockSubEntity{})
25 | query, _ := builder.drop()
26 |
27 | expected := "DROP TABLE IF EXISTS mock_sub_entity;"
28 | assert.Equal(expected, query)
29 | }
30 |
31 | func TestShouldBuildQueryForInsert(t *testing.T) {
32 | assert := assert.New(t)
33 |
34 | entity := &MockSubEntity{
35 | ID: "12345",
36 | Name: "yang",
37 | Age: 18,
38 | CreatedAt: time.Now(),
39 | UpdatedAt: time.Now(),
40 | }
41 |
42 | builder := initQueryBuilder(MockSubEntity{})
43 | query, _ := builder.insert(entity)
44 |
45 | expected := "INSERT INTO mock_sub_entity ('id','name','age') VALUES ('12345','yang',18);"
46 | assert.Equal(expected, query)
47 | }
48 |
49 | func TestShouldBuildQueryForFindByID(t *testing.T) {
50 | assert := assert.New(t)
51 |
52 | builder := initQueryBuilder(MockSubEntity{})
53 | query, _ := builder.find("12345")
54 |
55 | expected := "SELECT 'id','name','age' FROM mock_sub_entity WHERE id='12345';"
56 | assert.Equal(expected, query)
57 | }
58 |
--------------------------------------------------------------------------------
/dao/client.go:
--------------------------------------------------------------------------------
1 | package dao
2 |
3 | import (
4 | "database/sql"
5 |
6 | "github.com/flowci/flow-agent-x/util"
7 |
8 | _ "github.com/mattn/go-sqlite3"
9 | )
10 |
11 | const (
12 | //file = "agent.db"
13 | )
14 |
15 | // Client for sqlite3
16 | type Client struct {
17 | dbPath string
18 | db *sql.DB
19 | }
20 |
21 | func NewInstance(path string) (*Client, error) {
22 | instance := &Client{
23 | dbPath: path,
24 | }
25 |
26 | db, err := sql.Open("sqlite3", path)
27 |
28 | if util.HasError(err) {
29 | return nil, err
30 | }
31 |
32 | instance.db = db
33 | return instance, nil
34 | }
35 |
36 | func (c *Client) Close() {
37 | if c.db != nil {
38 | c.db.Close()
39 | }
40 | }
41 |
42 | func (c *Client) Create(entity interface{}) error {
43 | builder := QueryBuilder{
44 | entity: entity,
45 | }
46 |
47 | sqlStmt, err := builder.create()
48 | if util.HasError(err) {
49 | return err
50 | }
51 |
52 | _, err = c.db.Exec(sqlStmt)
53 | return err
54 | }
55 |
--------------------------------------------------------------------------------
/dao/client_test.go:
--------------------------------------------------------------------------------
1 | package dao
2 |
3 | import (
4 | "io/ioutil"
5 | "os"
6 | "testing"
7 |
8 | "github.com/stretchr/testify/assert"
9 | )
10 |
11 | func TestShouldCreateTable(t *testing.T) {
12 | assert := assert.New(t)
13 |
14 | dir, _ := ioutil.TempDir("", "t")
15 | defer os.RemoveAll(dir)
16 |
17 | // dbPath := path.Join(dir, "test.db")
18 |
19 | client, err := NewInstance("/Users/yang/test.db")
20 | assert.Nil(err)
21 | assert.NotNil(client)
22 |
23 | entity := &MockSubEntity{}
24 | err = client.Create(entity)
25 | assert.Nil(err)
26 | }
27 |
--------------------------------------------------------------------------------
/dao/entity.go:
--------------------------------------------------------------------------------
1 | package dao
2 |
3 | import (
4 | "reflect"
5 | "strconv"
6 | "strings"
7 |
8 | "github.com/flowci/flow-agent-x/util"
9 | )
10 |
11 | const (
12 | tag = "db"
13 | tagSeparator = ","
14 | valSeparator = "="
15 |
16 | //keyFieldColumn = "column"
17 | //keyFieldNullable = "nullable"
18 | )
19 |
20 | var (
21 | typeMapping = map[reflect.Kind]string{
22 | reflect.Int: "INTEGER",
23 | reflect.String: "TEXT",
24 | }
25 | )
26 |
27 | type EntityColumn struct {
28 | Field reflect.StructField
29 | Column string
30 | Nullable bool
31 | Pk bool
32 | }
33 |
34 | func (f *EntityColumn) toQuery() (string, error) {
35 | t := typeMapping[f.Field.Type.Kind()]
36 |
37 | if util.IsEmptyString(t) {
38 | return util.EmptyStr, ErrorDBTypeNotAvailable
39 | }
40 |
41 | if f.Pk && f.Nullable {
42 | return util.EmptyStr, ErrorPrimaryKeyCannotBeNull
43 | }
44 |
45 | var query strings.Builder
46 | query.Grow(30)
47 | query.WriteString(f.Column)
48 | query.WriteByte(' ')
49 | query.WriteString(t)
50 |
51 | if !f.Nullable {
52 | query.WriteString(" NOT NULL")
53 | }
54 |
55 | if f.Pk {
56 | query.WriteString(" PRIMARY KEY")
57 | }
58 |
59 | return query.String(), nil
60 | }
61 |
62 | func parseEntityColumn(field reflect.StructField) *EntityColumn {
63 | val := field.Tag.Get(tag)
64 |
65 | if util.IsEmptyString(val) {
66 | return nil
67 | }
68 |
69 | count := 0
70 | entityField := &EntityColumn{
71 | Field: field,
72 | Nullable: true,
73 | Pk: false,
74 | }
75 |
76 | items := strings.Split(val, tagSeparator)
77 |
78 | for _, item := range items {
79 | kv := strings.Split(item, valSeparator)
80 |
81 | if len(kv) != 2 {
82 | continue
83 | }
84 |
85 | key := kv[0]
86 | val := kv[1]
87 | count++
88 |
89 | fieldVal := reflect.ValueOf(entityField).Elem()
90 | fieldOfEntityField := fieldVal.FieldByName(capitalFirstChar(key))
91 |
92 | if fieldOfEntityField.Type().Kind() == reflect.String {
93 | fieldOfEntityField.SetString(val)
94 | }
95 |
96 | if fieldOfEntityField.Type().Kind() == reflect.Bool {
97 | b, _ := strconv.ParseBool(val)
98 | fieldOfEntityField.SetBool(b)
99 | }
100 | }
101 |
102 | // no valid entity field
103 | if count == 0 {
104 | return nil
105 | }
106 |
107 | return entityField
108 | }
109 |
--------------------------------------------------------------------------------
/dao/entity_test.go:
--------------------------------------------------------------------------------
1 | package dao
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/flowci/flow-agent-x/util"
7 |
8 | "github.com/stretchr/testify/assert"
9 | )
10 |
11 | func TestShouldPrimaryKeyNotNullForEntityField(t *testing.T) {
12 | assert := assert.New(t)
13 | nameField, _ := util.GetType(MockSubEntity{}).FieldByName("Name")
14 |
15 | field := &EntityColumn{
16 | Field: nameField,
17 | Column: "name",
18 | Pk: true,
19 | Nullable: true,
20 | }
21 |
22 | _, err := field.toQuery()
23 | assert.NotNil(err)
24 | assert.Equal(ErrorPrimaryKeyCannotBeNull, err)
25 | }
26 |
27 | func TestShouldGenPrimaryKeyQueryForEntityField(t *testing.T) {
28 | assert := assert.New(t)
29 | nameField, _ := util.GetType(MockSubEntity{}).FieldByName("Name")
30 |
31 | field := &EntityColumn{
32 | Field: nameField,
33 | Column: "name",
34 | Pk: true,
35 | }
36 |
37 | q, err := field.toQuery()
38 | assert.Nil(err)
39 | assert.Equal("name TEXT NOT NULL PRIMARY KEY", q)
40 | }
41 |
42 | func TestShouldGenColumnQueryForEntityField(t *testing.T) {
43 | assert := assert.New(t)
44 | nameField, _ := util.GetType(MockSubEntity{}).FieldByName("Name")
45 |
46 | field := &EntityColumn{
47 | Field: nameField,
48 | Column: "name",
49 | Nullable: true,
50 | }
51 |
52 | q, err := field.toQuery()
53 | assert.Nil(err)
54 | assert.Equal("name TEXT", q)
55 | }
56 |
--------------------------------------------------------------------------------
/dao/errors.go:
--------------------------------------------------------------------------------
1 | package dao
2 |
3 | import "errors"
4 |
5 | var (
6 | ErrorNotEntity = errors.New("db: the instance is not entity")
7 | ErrorDBTypeNotAvailable = errors.New("db: db type not available")
8 | ErrorPrimaryKeyCannotBeNull = errors.New("db: primary key cannot set to null")
9 | ErrorUnsupporttedDataType = errors.New("db: the data type not supported yet")
10 | )
11 |
--------------------------------------------------------------------------------
/dao/helper.go:
--------------------------------------------------------------------------------
1 | package dao
2 |
3 | import (
4 | "strings"
5 | "unicode"
6 | )
7 |
8 | // change camel string to string with '_'
9 | func flatCamelString(v string) string {
10 | var builder strings.Builder
11 | builder.Grow(len(v) + 5)
12 |
13 | for i, c := range v {
14 | r := rune(c)
15 |
16 | if unicode.IsUpper(r) {
17 | r = unicode.ToLower(r)
18 |
19 | if i > 0 {
20 | builder.WriteByte('_')
21 | }
22 | }
23 |
24 | builder.WriteByte(byte(r))
25 | }
26 |
27 | return builder.String()
28 | }
29 |
30 | func capitalFirstChar(v string) string {
31 | bytes := []byte(v)
32 | bytes[0] = byte(unicode.ToUpper(rune(bytes[0])))
33 | return string(bytes)
34 | }
35 |
--------------------------------------------------------------------------------
/dao/helper_test.go:
--------------------------------------------------------------------------------
1 | package dao
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/stretchr/testify/assert"
7 | )
8 |
9 | func TestShouldFlatCamelString(t *testing.T) {
10 | assert := assert.New(t)
11 |
12 | str := flatCamelString("MockSuperEntity")
13 | assert.Equal("mock_super_entity", str)
14 | }
15 |
16 | func TestShouldCapitalFirstChar(t *testing.T) {
17 | assert := assert.New(t)
18 |
19 | str := capitalFirstChar("column")
20 | assert.Equal("Column", str)
21 | }
22 |
--------------------------------------------------------------------------------
/debug.yaml:
--------------------------------------------------------------------------------
1 | ## Docker-Compose file is used to start dependent services
2 |
3 | version: '3'
4 | services:
5 | zk:
6 | image: zookeeper:3.4
7 | container_name: flowci-agent-debug-zk
8 | ports:
9 | - "2181:2181"
10 |
11 | rabbitmq:
12 | image: rabbitmq:3-management
13 | container_name: flowci-agent-debug-rabbitmq
14 | ports:
15 | - "5672:5672"
16 | - "15672:15672"
--------------------------------------------------------------------------------
/domain/agent.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | // AgentStatus string of agent status
4 | type AgentStatus string
5 |
6 | const (
7 | // AgentOffline offline status
8 | AgentOffline AgentStatus = "OFFLINE"
9 |
10 | // AgentBusy busy status
11 | AgentBusy AgentStatus = "BUSY"
12 |
13 | // AgentIdle idle status
14 | AgentIdle AgentStatus = "IDLE"
15 | )
16 |
17 | type (
18 | // AgentProfile token signed at server side
19 | AgentProfile struct {
20 | CpuNum int `json:"cpuNum"`
21 | CpuUsage float64 `json:"cpuUsage"`
22 | TotalMemory uint64 `json:"totalMemory"`
23 | FreeMemory uint64 `json:"freeMemory"`
24 | TotalDisk uint64 `json:"totalDisk"`
25 | FreeDisk uint64 `json:"freeDisk"`
26 | }
27 |
28 | // AgentInit request data to get settings to server
29 | AgentInit struct {
30 | IsK8sCluster bool `json:"isK8sCluster"`
31 | IsDocker bool `json:"isDocker"`
32 | Token string `json:"token"`
33 | Port int `json:"port"`
34 | Os string `json:"os"`
35 | Status string `json:"status"`
36 | }
37 |
38 | // AgentConfig response body of AgentInit from server
39 | AgentConfig struct {
40 | ExitOnIdle int `json:"exitOnIdle"` // 0 for don't exit agent on idle
41 | }
42 |
43 | AgentConfigResponse struct {
44 | Response
45 | Data *AgentConfig
46 | }
47 | )
48 |
49 | func (r *AgentConfigResponse) IsOk() bool {
50 | return r.Code == ok
51 | }
52 |
53 | func (r *AgentConfigResponse) GetMessage() string {
54 | return r.Message
55 | }
56 |
--------------------------------------------------------------------------------
/domain/cmd.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | type CmdType string
4 |
5 | type CmdStatus string
6 |
7 | const (
8 | CmdTypeShell CmdType = "SHELL"
9 | CmdTypeTty CmdType = "TTY"
10 | CmdTypeKill CmdType = "KILL"
11 | CmdTypeClose CmdType = "CLOSE"
12 | )
13 |
14 | const (
15 | CmdStatusPending CmdStatus = "PENDING"
16 | CmdStatusRunning CmdStatus = "RUNNING"
17 | CmdStatusSuccess CmdStatus = "SUCCESS"
18 | CmdStatusSkipped CmdStatus = "SKIPPED"
19 | CmdStatusException CmdStatus = "EXCEPTION"
20 | CmdStatusKilled CmdStatus = "KILLED"
21 | CmdStatusTimeout CmdStatus = "TIMEOUT"
22 | )
23 |
24 | var (
25 | shellOutInd = []byte{1}
26 | ttyOutInd = []byte{2}
27 | )
28 |
29 | const (
30 | // CmdExitCodeUnknown default exit code
31 | CmdExitCodeUnknown = -1
32 |
33 | // CmdExitCodeTimeOut exit code for timeout
34 | CmdExitCodeTimeOut = -100
35 |
36 | // CmdExitCodeKilled exit code for killed
37 | CmdExitCodeKilled = -1
38 |
39 | // CmdExitCodeSuccess exit code for command executed successfully
40 | CmdExitCodeSuccess = 0
41 | )
42 |
43 | type (
44 | CmdIn struct {
45 | Type CmdType `json:"type"`
46 | }
47 |
48 | CmdOut interface {
49 | ToBytes() []byte
50 | }
51 | )
52 |
--------------------------------------------------------------------------------
/domain/cmd_shell.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | import (
4 | "encoding/json"
5 | "time"
6 | )
7 |
8 | type (
9 | Cache struct {
10 | Key string `json:"key"`
11 | Paths []string `json:"paths"`
12 | }
13 |
14 | ShellIn struct {
15 | CmdIn
16 | ID string `json:"id"`
17 | FlowId string `json:"flowId"`
18 | JobId string `json:"jobId"`
19 | AllowFailure bool `json:"allowFailure"`
20 | Plugin string `json:"plugin"`
21 | Cache *Cache `json:"cache"`
22 | Dockers []*DockerOption `json:"dockers"`
23 | Bash []string `json:"bash"`
24 | Pwsh []string `json:"pwsh"`
25 | Retry int `json:"retry"`
26 | Timeout int `json:"timeout"`
27 | Inputs Variables `json:"inputs"`
28 | EnvFilters []string `json:"envFilters"`
29 | Secrets []string `json:"secrets"` // secret name list
30 | Configs []string `json:"configs"` // config name list
31 | }
32 |
33 | ShellOut struct {
34 | ID string `json:"id"`
35 | ProcessId int `json:"processId"`
36 | Containers []string `json:"containers"` // container ids applied for shell
37 | Status CmdStatus `json:"status"`
38 | Code int `json:"code"`
39 | Output Variables `json:"output"`
40 | StartAt time.Time `json:"startAt"`
41 | FinishAt time.Time `json:"finishAt"`
42 | Error string `json:"error"`
43 | LogSize int64 `json:"logSize"`
44 | }
45 |
46 | ShellLog struct {
47 | JobId string `json:"jobId"`
48 | StepId string `json:"stepId"`
49 | Log string `json:"log"` // b64
50 | }
51 | )
52 |
53 | // ===================================
54 | // ShellIn Methods
55 | // ===================================
56 |
57 | func (in *ShellIn) HasSecrets() bool {
58 | return in.Secrets != nil && len(in.Secrets) > 0
59 | }
60 |
61 | func (in *ShellIn) HasConfigs() bool {
62 | return in.Configs != nil && len(in.Configs) > 0
63 | }
64 |
65 | func (in *ShellIn) HasCache() bool {
66 | return in.Cache != nil
67 | }
68 |
69 | func (in *ShellIn) HasPlugin() bool {
70 | return in.Plugin != ""
71 | }
72 |
73 | func (in *ShellIn) HasDockerOption() bool {
74 | return in.Dockers != nil && len(in.Dockers) > 0
75 | }
76 |
77 | func (in *ShellIn) HasEnvFilters() bool {
78 | if in.EnvFilters == nil {
79 | return false
80 | }
81 |
82 | return len(in.EnvFilters) != 0
83 | }
84 |
85 | func (in *ShellIn) VarsToStringArray() []string {
86 | if !NilOrEmpty(in.Inputs) {
87 | return in.Inputs.ToStringArray()
88 | }
89 |
90 | return []string{}
91 | }
92 |
93 | func NewShellOutput(in *ShellIn) *ShellOut {
94 | return &ShellOut{
95 | ID: in.ID,
96 | Code: CmdExitCodeUnknown,
97 | Status: CmdStatusPending,
98 | Output: NewVariables(),
99 | }
100 | }
101 |
102 | // ===================================
103 | // ShellOut Methods
104 | // ===================================
105 |
106 | func (e *ShellOut) IsFinishStatus() bool {
107 | switch e.Status {
108 | case CmdStatusKilled:
109 | return true
110 | case CmdStatusTimeout:
111 | return true
112 | case CmdStatusException:
113 | return true
114 | case CmdStatusSuccess:
115 | return true
116 | default:
117 | return false
118 | }
119 | }
120 |
121 | func (e *ShellOut) ToBytes() []byte {
122 | data, _ := json.Marshal(e)
123 | return append(shellOutInd, data...)
124 | }
125 |
--------------------------------------------------------------------------------
/domain/cmd_tty.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | import (
4 | "encoding/json"
5 | )
6 |
7 | const (
8 | TtyActionOpen = "OPEN"
9 | TtyActionClose = "CLOSE"
10 | TtyActionShell = "SHELL"
11 | )
12 |
13 | type (
14 | TtyIn struct {
15 | CmdIn
16 | ID string `json:"id"`
17 | Action string `json:"action"`
18 | Input string `json:"input"`
19 | }
20 |
21 | // Open, Close control action response
22 | TtyOut struct {
23 | ID string `json:"id"`
24 | Action string `json:"action"`
25 | IsSuccess bool `json:"success"`
26 | Error string `json:"error"`
27 | }
28 |
29 | TtyLog struct {
30 | ID string `json:"id"`
31 | Log string `json:"log"`
32 | }
33 | )
34 |
35 | // ===================================
36 | // TtyOut Methods
37 | // ===================================
38 |
39 | func (obj *TtyOut) ToBytes() []byte {
40 | data, _ := json.Marshal(obj)
41 | return append(ttyOutInd, data...)
42 | }
43 |
--------------------------------------------------------------------------------
/domain/config.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | const (
4 | ConfigCategorySmtp = "SMTP"
5 | ConfigCategoryText = "TEXT"
6 | )
7 |
8 | type (
9 | Config interface {
10 | GetName() string
11 | GetCategory() string
12 | ToEnvs() map[string]string
13 | ConfigMarker()
14 | }
15 |
16 | ConfigBase struct {
17 | Name string
18 | Category string
19 | }
20 |
21 | ConfigResponse struct {
22 | Response
23 | Data *ConfigBase `json:"data"`
24 | }
25 | )
26 |
27 | func (c *ConfigBase) GetName() string {
28 | return c.Name
29 | }
30 |
31 | func (c *ConfigBase) GetCategory() string {
32 | return c.Category
33 | }
34 |
35 | func (c *ConfigBase) ConfigMarker() {
36 | // placeholder
37 | }
38 |
--------------------------------------------------------------------------------
/domain/config_smtp.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | import "strconv"
4 |
5 | type SmtpConfig struct {
6 | ConfigBase
7 | Server string
8 | Port int
9 | SecureType string `json:"secure"`
10 | Auth *SimpleAuthPair
11 | }
12 |
13 | func (c *SmtpConfig) ToEnvs() map[string]string {
14 | return map[string]string{
15 | c.GetName() + "_SERVER": c.Server,
16 | c.GetName() + "_PORT": strconv.Itoa(c.Port),
17 | c.GetName() + "_SECURE_TYPE": c.SecureType,
18 | c.GetName() + "_AUTH_USERNAME": c.Auth.Username,
19 | c.GetName() + "_AUTH_PASSWORD": c.Auth.Password,
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/domain/config_text.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | type TextConfig struct {
4 | ConfigBase
5 | Text string
6 | }
7 |
8 | func (c *TextConfig) ToEnvs() map[string]string {
9 | return map[string]string{
10 | c.GetName(): c.Text,
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/domain/docker.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | import (
4 | "fmt"
5 | "github.com/flowci/flow-agent-x/util"
6 | "strings"
7 | )
8 |
9 | type (
10 | // DockerVolume volume will mount to step docker
11 | DockerVolume struct {
12 | Name string // volume name
13 | Dest string // dest path
14 | Script string // script file name to execute
15 | Image string // image contain volume
16 | Init string // init script /ws/{init} in image that will copy required data to /target
17 | }
18 | )
19 |
20 | func (v *DockerVolume) HasImage() bool {
21 | return v.Image != ""
22 | }
23 |
24 | func (v *DockerVolume) InitScriptInImage() string {
25 | return fmt.Sprintf("/ws/%s", v.Init)
26 | }
27 |
28 | func (v *DockerVolume) DefaultTargetInImage() string {
29 | return "/target"
30 | }
31 |
32 | func (v *DockerVolume) ScriptPath() string {
33 | return fmt.Sprintf("%s/%s", v.Dest, v.Script)
34 | }
35 |
36 | func (v *DockerVolume) ToBindStr() string {
37 | return fmt.Sprintf("%s:%s", v.Name, v.Dest)
38 | }
39 |
40 | // NewFromString parse string name=xxx,dest=xxx,script=xxx;name=xxx,dest=xxx,script=xxx;...
41 | func NewVolumesFromString(val string) []*DockerVolume {
42 | var volumes []*DockerVolume
43 |
44 | if util.IsEmptyString(val) {
45 | return volumes
46 | }
47 |
48 | tokens := strings.Split(val, ";")
49 | if len(tokens) == 0 {
50 | return volumes
51 | }
52 |
53 | getValue := func(val string) string {
54 | pair := strings.Split(val, "=")
55 |
56 | if len(pair) != 2 {
57 | panic(fmt.Errorf("'%s' is invalid volume string, must be key=value pair", val))
58 | }
59 |
60 | return pair[1]
61 | }
62 |
63 | for _, token := range tokens {
64 | if util.IsEmptyString(token) {
65 | continue
66 | }
67 |
68 | fields := strings.Split(token, ",")
69 | if len(fields) != 5 {
70 | panic(fmt.Errorf("'%s' is invalid volume string, fields must contain name,dest,script,image,init", token))
71 | }
72 |
73 | name := fields[0]
74 | dest := fields[1]
75 | script := fields[2]
76 | image := fields[3]
77 | initScript := fields[4]
78 |
79 | volumes = append(volumes, &DockerVolume{
80 | Name: getValue(name),
81 | Dest: getValue(dest),
82 | Script: getValue(script),
83 | Image: getValue(image),
84 | Init: getValue(initScript),
85 | })
86 | }
87 |
88 | return volumes
89 | }
90 |
--------------------------------------------------------------------------------
/domain/docker_config.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | import "github.com/docker/docker/api/types/container"
4 |
5 | type DockerConfig struct {
6 | Name string
7 | Auth *SimpleAuthPair
8 | Config *container.Config
9 | Host *container.HostConfig
10 | IsStop bool
11 | IsDelete bool
12 |
13 | ContainerID string // try to resume if container id is existed
14 | }
15 |
16 | func (c *DockerConfig) HasEntrypoint() bool {
17 | return c.Config.Entrypoint != nil && len(c.Config.Entrypoint) > 0
18 | }
19 |
--------------------------------------------------------------------------------
/domain/docker_option.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | import (
4 | "github.com/docker/docker/api/types/container"
5 | "github.com/docker/go-connections/nat"
6 | "github.com/flowci/flow-agent-x/util"
7 | )
8 |
9 | type (
10 | DockerOption struct {
11 | Image string `json:"image"`
12 | Auth string `json:"auth"`
13 | Name string `json:"name"`
14 | Entrypoint []string `json:"entrypoint"` // host:container
15 | Command []string `json:"command"`
16 | Ports []string `json:"ports"`
17 | Network string `json:"network"`
18 | Environment Variables `json:"environment"`
19 | User string `json:"user"`
20 | IsRuntime bool `json:"isRuntime"`
21 | IsStopContainer bool `json:"isStopContainer"`
22 | IsDeleteContainer bool `json:"isDeleteContainer"`
23 | ContainerID string // try to resume if container id is existed
24 |
25 | AuthContent *SimpleAuthPair // the real auth secret from 'auth' name
26 | }
27 | )
28 |
29 | func (d *DockerOption) HasAuth() bool {
30 | return d.Auth != ""
31 | }
32 |
33 | func (d *DockerOption) ToRuntimeConfig(vars Variables, workingDir string, binds []string) *DockerConfig {
34 | return d.toConfig(vars, workingDir, binds, true)
35 | }
36 |
37 | func (d *DockerOption) ToConfig() *DockerConfig {
38 | return d.toConfig(nil, "", nil, false)
39 | }
40 |
41 | func (d *DockerOption) SetDefaultNetwork(network string) {
42 | if d.Network == "bridge" || d.Network == "" {
43 | d.Network = network
44 | }
45 | }
46 |
47 | func (d *DockerOption) toConfig(vars Variables, workingDir string, binds []string, enableInput bool) (config *DockerConfig) {
48 | portSet, portMap, err := nat.ParsePortSpecs(d.Ports)
49 | util.PanicIfErr(err)
50 |
51 | vars = ConnectVars(vars, d.Environment)
52 |
53 | config = &DockerConfig{
54 | Name: d.Name,
55 | Config: &container.Config{
56 | Image: util.ParseStringWithSource(d.Image, vars),
57 | Env: vars.ToStringArray(),
58 | Entrypoint: d.Entrypoint,
59 | Cmd: d.Command,
60 | ExposedPorts: portSet,
61 | User: d.User,
62 | Tty: false,
63 | AttachStdin: enableInput,
64 | AttachStderr: enableInput,
65 | AttachStdout: enableInput,
66 | OpenStdin: enableInput,
67 | StdinOnce: enableInput,
68 | },
69 | Host: &container.HostConfig{
70 | NetworkMode: container.NetworkMode(d.Network),
71 | PortBindings: portMap,
72 | },
73 | IsStop: d.IsStopContainer,
74 | IsDelete: d.IsDeleteContainer,
75 | ContainerID: d.ContainerID,
76 | Auth: d.AuthContent,
77 | }
78 |
79 | if util.HasString(workingDir) {
80 | config.Config.WorkingDir = workingDir
81 | }
82 |
83 | if binds != nil {
84 | config.Host.Binds = binds
85 | }
86 |
87 | return
88 | }
89 |
--------------------------------------------------------------------------------
/domain/docker_test.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | import (
4 | "fmt"
5 | "github.com/stretchr/testify/assert"
6 | "testing"
7 | )
8 |
9 | func TestShouldParseString(t *testing.T) {
10 | assert := assert.New(t)
11 |
12 | v1 := "name=1,dest=$HOME/ws,script=init.sh,image=nginx:1,init=in-container-init.sh"
13 | v2 := "name=2,dest=$HOME/ws1,script=init1.sh,image=ubuntu:18.04,init=test.sh"
14 |
15 | volumes := NewVolumesFromString(fmt.Sprintf("%s;%s", v1, v2))
16 | assert.Equal(2, len(volumes))
17 |
18 | v := volumes[0]
19 | assert.Equal("1", v.Name)
20 | assert.Equal("$HOME/ws", v.Dest)
21 | assert.Equal("init.sh", v.Script)
22 | assert.Equal("nginx:1", v.Image)
23 | assert.Equal("in-container-init.sh", v.Init)
24 |
25 | v = volumes[1]
26 | assert.Equal("2", v.Name)
27 | assert.Equal("$HOME/ws1", v.Dest)
28 | assert.Equal("init1.sh", v.Script)
29 | assert.Equal("ubuntu:18.04", v.Image)
30 | assert.Equal("test.sh", v.Init)
31 | }
--------------------------------------------------------------------------------
/domain/events.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | type AppEvent string
4 |
5 | const EventOnIdle = AppEvent("EventOnIdle")
6 | const EventOnBusy = AppEvent("EventOnBusy")
7 |
--------------------------------------------------------------------------------
/domain/job_cache.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | type JobCache struct {
4 | Id string `json:"id"`
5 | FlowId string `json:"flowId"`
6 | JobId string `json:"jobId"`
7 | Key string `json:"key"`
8 | Os string `json:"os"`
9 | Files []string `json:"files"`
10 | }
11 |
12 | type JobCacheResponse struct {
13 | Response
14 | Data *JobCache
15 | }
16 |
17 | func (r *JobCacheResponse) IsOk() bool {
18 | return r.Code == ok
19 | }
20 |
21 | func (r *JobCacheResponse) GetMessage() string {
22 | return r.Message
23 | }
24 |
--------------------------------------------------------------------------------
/domain/k8s_config.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | type K8sConfig struct {
4 | Enabled bool
5 | InCluster bool
6 | Namespace string
7 | PodName string
8 | PodIp string
9 | }
10 |
--------------------------------------------------------------------------------
/domain/response.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | import "encoding/json"
4 |
5 | const (
6 | ok = 200
7 | )
8 |
9 | type (
10 | ResponseMessage interface {
11 | IsOk() bool
12 | GetMessage() string
13 | }
14 |
15 | // Response the base response message struct
16 | Response struct {
17 | Code int
18 | Message string
19 | }
20 |
21 | ResponseRaw struct {
22 | Raw json.RawMessage `json:"data"`
23 | }
24 | )
25 |
26 | func (r *Response) IsOk() bool {
27 | return r.Code == ok
28 | }
29 |
30 | func (r *Response) GetMessage() string {
31 | return r.Message
32 | }
33 |
--------------------------------------------------------------------------------
/domain/secret.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | const (
4 | SecretCategoryAuth = "AUTH"
5 | SecretCategorySshRsa = "SSH_RSA"
6 | SecretCategoryToken = "TOKEN"
7 | SecretCategoryAndroidSign = "ANDROID_SIGN"
8 | SecretCategoryKubeConfig = "KUBE_CONFIG"
9 | )
10 |
11 | type (
12 | Secret interface {
13 | GetName() string
14 | GetCategory() string
15 | ToEnvs() map[string]string
16 | SecretMarker()
17 | }
18 |
19 | SecretBase struct {
20 | Name string `json:"name"`
21 | Category string `json:"category"`
22 | }
23 |
24 | SecretField struct {
25 | Data string `json:"data"`
26 | }
27 |
28 | SecretResponse struct {
29 | Response
30 | Data *SecretBase `json:"data"`
31 | }
32 | )
33 |
34 | func (s *SecretBase) GetName() string {
35 | return s.Name
36 | }
37 |
38 | func (s *SecretBase) GetCategory() string {
39 | return s.Category
40 | }
41 |
42 | func (s *SecretBase) SecretMarker() {
43 | // placeholder
44 | }
45 |
--------------------------------------------------------------------------------
/domain/secret_auth.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | type (
4 | SimpleAuthPair struct {
5 | Username string `json:"username"`
6 | Password string `json:"password"`
7 | }
8 |
9 | AuthSecret struct {
10 | SecretBase
11 | Pair *SimpleAuthPair `json:"pair"`
12 | }
13 | )
14 |
15 | func (s *AuthSecret) ToEnvs() map[string]string {
16 | return map[string]string{
17 | s.GetName() + "_USERNAME": s.Pair.Username,
18 | s.GetName() + "_PASSWORD": s.Pair.Password,
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/domain/secret_rsa.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | type (
4 | SimpleKeyPair struct {
5 | PublicKey string `json:"publicKey"`
6 | PrivateKey string `json:"privateKey"`
7 | }
8 |
9 | RSASecret struct {
10 | SecretBase
11 | Pair *SimpleKeyPair `json:"pair"`
12 | MD5FingerPrint string `json:"md5Fingerprint"`
13 | }
14 | )
15 |
16 | func (s *RSASecret) ToEnvs() map[string]string {
17 | return map[string]string{
18 | s.GetName() + "_PUBLIC_KEY": s.Pair.PublicKey,
19 | s.GetName() + "_PRIVATE_KEY": s.Pair.PrivateKey,
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/domain/secret_test.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | import (
4 | "github.com/stretchr/testify/assert"
5 | "testing"
6 | )
7 |
8 | func TestShouldGetVarsOfAuthSecret(t *testing.T) {
9 | assert := assert.New(t)
10 |
11 | secret := &AuthSecret{
12 | SecretBase: SecretBase{
13 | Name: "MyAuth",
14 | Category: SecretCategoryAuth,
15 | },
16 | Pair: &SimpleAuthPair{
17 | Username: "admin",
18 | Password: "12345",
19 | },
20 | }
21 |
22 | vars := secret.ToEnvs()
23 | assert.NotNil(vars)
24 | assert.Equal(2, len(vars))
25 |
26 | assert.Equal("admin", vars["MyAuth_USERNAME"])
27 | assert.Equal("12345", vars["MyAuth_PASSWORD"])
28 | }
29 |
30 | func TestShouldGetVarsOfRsaSecret(t *testing.T) {
31 | assert := assert.New(t)
32 |
33 | secret := &RSASecret{
34 | SecretBase: SecretBase{
35 | Name: "MyRSA",
36 | Category: SecretCategorySshRsa,
37 | },
38 | Pair: &SimpleKeyPair{
39 | PublicKey: "publicAdmin",
40 | PrivateKey: "privateAdmin",
41 | },
42 | }
43 |
44 | vars := secret.ToEnvs()
45 | assert.NotNil(vars)
46 | assert.Equal(2, len(vars))
47 |
48 | assert.Equal("publicAdmin", vars["MyRSA_PUBLIC_KEY"])
49 | assert.Equal("privateAdmin", vars["MyRSA_PRIVATE_KEY"])
50 | }
51 |
52 | func TestShouldGetVarsOfTokenSecret(t *testing.T) {
53 | assert := assert.New(t)
54 |
55 | secret := &TokenSecret{
56 | SecretBase: SecretBase{
57 | Name: "MyToken",
58 | Category: SecretCategoryToken,
59 | },
60 | Token: &SecretField{
61 | Data: "mytoken",
62 | },
63 | }
64 |
65 | vars := secret.ToEnvs()
66 | assert.NotNil(vars)
67 | assert.Equal(1, len(vars))
68 | assert.Equal("mytoken", vars["MyToken"])
69 | }
70 |
--------------------------------------------------------------------------------
/domain/secret_token.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | import "github.com/flowci/flow-agent-x/util"
4 |
5 | type (
6 | TokenSecret struct {
7 | SecretBase
8 | Token *SecretField `json:"token"`
9 | }
10 | )
11 |
12 | func (s *TokenSecret) ToEnvs() map[string]string {
13 | util.PanicIfNil(s.Token, "secret token content")
14 |
15 | return map[string]string{
16 | s.GetName(): s.Token.Data,
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/domain/variables.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | import (
4 | "fmt"
5 | "github.com/flowci/flow-agent-x/util"
6 | )
7 |
8 | const (
9 | VarServerUrl = "FLOWCI_SERVER_URL"
10 |
11 | VarAgentDebug = "FLOWCI_AGENT_DEBUG" // boolean
12 | VarAgentToken = "FLOWCI_AGENT_TOKEN"
13 | VarAgentPort = "FLOWCI_AGENT_PORT"
14 | VarAgentWorkspace = "FLOWCI_AGENT_WORKSPACE"
15 | VarAgentJobDir = "FLOWCI_AGENT_JOB_DIR"
16 | VarAgentPluginDir = "FLOWCI_AGENT_PLUGIN_DIR"
17 | VarAgentLogDir = "FLOWCI_AGENT_LOG_DIR"
18 | VarAgentVolumes = "FLOWCI_AGENT_VOLUMES"
19 | VarAgentDockerNetwork = "FLOWCI_AGENT_DOCKER_NETWORK"
20 | VarAgentDockerAuth = "FLOWCI_AGENT_DOCKER_AUTH" // for private docker repo auth
21 | VarAgentEnableProfile = "FLOWCI_AGENT_PROFILE_ENABLED" // boolean
22 | VarAgentFromDocker = "FLOWCI_DOCKER_AGENT" // boolean
23 |
24 | VarK8sEnabled = "FLOWCI_AGENT_K8S_ENABLED" // boolean
25 | VarK8sInCluster = "FLOWCI_AGENT_K8S_IN_CLUSTER" // boolean
26 |
27 | VarK8sNodeName = "K8S_NODE_NAME"
28 | VarK8sPodName = "K8S_POD_NAME"
29 | VarK8sPodIp = "K8S_POD_IP"
30 | VarK8sNamespace = "K8S_NAMESPACE"
31 |
32 | VarAgentIpPattern = "FLOWCI_AGENT_IP_%s" // ip address of agent host
33 | VarExportContainerIdPattern = "export CONTAINER_ID_%d=%s" // container id , d=index of dockers
34 | VarExportContainerIpPattern = "export CONTAINER_IP_%d=%s" // container ip , d=index of dockers
35 | )
36 |
37 | // Variables applied for environment variable as key, value
38 | type Variables map[string]string
39 |
40 | func NewVariables() Variables {
41 | return Variables{
42 | "_TYPE_": "_string_",
43 | }
44 | }
45 |
46 | // NilOrEmpty detect variable is nil or empty
47 | func NilOrEmpty(v Variables) bool {
48 | return v == nil || v.IsEmpty()
49 | }
50 |
51 | func ConnectVars(a Variables, b Variables) Variables {
52 | if a == nil {
53 | a = Variables{}
54 | }
55 |
56 | if b == nil {
57 | b = Variables{}
58 | }
59 |
60 | vars := make(Variables, a.Size()+b.Size())
61 | for k, val := range a {
62 | vars[k] = val
63 | }
64 |
65 | for k, val := range b {
66 | vars[k] = val
67 | }
68 |
69 | return vars
70 | }
71 |
72 | func (v Variables) Copy() Variables {
73 | copied := make(Variables, v.Size())
74 | for k, val := range v {
75 | copied[k] = val
76 | }
77 | return copied
78 | }
79 |
80 | func (v Variables) Size() int {
81 | return len(v)
82 | }
83 |
84 | // Resolve to gain actual value from env variables
85 | func (v Variables) Resolve() Variables {
86 | // resolve from system env vars
87 | for key, val := range v {
88 | val = util.ParseString(val)
89 | v[key] = val
90 | }
91 |
92 | // resolve from current env vars
93 | for key, val := range v {
94 | val = util.ParseStringWithSource(val, v)
95 | v[key] = val
96 | }
97 |
98 | return v
99 | }
100 |
101 | // ToStringArray convert variables map to key=value string array
102 | func (v Variables) ToStringArray() []string {
103 | array := make([]string, v.Size())
104 | index := 0
105 | for key, val := range v {
106 | array[index] = fmt.Sprintf("%s=%s", key, val)
107 | index++
108 | }
109 |
110 | return array
111 | }
112 |
113 | // IsEmpty to check is empty variables
114 | func (v Variables) IsEmpty() bool {
115 | return len(v) == 0
116 | }
117 |
118 | func (v Variables) AddMapVars(vars map[string]string) {
119 | for key, value := range vars {
120 | v[key] = value
121 | }
122 | }
123 |
--------------------------------------------------------------------------------
/domain/variables_test.go:
--------------------------------------------------------------------------------
1 | package domain
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "reflect"
7 | "testing"
8 |
9 | "github.com/stretchr/testify/assert"
10 | )
11 |
12 | func TestShouldToStringArray(t *testing.T) {
13 | assert := assert.New(t)
14 |
15 | variables := Variables{
16 | "hello": "world",
17 | }
18 |
19 | array := variables.ToStringArray()
20 | assert.NotNil(array)
21 | assert.Equal(1, len(array))
22 | assert.Equal("hello=world", array[0])
23 | }
24 |
25 | func TestShouldDeepCopy(t *testing.T) {
26 | assert := assert.New(t)
27 |
28 | variables := Variables{"hello": "world"}
29 | copied := variables.Copy()
30 | assert.True(reflect.DeepEqual(variables, copied))
31 | }
32 |
33 | func TestShouldToStringArrayWithEnvVariables(t *testing.T) {
34 | assert := assert.New(t)
35 |
36 | variables := Variables{
37 | "SAY_HELLO": "${USER} hello",
38 | }
39 |
40 | variables.Resolve()
41 | array := variables.ToStringArray()
42 | assert.NotNil(array)
43 | assert.Equal(fmt.Sprintf("SAY_HELLO=%s hello", os.Getenv("USER")), array[0])
44 | }
45 |
46 | func TestShouldToStringArrayWithNestedEnvVariables(t *testing.T) {
47 | assert := assert.New(t)
48 |
49 | variables := Variables{
50 | "NESTED_HELLO": "${SAY_HELLO} hello",
51 | "SAY_HELLO": "${USER} hello",
52 | }
53 |
54 | variables.Resolve()
55 | array := variables.ToStringArray()
56 | assert.NotNil(array)
57 | assert.Equal(2, len(array))
58 |
59 | assert.Equal(fmt.Sprintf("NESTED_HELLO=%s hello hello", os.Getenv("USER")), array[0])
60 | assert.Equal(fmt.Sprintf("SAY_HELLO=%s hello", os.Getenv("USER")), array[1])
61 | }
62 |
63 | func TestShouldConnectVariables(t *testing.T) {
64 | assert := assert.New(t)
65 |
66 | varA := Variables{
67 | "NESTED_HELLO": "${SAY_HELLO_A} hello",
68 | "SAY_HELLO_A": "hello A",
69 | }
70 |
71 | varB := Variables{
72 | "SAY_HELLO_B": "hello B",
73 | }
74 |
75 | vars := ConnectVars(varA, varB)
76 | assert.NotNil(vars)
77 | assert.Equal(3, vars.Size())
78 |
79 | vars.Resolve()
80 | assert.Equal("hello A hello", vars["NESTED_HELLO"])
81 | assert.Equal("hello A", vars["SAY_HELLO_A"])
82 | assert.Equal("hello B", vars["SAY_HELLO_B"])
83 | }
84 |
--------------------------------------------------------------------------------
/executor/bin_file.go:
--------------------------------------------------------------------------------
1 | package executor
2 |
3 | import "os"
4 |
5 | var binFiles = []*binFile{
6 | {
7 | name: "wait-for-it.sh",
8 | content: MustAsset("wait-for-it.sh"),
9 | permission: os.FileMode(0755),
10 | permissionStr: "0755",
11 | },
12 | }
13 |
14 | type binFile struct {
15 | name string
16 | content []byte
17 | permission os.FileMode
18 | permissionStr string
19 | }
20 |
--------------------------------------------------------------------------------
/executor/bindata.go:
--------------------------------------------------------------------------------
1 | // Code generated by go-bindata. (@generated) DO NOT EDIT.
2 |
3 | // Package main generated by go-bindata.// sources:
4 | // wait-for-it.sh
5 | package executor
6 |
7 | import (
8 | "bytes"
9 | "compress/gzip"
10 | "fmt"
11 | "io"
12 | "io/ioutil"
13 | "os"
14 | "path/filepath"
15 | "strings"
16 | "time"
17 | )
18 |
19 | func bindataRead(data []byte, name string) ([]byte, error) {
20 | gz, err := gzip.NewReader(bytes.NewBuffer(data))
21 | if err != nil {
22 | return nil, fmt.Errorf("read %q: %v", name, err)
23 | }
24 |
25 | var buf bytes.Buffer
26 | _, err = io.Copy(&buf, gz)
27 | clErr := gz.Close()
28 |
29 | if err != nil {
30 | return nil, fmt.Errorf("read %q: %v", name, err)
31 | }
32 | if clErr != nil {
33 | return nil, err
34 | }
35 |
36 | return buf.Bytes(), nil
37 | }
38 |
39 | type asset struct {
40 | bytes []byte
41 | info os.FileInfo
42 | }
43 |
44 | type bindataFileInfo struct {
45 | name string
46 | size int64
47 | mode os.FileMode
48 | modTime time.Time
49 | }
50 |
51 | // Name return file name
52 | func (fi bindataFileInfo) Name() string {
53 | return fi.name
54 | }
55 |
56 | // Size return file size
57 | func (fi bindataFileInfo) Size() int64 {
58 | return fi.size
59 | }
60 |
61 | // Mode return file mode
62 | func (fi bindataFileInfo) Mode() os.FileMode {
63 | return fi.mode
64 | }
65 |
66 | // ModTime return file modify time
67 | func (fi bindataFileInfo) ModTime() time.Time {
68 | return fi.modTime
69 | }
70 |
71 | // IsDir return file whether a directory
72 | func (fi bindataFileInfo) IsDir() bool {
73 | return fi.mode&os.ModeDir != 0
74 | }
75 |
76 | // Sys return file is sys mode
77 | func (fi bindataFileInfo) Sys() interface{} {
78 | return nil
79 | }
80 |
81 | var _waitForItSh = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x58\x6d\x53\xe3\xc8\x11\xfe\xae\x5f\xd1\x2b\x2b\x60\x93\xd5\x0a\x53\x75\x49\xc5\x9c\xb8\x73\x80\x03\x55\xb1\x0b\xc1\x76\xa5\x52\x14\x45\x09\xa9\x65\x4d\x21\xcf\x68\x67\x46\xbc\x1c\x38\xbf\x3d\x35\x23\xc9\x92\x25\xd9\x0b\xb9\x44\x5f\x40\x9a\x9e\x9e\xee\xa7\xbb\x9f\xe9\x76\xef\x93\x93\x09\xee\xdc\x13\xea\x20\x7d\x84\x7b\x5f\xc4\x46\x0f\x66\x02\x41\xc6\x44\x80\x08\x38\x49\x25\x48\x06\x12\x85\x04\x12\x81\x0f\x73\xf2\x88\x14\xa6\xc7\x57\x10\x33\x21\x9d\x94\x71\x09\x3e\x47\xf0\x1f\x7d\x92\xf8\xf7\x09\x1a\xc6\x3f\xc7\xde\xf4\xb7\xcb\x6b\x6f\x7a\x17\x2c\x42\xea\x2f\xd0\xb5\x5e\xf7\x7b\xbd\x3d\x67\x69\x18\x18\xc4\x0c\x39\xef\x0f\xe0\x55\xe9\xbb\xb9\x01\xab\x12\xff\xc7\xcc\x3b\x9d\x82\x4d\x11\x86\x70\x7b\x7b\x08\x32\x46\x0a\x6a\x07\x98\xd6\xaf\x26\x0c\x8f\x76\x0e\x0e\x21\x22\xb0\x34\x8c\x4c\xf8\x73\xec\x0f\x8c\x57\x03\x00\x20\xf0\x25\xfc\xfc\x33\xcc\x26\xe3\xb3\x53\x38\xda\x39\x30\x66\x6a\x79\xa4\xd7\xac\x96\x39\xda\xf2\x91\xb6\xfc\xc6\x16\xb7\x70\x63\x4b\x90\x64\x81\x2c\x93\xea\xc5\x86\x80\x2d\x16\x3e\x0d\xc1\xe7\x73\x71\xab\x95\xd8\x31\x9c\x5f\x4e\xa6\xf0\x06\xb6\xad\x76\xbb\xfa\x2d\x7f\xce\x99\x90\xc0\x38\x78\x57\x90\xd1\x10\xb9\x06\x2b\xdf\x95\xc2\xd5\xe5\x75\xbe\x4b\x1d\xe7\xea\xb7\xfc\x51\x08\x6a\x13\x1a\x7b\xb6\x3d\xe3\x44\x22\xa7\xbe\x24\x8f\x98\xbc\x7c\x86\x17\x96\x81\x48\x31\x20\xd1\x8b\x82\x4a\xbb\x05\xca\xee\x3c\x28\xa2\xf2\x33\xb7\x46\x68\x43\x84\xe4\x24\x90\x0d\xcd\x97\x34\x79\x01\x7c\xc6\x20\x93\x08\x22\xbb\x2f\x11\x20\x91\xd6\xac\xc3\x2f\xb2\x20\x40\x0c\x45\xae\xec\xbb\x56\xf6\x3d\x23\xd8\xd4\x05\x27\x8c\xee\x4a\x60\x99\x4c\x33\x65\xcf\x0b\x08\xe9\xcb\x4c\xc0\x02\x85\x0a\x4b\xa1\x40\xc2\xd4\xfb\x7a\x7a\x39\xcb\xe1\x29\xf0\x77\x8b\x6f\x3f\x84\x62\x9a\xcb\x03\xa1\x20\x30\x60\x34\x14\x9f\xe1\x77\xe4\x0c\x22\xc6\x81\xb2\x32\x9e\xf9\x51\x36\x1c\x5f\x7e\xfd\x3a\xfe\x76\x02\xe3\xeb\xb3\xc9\x9a\x9e\xd3\xc2\xe7\xd2\xe1\x27\x22\x63\x1d\x77\xf0\x23\xa9\xe2\x52\x7a\x1f\x11\x4a\x44\x8c\xc2\xd0\x49\xa6\xf5\xe2\x33\x91\x30\x34\x96\x86\xf1\xe4\x13\x79\x17\x31\xbe\x4a\xc8\x56\x6a\x97\xbe\xda\x73\x09\xfb\xab\xe4\x5e\xb9\x59\x94\x05\x98\xed\x6c\x1d\x81\xd2\x4e\xe8\xbc\x4b\x5d\xe1\xbb\xf6\xba\xb6\xac\xb2\x73\x54\x7b\x57\x79\x67\xe6\x36\x27\x02\x3f\x76\xea\x3b\x54\x6b\xd4\x54\x34\xfc\x12\xf7\xfc\xb0\x88\xe8\x3f\x95\xb0\x90\x3e\x97\x77\x52\xb8\x56\x3f\xf4\x25\xc2\x9f\xff\x24\x06\x5a\xe4\x29\x26\x09\x42\x5e\xb0\x21\x5b\x19\xd8\x82\xd1\x9b\xfc\x7d\x36\xf9\x17\xd8\xf8\xbd\x46\x11\x6b\xc9\x42\x03\xb0\x7f\x6f\x1a\x0c\x0d\x83\xd7\x76\x54\x4b\x1c\x45\x96\x48\xd7\xfa\xa5\x42\xa8\x0e\x97\x7a\xfa\x9a\x8e\x8e\xc0\x09\xf1\xd1\x91\x41\xea\x34\x4e\x72\x1a\x27\x0d\xe0\x48\x8b\xd2\x2c\x49\xe0\xe0\x68\x67\xf8\xee\xa3\x0b\xf0\x3a\x61\xc8\xa5\x35\x0c\x1d\xc9\xb4\xae\x19\x69\xd8\x85\xf8\xbb\x72\xe0\x47\x71\x27\xa2\xe2\xfd\xa2\x60\xac\x7e\xbf\x79\x36\xd8\x1d\x29\x30\x18\x94\xc9\x6b\xae\xd9\x73\xcf\xd1\x7f\xe8\x02\x41\x24\x88\x29\x0c\x8b\x14\xa1\x79\x58\x38\xca\x8c\xd3\x36\x34\xf5\xa2\xbc\x7b\xe2\x7e\x9a\x62\x55\x9c\x3d\xf0\x28\x30\xae\x69\x97\x81\xc8\x52\xcd\x97\x13\xef\xcc\xfb\x36\x85\x30\xe3\x2a\xeb\x8b\x3c\x1e\x41\x2c\x65\x3a\x72\x9c\x8c\x92\xe7\x2f\x42\xfa\xc1\x03\x3e\x07\xb1\x4f\xe7\xf8\x25\x60\x0b\xc7\x77\x7e\xfa\xeb\x5f\xfe\x76\xd0\x5d\xf3\xc5\x75\xd6\x9d\xab\xc5\x01\x75\x79\x95\xda\xaa\xb0\x7f\xbb\x18\x9f\x75\x15\xbb\xb5\xbf\x62\x5c\xdb\x0e\x62\x92\x84\xe5\x6d\xd4\x4c\xf7\xe2\xba\x69\x86\xab\xa2\xd9\x0e\xed\x3b\x6d\x7e\xf8\x6f\x6d\xfc\x3f\xd9\xd6\xa2\x93\x2b\xef\xc4\xb5\x3e\xe9\x8f\x92\xfb\x29\x98\x0f\x24\x49\xc0\x56\x61\xb4\xad\x35\x31\x13\xbc\x6f\x79\xd5\xab\xac\x80\xf5\xc5\x86\xd6\xeb\xd3\xc9\xec\x62\x5a\x96\x62\x2b\xaa\xf9\xb2\xee\x52\x3e\x4a\xe4\x25\x9e\x2c\x08\x32\xce\x31\x2c\x4a\xe6\x7f\xc9\xef\x05\x46\xed\xba\xc8\xcd\x56\x75\xd1\x83\x94\xb3\x00\x85\x50\xf7\x5c\xb6\x40\x2a\x85\x91\xf3\xaf\x72\xb4\xb7\xba\xa2\x8c\x82\x88\x03\x5f\x20\x98\xd6\xd0\x04\x52\xf9\xb9\x37\xda\x83\x8a\x48\xaa\x63\x54\xc8\x75\x7c\xfb\xd6\xeb\xd0\x71\x46\x0e\x2c\xbb\xc4\x94\x03\xae\xf5\xda\xde\x77\xb3\x7f\xbb\xec\x90\x57\x0e\x76\xcb\x0f\x6b\xf2\x22\x26\x91\x84\x8a\x62\x0f\x0f\x57\xff\x16\x49\xd9\x65\xcb\xf1\xb9\x77\x71\xe2\x0e\xdf\xa5\xa5\xd6\xf5\x74\xa9\xd2\x15\xff\x4e\x55\xf5\x6e\xac\x4b\xd7\x64\x7a\xed\x1d\xbf\x57\x59\xbc\x11\x64\xd3\x3a\x30\x37\x5f\x24\xba\x22\x5d\x17\x4c\xb3\x6a\xb8\x35\x03\x1f\xae\x51\xaf\x3e\xf9\xa0\x1b\x57\x5d\xe4\x7b\x5b\xce\x7f\x1d\xf6\xf6\xdc\xa5\xf9\x2e\x3f\xd2\x2e\x3d\x3a\xf8\xdb\xfd\xd0\x14\xf2\xc7\xfc\xd0\x59\xdb\xe9\x47\x71\xfe\x47\xfc\xe8\x8c\x68\x51\xd4\x3f\x70\xa5\x2c\xfd\x3f\xe6\x4d\x49\xa6\x9d\x0e\x55\x86\x7c\xc4\x27\x7b\xb0\x2e\xd6\x55\x4a\x17\x9e\xdb\x57\xb3\x5a\x25\xba\x7e\x9f\xaf\x67\x0e\x26\xb5\x78\xeb\x89\xae\x4b\xb0\xe6\xc2\x8a\x5d\x67\xf4\x81\xb2\x27\xba\x62\xb0\x11\x58\x43\x73\xbb\x2a\x14\x7e\x60\xe8\xc6\xc1\xc8\x31\x37\x1b\x75\x60\x16\x90\xbf\xbd\xad\x2d\x69\x6e\x6d\x44\xc3\x58\x33\xe6\x94\x73\xc6\x47\x7a\x1c\xa3\x88\xa1\x6a\x2a\x52\xce\x1e\x49\x88\xe0\x37\x86\xb2\x62\x94\xfe\x92\xdb\x9a\xdb\x19\x91\xfa\xc8\x5c\xc6\xa6\x4e\x76\xc5\xb7\x91\x3d\xfc\x69\x69\xb4\x18\xa2\x2e\x99\x7f\x1a\xd9\xfb\x75\xb9\x9c\xe0\xea\x62\xfa\x4b\x43\x2a\xe7\xae\xba\x94\xfe\xa2\xa5\x8c\x1e\x1c\xc7\x18\x3c\xe8\x7e\x09\x51\x8f\x85\xe5\x00\x26\x20\xe2\x6c\x01\xf7\x99\x78\xb9\x67\xcf\xbf\xb4\x5d\xb9\xbb\x1a\x4f\xcf\x5d\xab\x2f\x5f\x52\x54\x23\x71\xb1\x73\xb0\x59\x92\xa3\x9f\xa4\xbe\x8c\x3b\xca\x42\x8b\xc0\x41\xad\xa9\x7e\x7b\x03\x8e\x7e\x98\x10\xfa\x00\x76\xb4\x69\xcb\xa0\x8e\x71\xbd\x89\x71\x4d\xd3\xd8\x54\x83\xf9\x61\xee\xbf\xc1\x2c\xbc\x6b\x64\x40\x73\x2e\x29\xc8\xba\x04\x8b\x44\x25\x2a\x2b\xb4\x32\x81\x42\x8d\xbe\x51\xe2\xcf\x0b\xd9\x3e\xc7\x00\xa9\x84\x71\x92\x12\x8a\xf0\x88\x5c\x10\x46\x85\xea\x71\x77\xe5\xaa\x3b\xb5\xf5\x24\xbd\x60\x1c\x07\x65\x5b\x52\xea\xdc\xc9\xc1\x10\x32\x54\x6f\x6f\x30\xe7\x98\xaa\x9b\xca\x46\xd8\xb5\x25\xec\x36\x3a\x94\x4d\x30\xd8\xd5\xd0\xb6\x6a\x02\x5b\x0e\xee\xeb\x6c\x6d\xe1\xa5\xf3\xa9\x6b\xb6\x2d\x7b\xf0\xad\x2d\x96\x9e\xa3\xdb\xad\xca\xca\x8a\x8f\x0e\xd3\xcd\xc6\xbf\xc3\xf5\x86\x01\xf5\xa6\x77\xcd\xe4\x6d\xbb\x22\xb2\x01\x8c\x0b\x0f\x3e\xb5\xe9\x62\x7b\x23\xb9\xb3\x53\x5f\xca\xcb\x78\xd3\xe8\xb0\xb5\xc7\x2c\x7e\xde\x59\xb0\x10\x3f\x03\xc7\x28\x13\x7a\x9c\x61\xf5\x5f\x77\x8a\xfe\xaf\x22\xcc\x0d\x11\x80\xaa\xa1\x54\xbb\xc1\x5c\xe3\x90\x0b\xef\xe6\xd7\xdb\xa5\x59\xc5\x69\x83\x96\x88\xfc\x27\x00\x00\xff\xff\xcc\x96\x0f\xcf\x67\x14\x00\x00")
82 |
83 | func waitForItShBytes() ([]byte, error) {
84 | return bindataRead(
85 | _waitForItSh,
86 | "wait-for-it.sh",
87 | )
88 | }
89 |
90 | func waitForItSh() (*asset, error) {
91 | bytes, err := waitForItShBytes()
92 | if err != nil {
93 | return nil, err
94 | }
95 |
96 | info := bindataFileInfo{name: "wait-for-it.sh", size: 5223, mode: os.FileMode(493), modTime: time.Unix(1595322570, 0)}
97 | a := &asset{bytes: bytes, info: info}
98 | return a, nil
99 | }
100 |
101 | // Asset loads and returns the asset for the given name.
102 | // It returns an error if the asset could not be found or
103 | // could not be loaded.
104 | func Asset(name string) ([]byte, error) {
105 | canonicalName := strings.Replace(name, "\\", "/", -1)
106 | if f, ok := _bindata[canonicalName]; ok {
107 | a, err := f()
108 | if err != nil {
109 | return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
110 | }
111 | return a.bytes, nil
112 | }
113 | return nil, fmt.Errorf("Asset %s not found", name)
114 | }
115 |
116 | // MustAsset is like Asset but panics when Asset would return an error.
117 | // It simplifies safe initialization of global variables.
118 | func MustAsset(name string) []byte {
119 | a, err := Asset(name)
120 | if err != nil {
121 | panic("asset: Asset(" + name + "): " + err.Error())
122 | }
123 |
124 | return a
125 | }
126 |
127 | // AssetInfo loads and returns the asset info for the given name.
128 | // It returns an error if the asset could not be found or
129 | // could not be loaded.
130 | func AssetInfo(name string) (os.FileInfo, error) {
131 | canonicalName := strings.Replace(name, "\\", "/", -1)
132 | if f, ok := _bindata[canonicalName]; ok {
133 | a, err := f()
134 | if err != nil {
135 | return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
136 | }
137 | return a.info, nil
138 | }
139 | return nil, fmt.Errorf("AssetInfo %s not found", name)
140 | }
141 |
142 | // AssetNames returns the names of the assets.
143 | func AssetNames() []string {
144 | names := make([]string, 0, len(_bindata))
145 | for name := range _bindata {
146 | names = append(names, name)
147 | }
148 | return names
149 | }
150 |
151 | // _bindata is a table, holding each asset generator, mapped to its name.
152 | var _bindata = map[string]func() (*asset, error){
153 | "wait-for-it.sh": waitForItSh,
154 | }
155 |
156 | // AssetDir returns the file names below a certain
157 | // directory embedded in the file by go-bindata.
158 | // For example if you run go-bindata on data/... and data contains the
159 | // following hierarchy:
160 | // data/
161 | // foo.txt
162 | // img/
163 | // a.png
164 | // b.png
165 | // then AssetDir("data") would return []string{"foo.txt", "img"}
166 | // AssetDir("data/img") would return []string{"a.png", "b.png"}
167 | // AssetDir("foo.txt") and AssetDir("nonexistent") would return an error
168 | // AssetDir("") will return []string{"data"}.
169 | func AssetDir(name string) ([]string, error) {
170 | node := _bintree
171 | if len(name) != 0 {
172 | canonicalName := strings.Replace(name, "\\", "/", -1)
173 | pathList := strings.Split(canonicalName, "/")
174 | for _, p := range pathList {
175 | node = node.Children[p]
176 | if node == nil {
177 | return nil, fmt.Errorf("Asset %s not found", name)
178 | }
179 | }
180 | }
181 | if node.Func != nil {
182 | return nil, fmt.Errorf("Asset %s not found", name)
183 | }
184 | rv := make([]string, 0, len(node.Children))
185 | for childName := range node.Children {
186 | rv = append(rv, childName)
187 | }
188 | return rv, nil
189 | }
190 |
191 | type bintree struct {
192 | Func func() (*asset, error)
193 | Children map[string]*bintree
194 | }
195 |
196 | var _bintree = &bintree{nil, map[string]*bintree{
197 | "wait-for-it.sh": &bintree{waitForItSh, map[string]*bintree{}},
198 | }}
199 |
200 | // RestoreAsset restores an asset under the given directory
201 | func RestoreAsset(dir, name string) error {
202 | data, err := Asset(name)
203 | if err != nil {
204 | return err
205 | }
206 | info, err := AssetInfo(name)
207 | if err != nil {
208 | return err
209 | }
210 | err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
211 | if err != nil {
212 | return err
213 | }
214 | err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
215 | if err != nil {
216 | return err
217 | }
218 | err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
219 | if err != nil {
220 | return err
221 | }
222 | return nil
223 | }
224 |
225 | // RestoreAssets restores an asset under the given directory recursively
226 | func RestoreAssets(dir, name string) error {
227 | children, err := AssetDir(name)
228 | // File
229 | if err != nil {
230 | return RestoreAsset(dir, name)
231 | }
232 | // Dir
233 | for _, child := range children {
234 | err = RestoreAssets(dir, filepath.Join(name, child))
235 | if err != nil {
236 | return err
237 | }
238 | }
239 | return nil
240 | }
241 |
242 | func _filePath(dir, name string) string {
243 | canonicalName := strings.Replace(name, "\\", "/", -1)
244 | return filepath.Join(append([]string{dir}, strings.Split(canonicalName, "/")...)...)
245 | }
246 |
--------------------------------------------------------------------------------
/executor/docker_test.go:
--------------------------------------------------------------------------------
1 | package executor
2 |
3 | import (
4 | "encoding/base64"
5 | "github.com/flowci/flow-agent-x/config"
6 | "github.com/flowci/flow-agent-x/domain"
7 | "github.com/flowci/flow-agent-x/util"
8 | "github.com/stretchr/testify/assert"
9 | "testing"
10 | "time"
11 | )
12 |
13 | func init() {
14 | app := config.GetInstance()
15 | app.Workspace = getTestDataDir()
16 | util.EnableDebugLog()
17 | }
18 |
19 | func TestShouldCheckRepoLinkIsDockerHub(t *testing.T) {
20 | assert := assert.New(t)
21 | assert.True(isDockerHubImage("flowci/core"))
22 | assert.False(isDockerHubImage("http://localhost:8080/flowci/core"))
23 | assert.False(isDockerHubImage("mcr.microsoft.com/dotnet/core/sdk:2.2.301"))
24 | }
25 |
26 | func TestShouldExecInDocker(t *testing.T) {
27 | assert := assert.New(t)
28 | cmd := createDockerTestCmd()
29 | shouldExecCmd(assert, cmd)
30 | }
31 |
32 | func TestShouldExecWithErrorInDocker(t *testing.T) {
33 | assert := assert.New(t)
34 | cmd := createDockerTestCmd()
35 | shouldExecWithError(assert, cmd)
36 | }
37 |
38 | func TestShouldExecWithErrorIfAllowFailureWithinDocker(t *testing.T) {
39 | assert := assert.New(t)
40 | cmd := createDockerTestCmd()
41 | shouldExecWithErrorButAllowFailure(assert, cmd)
42 | }
43 |
44 | func TestShouldExitWithTimeoutInDocker(t *testing.T) {
45 | assert := assert.New(t)
46 | cmd := createDockerTestCmd()
47 | shouldExecButTimeOut(assert, cmd)
48 | }
49 |
50 | func TestShouldExitByKillInDocker(t *testing.T) {
51 | assert := assert.New(t)
52 | cmd := createDockerTestCmd()
53 | shouldExecButKilled(assert, cmd)
54 | }
55 |
56 | func TestShouldReuseContainer(t *testing.T) {
57 | assert := assert.New(t)
58 |
59 | // run cmd in container
60 | cmd := createDockerTestCmd()
61 |
62 | cmd.Dockers[0].IsStopContainer = true
63 | cmd.Dockers[0].IsDeleteContainer = false
64 |
65 | result := shouldExecCmd(assert, cmd)
66 | assert.Equal(1, len(result.Containers))
67 |
68 | // run cmd in container from first step
69 | cmd = createDockerTestCmd()
70 | cmd.Dockers[0].ContainerID = result.Containers[0]
71 | cmd.Dockers[0].IsStopContainer = true
72 | cmd.Dockers[0].IsDeleteContainer = true
73 |
74 | resultFromReuse := shouldExecCmd(assert, cmd)
75 | assert.Equal(1, len(resultFromReuse.Containers))
76 | assert.Equal(result.Containers[0], resultFromReuse.Containers[0])
77 | }
78 |
79 | func TestShouldStartDockerInteract(t *testing.T) {
80 | assert := assert.New(t)
81 |
82 | executor := newExecutor(&domain.ShellIn{
83 | ID: "test111",
84 | FlowId: "test111",
85 | Bash: []string{
86 | "echo hello",
87 | "sleep 9999",
88 | },
89 | Dockers: []*domain.DockerOption{
90 | {
91 | Image: "ubuntu:18.04",
92 | IsRuntime: true,
93 | },
94 | },
95 | Timeout: 9999,
96 | }, false)
97 |
98 | dockerExecutor := executor.(*dockerExecutor)
99 | assert.NotNil(dockerExecutor)
100 |
101 | err := dockerExecutor.Init()
102 | assert.NoError(err)
103 |
104 | go func() {
105 | for {
106 | log, ok := <-executor.TtyOut()
107 | if !ok {
108 | return
109 | }
110 | content, _ := base64.StdEncoding.DecodeString(log)
111 | util.LogDebug("------ %s", content)
112 | }
113 | }()
114 |
115 | go func() {
116 | time.Sleep(2 * time.Second)
117 | executor.TtyIn() <- "echo helloworld...\n"
118 | time.Sleep(2 * time.Second)
119 | executor.TtyIn() <- "sleep 9999\n"
120 | time.Sleep(2 * time.Second)
121 | executor.TtyIn() <- "exit\n"
122 | }()
123 |
124 | // docker should start container for cmd before tty
125 | go func() {
126 | err := executor.Start()
127 | assert.NoError(err)
128 | }()
129 |
130 | for {
131 | if dockerExecutor.runtime().ContainerID == "" {
132 | time.Sleep(1 * time.Second)
133 | continue
134 | }
135 | break
136 | }
137 |
138 | // kill after 10 seconds
139 | go func() {
140 | time.Sleep(10 * time.Second)
141 | executor.StopTty()
142 | }()
143 |
144 | err = executor.StartTty("fakeId", func(ttyId string) {
145 | util.LogDebug("Tty started")
146 | })
147 | assert.NoError(err)
148 | assert.False(executor.IsInteracting())
149 | }
150 |
151 | func TestShouldRunWithTwoContainers(t *testing.T) {
152 | assert := assert.New(t)
153 |
154 | cmd := createDockerTestCmd()
155 | cmd.Dockers = append(cmd.Dockers, &domain.DockerOption{
156 | Image: "mysql:5.6",
157 | Environment: map[string]string{
158 | "MYSQL_ROOT_PASSWORD": "test",
159 | },
160 | Ports: []string{"3306:3306"},
161 | IsDeleteContainer: true,
162 | })
163 | cmd.Dockers = append(cmd.Dockers, &domain.DockerOption{
164 | Image: "mysql:5.6",
165 | Command: []string{"mysql", "-h127.0.0.1", "-uroot", "-ptest"},
166 | IsDeleteContainer: true,
167 | })
168 |
169 | executor := newExecutor(cmd, false)
170 | assert.NoError(executor.Init())
171 |
172 | err := executor.Start()
173 | assert.NoError(err)
174 |
175 | r := executor.GetResult()
176 | assert.Equal(3, len(r.Containers))
177 | }
178 |
179 | func createDockerTestCmd() *domain.ShellIn {
180 | return &domain.ShellIn{
181 | CmdIn: domain.CmdIn{
182 | Type: domain.CmdTypeShell,
183 | },
184 | FlowId: "flowid", // same as dir flowid in _testdata
185 | ID: "1-1-1",
186 | Dockers: []*domain.DockerOption{
187 | {
188 | Image: "ubuntu:18.04",
189 | IsDeleteContainer: true,
190 | IsStopContainer: true,
191 | IsRuntime: true,
192 | },
193 | },
194 | Bash: []string{
195 | "echo bbb",
196 | "sleep 5",
197 | ">&2 echo $INPUT_VAR",
198 | "export FLOW_VVV=flowci",
199 | "export FLOW_AAA=flow...",
200 | },
201 | Inputs: domain.Variables{"INPUT_VAR": "aaa"},
202 | Timeout: 1800,
203 | EnvFilters: []string{"FLOW_"},
204 | }
205 | }
206 |
--------------------------------------------------------------------------------
/executor/docker_util.go:
--------------------------------------------------------------------------------
1 | package executor
2 |
3 | import (
4 | "archive/tar"
5 | "bufio"
6 | "bytes"
7 | "github.com/flowci/flow-agent-x/util"
8 | "io"
9 | "os"
10 | "path/filepath"
11 | "strings"
12 | )
13 |
14 | const (
15 | dockerHeaderSize = 8
16 | dockerHeaderPrefixSize = 4 // [STREAM_TYPE, 0, 0 ,0, ....]
17 | )
18 |
19 | var (
20 | dockerStdInHeaderPrefix = []byte{1, 0, 0, 0}
21 | dockerStdErrHeaderPrefix = []byte{2, 0, 0, 0}
22 | )
23 |
24 | // is repo link belong to the format /
25 | func isDockerHubImage(image string) bool {
26 | items := strings.Split(image, "/")
27 | if len(items) <= 2 {
28 | return true
29 | }
30 | return false
31 | }
32 |
33 | func removeDockerHeader(in []byte) []byte {
34 | if len(in) < dockerHeaderSize {
35 | return in
36 | }
37 |
38 | if bytes.Compare(in[:dockerHeaderPrefixSize], dockerStdInHeaderPrefix) == 0 {
39 | return in[dockerHeaderSize:]
40 | }
41 |
42 | if bytes.Compare(in[:dockerHeaderPrefixSize], dockerStdErrHeaderPrefix) == 0 {
43 | return in[dockerHeaderSize:]
44 | }
45 |
46 | return in
47 | }
48 |
49 | func untarFromReader(tarReader io.Reader, dest string) error {
50 | reader := tar.NewReader(tarReader)
51 | for {
52 | header, err := reader.Next()
53 | if err == io.EOF {
54 | break
55 | }
56 |
57 | if err != nil {
58 | return err
59 | }
60 |
61 | fileInfo := header.FileInfo()
62 | target := dest + util.UnixPathSeparator + header.Name
63 |
64 | if fileInfo.IsDir() {
65 | if err := os.MkdirAll(target, 0755); err != nil {
66 | return err
67 | }
68 | continue
69 | }
70 |
71 | f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
72 | if err != nil {
73 | return err
74 | }
75 |
76 | if _, err := io.Copy(f, reader); err != nil {
77 | return err
78 | }
79 |
80 | f.Close()
81 | }
82 |
83 | return nil
84 | }
85 |
86 | // tar dir, ex: abc/.. output is archived content .. in dir
87 | func tarArchiveFromPath(path string) (io.Reader, error) {
88 | var buf bytes.Buffer
89 | tw := tar.NewWriter(&buf)
90 | dir := filepath.Dir(path)
91 |
92 | ok := filepath.Walk(path, func(file string, fi os.FileInfo, err error) (out error) {
93 | defer util.RecoverPanic(func(e error) {
94 | out = e
95 | })
96 |
97 | util.PanicIfErr(err)
98 |
99 | header, err := tar.FileInfoHeader(fi, fi.Name())
100 | util.PanicIfErr(err)
101 |
102 | relativeDir := strings.Replace(file, dir, "", -1)
103 | header.Name = strings.TrimPrefix(relativeDir, string(filepath.Separator))
104 |
105 | // convert path to linux path
106 | if util.IsWindows() {
107 | header.Name = strings.ReplaceAll(header.Name, util.WinPathSeparator, util.UnixPathSeparator)
108 | }
109 |
110 | err = tw.WriteHeader(header)
111 | util.PanicIfErr(err)
112 |
113 | f, err := os.Open(file)
114 | util.PanicIfErr(err)
115 |
116 | if fi.IsDir() {
117 | return
118 | }
119 |
120 | _, err = io.Copy(tw, f)
121 | util.PanicIfErr(err)
122 |
123 | err = f.Close()
124 | util.PanicIfErr(err)
125 |
126 | return
127 | })
128 |
129 | if ok != nil {
130 | return nil, ok
131 | }
132 |
133 | ok = tw.Close()
134 | if ok != nil {
135 | return nil, ok
136 | }
137 |
138 | return bufio.NewReader(&buf), nil
139 | }
140 |
--------------------------------------------------------------------------------
/executor/docker_util_test.go:
--------------------------------------------------------------------------------
1 | package executor
2 |
3 | import (
4 | "github.com/stretchr/testify/assert"
5 | "io/ioutil"
6 | "path/filepath"
7 | "testing"
8 | )
9 |
10 | func TestShouldTarAndUntarDir(t *testing.T) {
11 | assert := assert.New(t)
12 |
13 | dir := getTestDataDir()
14 |
15 | // tar flowid folder into reader
16 | reader, err := tarArchiveFromPath(filepath.Join(dir, "flowid"))
17 | assert.NoError(err)
18 | assert.NotNil(reader)
19 |
20 | // untar flowid folder into temp
21 | dest, err := ioutil.TempDir("", "test_tar_")
22 | assert.NoError(err)
23 |
24 | err = untarFromReader(reader, dest)
25 | assert.NoError(err)
26 | }
--------------------------------------------------------------------------------
/executor/executor.go:
--------------------------------------------------------------------------------
1 | package executor
2 |
3 | import (
4 | "context"
5 | "encoding/base64"
6 | "fmt"
7 | "github.com/flowci/flow-agent-x/domain"
8 | "github.com/flowci/flow-agent-x/util"
9 | "io"
10 | "sync"
11 | "sync/atomic"
12 | "time"
13 | )
14 |
15 | const (
16 | winPowerShell = "powershell.exe"
17 | linuxBash = "/bin/bash"
18 | //linuxBashShebang = "#!/bin/bash -i" // add -i enable to source .bashrc
19 |
20 | defaultChannelBufferSize = 1000
21 | defaultLogWaitingDuration = 5 * time.Second
22 | defaultReaderBufferSize = 8 * 1024 // 8k
23 | )
24 |
25 | type Executor interface {
26 | Init() error
27 |
28 | CacheDir() (string, string)
29 |
30 | CmdIn() *domain.ShellIn
31 |
32 | StartTty(ttyId string, onStarted func(ttyId string)) error
33 |
34 | StopTty()
35 |
36 | TtyId() string
37 |
38 | TtyIn() chan<- string
39 |
40 | TtyOut() <-chan string // b64 out
41 |
42 | IsInteracting() bool
43 |
44 | Start() error
45 |
46 | Stdout() <-chan string // b64 log
47 |
48 | Kill()
49 |
50 | Close()
51 |
52 | GetResult() *domain.ShellOut
53 | }
54 |
55 | type BaseExecutor struct {
56 | k8sConfig *domain.K8sConfig
57 |
58 | agentId string // should be agent token
59 | workspace string // agent workspace
60 | pluginDir string
61 | jobDir string // job workspace
62 |
63 | cacheInputDir string // downloaded cache temp dir
64 | cacheOutputDir string // temp dir that need to upload
65 |
66 | os string // current operation system
67 | context context.Context
68 | cancelFunc context.CancelFunc
69 |
70 | volumes []*domain.DockerVolume
71 | inCmd *domain.ShellIn
72 | result *domain.ShellOut
73 |
74 | vars domain.Variables // vars from input and in cmd
75 | secretVars domain.Variables
76 | configVars domain.Variables
77 |
78 | stdout chan string // output log
79 | stdOutWg sync.WaitGroup // init on subclasses
80 |
81 | ttyId string
82 | ttyIn chan string // b64 script
83 | ttyOut chan string // b64 log content
84 | ttyCtx context.Context
85 | ttyCancel context.CancelFunc
86 | }
87 |
88 | type Options struct {
89 | K8s *domain.K8sConfig
90 |
91 | AgentId string
92 | Parent context.Context
93 | Workspace string
94 | WorkspaceFromDockerVolume bool
95 | PluginDir string
96 | CacheSrcDir string
97 | Cmd *domain.ShellIn
98 | Vars domain.Variables
99 | SecretVars domain.Variables
100 | ConfigVars domain.Variables
101 | Volumes []*domain.DockerVolume
102 | }
103 |
104 | func NewExecutor(options Options) Executor {
105 | if options.Vars == nil {
106 | options.Vars = domain.NewVariables()
107 | }
108 |
109 | cmd := options.Cmd
110 | base := BaseExecutor{
111 | k8sConfig: options.K8s,
112 | agentId: options.AgentId,
113 | workspace: options.Workspace,
114 | pluginDir: options.PluginDir,
115 | cacheInputDir: options.CacheSrcDir,
116 | volumes: options.Volumes,
117 | stdout: make(chan string, defaultChannelBufferSize),
118 | inCmd: cmd,
119 | vars: domain.ConnectVars(options.Vars, cmd.Inputs),
120 | secretVars: options.SecretVars,
121 | configVars: options.ConfigVars,
122 | result: domain.NewShellOutput(cmd),
123 | ttyIn: make(chan string, defaultChannelBufferSize),
124 | ttyOut: make(chan string, defaultChannelBufferSize),
125 | }
126 |
127 | ctx, cancel := context.WithTimeout(options.Parent, time.Duration(cmd.Timeout)*time.Second)
128 | base.context = ctx
129 | base.cancelFunc = cancel
130 |
131 | if cmd.HasDockerOption() {
132 | return &dockerExecutor{
133 | BaseExecutor: base,
134 | wsFromDockerVolume: options.WorkspaceFromDockerVolume,
135 | }
136 | }
137 |
138 | return &shellExecutor{
139 | BaseExecutor: base,
140 | }
141 | }
142 |
143 | func (b *BaseExecutor) CacheDir() (input string, output string) {
144 | input = b.cacheInputDir
145 | output = b.cacheOutputDir
146 | return
147 | }
148 |
149 | func (b *BaseExecutor) CmdIn() *domain.ShellIn {
150 | return b.inCmd
151 | }
152 |
153 | func (b *BaseExecutor) TtyId() string {
154 | return b.ttyId
155 | }
156 |
157 | func (b *BaseExecutor) Stdout() <-chan string {
158 | return b.stdout
159 | }
160 |
161 | func (b *BaseExecutor) TtyIn() chan<- string {
162 | return b.ttyIn
163 | }
164 |
165 | func (b *BaseExecutor) TtyOut() <-chan string {
166 | return b.ttyOut
167 | }
168 |
169 | func (b *BaseExecutor) IsInteracting() bool {
170 | return b.ttyCtx != nil && b.ttyCancel != nil
171 | }
172 |
173 | func (b *BaseExecutor) GetResult() *domain.ShellOut {
174 | return b.result
175 | }
176 |
177 | func (b *BaseExecutor) Kill() {
178 | b.cancelFunc()
179 | }
180 |
181 | func (b *BaseExecutor) Close() {
182 | if len(b.stdout) > 0 {
183 | util.Wait(&b.stdOutWg, defaultLogWaitingDuration)
184 | }
185 |
186 | close(b.stdout)
187 | close(b.ttyIn)
188 | close(b.ttyOut)
189 |
190 | b.cancelFunc()
191 | }
192 |
193 | //====================================================================
194 | // private
195 | //====================================================================
196 |
197 | func (b *BaseExecutor) isK8sEnabled() bool {
198 | return b.k8sConfig != nil && b.k8sConfig.Enabled
199 | }
200 |
201 | func (b *BaseExecutor) writeCmd(stdin io.Writer, before, after func() []string, doScript func(string) string) {
202 | write := func(script string) {
203 | _, _ = io.WriteString(stdin, appendNewLine(script, b.os))
204 | util.LogDebug("----- exec: %s", script)
205 | }
206 |
207 | // source volume script
208 | if b.os != util.OSWin {
209 | write("set +e") // ignore source file failure
210 | for _, v := range b.volumes {
211 | if util.IsEmptyString(v.Script) {
212 | continue
213 | }
214 | write(fmt.Sprintf("source %s > /dev/null 2>&1", v.ScriptPath()))
215 | }
216 | }
217 |
218 | if before != nil {
219 | for _, script := range before() {
220 | write(script)
221 | }
222 | }
223 |
224 | // write shell script from cmd
225 | for _, script := range scriptForExitOnError(b.os) {
226 | write(script)
227 | }
228 |
229 | for _, script := range b.getScripts() {
230 | write(doScript(script))
231 | }
232 |
233 | if after != nil {
234 | for _, script := range after() {
235 | write(script)
236 | }
237 | }
238 |
239 | write("exit")
240 | }
241 |
242 | func (b *BaseExecutor) getScripts() []string {
243 | scripts := b.inCmd.Bash
244 | if b.os == util.OSWin {
245 | scripts = b.inCmd.Pwsh
246 | }
247 |
248 | isAllEmpty := true
249 | for _, script := range scripts {
250 | if !util.IsEmptyString(script) {
251 | isAllEmpty = false
252 | }
253 | }
254 |
255 | if isAllEmpty {
256 | panic(fmt.Errorf("agent: Missing bash or pwsh section in flow YAML"))
257 | }
258 |
259 | return scripts
260 | }
261 |
262 | func (b *BaseExecutor) writeLog(src io.Reader, inThread, doneOnWaitGroup bool) {
263 | write := func() {
264 | defer func() {
265 | if err := recover(); err != nil {
266 | util.LogWarn(err.(error).Error())
267 | }
268 |
269 | if doneOnWaitGroup {
270 | b.stdOutWg.Done()
271 | }
272 |
273 | util.LogDebug("[Exit]: StdOut/Err, log size = %d", b.result.LogSize)
274 | }()
275 |
276 | buf := make([]byte, defaultReaderBufferSize)
277 | for {
278 | select {
279 | case <-b.context.Done():
280 | return
281 | default:
282 | n, err := src.Read(buf)
283 | if err != nil {
284 | return
285 | }
286 |
287 | b.stdout <- base64.StdEncoding.EncodeToString(removeDockerHeader(buf[0:n]))
288 | atomic.AddInt64(&b.result.LogSize, int64(n))
289 | }
290 | }
291 | }
292 |
293 | if inThread {
294 | go write()
295 | return
296 | }
297 |
298 | write()
299 | }
300 |
301 | func (b *BaseExecutor) writeSingleLog(msg string) {
302 | b.stdout <- base64.StdEncoding.EncodeToString([]byte(msg + "\n"))
303 | }
304 |
305 | func (b *BaseExecutor) writeTtyIn(writer io.Writer) {
306 | for inputStr := range b.ttyIn {
307 | if inputStr == "\r" {
308 | inputStr = newLineForOs(b.os)
309 | }
310 |
311 | in := []byte(inputStr)
312 | _, err := writer.Write(in)
313 |
314 | if err != nil {
315 | util.LogIfError(err)
316 | return
317 | }
318 | }
319 | }
320 |
321 | func (b *BaseExecutor) writeTtyOut(reader io.Reader) {
322 | buf := make([]byte, defaultReaderBufferSize)
323 | for {
324 | n, err := reader.Read(buf)
325 | if err != nil {
326 | return
327 | }
328 | b.ttyOut <- base64.StdEncoding.EncodeToString(removeDockerHeader(buf[0:n]))
329 | }
330 | }
331 |
332 | func (b *BaseExecutor) toStartStatus(pid int) {
333 | b.result.Status = domain.CmdStatusRunning
334 | b.result.ProcessId = pid
335 | }
336 |
337 | func (b *BaseExecutor) toErrorStatus(err error) error {
338 | b.result.Status = domain.CmdStatusException
339 | b.result.Error = err.Error()
340 | b.result.FinishAt = time.Now()
341 | return err
342 | }
343 |
344 | func (b *BaseExecutor) toTimeOutStatus() {
345 | b.result.Status = domain.CmdStatusTimeout
346 | b.result.Code = domain.CmdExitCodeTimeOut
347 | b.result.FinishAt = time.Now()
348 | }
349 |
350 | func (b *BaseExecutor) toKilledStatus() {
351 | b.result.Status = domain.CmdStatusKilled
352 | b.result.Code = domain.CmdExitCodeKilled
353 | b.result.FinishAt = time.Now()
354 | }
355 |
356 | func (b *BaseExecutor) toFinishStatus(exitCode int) {
357 | b.result.FinishAt = time.Now()
358 | b.result.Code = exitCode
359 |
360 | if exitCode == 0 {
361 | b.result.Status = domain.CmdStatusSuccess
362 | return
363 | }
364 |
365 | // no exported environment since it's failure
366 | if b.inCmd.AllowFailure {
367 | b.result.Status = domain.CmdStatusSuccess
368 | return
369 | }
370 |
371 | _ = b.toErrorStatus(fmt.Errorf("exit status %d", exitCode))
372 | return
373 | }
374 |
--------------------------------------------------------------------------------
/executor/executor_test.go:
--------------------------------------------------------------------------------
1 | package executor
2 |
3 | import (
4 | "context"
5 | "encoding/base64"
6 | "github.com/flowci/flow-agent-x/config"
7 | "github.com/flowci/flow-agent-x/domain"
8 | "github.com/flowci/flow-agent-x/util"
9 | "github.com/stretchr/testify/assert"
10 | "path"
11 | "runtime"
12 | "time"
13 | )
14 |
15 | func printLog(stdout <-chan string) {
16 | for {
17 | item, ok := <-stdout
18 | if !ok {
19 | break
20 | }
21 |
22 | bytes, _ := base64.StdEncoding.DecodeString(item)
23 | util.LogDebug("[LOG]: %s", string(bytes))
24 | }
25 | }
26 | func getTestDataDir() string {
27 | _, filename, _, _ := runtime.Caller(0)
28 | base := path.Dir(filename)
29 | return path.Join(base, "_testdata")
30 | }
31 |
32 | func newExecutor(cmd *domain.ShellIn, k8s bool) Executor {
33 | ctx, _ := context.WithCancel(context.Background())
34 | app := config.GetInstance()
35 |
36 | options := Options{
37 | K8s: &domain.K8sConfig{
38 | Enabled: k8s,
39 | InCluster: false,
40 | },
41 | Parent: ctx,
42 | Workspace: app.Workspace,
43 | PluginDir: app.PluginDir,
44 | Cmd: cmd,
45 | Volumes: []*domain.DockerVolume{
46 | {
47 | Name: "pyenv",
48 | Script: "init.sh",
49 | Dest: "/ws/.pyenv",
50 | Image: "flowci/pyenv:1.3",
51 | Init: "init-pyenv-volume.sh",
52 | },
53 | },
54 | }
55 |
56 | return NewExecutor(options)
57 | }
58 |
59 | func shouldExecCmd(assert *assert.Assertions, cmd *domain.ShellIn) *domain.ShellOut {
60 | // when:
61 | executor := newExecutor(cmd, false)
62 | assert.NoError(executor.Init())
63 |
64 | go printLog(executor.Stdout())
65 |
66 | err := executor.Start()
67 | assert.NoError(err)
68 |
69 | // then:
70 | result := executor.GetResult()
71 | assert.Equal(0, result.Code)
72 | assert.True(result.LogSize > 0)
73 | assert.NotNil(result.FinishAt)
74 | assert.Equal("flowci", result.Output["FLOW_VVV"])
75 | assert.Equal("flow...", result.Output["FLOW_AAA"])
76 |
77 | return executor.GetResult()
78 | }
79 |
80 | func shouldExecWithError(assert *assert.Assertions, cmd *domain.ShellIn) {
81 | // init:
82 | cmd.AllowFailure = false
83 | cmd.Bash = []string{
84 | "notCommand should exit with error",
85 | "echo should_not_printed",
86 | }
87 | cmd.Pwsh = []string{
88 | "notCommand should exit with error",
89 | "echo should_not_printed",
90 | }
91 |
92 | // when:
93 | executor := newExecutor(cmd, false)
94 | assert.NoError(executor.Init())
95 |
96 | go printLog(executor.Stdout())
97 |
98 | err := executor.Start()
99 | assert.NoError(err)
100 |
101 | // then:
102 | result := executor.GetResult()
103 | assert.True(result.LogSize > 0)
104 | assert.True(result.Code != 0)
105 | assert.Equal(domain.CmdStatusException, result.Status)
106 | assert.NotNil(result.FinishAt)
107 | }
108 |
109 | func shouldExecWithErrorButAllowFailure(assert *assert.Assertions, cmd *domain.ShellIn) {
110 | // init:
111 | cmd.AllowFailure = true
112 | cmd.Bash = []string{"notCommand should exit with error"}
113 | cmd.Pwsh = []string{"notCommand should exit with error"}
114 |
115 | // when:
116 | executor := newExecutor(cmd, false)
117 | assert.NoError(executor.Init())
118 |
119 | go printLog(executor.Stdout())
120 |
121 | err := executor.Start()
122 | assert.NoError(err)
123 |
124 | // then:
125 | result := executor.GetResult()
126 | assert.True(result.LogSize > 0)
127 | assert.True(result.Code != 0)
128 | assert.Equal(domain.CmdStatusSuccess, result.Status)
129 | assert.NotNil(result.FinishAt)
130 | }
131 |
132 | func shouldExecButTimeOut(assert *assert.Assertions, cmd *domain.ShellIn) {
133 | // init:
134 | cmd.Timeout = 5
135 | cmd.Bash = []string{"echo $HOME", "sleep 9999", "echo ...."}
136 | cmd.Pwsh = []string{"echo ${HOME}", "sleep 9999", "echo ...."}
137 |
138 | // when:
139 | executor := newExecutor(cmd, false)
140 | assert.NoError(executor.Init())
141 |
142 | go printLog(executor.Stdout())
143 |
144 | err := executor.Start()
145 | assert.NoError(err)
146 |
147 | // then:
148 | result := executor.GetResult()
149 | assert.True(result.LogSize > 0)
150 | assert.Equal(domain.CmdStatusTimeout, result.Status)
151 | assert.Equal(domain.CmdExitCodeTimeOut, result.Code)
152 | assert.NotNil(result.FinishAt)
153 | }
154 |
155 | func shouldExecButKilled(assert *assert.Assertions, cmd *domain.ShellIn) {
156 | // init:
157 | cmd.Bash = []string{"echo $HOME", "sleep 9999", "echo ...."}
158 | cmd.Pwsh = []string{"echo ${HOME}", "sleep 9999", "echo ...."}
159 |
160 | // when:
161 | executor := newExecutor(cmd, false)
162 | assert.NoError(executor.Init())
163 |
164 | go printLog(executor.Stdout())
165 |
166 | time.AfterFunc(5*time.Second, func() {
167 | executor.Kill()
168 | })
169 |
170 | err := executor.Start()
171 | assert.NoError(err)
172 |
173 | // then:
174 | result := executor.GetResult()
175 | assert.True(result.LogSize > 0)
176 | assert.Equal(domain.CmdStatusKilled, result.Status)
177 | assert.Equal(domain.CmdExitCodeKilled, result.Code)
178 | assert.NotNil(result.FinishAt)
179 | }
180 |
--------------------------------------------------------------------------------
/executor/shell.go:
--------------------------------------------------------------------------------
1 | package executor
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "github.com/flowci/flow-agent-x/domain"
7 | "github.com/flowci/flow-agent-x/util"
8 | "io/ioutil"
9 | "os"
10 | "os/exec"
11 | "path/filepath"
12 | "runtime"
13 | "time"
14 | )
15 |
16 | type (
17 | shellExecutor struct {
18 | BaseExecutor
19 | command *exec.Cmd
20 | tty *exec.Cmd
21 | binDir string
22 | envFile string
23 | }
24 | )
25 |
26 | func (se *shellExecutor) Init() (out error) {
27 | defer util.RecoverPanic(func(e error) {
28 | out = e
29 | })
30 |
31 | se.os = runtime.GOOS
32 | se.result.StartAt = time.Now()
33 |
34 | if util.IsEmptyString(se.workspace) {
35 | se.workspace, _ = ioutil.TempDir("", "agent_")
36 | }
37 |
38 | // setup bin under workspace
39 | se.binDir = filepath.Join(se.workspace, "bin")
40 | err := os.MkdirAll(se.binDir, os.ModePerm)
41 | util.PanicIfErr(err)
42 |
43 | for _, f := range binFiles {
44 | path := filepath.Join(se.binDir, f.name)
45 | if !util.IsFileExists(path) {
46 | _ = ioutil.WriteFile(path, f.content, f.permission)
47 | }
48 | }
49 |
50 | // setup job dir under workspace
51 | se.jobDir = filepath.Join(se.workspace, util.ParseString(se.inCmd.FlowId))
52 | se.vars[domain.VarAgentJobDir] = se.jobDir
53 |
54 | err = os.MkdirAll(se.jobDir, os.ModePerm)
55 | util.PanicIfErr(err)
56 |
57 | se.vars.Resolve()
58 | se.copyCache()
59 | return nil
60 | }
61 |
62 | func (se *shellExecutor) Start() (out error) {
63 | // handle context error
64 | go func() {
65 | <-se.context.Done()
66 | err := se.context.Err()
67 |
68 | if err != nil {
69 | se.handleErrors(err)
70 | }
71 | }()
72 |
73 | for i := se.inCmd.Retry; i >= 0; i-- {
74 | out = se.doStart()
75 | r := se.result
76 |
77 | if r.Status == domain.CmdStatusException || out != nil {
78 | if i > 0 {
79 | se.writeSingleLog(">>>>>>> retry >>>>>>>")
80 | }
81 | continue
82 | }
83 |
84 | break
85 | }
86 |
87 | se.writeCache()
88 | return
89 | }
90 |
91 | func (se *shellExecutor) StopTty() {
92 | if se.IsInteracting() {
93 | _ = se.tty.Process.Kill()
94 | }
95 | }
96 |
97 | //====================================================================
98 | // private
99 | //====================================================================
100 |
101 | // copy cache to job dir if cache defined in cacheSrcDir
102 | func (se *shellExecutor) copyCache() {
103 | if !util.HasString(se.cacheInputDir) {
104 | return
105 | }
106 |
107 | files, err := ioutil.ReadDir(se.cacheInputDir)
108 | util.PanicIfErr(err)
109 |
110 | for _, f := range files {
111 | oldPath := filepath.Join(se.cacheInputDir, f.Name())
112 | newPath := filepath.Join(se.jobDir, f.Name())
113 |
114 | if util.IsFileExists(newPath) {
115 | _ = os.Remove(newPath)
116 | }
117 |
118 | // move cache from src dir to job dir
119 | err = os.Rename(oldPath, newPath)
120 |
121 | if err == nil {
122 | se.writeSingleLog(fmt.Sprintf("cache %s has been applied", f.Name()))
123 | } else {
124 | se.writeSingleLog(fmt.Sprintf("cache %s not applied: %s", f.Name(), err.Error()))
125 | }
126 |
127 | // remove cache from cache dir anyway
128 | _ = os.RemoveAll(oldPath)
129 | }
130 | }
131 |
132 | // write cache back to cacheSrcDir
133 | func (se *shellExecutor) writeCache() {
134 | if !se.inCmd.HasCache() {
135 | return
136 | }
137 |
138 | defer util.RecoverPanic(func(e error) {
139 | util.LogWarn(e.Error())
140 | })
141 |
142 | dir, err := ioutil.TempDir("", "_cache_output_")
143 | if err != nil {
144 | util.LogWarn(err.Error())
145 | return
146 | }
147 |
148 | se.cacheOutputDir = dir
149 | cache := se.inCmd.Cache
150 |
151 | for _, path := range cache.Paths {
152 | path = filepath.Clean(path)
153 | fullPath := filepath.Join(se.jobDir, path)
154 |
155 | info, exist := util.IsFileExistsAndReturnFileInfo(fullPath)
156 | if !exist {
157 | continue
158 | }
159 |
160 | newPath := filepath.Join(dir, path)
161 |
162 | if info.IsDir() {
163 | err := util.CopyDir(fullPath, newPath)
164 | util.PanicIfErr(err)
165 |
166 | util.LogDebug("dir %s write back to cache dir", newPath)
167 | continue
168 | }
169 |
170 | err := util.CopyFile(fullPath, newPath)
171 | util.PanicIfErr(err)
172 | util.LogDebug("file %s write back to cache dir", newPath)
173 | }
174 | }
175 |
176 | func (se *shellExecutor) exportEnv() {
177 | if util.IsEmptyString(se.envFile) {
178 | return
179 | }
180 |
181 | file, err := os.Open(se.envFile)
182 | if err != nil {
183 | return
184 | }
185 |
186 | defer file.Close()
187 | se.result.Output = readEnvFromReader(se.os, file, se.inCmd.EnvFilters)
188 | }
189 |
190 | func (se *shellExecutor) handleErrors(err error) {
191 | kill := func() {
192 | if se.command != nil {
193 | _ = se.command.Process.Kill()
194 | }
195 |
196 | if se.tty != nil {
197 | _ = se.tty.Process.Kill()
198 | }
199 | }
200 |
201 | util.LogWarn("handleError on shell: %s", err.Error())
202 |
203 | if err == context.DeadlineExceeded {
204 | util.LogDebug("Timeout..")
205 | kill()
206 | se.toTimeOutStatus()
207 | return
208 | }
209 |
210 | if err == context.Canceled {
211 | util.LogDebug("Cancel..")
212 | kill()
213 | se.toKilledStatus()
214 | return
215 | }
216 |
217 | _ = se.toErrorStatus(err)
218 | }
219 |
--------------------------------------------------------------------------------
/executor/shell_bash.go:
--------------------------------------------------------------------------------
1 | //go:build !windows
2 | // +build !windows
3 |
4 | package executor
5 |
6 | import (
7 | "context"
8 | "fmt"
9 | "github.com/creack/pty"
10 | "github.com/flowci/flow-agent-x/util"
11 | "io/ioutil"
12 | "os"
13 | "os/exec"
14 | )
15 |
16 | // Start run the cmd from domain.CmdIn
17 | func (se *shellExecutor) doStart() (out error) {
18 | defer func() {
19 | if err := recover(); err != nil {
20 | out = err.(error)
21 | se.handleErrors(out)
22 | }
23 | }()
24 |
25 | // init wait group fro StdOut and StdErr
26 | se.stdOutWg.Add(2)
27 |
28 | command := exec.Command(linuxBash)
29 | command.Dir = se.jobDir
30 | command.Env = append(os.Environ(), se.vars.ToStringArray()...)
31 | command.Env = append(command.Env, se.secretVars.ToStringArray()...)
32 | command.Env = append(command.Env, se.configVars.ToStringArray()...)
33 |
34 | stdin, err := command.StdinPipe()
35 | util.PanicIfErr(err)
36 |
37 | stdout, err := command.StdoutPipe()
38 | util.PanicIfErr(err)
39 |
40 | stderr, err := command.StderrPipe()
41 | util.PanicIfErr(err)
42 |
43 | defer func() {
44 | _ = stdin.Close()
45 | _ = stdout.Close()
46 | _ = stderr.Close()
47 | }()
48 |
49 | se.command = command
50 |
51 | // start command
52 | if err := command.Start(); err != nil {
53 | return se.toErrorStatus(err)
54 | }
55 |
56 | se.writeLog(stdout, true, true)
57 | se.writeLog(stderr, true, true)
58 | se.writeCmd(stdin, se.setupBin, se.writeEnv, func(script string) string {
59 | return script
60 | })
61 | se.toStartStatus(command.Process.Pid)
62 |
63 | // wait or timeout
64 | _ = command.Wait()
65 | util.LogDebug("[Done]: Shell for %s", se.inCmd.ID)
66 |
67 | se.exportEnv()
68 |
69 | // wait for tty if it's running
70 | if se.IsInteracting() {
71 | util.LogDebug("Tty is running, wait..")
72 | <-se.ttyCtx.Done()
73 | }
74 |
75 | if se.result.IsFinishStatus() {
76 | return nil
77 | }
78 |
79 | // to finish status
80 | se.toFinishStatus(getExitCode(command))
81 | return se.context.Err()
82 | }
83 |
84 | func (se *shellExecutor) StartTty(ttyId string, onStarted func(ttyId string)) (out error) {
85 | defer func() {
86 | if err := recover(); err != nil {
87 | out = err.(error)
88 | }
89 |
90 | se.tty = nil
91 | se.ttyId = ""
92 | }()
93 |
94 | if se.IsInteracting() {
95 | panic(fmt.Errorf("interaction is ongoning"))
96 | }
97 |
98 | c := exec.Command(linuxBash)
99 | c.Dir = se.jobDir
100 | c.Env = append(os.Environ(), se.vars.ToStringArray()...)
101 | c.Env = append(c.Env, se.secretVars.ToStringArray()...)
102 | c.Env = append(c.Env, se.configVars.ToStringArray()...)
103 |
104 | ptmx, err := pty.Start(c)
105 | util.PanicIfErr(err)
106 |
107 | se.tty = c
108 | se.ttyId = ttyId
109 | se.ttyCtx, se.ttyCancel = context.WithCancel(se.context)
110 |
111 | defer func() {
112 | _ = ptmx.Close()
113 | se.ttyCancel()
114 | se.ttyCtx = nil
115 | se.ttyCancel = nil
116 | }()
117 |
118 | onStarted(ttyId)
119 |
120 | go se.writeTtyIn(ptmx)
121 | go se.writeTtyOut(ptmx)
122 |
123 | _ = c.Wait()
124 | return
125 | }
126 |
127 | func (se *shellExecutor) setupBin() []string {
128 | return []string{fmt.Sprintf("export PATH=%s:$PATH", se.binDir)}
129 | }
130 |
131 | func (se *shellExecutor) writeEnv() []string {
132 | tmpFile, err := ioutil.TempFile("", "agent_env_")
133 | util.PanicIfErr(err)
134 |
135 | defer tmpFile.Close()
136 |
137 | se.envFile = tmpFile.Name()
138 | return []string{"env > " + tmpFile.Name()}
139 | }
140 |
--------------------------------------------------------------------------------
/executor/shell_bash_test.go:
--------------------------------------------------------------------------------
1 | //go:build !windows
2 | // +build !windows
3 |
4 | package executor
5 |
6 | import (
7 | "encoding/base64"
8 | "github.com/flowci/flow-agent-x/domain"
9 | "github.com/flowci/flow-agent-x/util"
10 | "github.com/stretchr/testify/assert"
11 | "testing"
12 | "time"
13 | )
14 |
15 | func init() {
16 | util.EnableDebugLog()
17 | }
18 |
19 | func TestShouldExecInBash(t *testing.T) {
20 | assert := assert.New(t)
21 | cmd := createBashTestCmd()
22 | //
23 | //ok, _ := hasPyenv()
24 | //assert.True(ok)
25 |
26 | shouldExecCmd(assert, cmd)
27 | }
28 |
29 | func TestShouldExecWithErrorInBash(t *testing.T) {
30 | assert := assert.New(t)
31 | cmd := createBashTestCmd()
32 | cmd.Retry = 1
33 | shouldExecWithError(assert, cmd)
34 | }
35 |
36 | func TestShouldExecWithErrorButAllowFailureInBash(t *testing.T) {
37 | assert := assert.New(t)
38 | cmd := createBashTestCmd()
39 | shouldExecWithErrorButAllowFailure(assert, cmd)
40 | }
41 |
42 | func TestShouldExecButTimeoutInBash(t *testing.T) {
43 | assert := assert.New(t)
44 | cmd := createBashTestCmd()
45 | shouldExecButTimeOut(assert, cmd)
46 | }
47 |
48 | func TestShouldExitByKillInBash(t *testing.T) {
49 | assert := assert.New(t)
50 | cmd := createBashTestCmd()
51 | shouldExecButKilled(assert, cmd)
52 | }
53 |
54 | func TestShouldStartBashInteract(t *testing.T) {
55 | assert := assert.New(t)
56 |
57 | executor := newExecutor(&domain.ShellIn{
58 | ID: "test111",
59 | FlowId: "test111",
60 | Bash: []string{
61 | "echo hello",
62 | },
63 | Timeout: 9999,
64 | }, false)
65 |
66 | go func() {
67 | for {
68 | log, ok := <-executor.TtyOut()
69 | if !ok {
70 | return
71 | }
72 |
73 | content, _ := base64.StdEncoding.DecodeString(log)
74 | util.LogDebug("------ %s", content)
75 | }
76 | }()
77 |
78 | go func() {
79 | time.Sleep(2 * time.Second)
80 | executor.TtyIn() <- "cd ~/\n"
81 | executor.TtyIn() <- "ls -l\n"
82 | time.Sleep(2 * time.Second)
83 | executor.TtyIn() <- string([]byte{4})
84 | }()
85 |
86 | err := executor.StartTty("fakeId", func(ttyId string) {
87 | util.LogDebug("Tty Started")
88 | })
89 | assert.NoError(err)
90 | assert.False(executor.IsInteracting())
91 | }
92 |
93 | func createBashTestCmd() *domain.ShellIn {
94 | return &domain.ShellIn{
95 | CmdIn: domain.CmdIn{
96 | Type: domain.CmdTypeShell,
97 | },
98 | ID: "1-1-1",
99 | Bash: []string{
100 | "set -e",
101 | "echo bbb",
102 | "sleep 5",
103 | ">&2 echo $INPUT_VAR",
104 | "export FLOW_VVV=flowci",
105 | "export FLOW_AAA=flow...",
106 | },
107 | Inputs: domain.Variables{"INPUT_VAR": "aaa"},
108 | Timeout: 1800,
109 | EnvFilters: []string{"FLOW_"},
110 | }
111 | }
112 |
--------------------------------------------------------------------------------
/executor/shell_powershell.go:
--------------------------------------------------------------------------------
1 | //go:build windows
2 | // +build windows
3 |
4 | package executor
5 |
6 | import (
7 | "context"
8 | "fmt"
9 | "github.com/flowci/flow-agent-x/util"
10 | "io"
11 | "io/ioutil"
12 | "os"
13 | "os/exec"
14 | )
15 |
16 | func (se *shellExecutor) doStart() (out error) {
17 | defer func() {
18 | if err := recover(); err != nil {
19 | out = err.(error)
20 | se.handleErrors(out)
21 | }
22 | }()
23 |
24 | path, err := exec.LookPath(winPowerShell)
25 | util.PanicIfErr(err)
26 |
27 | ps1File := se.writeScriptToTmpFile()
28 | defer func() {
29 | _ = os.Remove(ps1File)
30 | }()
31 |
32 | // init wait group fro StdOut and StdErr
33 | se.stdOutWg.Add(2)
34 |
35 | command := exec.Command(path, "-NoLogo", "-NoProfile", "-NonInteractive", "-File", ps1File)
36 | command.Dir = se.jobDir
37 | command.Env = append(os.Environ(), se.vars.ToStringArray()...)
38 | command.Env = append(command.Env, se.secretVars.ToStringArray()...)
39 | command.Env = append(command.Env, se.configVars.ToStringArray()...)
40 |
41 | stdout, err := command.StdoutPipe()
42 | util.PanicIfErr(err)
43 |
44 | stderr, err := command.StderrPipe()
45 | util.PanicIfErr(err)
46 |
47 | se.command = command
48 |
49 | defer func() {
50 | _ = stdout.Close()
51 | _ = stderr.Close()
52 | }()
53 |
54 | // handle context error
55 | go func() {
56 | <-se.context.Done()
57 | err := se.context.Err()
58 |
59 | if err != nil {
60 | se.handleErrors(err)
61 | }
62 | }()
63 |
64 | // start command
65 | if err := command.Start(); err != nil {
66 | return se.toErrorStatus(err)
67 | }
68 |
69 | se.writeLog(stdout, true, true)
70 | se.writeLog(stderr, true, true)
71 | se.toStartStatus(command.Process.Pid)
72 |
73 | // wait or timeout
74 | _ = command.Wait()
75 | util.LogDebug("[Done]: Shell for %s", se.inCmd.ID)
76 |
77 | se.exportEnv()
78 |
79 | // wait for tty if it's running
80 | if se.IsInteracting() {
81 | util.LogDebug("Tty is running, wait..")
82 | <-se.ttyCtx.Done()
83 | }
84 |
85 | if se.result.IsFinishStatus() {
86 | return nil
87 | }
88 |
89 | // to finish status
90 | se.toFinishStatus(getExitCode(command))
91 | return se.context.Err()
92 | }
93 |
94 | func (se *shellExecutor) StartTty(ttyId string, onStarted func(ttyId string)) (out error) {
95 | defer func() {
96 | if err := recover(); err != nil {
97 | out = err.(error)
98 | }
99 |
100 | se.tty = nil
101 | se.ttyId = ""
102 | }()
103 |
104 | if se.IsInteracting() {
105 | panic(fmt.Errorf("interaction is ongoning"))
106 | }
107 |
108 | path, err := exec.LookPath(winPowerShell)
109 | util.PanicIfErr(err)
110 |
111 | c := exec.Command(path, "-NoLogo", "-NoProfile")
112 | c.Dir = se.jobDir
113 | c.Env = append(os.Environ(), se.vars.ToStringArray()...)
114 | c.Env = append(c.Env, se.secretVars.ToStringArray()...)
115 | c.Env = append(c.Env, se.configVars.ToStringArray()...)
116 |
117 | stdin, err := c.StdinPipe()
118 | util.PanicIfErr(err)
119 |
120 | stdout, err := c.StdoutPipe()
121 | util.PanicIfErr(err)
122 |
123 | stderr, err := c.StderrPipe()
124 | util.PanicIfErr(err)
125 |
126 | se.tty = c
127 | se.ttyId = ttyId
128 | se.ttyCtx, se.ttyCancel = context.WithCancel(se.context)
129 |
130 | defer func() {
131 | _ = stdin.Close()
132 | _ = stdout.Close()
133 | _ = stderr.Close()
134 |
135 | se.ttyCancel()
136 | se.ttyCtx = nil
137 | se.ttyCancel = nil
138 | }()
139 |
140 | if err := c.Start(); err != nil {
141 | return se.toErrorStatus(err)
142 | }
143 |
144 | onStarted(ttyId)
145 |
146 | go se.writeTtyIn(stdin)
147 | go se.writeTtyOut(io.MultiReader(stdout, stderr))
148 |
149 | _ = c.Wait()
150 | return
151 | }
152 |
153 | func (se *shellExecutor) setupBin() []string {
154 | return []string{fmt.Sprintf("$Env:PATH += \";%s\"", se.binDir)}
155 | }
156 |
157 | func (se *shellExecutor) writeEnv() []string {
158 | tmpFile, err := ioutil.TempFile("", "agent_env_")
159 | util.PanicIfErr(err)
160 |
161 | defer tmpFile.Close()
162 |
163 | se.envFile = tmpFile.Name()
164 | return []string{"gci env: > " + tmpFile.Name()}
165 | }
166 |
167 | func (se *shellExecutor) writeScriptToTmpFile() string {
168 | tempScriptFile, err := ioutil.TempFile("", "agent_tmp_script_")
169 | util.PanicIfErr(err)
170 | _ = tempScriptFile.Close()
171 |
172 | psTmpFile := tempScriptFile.Name() + ".ps1"
173 |
174 | err = os.Rename(tempScriptFile.Name(), psTmpFile)
175 | util.PanicIfErr(err)
176 |
177 | // open tmp ps file
178 | file, err := os.OpenFile(psTmpFile, os.O_RDWR, 0)
179 | util.PanicIfErr(err)
180 |
181 | doScript := func(script string) string {
182 | return script
183 | }
184 |
185 | se.writeCmd(file, se.setupBin, se.writeEnv, doScript)
186 |
187 | _ = file.Close()
188 | return file.Name()
189 | }
190 |
--------------------------------------------------------------------------------
/executor/shell_powershell_test.go:
--------------------------------------------------------------------------------
1 | //go:build windows
2 | // +build windows
3 |
4 | package executor
5 |
6 | import (
7 | "encoding/base64"
8 | "github.com/flowci/flow-agent-x/domain"
9 | "github.com/flowci/flow-agent-x/util"
10 | "github.com/stretchr/testify/assert"
11 | "testing"
12 | "time"
13 | )
14 |
15 | func TestShouldExecInPowerShell(t *testing.T) {
16 | assert := assert.New(t)
17 |
18 | cmdIn := createPowerShellTestCmd()
19 | shouldExecCmd(assert, cmdIn)
20 | }
21 |
22 | func TestShouldExecWithErrorInPowerShell(t *testing.T) {
23 | assert := assert.New(t)
24 | cmd := createPowerShellTestCmd()
25 | shouldExecWithError(assert, cmd)
26 | }
27 |
28 | func TestShouldExecWithErrorButAllowFailureInPowerShell(t *testing.T) {
29 | assert := assert.New(t)
30 | cmd := createPowerShellTestCmd()
31 | shouldExecWithErrorButAllowFailure(assert, cmd)
32 | }
33 |
34 | func TestShouldExecButTimeoutInPowerShell(t *testing.T) {
35 | assert := assert.New(t)
36 | cmd := createPowerShellTestCmd()
37 | shouldExecButTimeOut(assert, cmd)
38 | }
39 |
40 | func TestShouldExitByKillInPowerShell(t *testing.T) {
41 | assert := assert.New(t)
42 | cmd := createPowerShellTestCmd()
43 | shouldExecButKilled(assert, cmd)
44 | }
45 |
46 | func TestShouldStartTtyInPowerShell(t *testing.T) {
47 | assert := assert.New(t)
48 |
49 | executor := newExecutor(&domain.ShellIn{
50 | ID: "test111",
51 | FlowId: "test111",
52 | Scripts: []string{
53 | "echo hello",
54 | },
55 | Timeout: 9999,
56 | }, false)
57 |
58 | go func() {
59 | for {
60 | log, ok := <-executor.TtyOut()
61 | if !ok {
62 | return
63 | }
64 |
65 | content, _ := base64.StdEncoding.DecodeString(log)
66 | util.LogDebug("------ %s", content)
67 | }
68 | }()
69 |
70 | go func() {
71 | time.Sleep(2 * time.Second)
72 | executor.TtyIn() <- "cd ~/\r\n"
73 | executor.TtyIn() <- "ls\r\n"
74 | time.Sleep(2 * time.Second)
75 | executor.TtyIn() <- "exit\r\n"
76 | }()
77 |
78 | err := executor.StartTty("fakeId", func(ttyId string) {
79 | util.LogDebug("Tty Started")
80 | })
81 | assert.NoError(err)
82 | assert.False(executor.IsInteracting())
83 | }
84 |
85 | func createPowerShellTestCmd() *domain.ShellIn {
86 | return &domain.ShellIn{
87 | CmdIn: domain.CmdIn{
88 | Type: domain.CmdTypeShell,
89 | },
90 | ID: "1-1-1",
91 | Scripts: []string{
92 | "echo bbb",
93 | "sleep 5",
94 | "$env:FLOW_VVV=\"flowci\"",
95 | "$env:FLOW_AAA=\"flow...\"",
96 | },
97 | Inputs: domain.Variables{"INPUT_VAR": "aaa"},
98 | Timeout: 1800,
99 | EnvFilters: []string{"FLOW_"},
100 | }
101 | }
102 |
--------------------------------------------------------------------------------
/executor/util.go:
--------------------------------------------------------------------------------
1 | package executor
2 |
3 | import (
4 | "github.com/flowci/flow-agent-x/domain"
5 | "github.com/flowci/flow-agent-x/util"
6 | "io"
7 | "os/exec"
8 | "strings"
9 | "syscall"
10 | )
11 |
12 | func getExitCode(cmd *exec.Cmd) int {
13 | ws := cmd.ProcessState.Sys().(syscall.WaitStatus)
14 | return ws.ExitStatus()
15 | }
16 |
17 | func matchEnvFilter(env string, filters []string) bool {
18 | for _, filter := range filters {
19 | if strings.HasPrefix(env, filter) {
20 | return true
21 | }
22 | }
23 | return false
24 | }
25 |
26 | func scriptForExitOnError(os string) []string {
27 | if os == util.OSWin {
28 | return []string{"$ErrorActionPreference = \"Stop\""}
29 | }
30 |
31 | return []string{"set -e"}
32 | }
33 |
34 | func appendNewLine(script, os string) string {
35 | newLine := newLineForOs(os)
36 |
37 | if !strings.HasSuffix(script, newLine) {
38 | script += newLine
39 | }
40 |
41 | return script
42 | }
43 |
44 | func newLineForOs(os string) string {
45 | if os == util.OSWin {
46 | return util.WinNewLine
47 | }
48 |
49 | return util.UnixNewLine
50 | }
51 |
52 | func readEnvFromReader(os string, r io.Reader, filters []string) domain.Variables {
53 | if os == util.OSWin {
54 | return readEnvFromReaderForWin(r, filters)
55 | }
56 |
57 | return readEnvFromReaderForUnix(r, filters)
58 | }
59 |
--------------------------------------------------------------------------------
/executor/util_unix.go:
--------------------------------------------------------------------------------
1 | package executor
2 |
3 | import (
4 | "bufio"
5 | "github.com/flowci/flow-agent-x/domain"
6 | "github.com/flowci/flow-agent-x/util"
7 | "io"
8 | "strings"
9 | )
10 |
11 | func readEnvFromReaderForUnix(r io.Reader, filters []string) domain.Variables {
12 | reader := bufio.NewReader(r)
13 | output := domain.NewVariables()
14 |
15 | for {
16 | line, err := reader.ReadString(util.LineBreak)
17 | if err != nil {
18 | return output
19 | }
20 |
21 | line = strings.TrimSpace(line)
22 | if ok, key, val := getEnvKeyAndValForUnix(line); ok {
23 | if matchEnvFilter(key, filters) {
24 | output[key] = val
25 | }
26 | }
27 | }
28 | }
29 |
30 | func getEnvKeyAndValForUnix(line string) (ok bool, key, val string) {
31 | index := strings.IndexAny(line, "=")
32 | if index == -1 {
33 | ok = false
34 | return
35 | }
36 |
37 | key = line[0:index]
38 | val = line[index+1:]
39 | ok = true
40 | return
41 | }
42 |
--------------------------------------------------------------------------------
/executor/util_win.go:
--------------------------------------------------------------------------------
1 | package executor
2 |
3 | import (
4 | "bufio"
5 | "encoding/binary"
6 | "github.com/flowci/flow-agent-x/domain"
7 | "github.com/flowci/flow-agent-x/util"
8 | "io"
9 | "strings"
10 | )
11 |
12 | var (
13 | winUTF16Dash = []byte{0, 45, 0, 45, 0, 45, 0, 45} // ----
14 | winUTF16CR = []byte{0, 13} // \r
15 | winNUL = []byte{0}
16 | )
17 |
18 | func readEnvFromReaderForWin(r io.Reader, filters []string) domain.Variables {
19 | reader := bufio.NewReaderSize(r, 1024*8)
20 | output := domain.NewVariables()
21 | process := false
22 |
23 | for {
24 | line, _, err := reader.ReadLine()
25 | if err != nil {
26 | return output
27 | }
28 |
29 | if util.IsByteStartWith(line, winUTF16Dash) {
30 | process = true
31 | continue
32 | }
33 |
34 | if process {
35 | line = util.BytesTrimRight(line, winNUL)
36 | line = util.BytesTrimRight(line, winUTF16CR)
37 | if len(line) == 0 {
38 | continue
39 | }
40 |
41 | strLine := util.UTF16BytesToString(line, binary.BigEndian)
42 |
43 | if ok, key, val := getEnvKeyAndValForWin(strLine); ok {
44 | if matchEnvFilter(key, filters) {
45 | output[key] = val
46 | }
47 | }
48 | }
49 | }
50 | }
51 |
52 | func getEnvKeyAndValForWin(line string) (ok bool, key, val string) {
53 | index := strings.IndexByte(line, ' ')
54 | if index == -1 {
55 | ok = false
56 | return
57 | }
58 |
59 | key = line[0:index]
60 | val = strings.Trim(line[index+1:], " ")
61 | ok = true
62 | return
63 | }
64 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/flowci/flow-agent-x
2 |
3 | go 1.17
4 |
5 | require (
6 | github.com/Microsoft/go-winio v0.4.14 // indirect
7 | github.com/creack/pty v1.1.11
8 | github.com/docker/distribution v2.7.1+incompatible // indirect
9 | github.com/docker/docker v1.13.1
10 | github.com/docker/go-connections v0.4.0
11 | github.com/docker/go-units v0.4.0 // indirect
12 | github.com/dustin/go-humanize v1.0.0
13 | github.com/emirpasic/gods v1.12.0 // indirect
14 | github.com/gin-contrib/pprof v1.3.0
15 | github.com/gin-gonic/gin v1.6.2
16 | github.com/golang/protobuf v1.3.5 // indirect
17 | github.com/google/uuid v1.1.2
18 | github.com/gorilla/websocket v1.4.2
19 | github.com/mattn/go-sqlite3 v1.10.0
20 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
21 | github.com/modern-go/reflect2 v1.0.1 // indirect
22 | github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
23 | github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec
24 | github.com/shirou/gopsutil/v3 v3.22.1
25 | github.com/sirupsen/logrus v1.8.1
26 | github.com/streadway/amqp v0.0.0-20181205114330-a314942b2fd9
27 | github.com/stretchr/testify v1.7.0
28 | github.com/urfave/cli v1.20.0
29 | golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 // indirect
30 | golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 // indirect
31 | golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect
32 | gopkg.in/src-d/go-billy.v4 v4.3.0 // indirect
33 | gopkg.in/src-d/go-git.v4 v4.8.1
34 | )
35 |
36 | require (
37 | github.com/davecgh/go-spew v1.1.1 // indirect
38 | github.com/gin-contrib/sse v0.1.0 // indirect
39 | github.com/go-ole/go-ole v1.2.6 // indirect
40 | github.com/go-playground/locales v0.13.0 // indirect
41 | github.com/go-playground/universal-translator v0.17.0 // indirect
42 | github.com/go-playground/validator/v10 v10.2.0 // indirect
43 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
44 | github.com/json-iterator/go v1.1.9 // indirect
45 | github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e // indirect
46 | github.com/leodido/go-urn v1.2.0 // indirect
47 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
48 | github.com/mattn/go-isatty v0.0.12 // indirect
49 | github.com/mitchellh/go-homedir v1.0.0 // indirect
50 | github.com/pelletier/go-buffruneio v0.2.0 // indirect
51 | github.com/pkg/errors v0.8.1 // indirect
52 | github.com/pmezard/go-difflib v1.0.0 // indirect
53 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
54 | github.com/sergi/go-diff v1.0.0 // indirect
55 | github.com/src-d/gcfg v1.4.0 // indirect
56 | github.com/stretchr/objx v0.1.1 // indirect
57 | github.com/tklauser/go-sysconf v0.3.9 // indirect
58 | github.com/tklauser/numcpus v0.3.0 // indirect
59 | github.com/ugorji/go/codec v1.1.7 // indirect
60 | github.com/xanzy/ssh-agent v0.2.0 // indirect
61 | github.com/yusufpapurcu/wmi v1.2.2 // indirect
62 | golang.org/x/text v0.3.2 // indirect
63 | gopkg.in/warnings.v0 v0.1.2 // indirect
64 | gopkg.in/yaml.v2 v2.2.8 // indirect
65 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
66 | )
67 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
2 | github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
3 | github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
4 | github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
5 | github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
6 | github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
7 | github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
8 | github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
9 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
10 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
11 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
12 | github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
13 | github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
14 | github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
15 | github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
16 | github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
17 | github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
18 | github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
19 | github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
20 | github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
21 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
22 | github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
23 | github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
24 | github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
25 | github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
26 | github.com/gin-contrib/pprof v1.3.0 h1:G9eK6HnbkSqDZBYbzG4wrjCsA4e+cvYAHUZw6W+W9K0=
27 | github.com/gin-contrib/pprof v1.3.0/go.mod h1:waMjT1H9b179t3CxuG1cV3DHpga6ybizwfBaM5OXaB0=
28 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
29 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
30 | github.com/gin-gonic/gin v1.6.2 h1:88crIK23zO6TqlQBt+f9FrPJNKm9ZEr7qjp9vl/d5TM=
31 | github.com/gin-gonic/gin v1.6.2/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
32 | github.com/gliderlabs/ssh v0.1.1 h1:j3L6gSLQalDETeEg/Jg0mGY0/y/N6zI2xX1978P0Uqw=
33 | github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
34 | github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
35 | github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
36 | github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
37 | github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
38 | github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
39 | github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
40 | github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
41 | github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
42 | github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
43 | github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
44 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
45 | github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
46 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
47 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
48 | github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
49 | github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
50 | github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
51 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
52 | github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
53 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
54 | github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
55 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
56 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
57 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
58 | github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
59 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
60 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
61 | github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8=
62 | github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
63 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
64 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
65 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
66 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
67 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
68 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
69 | github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
70 | github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
71 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
72 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
73 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
74 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
75 | github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
76 | github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
77 | github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
78 | github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
79 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
80 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
81 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
82 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
83 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
84 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
85 | github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
86 | github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
87 | github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA=
88 | github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
89 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
90 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
91 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
92 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
93 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
94 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
95 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
96 | github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec h1:6ncX5ko6B9LntYM0YBRXkiSaZMmLYeZ/NWcmeB43mMY=
97 | github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
98 | github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
99 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
100 | github.com/shirou/gopsutil/v3 v3.22.1 h1:33y31Q8J32+KstqPfscvFwBlNJ6xLaBy4xqBXzlYV5w=
101 | github.com/shirou/gopsutil/v3 v3.22.1/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
102 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
103 | github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
104 | github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
105 | github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
106 | github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
107 | github.com/streadway/amqp v0.0.0-20181205114330-a314942b2fd9 h1:37QTz/gdHBLQcsmgMTnQDSWCtKzJ7YnfI2M2yTdr4BQ=
108 | github.com/streadway/amqp v0.0.0-20181205114330-a314942b2fd9/go.mod h1:1WNBiOZtZQLpVAyu0iTduoJL9hEsMloAK5XWrtW0xdY=
109 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
110 | github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
111 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
112 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
113 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
114 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
115 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
116 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
117 | github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
118 | github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
119 | github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
120 | github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
121 | github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
122 | github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
123 | github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
124 | github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
125 | github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
126 | github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
127 | github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro=
128 | github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
129 | github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
130 | github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
131 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
132 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
133 | golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo=
134 | golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
135 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
136 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
137 | golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
138 | golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
139 | golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
140 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
141 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
142 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
143 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
144 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
145 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
146 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
147 | golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
148 | golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
149 | golang.org/x/sys v0.0.0-20220111092808-5a964db01320 h1:0jf+tOCoZ3LyutmCOWpVni1chK4VfFLhRsDK7MhqGRY=
150 | golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
151 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
152 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
153 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
154 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
155 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
156 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
157 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
158 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
159 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
160 | gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
161 | gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek=
162 | gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
163 | gopkg.in/src-d/go-git-fixtures.v3 v3.1.1 h1:XWW/s5W18RaJpmo1l0IYGqXKuJITWRFuA45iOf1dKJs=
164 | gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
165 | gopkg.in/src-d/go-git.v4 v4.8.1 h1:aAyBmkdE1QUUEHcP4YFCGKmsMQRAuRmUcPEQR7lOAa0=
166 | gopkg.in/src-d/go-git.v4 v4.8.1/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk=
167 | gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
168 | gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
169 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
170 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
171 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
172 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
173 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
174 |
--------------------------------------------------------------------------------
/service/cache_manager.go:
--------------------------------------------------------------------------------
1 | package service
2 |
3 | import (
4 | "encoding/base64"
5 | "fmt"
6 | "github.com/dustin/go-humanize"
7 | "github.com/flowci/flow-agent-x/api"
8 | "github.com/flowci/flow-agent-x/domain"
9 | "github.com/flowci/flow-agent-x/util"
10 | "io/ioutil"
11 | "path/filepath"
12 | )
13 |
14 | type CacheManager struct {
15 | client api.Client
16 | }
17 |
18 | type progressWriter struct {
19 | total uint64
20 | client api.Client
21 | cmdIn *domain.ShellIn
22 | }
23 |
24 | // Download download cache into a temp dir and return
25 | func (cm *CacheManager) Download(cmdIn *domain.ShellIn) string {
26 | defer util.RecoverPanic(func(e error) {
27 | util.LogWarn(e.Error())
28 | })
29 |
30 | cm.Resolve(cmdIn)
31 |
32 | cache := cm.client.CacheGet(cmdIn.JobId, cmdIn.Cache.Key)
33 | sendLog(cm.client, cmdIn, fmt.Sprintf("Start to download cache %s", cache.Key))
34 |
35 | writer := &progressWriter{
36 | client: cm.client,
37 | cmdIn: cmdIn,
38 | }
39 |
40 | cacheDir, err := ioutil.TempDir("", "cache_")
41 | util.PanicIfErr(err)
42 |
43 | for _, file := range cache.Files {
44 | sendLog(cm.client, cmdIn, fmt.Sprintf("---> cache %s", file))
45 | cm.client.CacheDownload(cache.Id, cacheDir, file, writer)
46 | }
47 |
48 | sendLog(cm.client, cmdIn, "All cached files downloaded")
49 | util.LogDebug("cache src file loaded at %s", cacheDir)
50 | return cacheDir
51 | }
52 |
53 | // Upload upload all files/dirs from cache dir
54 | func (cm *CacheManager) Upload(cmdIn *domain.ShellIn, cacheDir string) {
55 | fileInfos, err := ioutil.ReadDir(cacheDir)
56 | if err != nil {
57 | util.LogWarn(err.Error())
58 | return
59 | }
60 |
61 | if len(fileInfos) == 0 {
62 | return
63 | }
64 |
65 | files := make([]string, len(fileInfos))
66 | for i, fileInfo := range fileInfos {
67 | files[i] = filepath.Join(cacheDir, fileInfo.Name())
68 | }
69 |
70 | sendLog(cm.client, cmdIn, fmt.Sprintf("Start to upload cache %s", cmdIn.Cache.Key))
71 |
72 | err = cm.client.CachePut(cmdIn.JobId, cmdIn.Cache.Key, cacheDir, files)
73 | if err != nil {
74 | sendLog(cm.client, cmdIn, fmt.Sprintf("Unable to cache %s : %s", cmdIn.Cache.Key, err.Error()))
75 | return
76 | }
77 |
78 | sendLog(cm.client, cmdIn, fmt.Sprintf("Cache %s uploaded", cmdIn.Cache.Key))
79 | }
80 |
81 | // Resolve resolve env vars in cache key and paths
82 | func (cm *CacheManager) Resolve(cmdIn *domain.ShellIn) {
83 | cache := cmdIn.Cache
84 | cache.Key = util.ParseStringWithSource(cache.Key, cmdIn.Inputs)
85 |
86 | for i, p := range cache.Paths {
87 | cache.Paths[i] = util.ParseStringWithSource(p, cmdIn.Inputs)
88 | }
89 | }
90 |
91 | func (pw *progressWriter) Write(p []byte) (int, error) {
92 | n := len(p)
93 | pw.total += uint64(n)
94 |
95 | text := fmt.Sprintf("Downloading... %s complete", humanize.Bytes(pw.total))
96 | sendLog(pw.client, pw.cmdIn, text)
97 |
98 | return n, nil
99 | }
100 |
101 | func sendLog(client api.Client, cmdIn *domain.ShellIn, text string) {
102 | b64 := base64.StdEncoding.EncodeToString([]byte(text + "\n"))
103 | client.SendShellLog(cmdIn.JobId, cmdIn.ID, b64)
104 | }
105 |
--------------------------------------------------------------------------------
/service/cmd_service.go:
--------------------------------------------------------------------------------
1 | package service
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "net"
7 | "strconv"
8 | "sync"
9 | "time"
10 |
11 | "bufio"
12 | "encoding/base64"
13 | "github.com/google/uuid"
14 | "os"
15 | "path/filepath"
16 |
17 | "github.com/flowci/flow-agent-x/config"
18 | "github.com/flowci/flow-agent-x/domain"
19 | "github.com/flowci/flow-agent-x/executor"
20 | "github.com/flowci/flow-agent-x/util"
21 | )
22 |
23 | type (
24 | // CmdService receive and execute cmd
25 | CmdService struct {
26 | pluginManager *PluginManager
27 | cacheManager *CacheManager
28 |
29 | cmdIn <-chan []byte
30 |
31 | executor executor.Executor
32 | mux sync.Mutex
33 | }
34 | )
35 |
36 | // IsRunning check is available to run cmd
37 | func (s *CmdService) IsRunning() bool {
38 | return s.executor != nil
39 | }
40 |
41 | // Execute execute cmd according to the type
42 | func (s *CmdService) Execute(bytes []byte) error {
43 | var in domain.CmdIn
44 | err := json.Unmarshal(bytes, &in)
45 | util.PanicIfErr(err)
46 |
47 | switch in.Type {
48 | case domain.CmdTypeShell:
49 | var shell domain.ShellIn
50 | err := json.Unmarshal(bytes, &shell)
51 | util.PanicIfErr(err)
52 | return s.execShell(&shell)
53 | case domain.CmdTypeTty:
54 | s.execTty(bytes)
55 | return nil
56 | case domain.CmdTypeKill:
57 | return s.execKill()
58 | case domain.CmdTypeClose:
59 | return s.execClose()
60 | default:
61 | return ErrorCmdUnsupportedType
62 | }
63 | }
64 |
65 | func (s *CmdService) start() {
66 | go func() {
67 | defer util.LogDebug("[Exit]: Rabbit mq consumer")
68 |
69 | for {
70 | select {
71 | case bytes, ok := <-s.cmdIn:
72 | if !ok {
73 | break
74 | }
75 |
76 | util.LogDebug("Received a message: %s", bytes)
77 | err := s.Execute(bytes)
78 | if err != nil {
79 | util.LogDebug(err.Error())
80 | }
81 | case <-time.After(time.Second * 10):
82 | util.LogDebug("...")
83 | }
84 | }
85 | }()
86 | }
87 |
88 | func (s *CmdService) release() {
89 | if s.executor != nil {
90 | s.executor.Close()
91 | s.executor = nil
92 |
93 | cm := config.GetInstance()
94 | cm.FireEvent(domain.EventOnIdle)
95 | }
96 |
97 | util.LogDebug("[Exit]: cmd been executed and service is available !")
98 | }
99 |
100 | func (s *CmdService) execShell(in *domain.ShellIn) (out error) {
101 | defer func() {
102 | if err := recover(); err != nil {
103 | out = err.(error)
104 | s.failureBeforeExecute(in, out)
105 | s.release() // release current executor if error
106 | }
107 | }()
108 |
109 | cm := config.GetInstance()
110 |
111 | s.mux.Lock()
112 | defer s.mux.Unlock()
113 |
114 | if s.IsRunning() {
115 | return ErrorCmdIsRunning
116 | }
117 |
118 | err := initShellCmd(in)
119 | util.PanicIfErr(err)
120 |
121 | cm.FireEvent(domain.EventOnBusy)
122 |
123 | if in.HasPlugin() {
124 | err := s.pluginManager.Load(in.Plugin)
125 | util.PanicIfErr(err)
126 | }
127 |
128 | // all cache will move to job dir after started
129 | cacheSrcDir := ""
130 | if in.HasCache() {
131 | cacheSrcDir = s.cacheManager.Download(in)
132 | }
133 |
134 | s.loadSecretForDocker(in)
135 |
136 | s.executor = executor.NewExecutor(executor.Options{
137 | K8s: &domain.K8sConfig{
138 | Enabled: cm.K8sEnabled,
139 | InCluster: cm.K8sCluster,
140 | Namespace: cm.K8sNamespace,
141 | PodName: cm.K8sPodName,
142 | PodIp: cm.K8sPodIp,
143 | },
144 | AgentId: cm.Token,
145 | Parent: cm.AppCtx,
146 | Workspace: cm.Workspace,
147 | WorkspaceFromDockerVolume: cm.IsFromDocker,
148 | PluginDir: cm.PluginDir,
149 | CacheSrcDir: cacheSrcDir,
150 | Cmd: in,
151 | Vars: s.initEnv(),
152 | SecretVars: s.initSecretEnv(in),
153 | ConfigVars: s.initConfigEnv(in),
154 | Volumes: cm.Volumes,
155 | })
156 |
157 | err = s.executor.Init()
158 | util.PanicIfErr(err)
159 |
160 | s.startLogConsumer()
161 |
162 | go func() {
163 | defer func() {
164 | input, output := s.executor.CacheDir()
165 | os.RemoveAll(input)
166 | os.RemoveAll(output)
167 |
168 | s.release()
169 | }()
170 |
171 | _ = s.executor.Start()
172 |
173 | // write all files in srcCache back to cache
174 | _, output := s.executor.CacheDir()
175 | s.cacheManager.Upload(in, output)
176 |
177 | result := s.executor.GetResult()
178 | util.LogInfo("Cmd '%s' been executed with exit code %d", result.ID, result.Code)
179 | cm.Client.SendCmdOut(result)
180 | }()
181 |
182 | return nil
183 | }
184 |
185 | func (s *CmdService) initEnv() domain.Variables {
186 | config := config.GetInstance()
187 |
188 | vars := domain.NewVariables()
189 | vars[domain.VarAgentPluginDir] = config.PluginDir
190 | vars[domain.VarServerUrl] = config.Server
191 | vars[domain.VarAgentToken] = config.Token
192 | vars[domain.VarAgentPort] = strconv.Itoa(config.Port)
193 | vars[domain.VarAgentWorkspace] = config.Workspace
194 | vars[domain.VarAgentPluginDir] = config.PluginDir
195 | vars[domain.VarAgentLogDir] = config.LoggingDir
196 |
197 | // write env for interface ip on of agent host
198 | interfaces, err := net.Interfaces()
199 | if err != nil {
200 | return vars
201 | }
202 |
203 | for _, iface := range interfaces {
204 | addrs, err := iface.Addrs()
205 | if err != nil {
206 | continue
207 | }
208 |
209 | for _, addr := range addrs {
210 | var ip net.IP
211 | switch v := addr.(type) {
212 | case *net.IPNet:
213 | ip = v.IP
214 | case *net.IPAddr:
215 | ip = v.IP
216 | }
217 |
218 | key := fmt.Sprintf(domain.VarAgentIpPattern, iface.Name)
219 | vars[key] = ip.String()
220 | break
221 | }
222 | }
223 |
224 | return vars
225 | }
226 |
227 | // initSecretEnv load secret value as environment variables
228 | func (s *CmdService) initSecretEnv(in *domain.ShellIn) domain.Variables {
229 | vars := domain.NewVariables()
230 |
231 | if !in.HasSecrets() {
232 | return vars
233 | }
234 |
235 | api := config.GetInstance().Client
236 | for _, name := range in.Secrets {
237 | secret, err := api.GetSecret(name)
238 | util.PanicIfErr(err)
239 | vars.AddMapVars(secret.ToEnvs())
240 | }
241 |
242 | return vars
243 | }
244 |
245 | // initConfigEnv load config value as environment variables
246 | func (s *CmdService) initConfigEnv(in *domain.ShellIn) domain.Variables {
247 | vars := domain.NewVariables()
248 |
249 | if !in.HasConfigs() {
250 | return vars
251 | }
252 |
253 | api := config.GetInstance().Client
254 | for _, name := range in.Configs {
255 | config, err := api.GetConfig(name)
256 | util.PanicIfErr(err)
257 | vars.AddMapVars(config.ToEnvs())
258 | }
259 |
260 | return vars
261 | }
262 |
263 | func (s *CmdService) execTty(bytes []byte) {
264 | var in domain.TtyIn
265 | err := json.Unmarshal(bytes, &in)
266 | if err != nil {
267 | util.LogWarn("Unable to decode message to TtyIn")
268 | return
269 | }
270 |
271 | response := &domain.TtyOut{
272 | ID: in.ID,
273 | Action: in.Action,
274 | }
275 |
276 | appConfig := config.GetInstance()
277 |
278 | defer func() {
279 | if err := recover(); err != nil {
280 | response.IsSuccess = false
281 | response.Error = err.(error).Error()
282 | appConfig.Client.SendCmdOut(response)
283 | }
284 | }()
285 |
286 | e := s.executor
287 | if !s.IsRunning() {
288 | panic(fmt.Errorf("No running cmd"))
289 | }
290 |
291 | switch in.Action {
292 | case domain.TtyActionOpen:
293 | if e.IsInteracting() {
294 | response.IsSuccess = true
295 | appConfig.Client.SendCmdOut(response)
296 | return
297 | }
298 |
299 | go func() {
300 | err = e.StartTty(in.ID, func(ttyId string) {
301 | response.IsSuccess = true
302 | appConfig.Client.SendCmdOut(response)
303 | })
304 |
305 | if err != nil {
306 | response.IsSuccess = false
307 | response.Error = err.Error()
308 | appConfig.Client.SendCmdOut(response)
309 | return
310 | }
311 |
312 | // send close action when exit
313 | response.Action = domain.TtyActionClose
314 | response.IsSuccess = true
315 | appConfig.Client.SendCmdOut(response)
316 | }()
317 | case domain.TtyActionShell:
318 | if !e.IsInteracting() {
319 | panic(fmt.Errorf("Tty not started, please send open cmd"))
320 | }
321 |
322 | e.TtyIn() <- in.Input
323 | case domain.TtyActionClose:
324 | if !e.IsInteracting() {
325 | panic(fmt.Errorf("Tty not started, please send open cmd"))
326 | }
327 |
328 | // close action response send on exit
329 | e.StopTty()
330 | }
331 | }
332 |
333 | func (s *CmdService) execKill() error {
334 | if s.IsRunning() {
335 | s.executor.Kill()
336 | }
337 | return nil
338 | }
339 |
340 | func (s *CmdService) execClose() error {
341 | if s.IsRunning() {
342 | s.executor.Kill()
343 | }
344 |
345 | config := config.GetInstance()
346 | config.Cancel()
347 |
348 | return nil
349 | }
350 |
351 | func (s *CmdService) failureBeforeExecute(in *domain.ShellIn, err error) {
352 | result := &domain.ShellOut{
353 | ID: in.ID,
354 | Status: domain.CmdStatusException,
355 | Error: err.Error(),
356 | StartAt: time.Now(),
357 | FinishAt: time.Now(),
358 | }
359 |
360 | appConfig := config.GetInstance()
361 | appConfig.Client.SendCmdOut(result)
362 | }
363 |
364 | func (s *CmdService) startLogConsumer() {
365 | apiClient := config.GetInstance().Client
366 | loggingDir := config.GetInstance().LoggingDir
367 | executor := s.executor
368 |
369 | consumeShellLog := func() {
370 |
371 | // init path for shell, log and raw log
372 | logPath := filepath.Join(loggingDir, executor.CmdIn().ID+".log")
373 | f, _ := os.Create(logPath)
374 | logFileWriter := bufio.NewWriter(f)
375 |
376 | defer func() {
377 | // upload log after flush!!
378 | _ = logFileWriter.Flush()
379 | _ = f.Close()
380 |
381 | err := apiClient.UploadLog(logPath)
382 | util.LogIfError(err)
383 | util.LogDebug("[Exit]: LogConsumer")
384 | }()
385 |
386 | for b64Log := range executor.Stdout() {
387 |
388 | // write to file
389 | log, err := base64.StdEncoding.DecodeString(b64Log)
390 | if err == nil {
391 | _, _ = logFileWriter.Write(log)
392 | util.LogDebug("[ShellLog]: %s", string(log))
393 | }
394 |
395 | jobId := executor.CmdIn().JobId
396 | stepId := executor.CmdIn().ID
397 | apiClient.SendShellLog(jobId, stepId, b64Log)
398 | }
399 | }
400 |
401 | consumeTtyLog := func() {
402 | apiClient := config.GetInstance().Client
403 | for b64Log := range executor.TtyOut() {
404 | apiClient.SendTtyLog(executor.TtyId(), b64Log)
405 | }
406 | }
407 |
408 | go consumeShellLog()
409 | go consumeTtyLog()
410 | }
411 |
412 | func (s *CmdService) loadSecretForDocker(in *domain.ShellIn) {
413 | if !in.HasDockerOption() {
414 | return
415 | }
416 |
417 | cm := config.GetInstance()
418 | for _, option := range in.Dockers {
419 | if option.HasAuth() {
420 | secret, err := cm.Client.GetSecret(option.Auth)
421 | util.PanicIfErr(err)
422 |
423 | auth, ok := secret.(*domain.AuthSecret)
424 | if !ok {
425 | panic(fmt.Errorf("the secret '%s' is invalid, the secret category should be 'Auth pair'", option.Auth))
426 | }
427 |
428 | option.AuthContent = auth.Pair
429 | }
430 | }
431 | }
432 |
433 | // ---------------------------------
434 | // Utils
435 | // ---------------------------------
436 |
437 | func initShellCmd(in *domain.ShellIn) error {
438 | // init cmd id if undefined
439 | if util.IsEmptyString(in.ID) {
440 | in.ID = uuid.New().String()
441 | }
442 |
443 | // init inputs if undefined
444 | if in.Inputs == nil {
445 | in.Inputs = make(domain.Variables, 10)
446 | }
447 |
448 | return nil
449 | }
450 |
--------------------------------------------------------------------------------
/service/errors.go:
--------------------------------------------------------------------------------
1 | package service
2 |
3 | import "errors"
4 |
5 | var (
6 | ErrorCmdIsRunning = errors.New("agent: cmd is running, service not available")
7 | ErrorCmdUnsupportedType = errors.New("agent: unsupported cmd type")
8 |
9 | ErrorCmdScriptIsPersented = errors.New("agent: the scripts should be empty for session open")
10 | ErrorCmdMissingSessionID = errors.New("agent: the session id is required for cmd")
11 | ErrorCmdSessionNotFound = errors.New("agent: session not found")
12 | ErrorCmdSessionMissingScripts = errors.New("agent: script is missing")
13 | )
14 |
--------------------------------------------------------------------------------
/service/factory.go:
--------------------------------------------------------------------------------
1 | package service
2 |
3 | import (
4 | "github.com/flowci/flow-agent-x/config"
5 | "strings"
6 | "sync"
7 | )
8 |
9 | var (
10 | singleton *CmdService
11 | once sync.Once
12 | )
13 |
14 | // GetCmdService get singleton of cmd service
15 | func GetCmdService() *CmdService {
16 | appConfig := config.GetInstance()
17 | cmdIn := appConfig.Client.GetCmdIn()
18 |
19 | once.Do(func() {
20 | singleton = &CmdService{
21 | pluginManager: NewPluginManager(appConfig.PluginDir, appConfig.Server),
22 | cacheManager: NewCacheManager(),
23 | cmdIn: cmdIn,
24 | }
25 | singleton.start()
26 | })
27 |
28 | return singleton
29 | }
30 |
31 | func NewCacheManager() *CacheManager {
32 | appConfig := config.GetInstance()
33 | return &CacheManager{
34 | client: appConfig.Client,
35 | }
36 | }
37 |
38 | func NewPluginManager(dir, server string) *PluginManager {
39 | return &PluginManager{
40 | dir: dir,
41 | server: strings.TrimRight(server, "/"),
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/service/plugin_manager.go:
--------------------------------------------------------------------------------
1 | package service
2 |
3 | import (
4 | "github.com/flowci/flow-agent-x/util"
5 | git "gopkg.in/src-d/go-git.v4"
6 | "os"
7 | "path/filepath"
8 | )
9 |
10 | type PluginManager struct {
11 | // file dir to store plugin
12 | dir string
13 |
14 | // server url
15 | server string
16 | }
17 |
18 | func (p *PluginManager) Load(name string) error {
19 | url := p.server + "/git/plugins/" + name
20 | dir := filepath.Join(p.dir, name)
21 |
22 | util.LogInfo("agent: clone plugin '%s' to '%s'", url, dir)
23 |
24 | err := p.clone(dir, url)
25 |
26 | if err == git.ErrRepositoryAlreadyExists {
27 | return p.pull(dir, url)
28 | }
29 |
30 | return err
31 | }
32 |
33 | func (p *PluginManager) clone(dir, url string) error {
34 | options := &git.CloneOptions{
35 | URL: url,
36 | Progress: os.Stdout,
37 | }
38 |
39 | _, err := git.PlainClone(dir, false, options)
40 | return err
41 | }
42 |
43 | func (p *PluginManager) pull(dir, url string) (out error) {
44 | defer util.RecoverPanic(func(e error) {
45 | out = e
46 | })
47 |
48 | repo, err := git.PlainOpen(dir)
49 | util.PanicIfErr(err)
50 |
51 | workTree, err := repo.Worktree()
52 | util.PanicIfErr(err)
53 |
54 | // update remote url if url been changed
55 | remote, err := repo.Remote("origin")
56 | util.PanicIfErr(err)
57 |
58 | config := remote.Config()
59 | if config.URLs[0] != url {
60 | err = repo.DeleteRemote("origin")
61 | util.PanicIfErr(err)
62 |
63 | config.URLs[0] = url
64 | _, err = repo.CreateRemote(config)
65 | util.PanicIfErr(err)
66 | }
67 |
68 | err = workTree.Pull(&git.PullOptions{RemoteName: "origin"})
69 | if err == git.NoErrAlreadyUpToDate {
70 | return nil
71 | }
72 |
73 | return err
74 | }
75 |
--------------------------------------------------------------------------------
/util/common.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "encoding/binary"
5 | "fmt"
6 | "os"
7 | "runtime"
8 | "strings"
9 | "unicode/utf16"
10 | "unicode/utf8"
11 | )
12 |
13 | const (
14 | UnixNewLine = "\n"
15 | UnixPathSeparator = "/"
16 |
17 | WinNewLine = "\r\n"
18 | WinPathSeparator = "\\"
19 |
20 | LineBreak = '\n'
21 | EmptyStr = ""
22 |
23 | OSWin = "windows"
24 | OSLinux = "linux"
25 | OSMac = "darwin"
26 | )
27 |
28 | var (
29 | HomeDir = ""
30 | )
31 |
32 | func init() {
33 | HomeDir, _ = os.UserHomeDir()
34 | }
35 |
36 | func OS() string {
37 | if IsMac() {
38 | return "MAC"
39 | }
40 |
41 | if IsWindows() {
42 | return "WIN"
43 | }
44 |
45 | if IsLinux() {
46 | return "LINUX"
47 | }
48 |
49 | return "UNKNOWN"
50 | }
51 |
52 | func IsMac() bool {
53 | return runtime.GOOS == OSMac
54 | }
55 |
56 | func IsLinux() bool {
57 | return runtime.GOOS == OSLinux
58 | }
59 |
60 | func IsWindows() bool {
61 | return runtime.GOOS == OSWin
62 | }
63 |
64 | func PointerBoolean(val bool) *bool {
65 | p := val
66 | return &p
67 | }
68 |
69 | func IndexOfFirstSpace(str string) int {
70 | for i := 0; i < len(str); i++ {
71 | if i == ' ' {
72 | return i
73 | }
74 | }
75 | return -1
76 | }
77 |
78 | // ParseString parse string which include system env variable
79 | func ParseString(src string) string {
80 | return parseVariablesFrom(src, os.Getenv)
81 | }
82 |
83 | func ParseStringWithSource(src string, source map[string]string) string {
84 | return parseVariablesFrom(src, func(env string) string {
85 | return source[env]
86 | })
87 | }
88 |
89 | // replace ${VAR} with actual variable value
90 | func parseVariablesFrom(src string, getVariable func(string) string) string {
91 | if IsEmptyString(src) {
92 | return src
93 | }
94 |
95 | for i := 0; i < len(src); i++ {
96 | if src[i] != '$' {
97 | continue
98 | }
99 |
100 | // left bracket index
101 | lIndex := i + 1
102 | if src[lIndex] != '{' {
103 | continue
104 | }
105 |
106 | // find right bracket index
107 | for rIndex := lIndex + 1; rIndex < len(src); rIndex++ {
108 | if src[rIndex] != '}' {
109 | continue
110 | }
111 |
112 | env := src[lIndex+1 : rIndex]
113 | val := getVariable(env)
114 |
115 | // do not replace if no value found
116 | if IsEmptyString(val) {
117 | break
118 | }
119 |
120 | src = strings.Replace(src, fmt.Sprintf("${%s}", env), val, -1)
121 | i = rIndex
122 | break
123 | }
124 | }
125 |
126 | return src
127 | }
128 |
129 | func GetEnv(env, def string) string {
130 | val, ok := os.LookupEnv(env)
131 | if ok {
132 | return val
133 | }
134 | return def
135 | }
136 |
137 | func ByteToMB(bytes uint64) uint64 {
138 | return (bytes / 1024) / 1024
139 | }
140 |
141 | func IsByteStartWith(src []byte, start []byte) bool {
142 | if len(src) < len(start) {
143 | return false
144 | }
145 |
146 | for i, c := range start {
147 | if src[i] != c {
148 | return false
149 | }
150 | }
151 |
152 | return true
153 | }
154 |
155 | func BytesTrimRight(src []byte, trim []byte) []byte {
156 | if len(src) < len(trim) {
157 | return src
158 | }
159 |
160 | canTrim := true
161 |
162 | j := len(trim) - 1
163 | for i := len(src) - 1; i >= len(src)-len(trim); i-- {
164 | if src[i] != trim[j] {
165 | canTrim = false
166 | break
167 | }
168 | j--
169 | }
170 |
171 | if canTrim {
172 | return src[0 : len(src)-len(trim)]
173 | }
174 |
175 | return src
176 | }
177 |
178 | func TrimLeftString(src, trim string) string {
179 | if len(src) < len(trim) {
180 | return src
181 | }
182 |
183 | index := -1
184 | for i := 0; i < len(trim); i++ {
185 | if src[i] == trim[i] {
186 | index = i
187 | continue
188 | }
189 | }
190 |
191 | if index == -1 {
192 | return ""
193 | }
194 |
195 | return src[index+1:]
196 | }
197 |
198 | func UTF16BytesToString(b []byte, o binary.ByteOrder) string {
199 | utf := make([]uint16, (len(b)+(2-1))/2)
200 |
201 | for i := 0; i+(2-1) < len(b); i += 2 {
202 | utf[i/2] = o.Uint16(b[i:])
203 | }
204 |
205 | if len(b)/2 < len(utf) {
206 | utf[len(utf)-1] = utf8.RuneError
207 | }
208 |
209 | return string(utf16.Decode(utf))
210 | }
211 |
--------------------------------------------------------------------------------
/util/common_test.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "encoding/binary"
5 | "os/user"
6 | "reflect"
7 | "testing"
8 |
9 | "github.com/stretchr/testify/assert"
10 | )
11 |
12 | type TestType struct {
13 | Name string
14 | }
15 |
16 | func TestShouldDetectPointerType(t *testing.T) {
17 | assert := assert.New(t)
18 |
19 | p := &TestType{}
20 | assert.True(IsPointerType(p))
21 | }
22 |
23 | func TestShouldGetTypeOfPointer(t *testing.T) {
24 | assert := assert.New(t)
25 |
26 | p := new(TestType)
27 | assert.Equal(reflect.TypeOf(TestType{}), GetType(p))
28 | }
29 |
30 | func TestShouldParseStringWithEnvVariable(t *testing.T) {
31 | assert := assert.New(t)
32 | usr, _ := user.Current()
33 |
34 | assert.Equal("hello", ParseString("hello"))
35 | assert.Equal(usr.HomeDir+"/hello", ParseString("${HOME}/hello"))
36 | assert.Equal("/test"+usr.HomeDir+"/hello", ParseString("/test${HOME}/hello"))
37 | assert.Equal(usr.HomeDir+usr.HomeDir, ParseString("${HOME}${HOME}"))
38 | }
39 |
40 | func TestShouldDecodeUTF16(t *testing.T) {
41 | assert := assert.New(t)
42 | assert.Equal("-", UTF16BytesToString([]byte{0, 45}, binary.BigEndian))
43 | }
44 |
45 | func TestShouldTrimLeftBytes(t *testing.T) {
46 | assert := assert.New(t)
47 | assert.Equal([]byte{0, 1, 2}, BytesTrimRight([]byte{0, 1, 2, 3, 4}, []byte{3, 4}))
48 | }
49 |
50 | func TestShouldTrimLeftString(t *testing.T) {
51 | assert := assert.New(t)
52 |
53 | p := "/var/folders/vz/__yhmfmd1j97t9gnslqm03780000gn/T/cache_362016357/cache_1"
54 | z := "/var/folders/vz/__yhmfmd1j97t9gnslqm03780000gn/T/cache_362016357"
55 |
56 | cacheName := TrimLeftString(p, z)
57 | assert.NotNil(cacheName)
58 | assert.Equal("/cache_1", cacheName)
59 | }
60 |
--------------------------------------------------------------------------------
/util/files.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "archive/zip"
5 | "fmt"
6 | "io"
7 | "io/ioutil"
8 | "os"
9 | "path/filepath"
10 | )
11 |
12 | func IsFileExists(path string) bool {
13 | if _, err := os.Stat(path); !os.IsNotExist(err) {
14 | return true
15 | }
16 | return false
17 | }
18 |
19 | func IsFileExistsAndReturnFileInfo(path string) (os.FileInfo, bool) {
20 | stat, err := os.Stat(path)
21 | if !os.IsNotExist(err) {
22 | return stat, true
23 | }
24 | return stat, false
25 | }
26 |
27 | // CopyFile The file will be created if it does not already exist. If the
28 | // destination file exists, the contents will be replaced
29 | func CopyFile(src, dst string) (errOut error) {
30 | defer RecoverPanic(func(e error) {
31 | errOut = e
32 | })
33 |
34 | in, err := os.Open(src)
35 | PanicIfErr(err)
36 |
37 | defer in.Close()
38 |
39 | out, err := os.Create(dst)
40 | PanicIfErr(err)
41 |
42 | defer func() {
43 | if e := out.Close(); e != nil {
44 | errOut = e
45 | }
46 | }()
47 |
48 | _, err = io.Copy(out, in)
49 | PanicIfErr(err)
50 |
51 | err = out.Sync()
52 | PanicIfErr(err)
53 |
54 | si, err := os.Stat(src)
55 | PanicIfErr(err)
56 |
57 | err = os.Chmod(dst, si.Mode())
58 | PanicIfErr(err)
59 |
60 | return
61 | }
62 |
63 | func CopyDir(src string, dst string) (err error) {
64 | defer RecoverPanic(nil)
65 |
66 | src = filepath.Clean(src)
67 | dst = filepath.Clean(dst)
68 |
69 | si, err := os.Stat(src)
70 | PanicIfErr(err)
71 |
72 | if !si.IsDir() {
73 | panic(fmt.Errorf("source is not a directory"))
74 | }
75 |
76 | _, err = os.Stat(dst)
77 | if err != nil && !os.IsNotExist(err) {
78 | return
79 | }
80 |
81 | if err == nil {
82 | panic(fmt.Errorf("destination already exists"))
83 | }
84 |
85 | err = os.MkdirAll(dst, si.Mode())
86 | PanicIfErr(err)
87 |
88 | entries, err := ioutil.ReadDir(src)
89 | PanicIfErr(err)
90 |
91 | for _, entry := range entries {
92 | srcPath := filepath.Join(src, entry.Name())
93 | dstPath := filepath.Join(dst, entry.Name())
94 |
95 | if entry.IsDir() {
96 | err = CopyDir(srcPath, dstPath)
97 | PanicIfErr(err)
98 | continue
99 | }
100 |
101 | // skip symlinks.
102 | if entry.Mode()&os.ModeSymlink != 0 {
103 | continue
104 | }
105 |
106 | err = CopyFile(srcPath, dstPath)
107 | PanicIfErr(err)
108 | }
109 |
110 | return
111 | }
112 |
113 | func Zip(src, dest, separator string) (out error) {
114 | defer RecoverPanic(func(e error) {
115 | out = e
116 | })
117 |
118 | outFile, err := os.Create(dest)
119 | PanicIfErr(err)
120 |
121 | defer outFile.Close()
122 |
123 | w := zip.NewWriter(outFile)
124 | addFiles(w, src, "", separator)
125 |
126 | err = w.Close()
127 | PanicIfErr(err)
128 | return
129 | }
130 |
131 | func Unzip(src string, dest string) (out error) {
132 | defer RecoverPanic(func(e error) {
133 | out = e
134 | })
135 |
136 | r, err := zip.OpenReader(src)
137 | PanicIfErr(err)
138 | defer r.Close()
139 |
140 | for _, f := range r.File {
141 | fpath := filepath.Join(dest, f.Name)
142 |
143 | if f.FileInfo().IsDir() {
144 | _ = os.MkdirAll(fpath, os.ModePerm)
145 | continue
146 | }
147 |
148 | err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm)
149 | PanicIfErr(err)
150 |
151 | outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
152 | PanicIfErr(err)
153 |
154 | rc, err := f.Open()
155 | PanicIfErr(err)
156 |
157 | _, err = io.Copy(outFile, rc)
158 | PanicIfErr(err)
159 |
160 | outFile.Close()
161 | rc.Close()
162 | }
163 |
164 | return
165 | }
166 |
167 | func addFiles(w *zip.Writer, basePath, baseInZip, separator string) {
168 | files, err := ioutil.ReadDir(basePath)
169 | PanicIfErr(err)
170 |
171 | for _, file := range files {
172 | srcFullPath := basePath + separator + file.Name()
173 |
174 | if !file.IsDir() {
175 | dat, err := ioutil.ReadFile(srcFullPath)
176 | if err != nil {
177 | LogWarn(err.Error())
178 | continue
179 | }
180 |
181 | f, err := w.Create(baseInZip + file.Name())
182 | PanicIfErr(err)
183 |
184 | _, err = f.Write(dat)
185 | PanicIfErr(err)
186 | continue
187 | }
188 |
189 | // recurse on dir
190 | inZip := baseInZip + file.Name() + separator
191 | addFiles(w, srcFullPath, inZip, separator)
192 | }
193 | }
194 |
--------------------------------------------------------------------------------
/util/http.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | const (
4 | HttpMimeJson = "application/json"
5 |
6 | HttpHeaderContentType = "Content-Type"
7 | HttpHeaderAgentToken = "AGENT-TOKEN"
8 | )
9 |
--------------------------------------------------------------------------------
/util/logger.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "fmt"
5 |
6 | logger "github.com/sirupsen/logrus"
7 | )
8 |
9 | func LogInit() {
10 | logger.SetFormatter(&logger.TextFormatter{
11 | DisableColors: false,
12 | FullTimestamp: true,
13 | })
14 | }
15 |
16 | func EnableDebugLog() {
17 | logger.SetLevel(logger.DebugLevel)
18 | }
19 |
20 | func LogIfError(err error) bool {
21 | if HasError(err) {
22 | logger.Error(err)
23 | return true
24 | }
25 |
26 | return false
27 | }
28 |
29 | func LogInfo(format string, a ...interface{}) {
30 | str := fmt.Sprintf(format, a...)
31 | logger.Info(str)
32 | }
33 |
34 | func LogDebug(format string, a ...interface{}) {
35 | str := fmt.Sprintf(format, a...)
36 | logger.Debug(str)
37 | }
38 |
39 | func LogWarn(format string, a ...interface{}) {
40 | str := fmt.Sprintf(format, a...)
41 | logger.Warn(str)
42 | }
43 |
--------------------------------------------------------------------------------
/util/obj.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "fmt"
5 | "log"
6 | "reflect"
7 | )
8 |
9 | func PanicIfErr(err error) {
10 | if err != nil {
11 | panic(err)
12 | }
13 | }
14 |
15 | func PanicIfNil(obj interface{}, msg string) {
16 | if obj == nil {
17 | panic(fmt.Errorf("unhandled nil pointer : %s", msg))
18 | }
19 | }
20 |
21 | func RecoverPanic(handler func(e error)) {
22 | if r := recover(); r != nil {
23 | if handler != nil {
24 | handler(r.(error))
25 | }
26 | }
27 | }
28 |
29 | func HasError(err error) bool {
30 | return err != nil
31 | }
32 |
33 | // FailOnError exit program with err
34 | func FailOnError(err error, msg string) {
35 | if err != nil {
36 | log.Fatalf("%s: %s", msg, err)
37 | }
38 | }
39 |
40 | // IsEmptyString to check input s is empty
41 | func IsEmptyString(s string) bool {
42 | return s == ""
43 | }
44 |
45 | func HasString(s string) bool {
46 | return s != ""
47 | }
48 |
49 | // IsPointerType to check the input v is pointer type
50 | func IsPointerType(v interface{}) bool {
51 | return reflect.ValueOf(v).Kind() == reflect.Ptr
52 | }
53 |
54 | // GetType get type of pointer
55 | func GetType(v interface{}) reflect.Type {
56 | if IsPointerType(v) {
57 | val := reflect.ValueOf(v)
58 | return val.Elem().Type()
59 | }
60 |
61 | return reflect.TypeOf(v)
62 | }
63 |
64 | func GetValue(v interface{}) reflect.Value {
65 | val := reflect.ValueOf(v)
66 |
67 | if val.Kind() == reflect.Ptr {
68 | return val.Elem()
69 | }
70 |
71 | return val
72 | }
73 |
--------------------------------------------------------------------------------
/util/sync.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "sync"
5 | "time"
6 | )
7 |
8 | func Wait(group *sync.WaitGroup, timeout time.Duration) bool {
9 | c := make(chan struct{})
10 |
11 | go func() {
12 | group.Wait()
13 | close(c)
14 | }()
15 |
16 | select {
17 | case <- c:
18 | return true
19 | case <-time.After(timeout):
20 | return false
21 | }
22 | }
--------------------------------------------------------------------------------
/util/sync_test.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "github.com/stretchr/testify/assert"
5 | "sync"
6 | "testing"
7 | "time"
8 | )
9 |
10 | func TestShouldWaitForSyncGroup(t *testing.T) {
11 | assert := assert.New(t)
12 |
13 | // init
14 | var g sync.WaitGroup
15 | g.Add(1)
16 |
17 | // when: start routing with 1s
18 | go func() {
19 | time.Sleep(1 * time.Second)
20 | g.Done()
21 | }()
22 |
23 | // then: should return true if not timeout
24 | r := Wait(&g, 5 * time.Second)
25 | assert.True(r)
26 | }
27 |
28 | func TestShouldTimeoutForSyncGroup(t *testing.T) {
29 | assert := assert.New(t)
30 |
31 | // init
32 | var g sync.WaitGroup
33 | g.Add(1)
34 |
35 | // when: start routing with 5s
36 | go func() {
37 | time.Sleep(5 * time.Second)
38 | g.Done()
39 | }()
40 |
41 | // then: should return false for timeout
42 | r := Wait(&g, 1 * time.Second)
43 | assert.False(r)
44 | }
45 |
--------------------------------------------------------------------------------
/util/zk.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "fmt"
5 | "time"
6 |
7 | "github.com/samuel/go-zookeeper/zk"
8 | )
9 |
10 | const (
11 | ZkNodeTypePersistent = int32(0)
12 | ZkNodeTypeEphemeral = int32(zk.FlagEphemeral)
13 |
14 | connectTimeout = 10 // seconds
15 | disconnectTimeout = 5 // seconds
16 | sessionTimeout = 1 // seconds
17 | )
18 |
19 | type (
20 | ZkCallbacks struct {
21 | OnDisconnected func()
22 | }
23 |
24 | ZkClient struct {
25 | Callbacks *ZkCallbacks
26 | conn *zk.Conn
27 | }
28 | )
29 |
30 | func NewZkClient() *ZkClient {
31 | return &ZkClient{
32 | Callbacks: &ZkCallbacks{},
33 | }
34 | }
35 |
36 | // Connect zookeeper host
37 | func (client *ZkClient) Connect(host string) error {
38 | // make connection of zk
39 | conn, events, err := zk.Connect([]string{host}, sessionTimeout*time.Second)
40 | if err != nil {
41 | return err
42 | }
43 |
44 | if !waitForConnection(events, connectTimeout) {
45 | return fmt.Errorf("zk server connection failed")
46 | }
47 |
48 | client.conn = conn
49 | go client.handleZkEvents(events)
50 | return nil
51 | }
52 |
53 | // Create create node with node type and data
54 | func (client *ZkClient) Create(path string, nodeType int32, data string) (string, error) {
55 | acl := zk.WorldACL(zk.PermAll)
56 | return client.conn.Create(path, []byte(data), nodeType, acl)
57 | }
58 |
59 | // Exist check the node is exist
60 | func (client *ZkClient) Exist(path string) (bool, error) {
61 | exist, _, err := client.conn.Exists(path)
62 | return exist, err
63 | }
64 |
65 | func (client *ZkClient) Data(path string) (string, error) {
66 | bytes, _, err := client.conn.Get(path)
67 | return string(bytes), err
68 | }
69 |
70 | func (client *ZkClient) Delete(path string) error {
71 | exist, _ := client.Exist(path)
72 |
73 | if !exist {
74 | return nil
75 | }
76 |
77 | return client.conn.Delete(path, 0)
78 | }
79 |
80 | // Close release connection
81 | func (client *ZkClient) Close() {
82 | if client.conn != nil {
83 | client.conn.Close()
84 | }
85 | }
86 |
87 | func (client *ZkClient) handleZkEvents(events <-chan zk.Event) {
88 | var disconnectTimer *time.Timer
89 |
90 | for event := range events {
91 | if event.State == zk.StateConnected {
92 | LogInfo("zk: re-connected")
93 | if disconnectTimer != nil {
94 | _ = disconnectTimer.Stop()
95 | }
96 | }
97 |
98 | if event.State == zk.StateDisconnected {
99 | LogDebug("zk: disconnected")
100 | disconnectTimer = time.NewTimer(disconnectTimeout * time.Second)
101 |
102 | // close zk conn and fire disconnect event after disconnect timeout
103 | go func() {
104 | <-disconnectTimer.C
105 | client.conn.Close()
106 | if client.Callbacks.OnDisconnected != nil {
107 | client.Callbacks.OnDisconnected()
108 | }
109 | }()
110 | }
111 | }
112 | }
113 |
114 | func waitForConnection(events <-chan zk.Event, seconds int) bool {
115 | for event := range events {
116 | if event.State == zk.StateConnected {
117 | return true
118 | }
119 |
120 | if seconds == 0 {
121 | break
122 | }
123 |
124 | time.Sleep(1 * time.Second)
125 | seconds = seconds - 1
126 | }
127 |
128 | return false
129 | }
130 |
--------------------------------------------------------------------------------
/util/zk_test.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/stretchr/testify/assert"
7 | )
8 |
9 | func TestShouldCreateNode(t *testing.T) {
10 | assert := assert.New(t)
11 |
12 | client := new(ZkClient)
13 | defer client.Close()
14 |
15 | connErr := client.Connect("127.0.0.1:2181")
16 | assert.Nil(connErr)
17 |
18 | // create
19 | targetPath := "/flow-test"
20 | path, nodeErr := client.Create(targetPath, ZkNodeTypeEphemeral, "hello")
21 | assert.Nil(nodeErr)
22 | assert.Equal(targetPath, path)
23 |
24 | // exist
25 | exist, existErr := client.Exist(targetPath)
26 | assert.Nil(existErr)
27 | assert.True(exist)
28 |
29 | // get data
30 | data, dataErr := client.Data(targetPath)
31 | assert.Nil(dataErr)
32 | assert.Equal("hello", data)
33 |
34 | // delete
35 | delErr := client.Delete(targetPath)
36 | assert.Nil(delErr)
37 | }
38 |
--------------------------------------------------------------------------------
/wait-for-it.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | # Use this script to test if a given TCP host/port are available
3 |
4 | WAITFORIT_cmdname=${0##*/}
5 |
6 | echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
7 |
8 | usage()
9 | {
10 | cat << USAGE >&2
11 | Usage:
12 | $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
13 | -h HOST | --host=HOST Host or IP under test
14 | -p PORT | --port=PORT TCP port under test
15 | Alternatively, you specify the host and port as host:port
16 | -s | --strict Only execute subcommand if the test succeeds
17 | -q | --quiet Don't output any status messages
18 | -t TIMEOUT | --timeout=TIMEOUT
19 | Timeout in seconds, zero for no timeout
20 | -- COMMAND ARGS Execute command with args after the test finishes
21 | USAGE
22 | exit 1
23 | }
24 |
25 | wait_for()
26 | {
27 | if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
28 | echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
29 | else
30 | echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
31 | fi
32 | WAITFORIT_start_ts=$(date +%s)
33 | while :
34 | do
35 | if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
36 | nc -z $WAITFORIT_HOST $WAITFORIT_PORT
37 | WAITFORIT_result=$?
38 | else
39 | (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
40 | WAITFORIT_result=$?
41 | fi
42 | if [[ $WAITFORIT_result -eq 0 ]]; then
43 | WAITFORIT_end_ts=$(date +%s)
44 | echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
45 | break
46 | fi
47 | sleep 1
48 | done
49 | return $WAITFORIT_result
50 | }
51 |
52 | wait_for_wrapper()
53 | {
54 | # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
55 | if [[ $WAITFORIT_QUIET -eq 1 ]]; then
56 | timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
57 | else
58 | timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
59 | fi
60 | WAITFORIT_PID=$!
61 | trap "kill -INT -$WAITFORIT_PID" INT
62 | wait $WAITFORIT_PID
63 | WAITFORIT_RESULT=$?
64 | if [[ $WAITFORIT_RESULT -ne 0 ]]; then
65 | echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
66 | fi
67 | return $WAITFORIT_RESULT
68 | }
69 |
70 | # process arguments
71 | while [[ $# -gt 0 ]]
72 | do
73 | case "$1" in
74 | *:* )
75 | WAITFORIT_hostport=(${1//:/ })
76 | WAITFORIT_HOST=${WAITFORIT_hostport[0]}
77 | WAITFORIT_PORT=${WAITFORIT_hostport[1]}
78 | shift 1
79 | ;;
80 | --child)
81 | WAITFORIT_CHILD=1
82 | shift 1
83 | ;;
84 | -q | --quiet)
85 | WAITFORIT_QUIET=1
86 | shift 1
87 | ;;
88 | -s | --strict)
89 | WAITFORIT_STRICT=1
90 | shift 1
91 | ;;
92 | -h)
93 | WAITFORIT_HOST="$2"
94 | if [[ $WAITFORIT_HOST == "" ]]; then break; fi
95 | shift 2
96 | ;;
97 | --host=*)
98 | WAITFORIT_HOST="${1#*=}"
99 | shift 1
100 | ;;
101 | -p)
102 | WAITFORIT_PORT="$2"
103 | if [[ $WAITFORIT_PORT == "" ]]; then break; fi
104 | shift 2
105 | ;;
106 | --port=*)
107 | WAITFORIT_PORT="${1#*=}"
108 | shift 1
109 | ;;
110 | -t)
111 | WAITFORIT_TIMEOUT="$2"
112 | if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
113 | shift 2
114 | ;;
115 | --timeout=*)
116 | WAITFORIT_TIMEOUT="${1#*=}"
117 | shift 1
118 | ;;
119 | --)
120 | shift
121 | WAITFORIT_CLI=("$@")
122 | break
123 | ;;
124 | --help)
125 | usage
126 | ;;
127 | *)
128 | echoerr "Unknown argument: $1"
129 | usage
130 | ;;
131 | esac
132 | done
133 |
134 | if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
135 | echoerr "Error: you need to provide a host and port to test."
136 | usage
137 | fi
138 |
139 | WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
140 | WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
141 | WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
142 | WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
143 |
144 | # Check to see if timeout is from busybox?
145 | WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
146 | WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
147 |
148 | WAITFORIT_BUSYTIMEFLAG=""
149 | if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
150 | WAITFORIT_ISBUSY=1
151 | # Check if busybox timeout uses -t flag
152 | # (recent Alpine versions don't support -t anymore)
153 | if timeout &>/dev/stdout | grep -q -e '-t '; then
154 | WAITFORIT_BUSYTIMEFLAG="-t"
155 | fi
156 | else
157 | WAITFORIT_ISBUSY=0
158 | fi
159 |
160 | if [[ $WAITFORIT_CHILD -gt 0 ]]; then
161 | wait_for
162 | WAITFORIT_RESULT=$?
163 | exit $WAITFORIT_RESULT
164 | else
165 | if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
166 | wait_for_wrapper
167 | WAITFORIT_RESULT=$?
168 | else
169 | wait_for
170 | WAITFORIT_RESULT=$?
171 | fi
172 | fi
173 |
174 | if [[ $WAITFORIT_CLI != "" ]]; then
175 | if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
176 | echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
177 | exit $WAITFORIT_RESULT
178 | fi
179 | exec "${WAITFORIT_CLI[@]}"
180 | else
181 | exit $WAITFORIT_RESULT
182 | fi
--------------------------------------------------------------------------------