├── .gitignore ├── CONTRIBUTORS.md ├── LICENSE-HEADER.txt ├── LICENSE.txt ├── Makefile ├── README.md ├── Vagrantfile ├── glide.lock ├── glide.yaml ├── lib ├── notify │ ├── notify.go │ └── slack.go ├── pipeline │ ├── pipeline.go │ └── pipeline_test.go └── task │ ├── task.go │ ├── task_test.go │ ├── wait_for.go │ └── wait_for_test.go ├── main.go ├── main_test.go ├── scripts ├── compile.sh └── package.sh └── version.go /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | vendor/ 3 | .vagrant/ 4 | pkg/ 5 | 6 | -------------------------------------------------------------------------------- /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | # Contributors 2 | * Naoki Ainoya 3 | * Takahiko Ito -------------------------------------------------------------------------------- /LICENSE-HEADER.txt: -------------------------------------------------------------------------------- 1 | walter: a deployment pipeline template 2 | Copyright (C) 2014 Recruit Technologies Co., Ltd. and contributors 3 | (see CONTRIBUTORS.md) 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | Copyright 2014 Recruit Technologies Co., Ltd. and contributors 180 | (see CONTRIBUTORS.md) 181 | 182 | Licensed under the Apache License, Version 2.0 (the "License"); 183 | you may not use this file except in compliance with the License. 184 | You may obtain a copy of the License at 185 | 186 | http://www.apache.org/licenses/LICENSE-2.0 187 | 188 | Unless required by applicable law or agreed to in writing, software 189 | distributed under the License is distributed on an "AS IS" BASIS, 190 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 191 | See the License for the specific language governing permissions and 192 | limitations under the License. 193 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | NAME := walter 2 | VERSION := $(shell grep 'Version string' version.go | sed -E 's/.*"(.+)"$$/\1/') 3 | REVISION := $(shell git rev-parse --short HEAD) 4 | LDFLAGS := -X 'main.GitCommit=$(REVISION)' 5 | 6 | setup: 7 | go get github.com/Masterminds/glide 8 | go get github.com/golang/lint/golint 9 | go get golang.org/x/tools/cmd/goimports 10 | go get github.com/mitchellh/gox 11 | 12 | deps: setup 13 | glide install 14 | 15 | test: deps lint 16 | go test $$(glide novendor) 17 | go test -race $$(glide novendor) 18 | 19 | lint: setup 20 | go vet $$(glide novendor) 21 | for pkg in $$(glide novendor -x); do \ 22 | golint -set_exit_status $$pkg || exit $$?; \ 23 | done 24 | 25 | fmt: setup 26 | goimports -w $$(glide nv -x) 27 | 28 | build: deps 29 | go build -ldflags "$(LDFLAGS)" -o bin/$(NAME) 30 | 31 | clean: 32 | rm $(GOPATH)/bin/$(NAME) 33 | rm bin/$(NAME) 34 | 35 | package: deps 36 | @sh -c "'$(CURDIR)/scripts/package.sh'" 37 | 38 | ghr: 39 | ghr -u walter-cd $(VERSION) pkg/dist/$(VERSION) 40 | 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Walter 2 | ====== 3 | 4 |

5 | 6 |

7 | 8 | [![wercker status](https://app.wercker.com/status/4fcb4b110909fc45775d12641f5cf037/m "wercker status")](https://app.wercker.com/project/bykey/4fcb4b110909fc45775d12641f5cf037) 9 | 10 | Walter is a tiny deployment pipeline tool. 11 | 12 | ---- 13 | 14 | Blogs 15 | ===== 16 | 17 | * http://ainoya.io/walter 18 | * http://walter-cd.net 19 | 20 | 21 | ---- 22 | 23 | Overview 24 | ======== 25 | 26 | Walter is a simple command line tool that automates build, test and deployment of applications or servers. 27 | 28 | ---- 29 | 30 | Getting Started 31 | =============== 32 | 33 | 34 | How to install 35 | -------------- 36 | 37 | Get a binary file from [GitHub Releases](https://github.com/walter-cd/walter/releases) and place it in `$PATH`. 38 | 39 | 40 | Writing your pipeline 41 | -------------------- 42 | 43 | Write your command pipeline in `pipeline.yml`. 44 | 45 | 46 | ```yaml 47 | build: 48 | tasks: 49 | - name: setup build 50 | command: echo "setting up ..." 51 | - name: run build 52 | command: echo "building ..." 53 | cleanup: 54 | - name: cleanup build 55 | command: echo "cleanup build ..." 56 | 57 | deploy: 58 | tasks: 59 | - name: run deploy 60 | command: echo "deploying ..." 61 | cleanup: 62 | - name: cleanup 63 | command: echo "cleanup deploy ..." 64 | ``` 65 | 66 | 67 | Run walter 68 | ---------- 69 | 70 | ``` 71 | $ walter -build -deploy 72 | INFO[0000] Build started 73 | INFO[0000] [setup build] Start task 74 | INFO[0000] [setup build] setting up ... 75 | INFO[0000] [setup build] End task 76 | INFO[0000] [run build] Start task 77 | INFO[0000] [run build] building ... 78 | INFO[0000] [run build] End task 79 | INFO[0000] Build succeeded 80 | INFO[0000] Build cleanup started 81 | INFO[0000] [cleanup build] Start task 82 | INFO[0000] [cleanup build] cleanup build ... 83 | INFO[0000] [cleanup build] End task 84 | INFO[0000] Build cleanup succeeded 85 | INFO[0000] Deploy started 86 | INFO[0000] [run deploy] Start task 87 | INFO[0000] [run deploy] deploying ... 88 | INFO[0000] [run deploy] End task 89 | INFO[0000] Deploy succeeded 90 | INFO[0000] [cleanup] Start task 91 | INFO[0000] [cleanup] cleanup deploy ... 92 | INFO[0000] [cleanup] End task 93 | INFO[0000] Deploy cleanup succeeded 94 | ``` 95 | 96 | That's it. 97 | 98 | ---- 99 | 100 | Other features 101 | ============== 102 | 103 | Environment variables 104 | --------------------- 105 | 106 | You can use environment variables. 107 | 108 | ```yaml 109 | deploy: 110 | tasks: 111 | - name: release files 112 | command: ghr -token $GITHUB_TOKEN $VERSION pkg/dist/$VERSION 113 | ``` 114 | 115 | 116 | Working directory 117 | ----------------- 118 | 119 | You can specify a working directory of a task. 120 | 121 | ```yaml 122 | build: 123 | tasks: 124 | - name: list files under /tmp 125 | command: ls 126 | directory: /tmp 127 | ``` 128 | 129 | Conditions to run tasks 130 | ---------- 131 | 132 | ```yaml 133 | build: 134 | tasks: 135 | - name: list files under /tmp 136 | command: ls 137 | only_if: test -d /tmp 138 | ``` 139 | 140 | 141 | Get stdout of a previous task 142 | ----------------------------- 143 | 144 | Tasks get stdout of a previous task through a pipe. 145 | 146 | ```yaml 147 | build: 148 | tasks: 149 | - name: setup build 150 | command: echo "setting up" 151 | - name: run build 152 | command: cat 153 | ``` 154 | 155 | The second "run build" task outputs "setting up". 156 | 157 | 158 | Parallel tasks 159 | -------------- 160 | 161 | You can define parallel tasks. 162 | 163 | ```yaml 164 | build: 165 | tasks: 166 | - name: parallel tasks 167 | parallel: 168 | - name: task 1 169 | command: echo task 1 170 | - name: task 2 171 | command: echo task 2 172 | - name: task 3 173 | command: echo task 3 174 | ``` 175 | 176 | You can also mix serial tasks in parallel tasks. 177 | 178 | ```yaml 179 | build: 180 | tasks: 181 | - name: parallel tasks 182 | parallel: 183 | - name: task 1 184 | command: echo task 1 185 | - name: task 2 186 | serial: 187 | - name: task 2-1 188 | command: echo task 2-1 189 | - name: task 2-2 190 | command: echo task 2-2 191 | - name: task 3 192 | command: echo task 3 193 | ``` 194 | 195 | 196 | Split pipeline definitions and include them 197 | ------------------------------------------- 198 | 199 | You can split pipeline definitions in other files and include them. 200 | 201 | **pipeline.yml** 202 | 203 | ```yaml 204 | build: 205 | tasks: 206 | - include: task1.yml 207 | - include: task2.yml 208 | ``` 209 | 210 | **task1.yml** 211 | 212 | ```yaml 213 | - name: task1 214 | command: echo task1 215 | ``` 216 | 217 | **task2.yaml** 218 | 219 | ```yaml 220 | - name: task2 221 | command: echo task2 222 | ``` 223 | 224 | You can also run single definition file. 225 | 226 | ``` 227 | $ walter -build -config task2.yml 228 | ``` 229 | 230 | 231 | Wait for some conditions 232 | ------------------------ 233 | 234 | You can make tasks wait for some conditions. 235 | 236 | ```yaml 237 | build: 238 | tasks: 239 | - name: launch solr 240 | command: bin/solr start 241 | - name: post data to solr index 242 | command: bin/post -d ~/tmp/foobar.js 243 | wait_for: 244 | host: localhost 245 | port: 8983 246 | state: ready 247 | ``` 248 | 249 | Available keys and values are these: 250 | 251 | 252 | | Key | Value (value type) | Description | 253 | |:--------|:--------------------|:----------------------------------------------------| 254 | | delay | second (float) | Seconds to wait after the previous stage finish | 255 | | host | host (string) | IP address or host name | 256 | | port | port number (int) | Port number | 257 | | file | file name (string) | File name| 258 | | state | state of the other key (string) | Two types(present/ready or absent/unready) of states are supported. | 259 | 260 | 261 | 262 | Notification 263 | ------------ 264 | 265 | Walter supports notification of task results to Slack. 266 | 267 | ```yaml 268 | notify: 269 | - type: slack 270 | channel: serverspec 271 | url: $SLACK_WEBHOOK_URL 272 | icon_url: http://example.jp/walter.jpg 273 | username: walter 274 | ``` 275 | 276 | Other services(ex. HipChat) are not supported currently. 277 | 278 | ---- 279 | 280 | Changes in v2 281 | ============= 282 | 283 | 284 | Pipeline definition format 285 | -------------------------- 286 | 287 | Pipeline definition in v1: 288 | 289 | ```yaml 290 | pipeline: 291 | - name: setup build 292 | type: command 293 | command: echo "setting up ..." 294 | - name: run build 295 | type: command 296 | command: echo "building ..." 297 | cleanup: 298 | - name: cleanup build 299 | type: command 300 | command: echo "cleanup build ..." 301 | 302 | ``` 303 | 304 | In v2: 305 | 306 | ```yaml 307 | build: 308 | tasks: 309 | - name: setup build 310 | command: echo "setting up ..." 311 | - name: run build 312 | command: echo "building ..." 313 | cleanup: 314 | - name: cleanup build 315 | command: echo "cleanup build ..." 316 | ``` 317 | 318 | Separate build and deploy phase 319 | ------------------------------- 320 | 321 | You can separate build and deploy phases in v2: 322 | 323 | ```yaml 324 | build: 325 | tasks: 326 | - name: setup build 327 | command: echo "setting up ..." 328 | - name: run build 329 | command: echo "building ..." 330 | cleanup: 331 | - name: cleanup build 332 | command: echo "cleanup build ..." 333 | 334 | deploy: 335 | tasks: 336 | - name: run deploy 337 | command: echo "deploying ..." 338 | cleanup: 339 | - name: cleanup 340 | command: echo "cleanup deploy ..." 341 | ``` 342 | 343 | You can run both phases at once or each phase separately. 344 | 345 | ``` 346 | # Run build and deploy phases 347 | $ walter -build -deploy 348 | 349 | # Run build phase only 350 | $ walter -build 351 | 352 | # Run deploy phase only 353 | $ walter -deploy 354 | ``` 355 | 356 | Format of wait_for 357 | ------------------ 358 | 359 | You must define parameters for `wait_for` in one line in v1: 360 | 361 | ```yaml 362 | pipeline: 363 | - name: launch solr 364 | command: bin/solr start 365 | - name: post data to solr index 366 | command: bin/post -d ~/tmp/foobar.js 367 | wait_for: host=localhost port=8983 state=ready 368 | ``` 369 | 370 | In v2, you must define parameters for `wait_for` with mapping of yaml. 371 | 372 | ``` 373 | build: 374 | tasks: 375 | - name: launch solr 376 | command: bin/solr start 377 | - name: post data to solr index 378 | command: bin/post -d ~/tmp/foobar.js 379 | wait_for: 380 | host: localhost 381 | port: 8983 382 | state: ready 383 | ``` 384 | 385 | 386 | Definition of notification 387 | -------------------------- 388 | 389 | In v1: 390 | 391 | ```yaml 392 | messenger: 393 | type: slack 394 | channel: serverspec 395 | url: $SLACK_WEBHOOK_URL 396 | icon_url: http://example.jp/walter.jpg 397 | username: walter 398 | ``` 399 | 400 | In v2: 401 | 402 | ```yaml 403 | notify: 404 | - type: slack 405 | channel: serverspec 406 | url: $SLACK_WEBHOOK_URL 407 | icon_url: http://example.jp/walter.jpg 408 | username: walter 409 | ``` 410 | 411 | The key `messenger` was changed to `notify` and you can define multiple notification definitions in v2. 412 | 413 | 414 | Special variables 415 | ----------------- 416 | 417 | Special variables like `__OUT`, `__ERR`, `__COMBINED` and `__RESULT` are obsoleted in v2. 418 | 419 | Tasks get stdout of a previous task through a pipe. 420 | 421 | ```yaml 422 | build: 423 | tasks: 424 | - name: setup build 425 | command: echo "setting up" 426 | - name: run build 427 | command: cat 428 | ``` 429 | 430 | The second "run build" task outputs "setting up". 431 | 432 | I think this is suffient for defining pipelines. Special variables bring complexity for pipelines. 433 | 434 | 435 | ---- 436 | 437 | Contributing 438 | ============ 439 | 440 | 1. Fork it 441 | 2. Create your feature branch (`git checkout -b my-new-feature`) 442 | 3. Commit your changes (`git commit -am 'Add some feature'`) 443 | 4. Push to the branch (`git push origin my-new-feature`) 444 | 5. Create new Pull Request 445 | 446 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure("2") do |config| 5 | config.vm.box = 'ubuntu/trusty64' 6 | 7 | config.vm.provision :shell, inline: <<-EOF 8 | echo Installing golang ... 9 | archive=go1.7.3.linux-amd64.tar.gz 10 | if [ ! -f $archive ]; then 11 | wget -q https://storage.googleapis.com/golang/$archive 12 | fi 13 | 14 | if [ ! -d /usr/local/go ]; then 15 | tar -C /usr/local -xzf $archive 16 | fi 17 | 18 | if ! grep GOPATH ~vagrant/.bashrc; then 19 | echo 'export PATH=$PATH:/usr/local/go/bin:$HOME/bin' >> ~vagrant/.bashrc 20 | echo 'export GOPATH=$HOME' >> ~vagrant/.bashrc 21 | fi 22 | 23 | if [ ! -d ~vagrant/src ]; then 24 | mkdir ~vagrant/src 25 | chown vagrant:vagrant ~vagrant/src 26 | fi 27 | 28 | echo Installing pkgconf, git and cmake ... 29 | apt-get update 30 | apt-get install -y pkgconf git cmake 31 | 32 | chown -R vagrant:vagrant ~vagrant/src 33 | 34 | echo Installing libgit2 ... 35 | if [ ! -f v0.24.2.tar.gz ]; then 36 | wget -q https://github.com/libgit2/libgit2/archive/v0.24.2.tar.gz 37 | tar zxvf v0.24.2.tar.gz 38 | cd libgit2-0.24.2 39 | mkdir build && cd build 40 | cmake .. 41 | cmake --build . 42 | cmake .. -DCMAKE_INSTALL_PREFIX=/usr 43 | cmake --build . --target install 44 | fi 45 | 46 | apt-get install -y apt-transport-https ca-certificates 47 | apt-get install -y linux-image-extra-$(uname -r) linux-image-extra-virtual 48 | apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D 49 | echo deb https://apt.dockerproject.org/repo ubuntu-trusty main > /etc/apt/sources.list.d/docker.list 50 | apt-get update 51 | apt-get install -y docker-engine 52 | EOF 53 | 54 | config.vm.synced_folder '.', '/home/vagrant/src/github.com/walter-cd/walter' 55 | end 56 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: a5f68bd6e842ed0caee42204380fec21ed1b698432404200c23221e4a7b35eb8 2 | updated: 2016-11-29T11:51:00.292205511+09:00 3 | imports: 4 | - name: github.com/go-yaml/yaml 5 | version: 31c299268d302dd0aa9a0dcf765a3d58971ac83f 6 | - name: github.com/google/go-github 7 | version: 2ae0bbb4f91336db239c582fe96deb8e9e6d43df 8 | subpackages: 9 | - github 10 | - name: github.com/google/go-querystring 11 | version: 9235644dd9e52eeae6fa48efd539fdc351a0af53 12 | subpackages: 13 | - query 14 | - name: github.com/hashicorp/go-version 15 | version: e96d3840402619007766590ecea8dd7af1292276 16 | - name: github.com/Sirupsen/logrus 17 | version: d26492970760ca5d33129d2d799e34be5c4782eb 18 | - name: github.com/tcnksm/go-latest 19 | version: 79c2c6c7fa60f1bd7e1f204606aae7b9082f7304 20 | - name: golang.org/x/net 21 | version: 5d997795f7bb1d2de5edc1f5d64af2562401c82d 22 | subpackages: 23 | - context 24 | - html 25 | - html/atom 26 | - name: golang.org/x/sys 27 | version: c200b10b5d5e122be351b67af224adc6128af5bf 28 | subpackages: 29 | - unix 30 | testImports: [] 31 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/walter-cd/walter 2 | import: [] 3 | -------------------------------------------------------------------------------- /lib/notify/notify.go: -------------------------------------------------------------------------------- 1 | package notify 2 | 3 | import ( 4 | "os" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/go-yaml/yaml" 9 | "github.com/walter-cd/walter/lib/task" 10 | ) 11 | 12 | type Notifier interface { 13 | Notify(*task.Task) error 14 | } 15 | 16 | type Notify struct { 17 | Notify []map[string]string 18 | } 19 | 20 | type Default struct{} 21 | 22 | func NewNotifiers(b []byte) ([]Notifier, error) { 23 | notify := Notify{} 24 | err := yaml.Unmarshal(b, ¬ify) 25 | 26 | var notifiers []Notifier 27 | for _, n := range notify.Notify { 28 | re := regexp.MustCompile(`\$[A-Z1-9\-_]+`) 29 | for k, v := range n { 30 | matches := re.FindAllString(v, -1) 31 | for _, m := range matches { 32 | env := os.Getenv(strings.TrimPrefix(m, "$")) 33 | n[k] = strings.Replace(n[k], m, env, -1) 34 | } 35 | } 36 | 37 | switch n["type"] { 38 | case "slack": 39 | notifiers = append(notifiers, NewSlack(n)) 40 | default: 41 | notifiers = append(notifiers, &Default{}) 42 | } 43 | } 44 | 45 | return notifiers, err 46 | } 47 | 48 | func (d *Default) Notify(t *task.Task) error { 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /lib/notify/slack.go: -------------------------------------------------------------------------------- 1 | package notify 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "net/http" 9 | 10 | log "github.com/Sirupsen/logrus" 11 | 12 | "github.com/walter-cd/walter/lib/task" 13 | ) 14 | 15 | type Slack struct { 16 | Channel string `json:"channel"` 17 | URL string `json:"-"` 18 | IconURL string `json:"icon_url"` 19 | UserName string `json:"username"` 20 | Text string `json:"text"` 21 | } 22 | 23 | type payload struct { 24 | Slack 25 | Attachments []attachment `json:"attachments"` 26 | } 27 | 28 | type attachment struct { 29 | Text string `json:"text"` 30 | Color string `json:"color"` 31 | } 32 | 33 | func NewSlack(m map[string]string) *Slack { 34 | s := &Slack{} 35 | s.Channel = m["channel"] 36 | s.URL = m["url"] 37 | s.IconURL = m["icon_url"] 38 | s.UserName = m["username"] 39 | return s 40 | } 41 | 42 | func (s Slack) Notify(t *task.Task) error { 43 | if s.Channel[0] != '#' { 44 | s.Channel = "#" + s.Channel 45 | } 46 | 47 | var message string 48 | var color string 49 | 50 | switch t.Status { 51 | case task.Succeeded: 52 | message = fmt.Sprintf("[%s] Succeeded", t.Name) 53 | color = "good" 54 | case task.Failed: 55 | message = fmt.Sprintf("[%s] Failed", t.Name) 56 | color = "danger" 57 | case task.Skipped: 58 | message = fmt.Sprintf("[%s] Skipped", t.Name) 59 | color = "warning" 60 | case task.Aborted: 61 | message = fmt.Sprintf("[%s] Aborted", t.Name) 62 | color = "warning" 63 | } 64 | 65 | a := attachment{ 66 | Text: message, 67 | Color: color, 68 | } 69 | 70 | p := payload{Slack: s, Attachments: []attachment{a}} 71 | j, _ := json.Marshal(p) 72 | buf := bytes.NewBuffer(j) 73 | 74 | log.Infof("[%s] Notify to Slack", t.Name) 75 | resp, err := http.Post(s.URL, "application/json", buf) 76 | if err != nil { 77 | e := fmt.Sprintf("[%s] Failed to notify to Slack: %s", t.Name, message) 78 | log.Errorf(e) 79 | return errors.New(e) 80 | } 81 | 82 | defer resp.Body.Close() 83 | 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /lib/pipeline/pipeline.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io/ioutil" 7 | "os" 8 | "regexp" 9 | "strings" 10 | "sync" 11 | 12 | log "github.com/Sirupsen/logrus" 13 | 14 | "golang.org/x/net/context" 15 | 16 | "github.com/go-yaml/yaml" 17 | "github.com/walter-cd/walter/lib/notify" 18 | "github.com/walter-cd/walter/lib/task" 19 | ) 20 | 21 | type Pipeline struct { 22 | Build Build 23 | Deploy Deploy 24 | Notifiers []notify.Notifier 25 | } 26 | 27 | type Build struct { 28 | Tasks Tasks 29 | Cleanup Tasks 30 | } 31 | 32 | type Deploy struct { 33 | Tasks Tasks 34 | Cleanup Tasks 35 | } 36 | 37 | type Tasks []*task.Task 38 | 39 | func Load(b []byte) (Pipeline, error) { 40 | p := Pipeline{} 41 | err := yaml.Unmarshal(b, &p) 42 | if err == nil { 43 | p.Notifiers, err = notify.NewNotifiers(b) 44 | return p, err 45 | } 46 | 47 | t := Tasks{} 48 | err = yaml.Unmarshal(b, &t) 49 | if err != nil { 50 | log.Error(err) 51 | } 52 | 53 | p.Build.Tasks = t 54 | return p, nil 55 | } 56 | 57 | func LoadFromFile(file string) (Pipeline, error) { 58 | data, err := ioutil.ReadFile(file) 59 | if err != nil { 60 | return Pipeline{}, err 61 | } 62 | return Load(data) 63 | } 64 | 65 | func (p *Pipeline) Run(build, deploy bool) int { 66 | failed := false 67 | 68 | if build { 69 | log.Info("Build started") 70 | ctx, cancel := context.WithCancel(context.Background()) 71 | err := p.runTasks(ctx, cancel, p.Build.Tasks, nil) 72 | if err != nil { 73 | log.Error("Build failed") 74 | failed = true 75 | } else { 76 | log.Info("Build succeeded") 77 | } 78 | 79 | log.Info("Build cleanup started") 80 | ctx, cancel = context.WithCancel(context.Background()) 81 | err = p.runTasks(ctx, cancel, p.Build.Cleanup, nil) 82 | if err != nil { 83 | log.Error("Build cleanup failed") 84 | failed = true 85 | } else { 86 | log.Info("Build cleanup succeeded") 87 | } 88 | 89 | if failed { 90 | return 1 91 | } 92 | } 93 | 94 | if deploy { 95 | log.Info("Deploy started") 96 | ctx, cancel := context.WithCancel(context.Background()) 97 | err := p.runTasks(ctx, cancel, p.Deploy.Tasks, nil) 98 | if err != nil { 99 | log.Error("Deploy failed") 100 | failed = true 101 | } else { 102 | log.Info("Deploy succeeded") 103 | } 104 | 105 | log.Info("Deploy cleanup started") 106 | ctx, cancel = context.WithCancel(context.Background()) 107 | err = p.runTasks(ctx, cancel, p.Deploy.Cleanup, nil) 108 | if err != nil { 109 | log.Error("Deploy cleanup failed") 110 | failed = true 111 | } else { 112 | log.Info("Deploy cleanup succeeded") 113 | } 114 | 115 | if failed { 116 | return 1 117 | } 118 | } 119 | 120 | return 0 121 | } 122 | 123 | func includeTasks(file string) (Tasks, error) { 124 | re := regexp.MustCompile(`\$[A-Z1-9\-_]+`) 125 | matches := re.FindAllString(file, -1) 126 | for _, m := range matches { 127 | env := os.Getenv(strings.TrimPrefix(m, "$")) 128 | file = strings.Replace(file, m, env, -1) 129 | } 130 | 131 | data, err := ioutil.ReadFile(file) 132 | tasks := Tasks{} 133 | if err != nil { 134 | return tasks, err 135 | } 136 | 137 | err = yaml.Unmarshal(data, &tasks) 138 | if err != nil { 139 | return tasks, err 140 | } 141 | 142 | return tasks, err 143 | } 144 | 145 | func (p *Pipeline) runTasks(ctx context.Context, cancel context.CancelFunc, tasks Tasks, prevTask *task.Task) error { 146 | failed := false 147 | for i, t := range tasks { 148 | if i > 0 { 149 | prevTask = tasks[i-1] 150 | } 151 | 152 | if t.Include != "" { 153 | include, err := includeTasks(t.Include) 154 | if err != nil { 155 | log.Error(err) 156 | return err 157 | } 158 | p.runTasks(ctx, cancel, include, prevTask) 159 | continue 160 | } 161 | 162 | if len(t.Parallel) > 0 { 163 | err := p.runParallel(ctx, cancel, t, prevTask) 164 | if err != nil { 165 | failed = true 166 | } 167 | continue 168 | } 169 | 170 | if len(t.Serial) > 0 { 171 | err := p.runSerial(ctx, cancel, t, prevTask) 172 | if err != nil { 173 | failed = true 174 | } 175 | continue 176 | } 177 | 178 | if failed || (i > 0 && tasks[i-1].Status == task.Failed) { 179 | t.Status = task.Skipped 180 | failed = true 181 | log.Warnf("[%s] Task skipped because previous task failed", t.Name) 182 | continue 183 | } 184 | 185 | err := t.Run(ctx, cancel, prevTask) 186 | if err != nil { 187 | failed = true 188 | log.Errorf("[%s] %s", t.Name, err) 189 | } 190 | 191 | for _, n := range p.Notifiers { 192 | n.Notify(t) 193 | } 194 | } 195 | 196 | if failed { 197 | return errors.New("One of the tasks failed") 198 | } 199 | 200 | return nil 201 | } 202 | 203 | func (p *Pipeline) runParallel(ctx context.Context, cancel context.CancelFunc, t *task.Task, prevTask *task.Task) error { 204 | 205 | var tasks Tasks 206 | for _, child := range t.Parallel { 207 | if child.Include != "" { 208 | include, err := includeTasks(child.Include) 209 | if err != nil { 210 | log.Error(err) 211 | return err 212 | } 213 | tasks = append(tasks, include...) 214 | } else { 215 | tasks = append(tasks, child) 216 | } 217 | } 218 | 219 | log.Infof("[%s] Start task", t.Name) 220 | 221 | var wg sync.WaitGroup 222 | for _, t := range tasks { 223 | wg.Add(1) 224 | go func(t *task.Task) { 225 | defer wg.Done() 226 | 227 | if len(t.Serial) > 0 { 228 | p.runSerial(ctx, cancel, t, prevTask) 229 | return 230 | } 231 | 232 | t.Run(ctx, cancel, prevTask) 233 | 234 | for _, n := range p.Notifiers { 235 | n.Notify(t) 236 | } 237 | }(t) 238 | } 239 | wg.Wait() 240 | 241 | t.Status = task.Succeeded 242 | 243 | t.Stdout = new(bytes.Buffer) 244 | t.Stderr = new(bytes.Buffer) 245 | t.CombinedOutput = new(bytes.Buffer) 246 | 247 | for _, child := range tasks { 248 | t.Stdout.Write(child.Stdout.Bytes()) 249 | t.Stderr.Write(child.Stderr.Bytes()) 250 | t.CombinedOutput.Write(child.CombinedOutput.Bytes()) 251 | if child.Status == task.Failed { 252 | t.Status = task.Failed 253 | } 254 | } 255 | 256 | if t.Status == task.Failed { 257 | return errors.New("One of parallel tasks failed") 258 | } else { 259 | log.Infof("[%s] End task", t.Name) 260 | return nil 261 | } 262 | } 263 | 264 | func (p *Pipeline) runSerial(ctx context.Context, cancel context.CancelFunc, t *task.Task, prevTask *task.Task) error { 265 | var tasks Tasks 266 | for _, child := range t.Serial { 267 | if child.Include != "" { 268 | include, err := includeTasks(child.Include) 269 | if err != nil { 270 | log.Error(err) 271 | } 272 | tasks = append(tasks, include...) 273 | } else { 274 | tasks = append(tasks, child) 275 | } 276 | } 277 | 278 | log.Infof("[%s] Start task", t.Name) 279 | 280 | p.runTasks(ctx, cancel, tasks, prevTask) 281 | t.Status = task.Succeeded 282 | for _, child := range tasks { 283 | if child.Status == task.Failed { 284 | t.Status = task.Failed 285 | } 286 | } 287 | 288 | t.Stdout = new(bytes.Buffer) 289 | t.Stderr = new(bytes.Buffer) 290 | t.CombinedOutput = new(bytes.Buffer) 291 | 292 | lastTask := tasks[len(tasks)-1] 293 | t.Stdout.Write(lastTask.Stdout.Bytes()) 294 | t.Stderr.Write(lastTask.Stderr.Bytes()) 295 | t.CombinedOutput.Write(lastTask.CombinedOutput.Bytes()) 296 | 297 | if t.Status == task.Failed { 298 | return errors.New("One of serial tasks failed") 299 | } else { 300 | log.Infof("[%s] End task", t.Name) 301 | return nil 302 | } 303 | } 304 | -------------------------------------------------------------------------------- /lib/pipeline/pipeline_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "golang.org/x/net/context" 9 | 10 | "github.com/walter-cd/walter/lib/task" 11 | ) 12 | 13 | func TestLoad(t *testing.T) { 14 | yaml := ` 15 | build: 16 | tasks: 17 | - name: command_stage_1 18 | command: echo "hello, world" 19 | - name: command_stage_2 20 | command: echo "hello, world, command_stage_2" 21 | - name: command_stage_3 22 | command: echo "hello, world, command_stage_3" 23 | ` 24 | p, err := Load([]byte(yaml)) 25 | 26 | fmt.Printf("%v", p) 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | 31 | } 32 | 33 | func TestSerialTasks(t *testing.T) { 34 | t1 := &task.Task{Name: "foo", Command: "echo foo"} 35 | t2 := &task.Task{Name: "bar", Command: "barbarbar"} 36 | t3 := &task.Task{Name: "baz", Command: "echo baz"} 37 | 38 | tasks := Tasks{t1, t2, t3} 39 | 40 | ctx, cancel := context.WithCancel(context.Background()) 41 | 42 | p := &Pipeline{} 43 | p.runTasks(ctx, cancel, tasks, nil) 44 | 45 | if t1.Status != task.Succeeded { 46 | t.Fatal("t1 should have succeeded") 47 | } 48 | 49 | if t2.Status != task.Failed { 50 | t.Fatal("t2 should have failed") 51 | } 52 | 53 | if t3.Status != task.Skipped { 54 | t.Fatalf("t2 should have beed skipped") 55 | } 56 | } 57 | 58 | func TestSerialAndParallelTasks(t *testing.T) { 59 | p1 := &task.Task{Name: "p1", Command: "sleep 1"} 60 | p2 := &task.Task{Name: "p2", Command: "p2p2p2p2"} 61 | p3 := &task.Task{Name: "p3", Command: "sleep 1"} 62 | 63 | t1 := &task.Task{Name: "foo", Command: "echo foo"} 64 | t2 := &task.Task{Name: "bar", Parallel: Tasks{p1, p2, p3}} 65 | t3 := &task.Task{Name: "baz", Command: "echo baz"} 66 | 67 | tasks := Tasks{t1, t2, t3} 68 | ctx, cancel := context.WithCancel(context.Background()) 69 | p := &Pipeline{} 70 | p.runTasks(ctx, cancel, tasks, nil) 71 | 72 | if p1.Status != task.Aborted { 73 | t.Fatal("p1 should have been aborted") 74 | } 75 | 76 | if p2.Status != task.Failed { 77 | t.Fatal("p2 should have been failed") 78 | } 79 | 80 | if p3.Status != task.Aborted { 81 | t.Fatal("p3 should have been aborted") 82 | } 83 | 84 | if t2.Status != task.Failed { 85 | t.Fatal("t2 should have failed") 86 | } 87 | 88 | if t3.Status != task.Skipped { 89 | t.Fatal("t3 should have skipped") 90 | } 91 | } 92 | 93 | func TestParallelOutput(t *testing.T) { 94 | c1 := &task.Task{Name: "c1", Command: "echo a"} 95 | c2 := &task.Task{Name: "c2", Command: "echo b 1>&2"} 96 | c3 := &task.Task{Name: "c3", Command: "echo c"} 97 | c4 := &task.Task{Name: "c1", Command: "echo d 1>&2"} 98 | c5 := &task.Task{Name: "c2", Command: "echo e"} 99 | c6 := &task.Task{Name: "c3", Command: "echo f 1>&2"} 100 | 101 | parent := &task.Task{Name: "parent", Parallel: Tasks{c1, c2, c3, c4, c5, c6}} 102 | 103 | ctx, cancel := context.WithCancel(context.Background()) 104 | p := &Pipeline{} 105 | p.runTasks(ctx, cancel, Tasks{parent}, nil) 106 | 107 | for i, v := range []string{"a", "c", "e"} { 108 | str := strings.Split(parent.Stdout.String(), "\n")[i] 109 | if str != v { 110 | t.Fatalf("parent.Stdout should contain %s, not %s", v, str) 111 | } 112 | } 113 | 114 | for i, v := range []string{"b", "d", "f"} { 115 | str := strings.Split(parent.Stderr.String(), "\n")[i] 116 | if str != v { 117 | t.Fatalf("parent.Stderr should contain %s, not %s", v, str) 118 | } 119 | } 120 | 121 | for i, v := range []string{"a", "b", "c", "d", "e", "f"} { 122 | str := strings.Split(parent.CombinedOutput.String(), "\n")[i] 123 | if str != v { 124 | t.Fatalf("parent.CombinedOutput should contain %s, not %s", v, str) 125 | } 126 | } 127 | } 128 | 129 | func TestSerialOutput(t *testing.T) { 130 | c1 := &task.Task{Name: "c1", Command: "echo a"} 131 | c2 := &task.Task{Name: "c2", Command: "echo b 1>&2"} 132 | c3 := &task.Task{Name: "c3", Command: "echo c"} 133 | c4 := &task.Task{Name: "c1", Command: "echo d 1>&2"} 134 | c5 := &task.Task{Name: "c2", Command: "echo e"} 135 | c6 := &task.Task{Name: "c3", Command: "echo f && echo g 1>&2"} 136 | 137 | parent := &task.Task{Name: "parent", Serial: Tasks{c1, c2, c3, c4, c5, c6}} 138 | 139 | ctx, cancel := context.WithCancel(context.Background()) 140 | p := &Pipeline{} 141 | p.runTasks(ctx, cancel, Tasks{parent}, nil) 142 | 143 | if !strings.Contains(parent.Stdout.String(), "f") { 144 | t.Fatal("stdout should contain f") 145 | } 146 | 147 | if !strings.Contains(parent.Stderr.String(), "g") { 148 | t.Fatal("stderr should contain g") 149 | } 150 | 151 | if !strings.Contains(parent.CombinedOutput.String(), "f") { 152 | t.Fatal("combined output should contain f") 153 | } 154 | 155 | if !strings.Contains(parent.CombinedOutput.String(), "g") { 156 | t.Fatal("combined output should contain g") 157 | } 158 | } 159 | 160 | func TestPipe(t *testing.T) { 161 | t1 := &task.Task{Name: "t1", Command: "echo \"a\nb\""} 162 | t2 := &task.Task{Name: "t2", Command: "cat"} 163 | 164 | ctx, cancel := context.WithCancel(context.Background()) 165 | p := &Pipeline{} 166 | p.runTasks(ctx, cancel, Tasks{t1, t2}, nil) 167 | 168 | if !strings.Contains(t2.Stdout.String(), "a") { 169 | t.Fatal("t2.Stdout should contain a") 170 | } 171 | 172 | if !strings.Contains(t2.Stdout.String(), "b") { 173 | t.Fatal("t2.Stdout should contain b") 174 | } 175 | } 176 | 177 | func TestPipeOfParallelTasks(t *testing.T) { 178 | t1 := &task.Task{Name: "t1", Command: "echo t1"} 179 | 180 | p1 := &task.Task{Name: "p1", Command: "cat"} 181 | p2 := &task.Task{Name: "p2", Command: "cat"} 182 | p3 := &task.Task{Name: "p3", Command: "echo p3"} 183 | 184 | t2 := &task.Task{Name: "t2", Parallel: Tasks{p1, p2, p3}} 185 | 186 | t3 := &task.Task{Name: "t3", Command: "cat"} 187 | 188 | ctx, cancel := context.WithCancel(context.Background()) 189 | p := &Pipeline{} 190 | p.runTasks(ctx, cancel, Tasks{t1, t2, t3}, nil) 191 | 192 | if !strings.Contains(p1.Stdout.String(), "t1") { 193 | t.Fatal("p1.Stdout should contain t1") 194 | } 195 | 196 | if !strings.Contains(p2.Stdout.String(), "t1") { 197 | t.Fatal("p2.Stdout should contain t1") 198 | } 199 | 200 | if !strings.Contains(t2.Stdout.String(), "t1") { 201 | t.Fatal("t2.Stdout should contain t1") 202 | } 203 | 204 | if !strings.Contains(t2.Stdout.String(), "p3") { 205 | t.Fatal("t2.Stdout should contain p3") 206 | } 207 | 208 | if !strings.Contains(t3.Stdout.String(), "t1") { 209 | t.Fatal("t3.Stdout should contain t1") 210 | } 211 | 212 | if !strings.Contains(t3.Stdout.String(), "p3") { 213 | t.Fatal("t3.Stdout should contain p3") 214 | } 215 | } 216 | 217 | func TestPipeOfSerialTasks(t *testing.T) { 218 | t1 := &task.Task{Name: "t1", Command: "echo t1"} 219 | 220 | s1 := &task.Task{Name: "s1", Command: "cat"} 221 | s2 := &task.Task{Name: "s2", Command: "cat"} 222 | s3 := &task.Task{Name: "s3", Command: "echo s3"} 223 | 224 | t2 := &task.Task{Name: "t2", Serial: Tasks{s1, s2, s3}} 225 | 226 | t3 := &task.Task{Name: "t3", Command: "cat"} 227 | 228 | ctx, cancel := context.WithCancel(context.Background()) 229 | p := &Pipeline{} 230 | p.runTasks(ctx, cancel, Tasks{t1, t2, t3}, nil) 231 | 232 | if !strings.Contains(s1.Stdout.String(), "t1") { 233 | t.Fatal("p1.Stdout should contain t1") 234 | } 235 | 236 | if !strings.Contains(s2.Stdout.String(), "t1") { 237 | t.Fatal("p2.Stdout should contain t1") 238 | } 239 | 240 | if !strings.Contains(t2.Stdout.String(), "s3") { 241 | t.Fatal("t2.Stdout should contain s3") 242 | } 243 | 244 | if !strings.Contains(t3.Stdout.String(), "s3") { 245 | t.Fatal("t3.Stdout should contain s3") 246 | } 247 | } 248 | 249 | func TestExitStatusSuccess(t *testing.T) { 250 | p := &Pipeline{} 251 | t1 := &task.Task{Command: "echo"} 252 | p.Build.Tasks = Tasks{t1} 253 | code := p.Run(true, true) 254 | if code != 0 { 255 | t.Fatalf("Exit code should be 0, not %d", code) 256 | } 257 | } 258 | 259 | func TestExitStatusFail(t *testing.T) { 260 | p := &Pipeline{} 261 | t1 := &task.Task{Command: "no_such_command"} 262 | p.Build.Tasks = Tasks{t1} 263 | code := p.Run(true, true) 264 | if code != 1 { 265 | t.Fatalf("Exit code should be 1, not %d", code) 266 | } 267 | } 268 | 269 | func TestExitStatusParallel(t *testing.T) { 270 | t1 := &task.Task{Command: "echo"} 271 | t2 := &task.Task{Command: "no_such_command"} 272 | 273 | p := &Pipeline{} 274 | p.Build.Tasks = Tasks{&task.Task{Parallel: Tasks{t1, t2}}} 275 | code := p.Run(true, true) 276 | if code != 1 { 277 | t.Fatalf("Exit code should be 1, not %d", code) 278 | } 279 | } 280 | 281 | func TestExitStatusSerial(t *testing.T) { 282 | t1 := &task.Task{Command: "echo"} 283 | t2 := &task.Task{Command: "no_such_command"} 284 | 285 | p := &Pipeline{} 286 | p.Build.Tasks = Tasks{&task.Task{Serial: Tasks{t1, t2}}} 287 | code := p.Run(true, true) 288 | if code != 1 { 289 | t.Fatalf("Exit code should be 1, not %d", code) 290 | } 291 | } 292 | 293 | func TestIncludeInParallel(t *testing.T) { 294 | tsk := &task.Task{ 295 | Name: "test include files in parallel task", 296 | Parallel: []*task.Task{&task.Task{Include: "foo.yml"}}, 297 | } 298 | 299 | p := &Pipeline{} 300 | p.Build.Tasks = Tasks{tsk} 301 | 302 | code := p.Run(true, true) 303 | if code != 1 { 304 | t.Fatalf("Exit code should be 1, not %d", code) 305 | } 306 | } 307 | -------------------------------------------------------------------------------- /lib/task/task.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io" 7 | "os" 8 | "os/exec" 9 | "regexp" 10 | "strings" 11 | "sync" 12 | "syscall" 13 | 14 | "golang.org/x/net/context" 15 | 16 | log "github.com/Sirupsen/logrus" 17 | ) 18 | 19 | const ( 20 | Init = iota 21 | Running 22 | Succeeded 23 | Failed 24 | Skipped 25 | Aborted 26 | ) 27 | 28 | type Task struct { 29 | Name string 30 | Command string 31 | Directory string 32 | Parallel []*Task 33 | Serial []*Task 34 | Stdout *bytes.Buffer 35 | Stderr *bytes.Buffer 36 | CombinedOutput *bytes.Buffer 37 | Status int 38 | Cmd *exec.Cmd 39 | Include string 40 | OnlyIf string `yaml:"only_if"` 41 | WaitFor *WaitFor `yaml:"wait_for"` 42 | } 43 | 44 | type outputHandler struct { 45 | task *Task 46 | writer io.Writer 47 | copy io.Writer 48 | mu *sync.Mutex 49 | } 50 | 51 | func (t *Task) Run(ctx context.Context, cancel context.CancelFunc, prevTask *Task) error { 52 | if t.Command == "" { 53 | return nil 54 | } 55 | 56 | if t.Directory != "" { 57 | re := regexp.MustCompile(`\$[A-Z1-9\-_]+`) 58 | matches := re.FindAllString(t.Directory, -1) 59 | for _, m := range matches { 60 | env := os.Getenv(strings.TrimPrefix(m, "$")) 61 | t.Directory = strings.Replace(t.Directory, m, env, -1) 62 | } 63 | } 64 | 65 | if t.OnlyIf != "" { 66 | cmd := exec.Command("sh", "-c", t.OnlyIf) 67 | cmd.Dir = t.Directory 68 | err := cmd.Run() 69 | 70 | if err != nil { 71 | log.Warnf("[%s] Skipped because only_if failed: %s", t.Name, err) 72 | return nil 73 | } 74 | } 75 | 76 | if t.WaitFor != nil { 77 | err := t.wait() 78 | if err != nil { 79 | return err 80 | } 81 | } 82 | 83 | log.Infof("[%s] Start task", t.Name) 84 | 85 | t.Cmd = exec.Command("sh", "-c", t.Command) 86 | t.Cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} 87 | t.Cmd.Dir = t.Directory 88 | 89 | if prevTask != nil && prevTask.Stdout != nil { 90 | t.Cmd.Stdin = bytes.NewBuffer(prevTask.Stdout.Bytes()) 91 | } 92 | 93 | t.Stdout = new(bytes.Buffer) 94 | t.Stderr = new(bytes.Buffer) 95 | t.CombinedOutput = new(bytes.Buffer) 96 | 97 | var mu sync.Mutex 98 | t.Cmd.Stdout = &outputHandler{t, t.Stdout, t.CombinedOutput, &mu} 99 | t.Cmd.Stderr = &outputHandler{t, t.Stderr, t.CombinedOutput, &mu} 100 | 101 | if err := t.Cmd.Start(); err != nil { 102 | t.Status = Failed 103 | return err 104 | } 105 | 106 | t.Status = Running 107 | 108 | go func(t *Task) { 109 | for { 110 | select { 111 | case <-ctx.Done(): 112 | if t.Status == Running { 113 | t.Status = Aborted 114 | t.Cmd.Process.Kill() 115 | pgid, err := syscall.Getpgid(t.Cmd.Process.Pid) 116 | if err == nil { 117 | syscall.Kill(-pgid, syscall.SIGTERM) 118 | } 119 | log.Warnf("[%s] aborted", t.Name) 120 | } 121 | return 122 | } 123 | } 124 | }(t) 125 | 126 | t.Cmd.Wait() 127 | 128 | if t.Cmd.ProcessState.Success() { 129 | t.Status = Succeeded 130 | } else if t.Status == Running { 131 | t.Status = Failed 132 | cancel() 133 | return errors.New("Task failed") 134 | } 135 | 136 | if t.Status == Succeeded { 137 | log.Infof("[%s] End task", t.Name) 138 | } 139 | 140 | return nil 141 | } 142 | 143 | func (o *outputHandler) Write(b []byte) (int, error) { 144 | log.Infof("[%s] %s", o.task.Name, strings.TrimSuffix(string(b), "\n")) 145 | 146 | o.mu.Lock() 147 | defer o.mu.Unlock() 148 | o.writer.Write(b) 149 | o.copy.Write(b) 150 | 151 | return len(b), nil 152 | } 153 | -------------------------------------------------------------------------------- /lib/task/task_test.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | "testing" 7 | 8 | "golang.org/x/net/context" 9 | ) 10 | 11 | func TestStdout(t *testing.T) { 12 | tsk := &Task{Name: "echo", Command: "echo hello"} 13 | 14 | ctx, cancel := context.WithCancel(context.Background()) 15 | 16 | err := tsk.Run(ctx, cancel, nil) 17 | 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | 22 | if !contains(tsk.Stdout, "hello") { 23 | t.Fatalf("tsk.Stdout is %s, it does not contain \"hello\"", tsk.Stdout) 24 | } 25 | 26 | if !contains(tsk.CombinedOutput, "hello") { 27 | t.Fatalf("tsk.CombinedOutput is %s, it does not contain \"hello\"", tsk.CombinedOutput) 28 | } 29 | } 30 | 31 | func TestStderr(t *testing.T) { 32 | tsk := &Task{Name: "echo", Command: "echo hello 1>&2"} 33 | 34 | ctx, cancel := context.WithCancel(context.Background()) 35 | 36 | err := tsk.Run(ctx, cancel, nil) 37 | 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | 42 | if !contains(tsk.Stderr, "hello") { 43 | t.Fatalf("tsk.Stderr is %s, it does not contain \"hello\"", tsk.Stderr) 44 | } 45 | 46 | if !contains(tsk.CombinedOutput, "hello") { 47 | t.Fatalf("tsk.CombinedOutput is %s, it does not contain \"hello\"", tsk.CombinedOutput) 48 | } 49 | } 50 | 51 | func TestStatus(t *testing.T) { 52 | tsk := &Task{Name: "command should succeed", Command: "echo foo"} 53 | 54 | ctx, cancel := context.WithCancel(context.Background()) 55 | err := tsk.Run(ctx, cancel, nil) 56 | if err != nil { 57 | t.Fatal(err) 58 | } 59 | if tsk.Status != Succeeded { 60 | t.Fatal("command not succeeded") 61 | } 62 | 63 | tsk = &Task{Name: "command should fail", Command: "no_such_command"} 64 | err = tsk.Run(ctx, cancel, nil) 65 | if err == nil { 66 | t.Fatal("tsk.Run() should return err") 67 | } 68 | 69 | if tsk.Status != Failed { 70 | t.Fatal("tsk.Status shoud be Fail") 71 | } 72 | } 73 | 74 | func contains(buf *bytes.Buffer, e string) bool { 75 | for _, a := range strings.Split(buf.String(), "\n") { 76 | if strings.Contains(a, e) { 77 | return true 78 | } 79 | } 80 | return false 81 | } 82 | -------------------------------------------------------------------------------- /lib/task/wait_for.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net" 7 | "os" 8 | "strconv" 9 | "time" 10 | 11 | log "github.com/Sirupsen/logrus" 12 | ) 13 | 14 | type WaitFor struct { 15 | Host string 16 | Port int 17 | File string 18 | State string 19 | Delay float64 20 | } 21 | 22 | func (t *Task) wait() error { 23 | err := t.WaitFor.validate() 24 | if err != nil { 25 | return err 26 | } 27 | t.WaitFor.wait(t) 28 | return nil 29 | } 30 | 31 | func (w *WaitFor) wait(t *Task) { 32 | switch { 33 | case w.Delay > 0.0: 34 | w.waitForDelay(t) 35 | case w.Port > 0: 36 | w.waitForPort(t) 37 | case w.File != "": 38 | w.waitForFile(t) 39 | } 40 | } 41 | 42 | func (w *WaitFor) waitForDelay(t *Task) { 43 | log.Infof("[%s] wait_for: %f seconds delay", t.Name, w.Delay) 44 | time.Sleep(time.Duration(w.Delay) * time.Second) 45 | return 46 | } 47 | 48 | func (w *WaitFor) waitForPort(t *Task) { 49 | log.Infof("[%s] wait_for: %s:%d %s", t.Name, w.Host, w.Port, w.State) 50 | if w.State == "ready" || w.State == "present" { 51 | for { 52 | if connected(w.Host, w.Port) { 53 | return 54 | } else { 55 | time.Sleep(10 * time.Millisecond) 56 | } 57 | } 58 | } 59 | 60 | if w.State == "unready" || w.State == "absent" { 61 | for { 62 | if !connected(w.Host, w.Port) { 63 | return 64 | } else { 65 | time.Sleep(10 * time.Millisecond) 66 | } 67 | } 68 | } 69 | } 70 | 71 | func connected(host string, port int) bool { 72 | conn, err := net.Dial("tcp", fmt.Sprintf("%s:%s", host, strconv.Itoa(port))) 73 | if err != nil { 74 | return false 75 | } 76 | defer conn.Close() 77 | return true 78 | } 79 | 80 | func (w *WaitFor) waitForFile(t *Task) { 81 | log.Infof("[%s] wait_for: file %s %s", t.Name, w.File, w.State) 82 | if w.State == "ready" || w.State == "present" { 83 | for { 84 | if isExist(w.File) { 85 | return 86 | } else { 87 | time.Sleep(10 * time.Millisecond) 88 | } 89 | } 90 | } 91 | 92 | if w.State == "unready" || w.State == "absent" { 93 | for { 94 | if !isExist(w.File) { 95 | return 96 | } else { 97 | time.Sleep(10 * time.Millisecond) 98 | } 99 | } 100 | } 101 | } 102 | 103 | func isExist(file string) bool { 104 | _, err := os.Stat(file) 105 | return err == nil 106 | } 107 | 108 | func (w *WaitFor) validate() error { 109 | switch { 110 | case w.Port != 0 && w.File != "": 111 | return errors.New("wait_for: cannot use port and file at the same time") 112 | case w.Port != 0 && w.Delay > 0.0: 113 | return errors.New("wait_for: cannot use port and delay at the same time") 114 | case w.File != "" && w.Delay > 0.0: 115 | return errors.New("wait_for: cannot use file and delay at the same time") 116 | case w.Delay < 0: 117 | return errors.New("wait_for: delay must be positive") 118 | case w.Port < 0: 119 | return errors.New("wait_for: port must be positive") 120 | case w.Port > 0 && w.Host == "": 121 | return errors.New("wait_for: cannot use port without host") 122 | case w.Port == 0 && w.Host != "": 123 | return errors.New("wait_for: cannot use host without port") 124 | case w.Delay == 0 && !includes([]string{"ready", "unready", "present", "absent"}, w.State): 125 | return errors.New("wait_for: state does not support " + w.State) 126 | case w.Port > 0 && w.State == "": 127 | return errors.New("wait_for: cannot use port without state") 128 | case w.File != "" && w.State == "": 129 | return errors.New("wait_for: cannot use file without state") 130 | } 131 | 132 | return nil 133 | } 134 | 135 | func includes(s []string, e string) bool { 136 | for _, a := range s { 137 | if a == e { 138 | return true 139 | } 140 | } 141 | return false 142 | } 143 | -------------------------------------------------------------------------------- /lib/task/wait_for_test.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import "testing" 4 | 5 | func TestValidateWaitFor(t *testing.T) { 6 | w := &WaitFor{} 7 | w.Port = 80 8 | w.File = "/tmp" 9 | err := w.validate() 10 | if err == nil { 11 | t.Fatalf("Error should be returned: %#v", w) 12 | } 13 | 14 | w = &WaitFor{} 15 | w.Port = 80 16 | w.Delay = 10 17 | err = w.validate() 18 | if err == nil { 19 | t.Fatalf("Error should be returned: %#v", w) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | 7 | log "github.com/Sirupsen/logrus" 8 | 9 | "github.com/walter-cd/walter/lib/pipeline" 10 | ) 11 | 12 | func main() { 13 | const defaultConfigFile = "pipeline.yml" 14 | 15 | var ( 16 | configFile string 17 | version bool 18 | build bool 19 | deploy bool 20 | ) 21 | 22 | flag.StringVar(&configFile, "config", defaultConfigFile, "file which define pipeline") 23 | flag.BoolVar(&version, "version", false, "print version string") 24 | flag.BoolVar(&build, "build", false, "run build") 25 | flag.BoolVar(&deploy, "deploy", false, "run deploy") 26 | 27 | flag.Parse() 28 | 29 | if version { 30 | log.Info(OutputVersion()) 31 | os.Exit(0) 32 | } 33 | 34 | if !build && !deploy { 35 | log.Error("specify -build and/or -deploy flags") 36 | os.Exit(1) 37 | } 38 | 39 | p, err := pipeline.LoadFromFile(configFile) 40 | if err != nil { 41 | log.Fatal(err) 42 | } 43 | 44 | os.Exit(p.Run(build, deploy)) 45 | } 46 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | -------------------------------------------------------------------------------- /scripts/compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | DIR=$(cd $(dirname ${0})/.. && pwd) 6 | cd ${DIR} 7 | 8 | XC_ARCH=${XC_ARCH:-386 amd64} 9 | XC_OS=${XC_OS:-darwin linux} 10 | 11 | COMMIT=`git describe --always` 12 | 13 | rm -rf pkg/ 14 | gox \ 15 | -ldflags "-X main.GitCommit=${COMMIT}" \ 16 | -parallel=5 \ 17 | -os="${XC_OS}" \ 18 | -arch="${XC_ARCH}" \ 19 | -output "pkg/{{.OS}}_{{.Arch}}/{{.Dir}}" 20 | -------------------------------------------------------------------------------- /scripts/package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | DIR=$(cd $(dirname ${0})/.. && pwd) 6 | cd ${DIR} 7 | 8 | VERSION=$(grep "const Version " version.go | sed -E 's/.*"(.+)"$/\1/') 9 | REPO="walter" 10 | 11 | # Run Compile 12 | ./scripts/compile.sh 13 | 14 | if [ -d pkg ];then 15 | rm -rf ./pkg/dist 16 | fi 17 | 18 | # Package all binary as .zip 19 | mkdir -p ./pkg/dist/${VERSION} 20 | for PLATFORM in $(find ./pkg -mindepth 1 -maxdepth 1 -type d); do 21 | PLATFORM_NAME=$(basename ${PLATFORM}) 22 | ARCHIVE_NAME=${REPO}_${VERSION}_${PLATFORM_NAME} 23 | 24 | if [ $PLATFORM_NAME = "dist" ]; then 25 | continue 26 | fi 27 | 28 | pushd ${PLATFORM} 29 | zip ${DIR}/pkg/dist/${VERSION}/${ARCHIVE_NAME}.zip ./* 30 | popd 31 | done 32 | 33 | # Generate shasum 34 | pushd ./pkg/dist/${VERSION} 35 | shasum -a 256 * > ./${VERSION}_SHASUMS 36 | popd 37 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "time" 7 | 8 | log "github.com/Sirupsen/logrus" 9 | 10 | latest "github.com/tcnksm/go-latest" 11 | ) 12 | 13 | // Name is application name 14 | const Name = "walter" 15 | 16 | // Version is application version 17 | const Version string = "v2.0.0" 18 | 19 | // GitCommit describes latest commit hash. 20 | // This is automatically extracted by git describe --always. 21 | var GitCommit string 22 | 23 | const defaultCheckTimeout = 2 * time.Second 24 | 25 | // OutputVersion display version number 26 | func OutputVersion() string { 27 | var buf bytes.Buffer 28 | fmt.Fprintf(&buf, "%s version %s", Name, Version) 29 | if len(GitCommit) != 0 { 30 | fmt.Fprintf(&buf, " (%s)", GitCommit) 31 | } 32 | fmt.Fprintf(&buf, "\n") 33 | 34 | // Check latest version is release or not. 35 | verCheckCh := make(chan *latest.CheckResponse) 36 | go func() { 37 | fixFunc := latest.DeleteFrontV() 38 | githubTag := &latest.GithubTag{ 39 | Owner: "walter-cd", 40 | Repository: "walter", 41 | FixVersionStrFunc: fixFunc, 42 | } 43 | 44 | res, err := latest.Check(githubTag, fixFunc(Version)) 45 | if err != nil { 46 | // Don't return error 47 | log.Debugf("[ERROR] Check lastet version is failed: %s", err) 48 | return 49 | } 50 | verCheckCh <- res 51 | }() 52 | 53 | select { 54 | case <-time.After(defaultCheckTimeout): 55 | case res := <-verCheckCh: 56 | if res.Outdated { 57 | fmt.Fprintf(&buf, 58 | "Latest version of walter is v%s, please upgrade!\n", 59 | res.Current) 60 | } 61 | } 62 | 63 | return buf.String() 64 | } 65 | --------------------------------------------------------------------------------