├── .github
└── workflows
│ ├── build.yml
│ └── release.yml
├── .gitignore
├── .gitmodules
├── .goreleaser.yml
├── LICENSE
├── Makefile
├── README.md
├── address.go
├── address_test.go
├── archive.go
├── archive_write_unix.go
├── archive_write_windows.go
├── bytes_stringer.go
├── callback_writer.go
├── command.go
├── connect.go
├── debug_writer.go
├── demo.gif
├── distributed_lock.go
├── distributed_lock_node.go
├── file.go
├── format.go
├── go.mod
├── go.sum
├── heartbeat.go
├── json_output_writer.go
├── key.go
├── locked_writer.go
├── log.go
├── main.go
├── main_unix.go
├── main_windows.go
├── multiwrite_closer.go
├── nop_closer.go
├── protocol_node_writer.go
├── remote_execution.go
├── remote_execution_node.go
├── remote_execution_runner.go
├── run_tests
├── runner_factory.go
├── sshagent_unix.go
├── sshagent_windows.go
├── status_bar_update_writer.go
├── sync.go
├── sync_protocol.go
├── tests
├── .gitignore
├── build.sh
├── deps.sh
├── orgalorg.sh
├── setup.sh
├── ssh.sh
├── teardown.sh
└── testcases
│ ├── auth
│ ├── can-authenticate-via-key-with-passphrase.test.sh
│ └── can-authenticate-via-password.test.sh
│ ├── can-skip-unreachable-servers-if-flag-is-given.test.sh
│ ├── commandline
│ ├── can-handle-common-mistakes-in-arguments.test.sh
│ └── will-read-hosts-from-file-if-argument-starts-with-slash-or-dot-slash.test.sh
│ ├── commands
│ ├── can-aggregate-errors-on-the-end-of-execution.test.sh
│ ├── can-escape-space-in-the-remote-command.test.sh
│ ├── can-run-remote-command-and-pass-stdin-to-it.test.sh
│ ├── can-run-remote-command.test.sh
│ ├── can-run-shell-command.test.sh
│ ├── should-properly-escape-shell-arguments.test.sh
│ ├── should-return-non-zero-exit-code-if-command-fails.test.sh
│ ├── should-use-default-user-directory-unless-specified.test.sh
│ ├── should-use-specified-working-directory.test.sh
│ └── will-flush-lines-so-they-do-not-interleave.test.sh
│ ├── locking
│ ├── can-acquire-global-lock.test.sh
│ ├── can-detect-if-lock-process-is-aborted-by-remote-host-after-acquire.test.sh
│ ├── lock.sh
│ └── will-continue-execution-when-lock-failed-if-flag-is-specified.test.sh
│ ├── newlines
│ ├── do-append-newlines-in-non-quite-mode-when-necessary.test.sh
│ └── do-not-append-newlines-if-they-are-not-present-in-remote-output-at-quiet-mode.test.sh
│ ├── sudo
│ ├── can-run-command-under-sudo.test.sh
│ ├── can-upload-files-under-sudo.test.sh
│ └── should-run-all-commands-under-sudo-if-specified.test.sh
│ ├── sync
│ ├── can-create-run-directory-under-sudo.test.sh
│ ├── can-pass-additional-args-to-sync-tool.test.sh
│ ├── can-pass-arguments-to-the-sync-tool-by-default.test.sh
│ ├── can-run-simple-command-after-files-sync-even-in-protocol-mode.test.sh
│ ├── can-run-simple-command-after-files-sync.test.sh
│ ├── can-run-two-steps-with-synchronization-in-between.test.sh
│ ├── can-upload-directory.test.sh
│ ├── can-upload-file-by-absolute-path-by-default.test.sh
│ ├── can-upload-single-file.test.sh
│ ├── can-upload-two-files.test.sh
│ ├── sync-tool-will-be-run-in-the-directory-with-uploaded-files.test.sh
│ ├── will-print-uploaded-files-in-verbose-mode.test.sh
│ ├── will-say-hello-to-sync-tool-at-start.test.sh
│ ├── will-send-list-of-nodes-to-sync-tool.test.sh
│ └── will-send-stderr-from-sync-tool-back-to-master.test.sh
│ └── verbosity
│ ├── can-log-stderr-and-stdout-from-remote-at-verbose-level-to-stderr.test.sh
│ ├── can-output-ip-in-response-from-remote-server-when-running-command.test.sh
│ ├── can-output-node-stdout-and-stderr-in-json-format.test.sh
│ ├── can-output-verbose-debug-info-in-json-format.test.sh
│ ├── can-print-debug-output.test.sh
│ ├── can-report-internal-errors-in-json-format.test.sh
│ ├── do-not-output-ip-in-quiet-mode.test.sh
│ └── will-output-hierarchical-errors-by-default.test.sh
├── themes.go
├── thread_pool.go
├── timeouts.go
├── vendor.bash
├── .gitignore
└── Makefile
└── verbosity.go
/.github/workflows/build.yml:
--------------------------------------------------------------------------------
1 | name: Build
2 |
3 | on: [push, pull_request, workflow_dispatch]
4 |
5 | jobs:
6 | build:
7 | runs-on: ubuntu-latest
8 | steps:
9 | - uses: actions/checkout@v4
10 |
11 | - name: Set up Go
12 | uses: actions/setup-go@v5
13 | with:
14 | go-version-file: "go.mod"
15 | id: go
16 |
17 | - name: Build
18 | run: go build -v .
19 |
20 | - name: Test
21 | run: go test -v .
22 |
23 | - name: Upload build artifact
24 | uses: actions/upload-artifact@v2
25 | with:
26 | name: build
27 | path: orgalorg
28 |
--------------------------------------------------------------------------------
/.github/workflows/release.yml:
--------------------------------------------------------------------------------
1 | name: Release
2 |
3 | on:
4 | push:
5 | tags:
6 | - '*'
7 |
8 | jobs:
9 | goreleaser:
10 | runs-on: ubuntu-latest
11 | steps:
12 | - uses: actions/checkout@v4
13 |
14 | - uses: actions/setup-go@v5
15 | with:
16 | go-version-file: "go.mod"
17 | -
18 | name: Run GoReleaser
19 | uses: goreleaser/goreleaser-action@v2
20 | with:
21 | version: latest
22 | args: release
23 | env:
24 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
25 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /lorg
2 | /.last-testcase
3 | /orgalorg
4 | /orgalorg.exe
5 | /.cover
6 | /coverage
7 |
--------------------------------------------------------------------------------
/.gitmodules:
--------------------------------------------------------------------------------
1 | [submodule "vendor.bash/github.com/reconquest/import.bash"]
2 | path = vendor.bash/github.com/reconquest/import.bash
3 | url = https://github.com/reconquest/import.bash
4 |
--------------------------------------------------------------------------------
/.goreleaser.yml:
--------------------------------------------------------------------------------
1 | project_name: orgalorg
2 | builds:
3 | - env:
4 | - CGO_ENABLED=0
5 | - GO111MODULE=on
6 | goos:
7 | - linux
8 | - darwin
9 | - windows
10 | goarch:
11 | - amd64
12 | flags:
13 | - -mod=readonly
14 | ldflags:
15 | - -X=main.version={{.Tag}}
16 | checksum:
17 | name_template: 'sha256sums.txt'
18 | algorithm: sha256
19 | snapshot:
20 | name_template: "{{ .Tag }}-next"
21 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2016 Stanislav Seletskiy
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | build:
2 | go build -mod=mod -ldflags -X=main.version=$$(git describe --tags --abbrev=6)
3 |
4 | test:
5 | ./run_tests -vvvv
6 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # orgalorg [](https://goreportcard.com/report/github.com/reconquest/orgalorg) [](https://raw.githubusercontent.com/reconquest/orgalorg/master/LICENSE)
2 |
3 |
4 | orgalorg can run command and upload files in parallel by SSH on many hosts
5 |
6 |
7 |
8 | 
9 |
10 |
11 | # Features
12 |
13 | * Zero-configuration. No config files. Everything is done via command line
14 | flags.
15 |
16 | * Running SSH commands or shell scripts on any number of hosts in parallel. All
17 | output from nodes will be returned back, keeping stdout and stderr streams
18 | mapping of original commands.
19 |
20 | * Synchronizing files and directories across cluster with prior global cluster
21 | locking.
22 | After synchronization is done, arbitrary command can be evaluated.
23 |
24 | * Synchronizing files and directories with subsequent run of complex multi-step
25 | scenario with steps synchronization across cluster.
26 |
27 | * User-friendly progress indication.
28 |
29 | * Both strict or loose modes of failover to be sure that everything will either
30 | fail on any error or try to complete, no matter of what.
31 |
32 | * Interactive password authentication as well as SSH public key authentication.
33 | Will use ssh-agent if present. On Windows, orgalorg can connect to
34 | **pageant** or **openssh agent**.
35 |
36 | * Ability to run commands through `sudo`.
37 |
38 | * Grouped mode of output, so stdout and stderr from nodes will be grouped by
39 | node name. Alternatively, output can be returned as soon as node returns
40 | something.
41 |
42 | # Installation
43 |
44 | ## go install
45 |
46 | ```bash
47 | go install github.com/reconquest/orgalorg@latest
48 | ```
49 |
50 | # Alternatives
51 |
52 | * ansible: intended to apply complex DSL-based scenarios of actions;
53 | orgalorg aimed only on running commands and synchronizing files in parallel.
54 | orgalorg can accept target hosts list on stdin and can provide realtime
55 | output from commands, which ansible can't do (like running `tail -f`).
56 | orgalorg also uses same argument semantic as `ssh`:
57 | `orgalorg ... -C tail -f '/var/log/*.log'` will do exactly the same.
58 |
59 | * clusterssh / cssh: will open number of xterm terminals to all nodes.
60 | orgalorg intended to use in batch mode, no GUI is assumed. orgalorg, however,
61 | can be used in interactive mode (see example section below).
62 |
63 | * pssh: buggy, uses binary ssh, which is not resource efficient.
64 | orgalorg uses native SSH protocol implementation, so safe and fast to use
65 | on thousand of nodes.
66 |
67 | * dsh / gsh / pdsh: not maintained.
68 |
69 | # Example usages
70 |
71 | `-o ...` in later examples will mean any supported combination of
72 | host-specification arguments, like
73 | `-o node1.example.com -o node2.example.com`.
74 |
75 | ## Evaluating command on hosts in parallel
76 |
77 | ```bash
78 | orgalorg -o ... -C uptime
79 | ```
80 |
81 | ## Evaluating command on hosts given by stdin
82 |
83 | `axfr` is a tool of your choice for retrieving domain information from your
84 | infrastructure DNS.
85 |
86 | ```bash
87 | axfr | grep phpnode | orgalorg -s -C uptime
88 | ```
89 |
90 | ## Evaluate command under root (passwordless sudo required)
91 |
92 | ```bash
93 | orgalorg -o ... -x -C whoami
94 | ```
95 |
96 | ## Tailing logs from many hosts in realtime
97 |
98 | ```bash
99 | orgalorg -o ... -C tail -f /var/log/syslog
100 | ```
101 |
102 | ## Copying SSH public key for remote authentication
103 |
104 | ```bash
105 | orgalorg -o ... -p -i ~/.ssh/id_rsa.pub -C tee -a ~/.ssh/authorized_keys
106 | ```
107 |
108 | ## Synchronizing configs and then reloading service (like nginx)
109 |
110 | ```bash
111 | orgalorg -o ... -xn 'systemctl reload nginx' -S /etc/nginx.conf
112 | ```
113 |
114 | ## Evaluating shell script
115 |
116 | ```bash
117 | orgalorg -o ... -i script.bash -C bash
118 | ```
119 |
120 | ## Install package on all nodes and get combined output from each node
121 |
122 | ```bash
123 | orgalorg -o ... -lx -C pacman -Sy my-package --noconfirm
124 | ```
125 |
126 | ## Evaluating shell oneliner
127 |
128 | ```bash
129 | orgalorg -o ... -C sleep '$(($RANDOM % 10))' '&&' echo done
130 | ```
131 |
132 | ## Running poor-man interactive parallel shell
133 |
134 | ```bash
135 | orgalorg -o ... -i /dev/stdin -C bash -s
136 | ```
137 |
138 | ## Obtaining global cluster lock
139 |
140 | ```bash
141 | orgalorg -o ... -L
142 | ```
143 |
144 | Next orgalorg calls will fail with message, that lock is already acquired,
145 | until first instance will be stopped.
146 |
147 | Useful for setting cluster into maintenance state.
148 |
149 | ## Obtaining global cluster lock on custom directory
150 |
151 | ```bash
152 | orgalorg -o ... -L -r /etc
153 | ```
154 |
155 | # Description
156 |
157 | orgalorg provides easy way of synchronizing files across cluster and running
158 | arbitrary SSH commands.
159 |
160 | orgalorg works through SSH & tar, so no unexpected protocol errors will arise.
161 |
162 | In default mode of operation (lately referred as sync mode) orgalorg will
163 | perform steps in the following order:
164 |
165 | 1. Acquire global cluster lock (check more detailed info above).
166 | 2. Create, upload and extract specified files in streaming mode to the
167 | specified nodes into temporary run directory.
168 | 3. Start synchronization tool on each node, that should relocate files from
169 | temporary run directory to the destination.
170 |
171 | So, orgalorg expected to work with third-party synchronization tool, that
172 | will do actual files relocation and can be quite intricate, **but orgalorg can
173 | work without that tool and perform simple files sync (more on this later)**.
174 |
175 | ## Global Cluster Lock
176 |
177 | Before doing anything else orgalorg will perform global cluster lock. That lock
178 | is acquired atomically, and no other orgalorg instance can acquire lock if it
179 | is already acquired.
180 |
181 | Locking is done via flock'ing specified file or directory on each of target
182 | nodes, and will fail, if flock fails on at least one node.
183 |
184 | Directory can be used as lock target as well as ordinary file. `--lock-file`
185 | can be used to specify lock target different from `/`.
186 |
187 | After acquiring lock, orgalorg will run heartbeat process, which will check,
188 | that lock is still intact. By default, that check will be performed every 10
189 | seconds. If at least one heartbeat is failed, then orgalorg will abort entire
190 | sync procedure.
191 |
192 | User can stop there by using `--lock` or `-L` flag, effectively transform
193 | orgalorg to the distributed locking tool.
194 |
195 | ## File Upload
196 |
197 | Files will be sent from local node to the amount of specified nodes.
198 |
199 | orgalorg will perform streaming transfer, so it's safe to synchronize large
200 | files without major memory consumption.
201 |
202 | By default, orgalorg will upload files to the temporary run directory. That
203 | behaviour can be changed by using `--root` or `-r` flag. Then, files will be
204 | uploaded to the specified directory.
205 |
206 | User can specify `--upload` or `-U` flag to transform orgalorg to the simple
207 | file upload tool. In that mode orgalorg will upload files to the specified
208 | directory and then exit.
209 |
210 | orgalorg preserves all file attributes while transfer as well as user and group
211 | IDs. That behaviour can be changed by using `--no-preserve-uid` and
212 | `--no-preseve-gid` command line options. These flags are ignored when orgalorg
213 | is ran from Windows.
214 |
215 | By default, orgalorg will keep source file paths as is, creating same directory
216 | layout on the target nodes. E.g., if orgalorg told to upload file `a` while
217 | current working directory is `/b/c/`, orgalorg will upload file to the
218 | `/b/c/a` on the remote nodes. That behaviour can be changed by
219 | specifying `--relative` or `-e` flag. Then, orgalorg will not preserve source
220 | file base directory.
221 |
222 | orgalorg will try to upload files under specified user (current user by
223 | default). However, if user has `NOPASSWD` record in the sudoers file on the
224 | remote nodes, `--sudo` or `-x` can be used to elevate to root before uploading
225 | files. It makes possible to login to the remote nodes under normal user and
226 | rewrite system files.
227 |
228 | ## Synchronization Tool
229 |
230 | After file upload orgalorg will execute synchronization tool
231 | (`/usr/lib/orgalorg/sync`). That tool is expected to relocate synced files from
232 | temporary directory to the target directory. However, that tool can perform
233 | arbitrary actions, like reloading system services.
234 |
235 | To specify custom synchronization tool user can use `--sync-cmd` or `-n` flag.
236 | Full shell syntax is supported in the argument to that option.
237 |
238 | Tool is also expected to communicate with orgalorg using sync protocol
239 | (described below), however, it's not required. If not specified, orgalorg will
240 | communicate with that tool using stdin/stdout streams. User can change that
241 | behaviour using `--simple` or `-m` flag, which will cause orgalorg to treat
242 | specified sync tool as simple shell command. User can even provide stdin
243 | to that program by using `--stdin` or `-i` flag.
244 |
245 | Tool can accept number of arguments, which can be specified by using `-g` or
246 | `--arg` flags.
247 |
248 | # Synchronization Protocol
249 |
250 | orgalorg will communicate with given sync tool using special sync protocol,
251 | which gives possibility to perform some actions with synchronization across
252 | entire cluster.
253 |
254 | orgalorg will start sync tool as it specified in the command line, without
255 | any modification.
256 |
257 | After start, orgalorg will communicate with running sync tool using stdin
258 | and stdout streams. stderr will be passed to user untouched.
259 |
260 | All communication messages should be prefixed by special prefix, which is
261 | send by orgalorg in the hello message. All lines on stdout that are not match
262 | given prefix will be printed as is, untouched.
263 |
264 | Communication begins from the hello message.
265 |
266 | ## Protocol
267 |
268 | ### HELLO
269 |
270 | `orgalorg -> sync tool`
271 |
272 | ```
273 | HELLO
274 | ```
275 |
276 | Start communication session. All further messages should be prefixed with given
277 | prefix.
278 |
279 | ### NODE
280 |
281 | `orgalorg -> sync tool`
282 |
283 | ```
284 | NODE [CURRENT]
285 | ```
286 |
287 | orgalorg will send node list to the sync tools on each running node.
288 |
289 | `CURRENT` flag will be present next to the node which is currently receiving
290 | protocol messages.
291 |
292 | ### START
293 |
294 | `orgalorg -> sync tool`
295 |
296 | ```
297 | START
298 | ```
299 |
300 | Start messages will be sent at the end of the nodes list and means that sync
301 | tool can start doing actions.
302 |
303 | ### SYNC
304 |
305 | `sync tool -> orgalorg`
306 |
307 | ```
308 | SYNC
309 | ```
310 |
311 | Sync tool can send sync messages after some steps are done to be sure, that
312 | every node in cluster are performing steps gradually, in order.
313 |
314 | When orgalorg receives sync message, it will be broadcasted to every connected
315 | sync tool.
316 |
317 | ### SYNC (broadcasted)
318 |
319 | `orgalorg -> sync tool`
320 |
321 | ```
322 | SYNC
323 | ```
324 |
325 | orgalorg will retransmit incoming sync message from one node to every connected
326 | node (including node, that is sending sync).
327 |
328 | Sync tools can wait for specific number of the incoming sync messages to
329 | continue to the next step of execution process.
330 |
331 | ## Example
332 |
333 | `<-` are outgoing messages (from orgalorg to sync tools).
334 |
335 | ```
336 | <- ORGALORG:132464327653 HELLO
337 | <- ORGALORG:132464327653 NODE [user@node1:22]
338 | <- ORGALORG:132464327653 NODE [user@node2:1234] CURRENT
339 | <- ORGALORG:132464327653 START
340 | -> (from node1) ORGALORG:132464327653 SYNC phase 1 completed
341 | <- ORGALORG:132464327653 SYNC [user@node1:22] phase 1 completed
342 | -> (from node2) ORGALORG:132464327653 SYNC phase 1 completed
343 | <- ORGALORG:132464327653 SYNC [user@node2:1234] phase 1 completed
344 | ```
345 |
346 | # Testing
347 |
348 | To run tests it's enough to:
349 |
350 | ```
351 | ./run_tests
352 | ```
353 |
354 | ## Requirements
355 |
356 | Testcases are run through [tests.sh](https://github.com/reconquest/tests.sh)
357 | library.
358 |
359 | For every testcase new set of temporary containers will be initialized through
360 | [hastur](https://github.com/seletskiy/hastur), so `systemd` is required for
361 | running test suite.
362 |
363 | orgalorg testcases are close to reality as possible, so orgalorg will really
364 | connect via SSH to cluster of containers in each testcase.
365 |
366 | ## Coverage
367 |
368 | Run following command to calculate total coverage (available after running
369 | testsuite):
370 |
371 | ```bash
372 | make coverage.total
373 | ```
374 |
375 | Current coverage level is something about **85%**.
376 |
--------------------------------------------------------------------------------
/address.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "regexp"
6 | "strconv"
7 |
8 | "github.com/reconquest/hierr-go"
9 | )
10 |
11 | var (
12 | hostRegexp = regexp.MustCompile(`^(?:([^@]+)@)?(.*?)(?::(\d+))?$`)
13 | )
14 |
15 | type address struct {
16 | user string
17 | domain string
18 | port int
19 |
20 | host string
21 | }
22 |
23 | func (address address) String() string {
24 | return fmt.Sprintf(
25 | "[%s@%s:%d]",
26 | address.user,
27 | address.domain,
28 | address.port,
29 | )
30 | }
31 |
32 | func parseAddress(
33 | host string, defaultUser string, defaultPort int,
34 | ) (address, error) {
35 | matches := hostRegexp.FindStringSubmatch(host)
36 |
37 | var (
38 | user = defaultUser
39 | domain = matches[2]
40 | rawPort = matches[3]
41 | port = defaultPort
42 | )
43 |
44 | if matches[1] != "" {
45 | user = matches[1]
46 | }
47 |
48 | if rawPort != "" {
49 | var err error
50 | port, err = strconv.Atoi(rawPort)
51 | if err != nil {
52 | return address{}, hierr.Errorf(
53 | err,
54 | `can't parse port number: '%s'`, rawPort,
55 | )
56 | }
57 | }
58 |
59 | return address{
60 | user: user,
61 | domain: domain,
62 | port: port,
63 | host: host,
64 | }, nil
65 | }
66 |
67 | func getUniqueAddresses(addresses []address) []address {
68 | result := []address{}
69 |
70 | for _, origin := range addresses {
71 | keep := true
72 |
73 | for _, another := range result {
74 | if origin.user != another.user {
75 | continue
76 | }
77 |
78 | if origin.domain != another.domain {
79 | continue
80 | }
81 |
82 | if origin.port != another.port {
83 | continue
84 | }
85 |
86 | keep = false
87 | }
88 |
89 | if keep {
90 | result = append(result, origin)
91 | }
92 | }
93 |
94 | return result
95 | }
96 |
--------------------------------------------------------------------------------
/address_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/stretchr/testify/assert"
7 | )
8 |
9 | func TestParseAddress_ParseValidDomainAddressWithDefaults(t *testing.T) {
10 | assertions := assert.New(t)
11 |
12 | tests := []struct {
13 | Host string
14 | DefaultUser string
15 | DefaultPort int
16 |
17 | ExpectedUser string
18 | ExpectedDomain string
19 | ExpectedPort int
20 | }{
21 | {"example.com", "_", 23,
22 | "_", "example.com", 23},
23 |
24 | {"user@example.com", "_", 23,
25 | "user", "example.com", 23},
26 |
27 | {"example.com:1234", "_", 23,
28 | "_", "example.com", 1234},
29 |
30 | {"user@example.com:1234", "_", 23,
31 | "user", "example.com", 1234},
32 |
33 | {"example.com:1234", "user", 23,
34 | "user", "example.com", 1234},
35 |
36 | {"user2@example.com:1234", "user", 23,
37 | "user2", "example.com", 1234},
38 | }
39 |
40 | for _, test := range tests {
41 | parsed, err := parseAddress(
42 | test.Host, test.DefaultUser, test.DefaultPort,
43 | )
44 |
45 | assertions.Nil(err)
46 | assertions.Equal(test.ExpectedUser, parsed.user)
47 | assertions.Equal(test.ExpectedPort, parsed.port)
48 | assertions.Equal(test.ExpectedDomain, parsed.domain)
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/archive.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "archive/tar"
5 | "io"
6 | "os"
7 | "path/filepath"
8 | "sync"
9 |
10 | "github.com/reconquest/hierr-go"
11 | "github.com/reconquest/lineflushwriter-go"
12 | "github.com/reconquest/prefixwriter-go"
13 | )
14 |
15 | func startArchiveReceivers(
16 | cluster *distributedLock,
17 | rootDir string,
18 | sudo bool,
19 | serial bool,
20 | ) (*remoteExecution, error) {
21 | command := []string{
22 | "mkdir", "-p", rootDir, "&&", "tar", "--directory", rootDir, "-x",
23 | }
24 |
25 | if verbose >= verbosityDebug {
26 | command = append(command, `--verbose`)
27 | }
28 |
29 | logMutex := &sync.Mutex{}
30 |
31 | runner := &remoteExecutionRunner{
32 | command: command,
33 | serial: serial,
34 | shell: defaultRemoteExecutionShell,
35 | sudo: sudo,
36 | }
37 |
38 | execution, err := runner.run(
39 | cluster,
40 | func(node *remoteExecutionNode) {
41 | node.stdout = lineflushwriter.New(
42 | prefixwriter.New(node.stdout, "{tar} "),
43 | logMutex,
44 | true,
45 | )
46 |
47 | node.stderr = lineflushwriter.New(
48 | prefixwriter.New(node.stderr, "{tar} "),
49 | logMutex,
50 | true,
51 | )
52 | },
53 | )
54 | if err != nil {
55 | return nil, hierr.Errorf(
56 | err,
57 | `can't start tar extraction command: '%v'`,
58 | command,
59 | )
60 | }
61 |
62 | return execution, nil
63 | }
64 |
65 | func archiveFilesToWriter(
66 | target io.WriteCloser,
67 | files []file,
68 | preserveUID, preserveGID bool,
69 | ) error {
70 | workDir, err := os.Getwd()
71 | if err != nil {
72 | return hierr.Errorf(
73 | err,
74 | `can't get current working directory`,
75 | )
76 | }
77 |
78 | status := &struct {
79 | Phase string
80 | Total int
81 | Fails int
82 | Success int
83 | Written bytesStringer
84 | Bytes bytesStringer
85 | }{
86 | Phase: "upload",
87 | Total: len(files),
88 | }
89 |
90 | setStatus(status)
91 |
92 | for _, file := range files {
93 | status.Bytes.Amount += file.size
94 | }
95 |
96 | archive := tar.NewWriter(target)
97 | stream := io.MultiWriter(archive, callbackWriter(
98 | func(data []byte) (int, error) {
99 | status.Written.Amount += len(data)
100 |
101 | drawStatus()
102 |
103 | return len(data), nil
104 | },
105 | ))
106 |
107 | for fileIndex, file := range files {
108 | infof(
109 | "%5d/%d sending file: '%s'",
110 | fileIndex+1,
111 | len(files),
112 | file.path,
113 | )
114 |
115 | err = writeFileToArchive(
116 | file.path,
117 | stream,
118 | archive,
119 | workDir,
120 | preserveUID,
121 | preserveGID,
122 | )
123 | if err != nil {
124 | return hierr.Errorf(
125 | err,
126 | `can't write file to archive: '%s'`,
127 | file.path,
128 | )
129 | }
130 |
131 | status.Success++
132 | }
133 |
134 | tracef("closing archive stream, %d files sent", len(files))
135 |
136 | err = archive.Close()
137 | if err != nil {
138 | return hierr.Errorf(
139 | err,
140 | `can't close tar stream`,
141 | )
142 | }
143 |
144 | err = target.Close()
145 | if err != nil {
146 | return hierr.Errorf(
147 | err,
148 | `can't close target stdin`,
149 | )
150 | }
151 |
152 | return nil
153 | }
154 |
155 | func getFilesList(relative bool, sources ...string) ([]file, error) {
156 | files := []file{}
157 |
158 | for _, source := range sources {
159 | err := filepath.Walk(
160 | source,
161 | func(path string, info os.FileInfo, err error) error {
162 | if err != nil {
163 | return err
164 | }
165 |
166 | if info.IsDir() {
167 | return nil
168 | }
169 |
170 | if !relative {
171 | path, err = filepath.Abs(path)
172 | if err != nil {
173 | return hierr.Errorf(
174 | err,
175 | `can't get absolute path for local file: '%s'`,
176 | path,
177 | )
178 | }
179 | }
180 |
181 | files = append(files, file{
182 | path: path,
183 | size: int(info.Size()),
184 | })
185 |
186 | return nil
187 | },
188 | )
189 | if err != nil {
190 | return nil, err
191 | }
192 | }
193 |
194 | return files, nil
195 | }
196 |
--------------------------------------------------------------------------------
/archive_write_unix.go:
--------------------------------------------------------------------------------
1 | //go:build !windows
2 |
3 | package main
4 |
5 | import (
6 | "archive/tar"
7 | "fmt"
8 | "io"
9 | "os"
10 | "path/filepath"
11 | "syscall"
12 |
13 | "github.com/reconquest/hierr-go"
14 | )
15 |
16 | func writeFileToArchive(
17 | fileName string,
18 | stream io.Writer,
19 | archive *tar.Writer,
20 | workDir string,
21 | preserveUID, preserveGID bool,
22 | ) error {
23 | fileInfo, err := os.Stat(fileName)
24 | if err != nil {
25 | return hierr.Errorf(
26 | err,
27 | `can't stat file for archiving: '%s`, fileName,
28 | )
29 | }
30 |
31 | // avoid tar warnings about leading slash
32 | tarFileName := fileName
33 | if tarFileName[0] == '/' {
34 | tarFileName = tarFileName[1:]
35 |
36 | fileName, err = filepath.Rel(workDir, fileName)
37 | if err != nil {
38 | return hierr.Errorf(
39 | err,
40 | `can't make relative path from: '%s'`,
41 | fileName,
42 | )
43 | }
44 | }
45 |
46 | header := &tar.Header{
47 | Name: tarFileName,
48 | Mode: int64(fileInfo.Sys().(*syscall.Stat_t).Mode),
49 | Size: fileInfo.Size(),
50 |
51 | ModTime: fileInfo.ModTime(),
52 | }
53 |
54 | if preserveUID {
55 | header.Uid = int(fileInfo.Sys().(*syscall.Stat_t).Uid)
56 | }
57 |
58 | if preserveGID {
59 | header.Gid = int(fileInfo.Sys().(*syscall.Stat_t).Gid)
60 | }
61 |
62 | tracef(
63 | hierr.Errorf(
64 | fmt.Sprintf(
65 | "size: %d bytes; mode: %o; uid/gid: %d/%d; modtime: %s",
66 | header.Size,
67 | header.Mode,
68 | header.Uid,
69 | header.Gid,
70 | header.ModTime,
71 | ),
72 | `local file: %s; remote file: %s`,
73 | fileName,
74 | tarFileName,
75 | ).Error(),
76 | )
77 |
78 | err = archive.WriteHeader(header)
79 |
80 | if err != nil {
81 | return hierr.Errorf(
82 | err,
83 | `can't write tar header for fileName: '%s'`, fileName,
84 | )
85 | }
86 |
87 | fileToArchive, err := os.Open(fileName)
88 | if err != nil {
89 | return hierr.Errorf(
90 | err,
91 | `can't open fileName for reading: '%s'`,
92 | fileName,
93 | )
94 | }
95 |
96 | _, err = io.Copy(stream, fileToArchive)
97 | if err != nil {
98 | return hierr.Errorf(
99 | err,
100 | `can't copy file to the archive: '%s'`,
101 | fileName,
102 | )
103 | }
104 |
105 | return nil
106 | }
107 |
--------------------------------------------------------------------------------
/archive_write_windows.go:
--------------------------------------------------------------------------------
1 | //go:build windows
2 |
3 | package main
4 |
5 | import (
6 | "archive/tar"
7 | "fmt"
8 | "io"
9 | "os"
10 | "path/filepath"
11 |
12 | "github.com/reconquest/hierr-go"
13 | )
14 |
15 | func writeFileToArchive(
16 | fileName string,
17 | stream io.Writer,
18 | archive *tar.Writer,
19 | workDir string,
20 | preserveUID, preserveGID bool,
21 | ) error {
22 | fileInfo, err := os.Stat(fileName)
23 | if err != nil {
24 | return hierr.Errorf(
25 | err,
26 | `can't stat file for archiving: '%s`, fileName,
27 | )
28 | }
29 |
30 | // avoid tar warnings about leading slash
31 | tarFileName := fileName
32 | if tarFileName[0] == '/' {
33 | tarFileName = tarFileName[1:]
34 |
35 | fileName, err = filepath.Rel(workDir, fileName)
36 | if err != nil {
37 | return hierr.Errorf(
38 | err,
39 | `can't make relative path from: '%s'`,
40 | fileName,
41 | )
42 | }
43 | }
44 |
45 | header := &tar.Header{
46 | Name: tarFileName,
47 | Size: fileInfo.Size(),
48 |
49 | ModTime: fileInfo.ModTime(),
50 | }
51 |
52 | tracef(
53 | hierr.Errorf(
54 | fmt.Sprintf(
55 | "size: %d bytes; modtime: %s",
56 | header.Size,
57 | header.ModTime,
58 | ),
59 | `local file: %s; remote file: %s`,
60 | fileName,
61 | tarFileName,
62 | ).Error(),
63 | )
64 |
65 | err = archive.WriteHeader(header)
66 |
67 | if err != nil {
68 | return hierr.Errorf(
69 | err,
70 | `can't write tar header for fileName: '%s'`, fileName,
71 | )
72 | }
73 |
74 | fileToArchive, err := os.Open(fileName)
75 | if err != nil {
76 | return hierr.Errorf(
77 | err,
78 | `can't open fileName for reading: '%s'`,
79 | fileName,
80 | )
81 | }
82 |
83 | _, err = io.Copy(stream, fileToArchive)
84 | if err != nil {
85 | return hierr.Errorf(
86 | err,
87 | `can't copy file to the archive: '%s'`,
88 | fileName,
89 | )
90 | }
91 |
92 | return nil
93 | }
94 |
--------------------------------------------------------------------------------
/bytes_stringer.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | )
6 |
7 | type bytesStringer struct {
8 | Amount int
9 | }
10 |
11 | func (stringer bytesStringer) String() string {
12 | amount := float64(stringer.Amount)
13 |
14 | suffixes := map[string]string{
15 | "b": "KiB",
16 | "KiB": "MiB",
17 | "MiB": "GiB",
18 | "GiB": "TiB",
19 | }
20 |
21 | suffix := "b"
22 | for amount >= 1024 {
23 | if newSuffix, ok := suffixes[suffix]; ok {
24 | suffix = newSuffix
25 | } else {
26 | break
27 | }
28 |
29 | amount /= 1024
30 | }
31 |
32 | return fmt.Sprintf("%.2f%s", amount, suffix)
33 | }
34 |
--------------------------------------------------------------------------------
/callback_writer.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | type (
4 | callbackWriter func([]byte) (int, error)
5 | )
6 |
7 | func (writer callbackWriter) Write(data []byte) (int, error) {
8 | return writer(data)
9 | }
10 |
--------------------------------------------------------------------------------
/command.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "io"
5 | "os"
6 | "sync"
7 |
8 | "github.com/reconquest/hierr-go"
9 | "github.com/reconquest/karma-go"
10 | "github.com/reconquest/lineflushwriter-go"
11 | "github.com/reconquest/prefixwriter-go"
12 | "github.com/reconquest/runcmd"
13 | "golang.org/x/crypto/ssh"
14 | )
15 |
16 | type remoteNodesMap map[*distributedLockNode]*remoteExecutionNode
17 |
18 | type remoteNodes struct {
19 | *sync.Mutex
20 |
21 | nodes remoteNodesMap
22 | }
23 |
24 | func (nodes *remoteNodes) Set(
25 | node *distributedLockNode,
26 | remote *remoteExecutionNode,
27 | ) {
28 | nodes.Lock()
29 | defer nodes.Unlock()
30 |
31 | nodes.nodes[node] = remote
32 | }
33 |
34 | func runRemoteExecution(
35 | lockedNodes *distributedLock,
36 | command []string,
37 | setupCallback func(*remoteExecutionNode),
38 | serial bool,
39 | term bool,
40 | ) (*remoteExecution, error) {
41 | var (
42 | stdins = []io.WriteCloser{}
43 |
44 | logLock = &sync.Mutex{}
45 | stdinsLock = &sync.Mutex{}
46 | outputLock = &sync.Mutex{}
47 |
48 | nodes = &remoteNodes{&sync.Mutex{}, remoteNodesMap{}}
49 | )
50 |
51 | if !serial {
52 | outputLock = nil
53 | }
54 |
55 | status := &struct {
56 | sync.Mutex
57 |
58 | Phase string
59 | Total int
60 | Fails int
61 | Success int
62 | }{
63 | Phase: `exec`,
64 | Total: len(lockedNodes.nodes),
65 | }
66 |
67 | setStatus(status)
68 |
69 | type nodeErr struct {
70 | err error
71 | node *distributedLockNode
72 | }
73 |
74 | errors := make(chan *nodeErr, 0)
75 | for _, node := range lockedNodes.nodes {
76 | go func(node *distributedLockNode) {
77 | pool.run(func() {
78 | tracef(
79 | "%s",
80 | hierr.Errorf(
81 | command,
82 | "%s starting command",
83 | node.String(),
84 | ).Error(),
85 | )
86 |
87 | remoteNode, err := runRemoteExecutionNode(
88 | node,
89 | command,
90 | term,
91 | logLock,
92 | outputLock,
93 | )
94 | if err != nil {
95 | errors <- &nodeErr{err, node}
96 |
97 | status.Lock()
98 | defer status.Unlock()
99 |
100 | status.Total--
101 | status.Fails++
102 |
103 | return
104 | }
105 |
106 | if setupCallback != nil {
107 | setupCallback(remoteNode)
108 | }
109 |
110 | remoteNode.command.SetStdout(remoteNode.stdout)
111 | remoteNode.command.SetStderr(remoteNode.stderr)
112 |
113 | err = remoteNode.command.Start()
114 | if err != nil {
115 | errors <- &nodeErr{
116 | hierr.Errorf(
117 | err,
118 | `can't start remote command`,
119 | ),
120 | node,
121 | }
122 |
123 | status.Lock()
124 | defer status.Unlock()
125 |
126 | status.Total--
127 | status.Fails++
128 |
129 | return
130 | }
131 |
132 | nodes.Set(node, remoteNode)
133 |
134 | stdinsLock.Lock()
135 | defer stdinsLock.Unlock()
136 |
137 | stdins = append(stdins, remoteNode.stdin)
138 |
139 | status.Lock()
140 | defer status.Unlock()
141 |
142 | status.Success++
143 |
144 | errors <- nil
145 | })
146 | }(node)
147 | }
148 |
149 | for range lockedNodes.nodes {
150 | err := <-errors
151 | if err != nil {
152 | return nil, hierr.Errorf(
153 | err.err,
154 | `%s remote execution failed`,
155 | err.node,
156 | )
157 | }
158 | }
159 |
160 | return &remoteExecution{
161 | stdin: &multiWriteCloser{stdins},
162 |
163 | nodes: nodes.nodes,
164 | }, nil
165 | }
166 |
167 | func runRemoteExecutionNode(
168 | node *distributedLockNode,
169 | command []string,
170 | term bool,
171 | logLock sync.Locker,
172 | outputLock sync.Locker,
173 | ) (*remoteExecutionNode, error) {
174 | remoteCommand := node.runner.Command(command[0], command[1:]...)
175 |
176 | if term {
177 | modes := ssh.TerminalModes{
178 | ssh.ECHO: 0, // disable echoing
179 | ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
180 | ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
181 | }
182 |
183 | err := remoteCommand.(*runcmd.RemoteCmd).GetSession().RequestPty(
184 | "xterm",
185 | 40,
186 | 80,
187 | modes,
188 | )
189 | if err != nil {
190 | return nil, karma.Format(
191 | err,
192 | "request for pseudo terminal failed",
193 | )
194 | }
195 | }
196 |
197 | stdoutBackend := io.Writer(os.Stdout)
198 | stderrBackend := io.Writer(os.Stderr)
199 |
200 | if format == outputFormatJSON {
201 | stdoutBackend = &jsonOutputWriter{
202 | stream: `stdout`,
203 | node: node.String(),
204 | host: node.address.host,
205 |
206 | output: os.Stdout,
207 | }
208 |
209 | stderrBackend = &jsonOutputWriter{
210 | stream: `stderr`,
211 | node: node.String(),
212 | host: node.address.host,
213 |
214 | output: os.Stderr,
215 | }
216 | }
217 |
218 | var stdout io.WriteCloser
219 | var stderr io.WriteCloser
220 | switch {
221 | case verbose == verbosityQuiet || format == outputFormatJSON:
222 | stdout = lineflushwriter.New(nopCloser{stdoutBackend}, logLock, false)
223 | stderr = lineflushwriter.New(nopCloser{stderrBackend}, logLock, false)
224 |
225 | case verbose == verbosityNormal:
226 | stdout = lineflushwriter.New(
227 | prefixwriter.New(
228 | nopCloser{stdoutBackend},
229 | node.address.domain+" ",
230 | ),
231 | logLock,
232 | true,
233 | )
234 |
235 | stderr = lineflushwriter.New(
236 | prefixwriter.New(
237 | nopCloser{stderrBackend},
238 | node.address.domain+" ",
239 | ),
240 | logLock,
241 | true,
242 | )
243 |
244 | default:
245 | stdout = lineflushwriter.New(
246 | prefixwriter.New(
247 | newDebugWriter(logger),
248 | "{cmd} "+node.String()+" ",
249 | ),
250 | logLock,
251 | false,
252 | )
253 |
254 | stderr = lineflushwriter.New(
255 | prefixwriter.New(
256 | newDebugWriter(logger),
257 | "{cmd} "+node.String()+" ",
258 | ),
259 | logLock,
260 | false,
261 | )
262 | }
263 |
264 | stdout = &statusBarUpdateWriter{stdout}
265 | stderr = &statusBarUpdateWriter{stderr}
266 |
267 | if outputLock != (*sync.Mutex)(nil) {
268 | sharedLock := newSharedLock(outputLock, 2)
269 |
270 | stdout = newLockedWriter(stdout, sharedLock)
271 | stderr = newLockedWriter(stderr, sharedLock)
272 | }
273 |
274 | stdin, err := remoteCommand.StdinPipe()
275 | if err != nil {
276 | return nil, hierr.Errorf(
277 | err,
278 | `can't get stdin from remote command`,
279 | )
280 | }
281 |
282 | return &remoteExecutionNode{
283 | node: node,
284 | command: remoteCommand,
285 |
286 | stdin: stdin,
287 | stdout: stdout,
288 | stderr: stderr,
289 | }, nil
290 | }
291 |
--------------------------------------------------------------------------------
/connect.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "sync"
6 | "sync/atomic"
7 | "time"
8 |
9 | "github.com/reconquest/hierr-go"
10 | )
11 |
12 | const (
13 | longConnectionWarningTimeout = 2 * time.Second
14 | )
15 |
16 | // connectToCluster tries to acquire atomic file lock on each of
17 | // specified remote nodes. lockFile is used to specify target lock file, it
18 | // must exist on every node. runnerFactory will be used to make connection
19 | // to remote node. If noLockFail is given, then only warning will be printed
20 | // if lock process has been failed.
21 | func connectToCluster(
22 | lockFile string,
23 | runnerFactory runnerFactory,
24 | addresses []address,
25 | noLock bool,
26 | noLockFail bool,
27 | noConnFail bool,
28 | heartbeat func(*distributedLockNode),
29 | ) (*distributedLock, error) {
30 | var (
31 | cluster = &distributedLock{}
32 |
33 | errors = make(chan error, 0)
34 |
35 | nodeAddMutex = &sync.Mutex{}
36 | )
37 |
38 | status := &struct {
39 | Phase string
40 | Total int64
41 | Fails int64
42 | Success int64
43 | }{
44 | Phase: `lock`,
45 | Total: int64(len(addresses)),
46 | }
47 |
48 | if noLock {
49 | status.Phase = `connect`
50 | }
51 |
52 | setStatus(status)
53 |
54 | for _, nodeAddress := range addresses {
55 | go func(nodeAddress address) {
56 | pool.run(func() {
57 | failed := false
58 |
59 | node, err := connectToNode(cluster, runnerFactory, nodeAddress)
60 | if err != nil {
61 | atomic.AddInt64(&status.Fails, 1)
62 | atomic.AddInt64(&status.Total, -1)
63 |
64 | if noConnFail {
65 | failed = true
66 | warningln(err)
67 | } else {
68 | errors <- err
69 | return
70 | }
71 | } else {
72 | if !noLock {
73 | err = node.lock(lockFile)
74 | if err != nil {
75 | if noLockFail {
76 | warningln(err)
77 | } else {
78 | errors <- err
79 | return
80 | }
81 | } else {
82 | go heartbeat(node)
83 | }
84 | }
85 | }
86 |
87 | textStatus := "established"
88 | if failed {
89 | textStatus = "failed"
90 | } else {
91 | atomic.AddInt64(&status.Success, 1)
92 |
93 | nodeAddMutex.Lock()
94 | defer nodeAddMutex.Unlock()
95 |
96 | cluster.nodes = append(cluster.nodes, node)
97 | }
98 |
99 | debugf(
100 | `%4d/%d (%d failed) connection %s: %s`,
101 | status.Success,
102 | status.Total,
103 | status.Fails,
104 | textStatus,
105 | nodeAddress,
106 | )
107 |
108 | errors <- nil
109 | })
110 | }(nodeAddress)
111 | }
112 |
113 | erronous := 0
114 | topError := hierr.Push(`can't connect to nodes`)
115 | for range addresses {
116 | err := <-errors
117 | if err != nil {
118 | erronous++
119 |
120 | topError = hierr.Push(topError, err)
121 | }
122 | }
123 |
124 | if erronous > 0 {
125 | return nil, hierr.Push(
126 | fmt.Errorf(
127 | `connection to %d of %d nodes failed`,
128 | erronous,
129 | len(addresses),
130 | ),
131 | topError,
132 | )
133 | }
134 |
135 | return cluster, nil
136 | }
137 |
138 | func connectToNode(
139 | cluster *distributedLock,
140 | runnerFactory runnerFactory,
141 | address address,
142 | ) (*distributedLockNode, error) {
143 | tracef(`connecting to address: '%s'`, address)
144 |
145 | done := make(chan struct{}, 0)
146 |
147 | go func() {
148 | select {
149 | case <-done:
150 | return
151 |
152 | case <-time.After(longConnectionWarningTimeout):
153 | warningf(
154 | "still connecting to address after %s: %s",
155 | longConnectionWarningTimeout,
156 | address,
157 | )
158 |
159 | <-done
160 | }
161 | }()
162 |
163 | defer func() {
164 | done <- struct{}{}
165 | }()
166 |
167 | runner, err := runnerFactory(address)
168 | if err != nil {
169 | return nil, hierr.Errorf(
170 | err,
171 | `can't connect to address: %s`,
172 | address,
173 | )
174 | }
175 |
176 | return &distributedLockNode{
177 | address: address,
178 | runner: runner,
179 | }, nil
180 | }
181 |
--------------------------------------------------------------------------------
/debug_writer.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "strings"
5 |
6 | "github.com/kovetskiy/lorg"
7 | )
8 |
9 | type debugWriter struct {
10 | log *lorg.Log
11 | }
12 |
13 | func newDebugWriter(log *lorg.Log) debugWriter {
14 | return debugWriter{
15 | log: log,
16 | }
17 | }
18 |
19 | func (writer debugWriter) Write(data []byte) (int, error) {
20 | writer.log.Debug(strings.TrimSuffix(string(data), "\n"))
21 |
22 | return len(data), nil
23 | }
24 |
25 | func (writer debugWriter) Close() error {
26 | return nil
27 | }
28 |
--------------------------------------------------------------------------------
/demo.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reconquest/orgalorg/17aad3570a15099fc52949ac4359b350f045ca84/demo.gif
--------------------------------------------------------------------------------
/distributed_lock.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | type distributedLock struct {
4 | nodes []*distributedLockNode
5 | }
6 |
--------------------------------------------------------------------------------
/distributed_lock_node.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bufio"
5 | "fmt"
6 | "io"
7 | "strings"
8 | "sync"
9 |
10 | "github.com/reconquest/hierr-go"
11 | "github.com/reconquest/lineflushwriter-go"
12 | "github.com/reconquest/prefixwriter-go"
13 | "github.com/reconquest/runcmd"
14 | )
15 |
16 | const (
17 | lockAcquiredString = `acquired`
18 | lockLockedString = `locked`
19 | )
20 |
21 | type distributedLockNode struct {
22 | address address
23 | runner runcmd.Runner
24 |
25 | connection *distributedLockConnection
26 | }
27 |
28 | func (node *distributedLockNode) String() string {
29 | return node.address.String()
30 | }
31 |
32 | type distributedLockConnection struct {
33 | stdin io.WriteCloser
34 | stdout io.Reader
35 | }
36 |
37 | func (node *distributedLockNode) lock(
38 | filename string,
39 | ) error {
40 | lockCommandLine := []string{
41 | "sh", "-c", fmt.Sprintf(
42 | `flock -nx %s -c 'printf "%s\n" && cat' || printf "%s\n"`,
43 | filename, lockAcquiredString, lockLockedString,
44 | ),
45 | }
46 |
47 | logMutex := &sync.Mutex{}
48 |
49 | traceln(hierr.Errorf(
50 | lockCommandLine,
51 | `%s running lock command`,
52 | node,
53 | ))
54 |
55 | lockCommand := node.runner.Command(
56 | lockCommandLine[0],
57 | lockCommandLine[1:]...,
58 | )
59 |
60 | stdout, err := lockCommand.StdoutPipe()
61 | if err != nil {
62 | return hierr.Errorf(
63 | err,
64 | `can't get control stdout pipe from lock process`,
65 | )
66 | }
67 |
68 | stderr := lineflushwriter.New(
69 | prefixwriter.New(
70 | newDebugWriter(logger),
71 | fmt.Sprintf("%s {flock} ", node.String()),
72 | ),
73 | logMutex,
74 | true,
75 | )
76 |
77 | lockCommand.SetStderr(stderr)
78 |
79 | stdin, err := lockCommand.StdinPipe()
80 | if err != nil {
81 | return hierr.Errorf(
82 | err,
83 | `can't get control stdin pipe to lock process`,
84 | )
85 | }
86 |
87 | err = lockCommand.Start()
88 | if err != nil {
89 | return hierr.Errorf(
90 | err,
91 | `%s can't start lock command: '%s`,
92 | node, lockCommandLine,
93 | )
94 | }
95 |
96 | line, err := bufio.NewReader(stdout).ReadString('\n')
97 | if err != nil {
98 | return hierr.Errorf(
99 | err,
100 | `%s can't read lock status line from lock process`,
101 | node,
102 | )
103 | }
104 |
105 | switch strings.TrimSpace(line) {
106 | case lockAcquiredString:
107 | // pass
108 |
109 | case lockLockedString:
110 | return fmt.Errorf(
111 | `%s can't acquire lock, `+
112 | `lock already obtained by another process `+
113 | `or unavailable`,
114 | node,
115 | )
116 |
117 | default:
118 | return fmt.Errorf(
119 | `%s unexpected reply string encountered `+
120 | `instead of '%s' or '%s': '%s'`,
121 | node, lockAcquiredString, lockLockedString,
122 | line,
123 | )
124 | }
125 |
126 | tracef(`lock acquired: '%s' on '%s'`, node, filename)
127 |
128 | node.connection = &distributedLockConnection{
129 | stdin: stdin,
130 | stdout: stdout,
131 | }
132 |
133 | return nil
134 | }
135 |
--------------------------------------------------------------------------------
/file.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | type file struct {
4 | path string
5 | size int
6 | }
7 |
--------------------------------------------------------------------------------
/format.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/reconquest/loreley"
5 | )
6 |
7 | type (
8 | outputFormat int
9 | )
10 |
11 | const (
12 | outputFormatText outputFormat = iota
13 | outputFormatJSON
14 | )
15 |
16 | func parseOutputFormat(
17 | args map[string]interface{},
18 | ) outputFormat {
19 |
20 | formatType := outputFormatText
21 | if args["--json"].(bool) {
22 | formatType = outputFormatJSON
23 | }
24 |
25 | return formatType
26 | }
27 |
28 | func parseColorMode(args map[string]interface{}) loreley.ColorizeMode {
29 | switch args["--color"].(string) {
30 | case "always":
31 | return loreley.ColorizeAlways
32 |
33 | case "auto":
34 | return loreley.ColorizeOnTTY
35 |
36 | case "never":
37 | return loreley.ColorizeNever
38 | }
39 |
40 | return loreley.ColorizeNever
41 | }
42 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/reconquest/orgalorg
2 |
3 | go 1.23.0
4 |
5 | require (
6 | github.com/Microsoft/go-winio v0.6.2
7 | github.com/ScaleFT/sshkeys v1.2.0
8 | github.com/davidmz/go-pageant v1.0.2
9 | github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815
10 | github.com/kovetskiy/lorg v1.2.0
11 | github.com/mattn/go-shellwords v1.0.12
12 | github.com/reconquest/barely v0.0.0-20211011075640-c4e789dc39f8
13 | github.com/reconquest/colorgful v0.0.0-20210914131800-3b7d32bf77e5
14 | github.com/reconquest/hierr-go v0.0.0-20170824213838-7d09c0176fd2
15 | github.com/reconquest/karma-go v1.5.0
16 | github.com/reconquest/lineflushwriter-go v0.0.0-20200921103343-b9b8d10a6851
17 | github.com/reconquest/loreley v0.0.0-20211011075601-29b1d7b0ad91
18 | github.com/reconquest/prefixwriter-go v0.0.0-20220109120116-49d119edab5d
19 | github.com/reconquest/runcmd v0.0.0-20210624093503-38dc6a8c98bd
20 | github.com/stretchr/testify v1.8.2
21 | github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78
22 | golang.org/x/crypto v0.26.0
23 | )
24 |
25 | require (
26 | github.com/davecgh/go-spew v1.1.1 // indirect
27 | github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect
28 | github.com/pmezard/go-difflib v1.0.0 // indirect
29 | github.com/reconquest/nopio-go v0.0.0-20161213101805-20796acb207f // indirect
30 | github.com/zazab/zhash v0.0.0-20221031090444-2b0d50417446 // indirect
31 | golang.org/x/sys v0.24.0 // indirect
32 | golang.org/x/term v0.23.0 // indirect
33 | gopkg.in/yaml.v3 v3.0.1 // indirect
34 | )
35 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
2 | github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
3 | github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
4 | github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
5 | github.com/ScaleFT/sshkeys v1.2.0 h1:5BRp6rTVIhJzXT3VcUQrKgXR8zWA3sOsNeuyW15WUA8=
6 | github.com/ScaleFT/sshkeys v1.2.0/go.mod h1:gxOHeajFfvGQh/fxlC8oOKBe23xnnJTif00IFFbiT+o=
7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
10 | github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
11 | github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
12 | github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU=
13 | github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0=
14 | github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ=
15 | github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
16 | github.com/kovetskiy/lorg v1.2.0 h1:wNIUT/VOhcjKOmizDClZLvchbKFGW+dzf9fQXbSVS5E=
17 | github.com/kovetskiy/lorg v1.2.0/go.mod h1:rdiamaIRUCkX9HtFZd0D9dQqUbad21hipHk+sat7Z6s=
18 | github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
19 | github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
20 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
21 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
22 | github.com/reconquest/barely v0.0.0-20211011075640-c4e789dc39f8 h1:nKTbEVRQCCTLOTL45sE3QZQVFOu0VroPaA/BobaGJGU=
23 | github.com/reconquest/barely v0.0.0-20211011075640-c4e789dc39f8/go.mod h1:s9u11Az+/tGw07dFMZItVz57LvjhdeGG8gLRladQ170=
24 | github.com/reconquest/colorgful v0.0.0-20210914131800-3b7d32bf77e5 h1:zQyJkHMuhL9DZpK2TgepcpW5eRbuyzgBEaE1TMsemJc=
25 | github.com/reconquest/colorgful v0.0.0-20210914131800-3b7d32bf77e5/go.mod h1:osXRwScBYylQ/ZvfwWXOIqGcjRCk+p0nc5oFnqqgVAY=
26 | github.com/reconquest/hierr-go v0.0.0-20170824213838-7d09c0176fd2 h1:v3ctQXyIHprCI7s42LGmMYbulnnpjeD4zcvq78D1ung=
27 | github.com/reconquest/hierr-go v0.0.0-20170824213838-7d09c0176fd2/go.mod h1:dF8sYs86hXr+kKjDVvxDZYMUsTm5yr0PFQqLer/xsrk=
28 | github.com/reconquest/karma-go v0.0.0-20200928103525-22da92476de6/go.mod h1:yuQiKpTdmXSX7E+h+3dD4jx09P/gHc67mRxN3eFLt7o=
29 | github.com/reconquest/karma-go v1.5.0 h1:Chn4LtauwnvKfz13ZbmGNrRLKO1NciExHQSOBOsQqt4=
30 | github.com/reconquest/karma-go v1.5.0/go.mod h1:52XRXXa2ec/VNrlCirwasdJfNmjI1O87q098gmqILh0=
31 | github.com/reconquest/lineflushwriter-go v0.0.0-20200921103343-b9b8d10a6851 h1:o2J6abTvFylmmepI2GgNSeZZlUUz4xz3UStWIY6EXLk=
32 | github.com/reconquest/lineflushwriter-go v0.0.0-20200921103343-b9b8d10a6851/go.mod h1:tpO+vNC9ppBvJErZXrQpfwiVDuza5k1nRWH7SrutKtY=
33 | github.com/reconquest/loreley v0.0.0-20211011075601-29b1d7b0ad91 h1:HuOjsuPj5mcsq/EagIovIS240SNUA+u4vLak5GcS12s=
34 | github.com/reconquest/loreley v0.0.0-20211011075601-29b1d7b0ad91/go.mod h1:nz+IRceapGhOTxO5cOJiu0Xc2qfEkEnR+qS6damNixI=
35 | github.com/reconquest/nopio-go v0.0.0-20161213101805-20796acb207f h1:IA86P8l4xRMNbd021H3fGgdIT/B5D/bvdT0xsMqOC/c=
36 | github.com/reconquest/nopio-go v0.0.0-20161213101805-20796acb207f/go.mod h1:xyNcU6XK6bQpR3pVBIHFPi4Xh6NV/FBKnWkYV8lwv1k=
37 | github.com/reconquest/prefixwriter-go v0.0.0-20220109120116-49d119edab5d h1:RXcAP8D7DgwBgECxBvTKurPpWU+yGGEPg+HkvrF7hWI=
38 | github.com/reconquest/prefixwriter-go v0.0.0-20220109120116-49d119edab5d/go.mod h1:vJ27/NK0BF5L4HaVVIQZRqOrGnCMRD0CraEjyPjK5HI=
39 | github.com/reconquest/runcmd v0.0.0-20210624093503-38dc6a8c98bd h1:Smgn0s4JKslWhFSzODN7jqNCNYbr+OPqF5tuqFCsReY=
40 | github.com/reconquest/runcmd v0.0.0-20210624093503-38dc6a8c98bd/go.mod h1:Za2j8JxxWX/f/NIdRd5wPhxQVGLnZUH4xcaAPUxXJz4=
41 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
42 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
43 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
44 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
45 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
46 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
47 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
48 | github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
49 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
50 | github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
51 | github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
52 | github.com/zazab/zhash v0.0.0-20221031090444-2b0d50417446 h1:75pcOSsb40+ub185cJI7g5uykl9Uu76rD5ONzK/4s40=
53 | github.com/zazab/zhash v0.0.0-20221031090444-2b0d50417446/go.mod h1:NtepZ8TEXErPsmQDMUoN72f8aIy4+xNinSJ3f1giess=
54 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
55 | golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
56 | golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
57 | golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
58 | golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
59 | golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
60 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
61 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
62 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
63 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
64 | golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
65 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
66 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
67 | golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
68 | golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
69 | golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
70 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
71 | golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
72 | golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
73 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
74 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
75 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
76 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
77 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
78 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
79 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
80 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
81 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
82 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
83 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
84 |
--------------------------------------------------------------------------------
/heartbeat.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bufio"
5 | "io"
6 | "strings"
7 | "sync"
8 | "time"
9 |
10 | "github.com/reconquest/hierr-go"
11 | "github.com/reconquest/runcmd"
12 | )
13 |
14 | const (
15 | heartbeatPing = "PING"
16 | )
17 |
18 | // heartbeat runs infinite process of sending test messages to the connected
19 | // node. All heartbeats to all nodes are connected to each other, so if one
20 | // heartbeat routine exits, all heartbeat routines will exit, because in that
21 | // case orgalorg can't guarantee global lock.
22 | func heartbeat(
23 | period time.Duration,
24 | node *distributedLockNode,
25 | canceler *sync.Cond,
26 | ) {
27 | abort := make(chan struct{}, 0)
28 |
29 | // Internal go-routine for listening abort broadcast and finishing current
30 | // heartbeat process.
31 | go func() {
32 | canceler.L.Lock()
33 | canceler.Wait()
34 | canceler.L.Unlock()
35 |
36 | abort <- struct{}{}
37 | }()
38 |
39 | // Finish finishes current go-routine and send abort broadcast to all
40 | // connected go-routines.
41 | finish := func(code int) {
42 | canceler.L.Lock()
43 | canceler.Broadcast()
44 | canceler.L.Unlock()
45 |
46 | <-abort
47 |
48 | if remote, ok := node.runner.(*runcmd.Remote); ok {
49 | tracef("%s closing connection", node.String())
50 | err := remote.CloseConnection()
51 | if err != nil {
52 | warningf(
53 | "%s",
54 | hierr.Errorf(
55 | err,
56 | "%s error while closing connection",
57 | node.String(),
58 | ),
59 | )
60 | }
61 | }
62 |
63 | exit(code)
64 | }
65 |
66 | ticker := time.Tick(period)
67 |
68 | // Infinite loop of heartbeating. It will send heartbeat message, wait
69 | // fraction of send timeout time and try to receive heartbeat response.
70 | // If no response received, heartbeat process aborts.
71 | for {
72 | _, err := io.WriteString(node.connection.stdin, heartbeatPing+"\n")
73 | if err != nil {
74 | errorf(
75 | "%s",
76 | hierr.Errorf(
77 | err,
78 | `%s can't send heartbeat`,
79 | node.String(),
80 | ),
81 | )
82 |
83 | finish(2)
84 | }
85 |
86 | select {
87 | case <-abort:
88 | return
89 |
90 | case <-ticker:
91 | // pass
92 | }
93 |
94 | ping, err := bufio.NewReader(node.connection.stdout).ReadString('\n')
95 | if err != nil {
96 | errorf(
97 | "%s",
98 | hierr.Errorf(
99 | err,
100 | `%s can't receive heartbeat`,
101 | node.String(),
102 | ),
103 | )
104 |
105 | finish(2)
106 | }
107 |
108 | if strings.TrimSpace(ping) != heartbeatPing {
109 | errorf(
110 | `%s received unexpected heartbeat ping: '%s'`,
111 | node.String(),
112 | ping,
113 | )
114 |
115 | finish(2)
116 | }
117 |
118 | tracef(`%s heartbeat`, node.String())
119 | }
120 | }
121 |
--------------------------------------------------------------------------------
/json_output_writer.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "io"
6 | )
7 |
8 | type jsonOutputWriter struct {
9 | stream string
10 | node string
11 | host string
12 |
13 | output io.Writer
14 | }
15 |
16 | func (writer *jsonOutputWriter) Write(data []byte) (int, error) {
17 | if len(data) == 0 {
18 | return 0, nil
19 | }
20 |
21 | message := map[string]interface{}{
22 | "stream": writer.stream,
23 | }
24 |
25 | if writer.node == "" {
26 | message["node"] = nil
27 | } else {
28 | message["node"] = writer.node
29 | }
30 |
31 | if writer.host == "" {
32 | message["host"] = nil
33 | } else {
34 | message["host"] = writer.host
35 | }
36 |
37 | message["body"] = string(data)
38 |
39 | jsonMessage, err := json.Marshal(message)
40 | if err != nil {
41 | return 0, err
42 | }
43 |
44 | _, err = writer.output.Write(append(jsonMessage, '\n'))
45 | if err != nil {
46 | return 0, err
47 | }
48 |
49 | return len(data), nil
50 | }
51 |
--------------------------------------------------------------------------------
/key.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/pem"
5 | "errors"
6 | "fmt"
7 | "io/ioutil"
8 | "os"
9 | "strings"
10 |
11 | "github.com/ScaleFT/sshkeys"
12 | "github.com/reconquest/hierr-go"
13 | "github.com/youmark/pkcs8"
14 | "golang.org/x/crypto/ssh"
15 | "golang.org/x/crypto/ssh/agent"
16 | "golang.org/x/crypto/ssh/terminal"
17 | )
18 |
19 | type sshKey struct {
20 | raw []byte
21 | block *pem.Block
22 | extra []byte
23 | private interface{}
24 | passphrase []byte
25 | }
26 |
27 | func (key *sshKey) validate() error {
28 | if len(key.extra) != 0 {
29 | return hierr.Errorf(
30 | errors.New(string(key.extra)),
31 | `extra data found in the SSH key`,
32 | )
33 | }
34 |
35 | return nil
36 | }
37 |
38 | func (key *sshKey) isOpenSSH() bool {
39 | return key.block.Type == "OPENSSH PRIVATE KEY"
40 | }
41 |
42 | func (key *sshKey) isPKCS8() bool {
43 | return key.block.Type == "ENCRYPTED PRIVATE KEY" ||
44 | key.block.Type == "PRIVATE KEY"
45 | }
46 |
47 | func (key *sshKey) isEncrypted() bool {
48 | if key.block.Type == "ENCRYPTED PRIVATE KEY" {
49 | return true
50 | }
51 |
52 | if strings.Contains(key.block.Headers["Proc-Type"], "ENCRYPTED") {
53 | return true
54 | }
55 |
56 | if key.isOpenSSH() {
57 | _, err := ssh.ParseRawPrivateKey([]byte(key.raw))
58 | return err != nil
59 | }
60 |
61 | return false
62 | }
63 |
64 | func (key *sshKey) parse() error {
65 | var err error
66 | switch {
67 | case key.isOpenSSH() && key.isEncrypted():
68 | key.private, err = sshkeys.ParseEncryptedRawPrivateKey(
69 | []byte(key.raw),
70 | key.passphrase,
71 | )
72 |
73 | case key.isPKCS8() && key.isEncrypted():
74 | key.private, err = pkcs8.ParsePKCS8PrivateKey(
75 | key.block.Bytes,
76 | key.passphrase,
77 | )
78 |
79 | case key.isPKCS8():
80 | key.private, err = pkcs8.ParsePKCS8PrivateKey(
81 | key.block.Bytes,
82 | nil,
83 | )
84 |
85 | case key.isEncrypted():
86 | key.private, err = ssh.ParseRawPrivateKeyWithPassphrase(
87 | []byte(key.raw),
88 | key.passphrase,
89 | )
90 |
91 | default:
92 | key.private, err = ssh.ParseRawPrivateKey(
93 | []byte(key.raw),
94 | )
95 | }
96 | return err
97 | }
98 |
99 | func readSSHKey(keyring agent.Agent, path string) error {
100 | var key sshKey
101 | var err error
102 |
103 | key.raw, err = ioutil.ReadFile(path)
104 | if err != nil {
105 | return hierr.Errorf(
106 | err,
107 | `can't read SSH key from file`,
108 | )
109 | }
110 |
111 | key.block, key.extra = pem.Decode(key.raw)
112 |
113 | err = key.validate()
114 | if err != nil {
115 | return err
116 | }
117 |
118 | if key.isEncrypted() {
119 | key.passphrase, err = readPassword(sshPassphrasePrompt)
120 | if err != nil {
121 | return hierr.Errorf(
122 | err,
123 | `can't read key passphrase`,
124 | )
125 | }
126 | }
127 |
128 | err = key.parse()
129 | if err != nil {
130 | if err == sshkeys.ErrIncorrectPassword {
131 | err = errors.New("invalid passphrase for private key specified")
132 | }
133 |
134 | return hierr.Errorf(
135 | err,
136 | "unable to parse ssh key",
137 | )
138 | }
139 |
140 | keyring.Add(agent.AddedKey{
141 | PrivateKey: key.private,
142 | Comment: "passed by orgalorg",
143 | })
144 |
145 | return nil
146 | }
147 |
148 | func readPassword(prompt string) ([]byte, error) {
149 | fmt.Fprintf(os.Stderr, prompt)
150 |
151 | tty, err := os.Open("/dev/tty")
152 | if err != nil {
153 | return nil, hierr.Errorf(
154 | err,
155 | `TTY is required for reading password, `+
156 | `but /dev/tty can't be opened`,
157 | )
158 | }
159 |
160 | password, err := terminal.ReadPassword(int(tty.Fd()))
161 | if err != nil {
162 | return nil, hierr.Errorf(
163 | err,
164 | `can't read password`,
165 | )
166 | }
167 |
168 | if prompt != "" {
169 | fmt.Fprintln(os.Stderr)
170 | }
171 |
172 | return password, nil
173 | }
174 |
--------------------------------------------------------------------------------
/locked_writer.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "io"
5 | "sync"
6 | )
7 |
8 | type sharedLock struct {
9 | sync.Locker
10 |
11 | held *struct {
12 | sync.Locker
13 |
14 | clients int
15 | locked bool
16 | }
17 | }
18 |
19 | func newSharedLock(lock sync.Locker, clients int) *sharedLock {
20 | return &sharedLock{
21 | Locker: lock,
22 |
23 | held: &struct {
24 | sync.Locker
25 |
26 | clients int
27 | locked bool
28 | }{
29 | Locker: &sync.Mutex{},
30 |
31 | clients: clients,
32 | locked: false,
33 | },
34 | }
35 | }
36 |
37 | func (mutex *sharedLock) Lock() {
38 | mutex.held.Lock()
39 | defer mutex.held.Unlock()
40 |
41 | if !mutex.held.locked {
42 | mutex.Locker.Lock()
43 |
44 | mutex.held.locked = true
45 | }
46 | }
47 |
48 | func (mutex *sharedLock) Unlock() {
49 | mutex.held.Lock()
50 | defer mutex.held.Unlock()
51 |
52 | mutex.held.clients--
53 |
54 | if mutex.held.clients == 0 && mutex.held.locked {
55 | mutex.held.locked = false
56 |
57 | mutex.Locker.Unlock()
58 | }
59 | }
60 |
61 | type lockedWriter struct {
62 | writer io.WriteCloser
63 |
64 | lock sync.Locker
65 | }
66 |
67 | func newLockedWriter(
68 | writer io.WriteCloser,
69 | lock sync.Locker,
70 | ) *lockedWriter {
71 | return &lockedWriter{
72 | writer: writer,
73 | lock: lock,
74 | }
75 | }
76 |
77 | func (writer *lockedWriter) Write(data []byte) (int, error) {
78 | writer.lock.Lock()
79 |
80 | return writer.writer.Write(data)
81 | }
82 |
83 | func (writer *lockedWriter) Close() error {
84 | writer.lock.Unlock()
85 |
86 | return writer.writer.Close()
87 | }
88 |
--------------------------------------------------------------------------------
/log.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "strings"
7 |
8 | "github.com/kovetskiy/lorg"
9 | "github.com/reconquest/hierr-go"
10 | )
11 |
12 | func setLoggerOutputFormat(logger *lorg.Log, format outputFormat) {
13 | if format == outputFormatJSON {
14 | logger.SetOutput(&jsonOutputWriter{
15 | stream: `stderr`,
16 | node: ``,
17 | output: os.Stderr,
18 | })
19 | }
20 | }
21 |
22 | func setLoggerVerbosity(level verbosity, logger *lorg.Log) {
23 | logger.SetLevel(lorg.LevelWarning)
24 |
25 | switch {
26 | case level >= verbosityTrace:
27 | logger.SetLevel(lorg.LevelTrace)
28 |
29 | case level >= verbosityDebug:
30 | logger.SetLevel(lorg.LevelDebug)
31 |
32 | case level >= verbosityNormal:
33 | logger.SetLevel(lorg.LevelInfo)
34 | }
35 | }
36 |
37 | func setLoggerStyle(logger *lorg.Log, style lorg.Formatter) {
38 | logger.SetFormat(style)
39 | logger.SetIndentLines(true)
40 |
41 | logger.SetShiftIndent(28)
42 | }
43 |
44 | func tracef(format string, args ...interface{}) {
45 | args = serializeErrors(args)
46 |
47 | logger.Tracef(format, args...)
48 |
49 | drawStatus()
50 | }
51 |
52 | func traceln(args ...interface{}) {
53 | tracef("%s", fmt.Sprint(serializeErrors(args)...))
54 | }
55 |
56 | func debugf(format string, args ...interface{}) {
57 | args = serializeErrors(args)
58 |
59 | logger.Debugf(format, args...)
60 |
61 | drawStatus()
62 | }
63 |
64 | func debugln(args ...interface{}) {
65 | debugf("%s", fmt.Sprint(serializeErrors(args)...))
66 | }
67 |
68 | func infof(format string, args ...interface{}) {
69 | args = serializeErrors(args)
70 |
71 | logger.Infof(format, args...)
72 |
73 | drawStatus()
74 | }
75 |
76 | func infoln(args ...interface{}) {
77 | infof("%s", fmt.Sprint(serializeErrors(args)...))
78 | }
79 |
80 | func warningf(format string, args ...interface{}) {
81 | args = serializeErrors(args)
82 |
83 | if verbose <= verbosityQuiet {
84 | return
85 | }
86 |
87 | logger.Warningf(format, args...)
88 |
89 | drawStatus()
90 | }
91 |
92 | func warningln(args ...interface{}) {
93 | warningf("%s", fmt.Sprint(serializeErrors(args)...))
94 | }
95 |
96 | func errorf(format string, args ...interface{}) {
97 | args = serializeErrors(args)
98 |
99 | logger.Errorf(format, args...)
100 | }
101 |
102 | func errorln(args ...interface{}) {
103 | errorf("%s", fmt.Sprint(serializeErrors(args)...))
104 | }
105 |
106 | func fatalf(format string, args ...interface{}) {
107 | args = serializeErrors(args)
108 |
109 | clearStatus()
110 |
111 | logger.Fatalf(format, args...)
112 |
113 | exit(1)
114 | }
115 |
116 | func fatalln(args ...interface{}) {
117 | fatalf("%s", fmt.Sprint(serializeErrors(args)...))
118 | }
119 |
120 | func serializeErrors(args []interface{}) []interface{} {
121 | for i, arg := range args {
122 | if err, ok := arg.(error); ok {
123 | args[i] = serializeError(err)
124 | }
125 | }
126 |
127 | return args
128 | }
129 |
130 | func setStatus(status interface{}) {
131 | if statusbar == nil {
132 | return
133 | }
134 |
135 | clearStatus()
136 |
137 | statusbar.SetStatus(status)
138 |
139 | drawStatus()
140 | }
141 |
142 | func shouldDrawStatus() bool {
143 | if statusbar == nil {
144 | return false
145 | }
146 |
147 | if format != outputFormatText {
148 | return false
149 | }
150 |
151 | if verbose <= verbosityQuiet {
152 | return false
153 | }
154 |
155 | return true
156 | }
157 |
158 | func drawStatus() {
159 | if !shouldDrawStatus() {
160 | return
161 | }
162 |
163 | err := statusbar.Render(os.Stderr)
164 | if err != nil {
165 | errorf(
166 | "%s", hierr.Errorf(
167 | err,
168 | `can't draw status bar`,
169 | ),
170 | )
171 | }
172 | }
173 |
174 | func clearStatus() {
175 | if !shouldDrawStatus() {
176 | return
177 | }
178 |
179 | statusbar.Clear(os.Stderr)
180 | }
181 |
182 | func serializeError(err error) string {
183 | if format == outputFormatText {
184 | return fmt.Sprint(err)
185 | }
186 |
187 | if hierarchicalError, ok := err.(hierr.Error); ok {
188 | serializedError := fmt.Sprint(hierarchicalError.Nested)
189 | switch nested := hierarchicalError.Nested.(type) {
190 | case error:
191 | serializedError = serializeError(nested)
192 |
193 | case []hierr.NestedError:
194 | serializeErrorParts := []string{}
195 |
196 | for _, nestedPart := range nested {
197 | serializedPart := fmt.Sprint(nestedPart)
198 | switch part := nestedPart.(type) {
199 | case error:
200 | serializedPart = serializeError(part)
201 |
202 | case string:
203 | serializedPart = part
204 | }
205 |
206 | serializeErrorParts = append(
207 | serializeErrorParts,
208 | serializedPart,
209 | )
210 | }
211 |
212 | serializedError = strings.Join(serializeErrorParts, "; ")
213 | }
214 |
215 | return hierarchicalError.Message + ": " + serializedError
216 | }
217 |
218 | return err.Error()
219 | }
220 |
--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bufio"
5 | "io"
6 | "os"
7 | "os/user"
8 | "path/filepath"
9 | "strconv"
10 | "strings"
11 | "sync"
12 | "time"
13 |
14 | "github.com/docopt/docopt-go"
15 | "github.com/kovetskiy/lorg"
16 | "github.com/mattn/go-shellwords"
17 | "github.com/reconquest/barely"
18 | "github.com/reconquest/hierr-go"
19 | "github.com/reconquest/loreley"
20 | "github.com/reconquest/runcmd"
21 | "golang.org/x/crypto/ssh/agent"
22 | )
23 |
24 | var version = "[manual build]"
25 |
26 | const usage = `orgalorg - files synchronization on many hosts.
27 |
28 | First of all, orgalorg will try to acquire global cluster lock by flock'ing
29 | file, specified by '--lock-file' on each host. If at least one flock fails,
30 | then orgalorg will stop, unless '-t' flag is specified.
31 |
32 | orgalorg will create tar-archive from specified files, keeping file attributes
33 | and ownerships, then upload archive in parallel to the specified hosts and
34 | unpack it in the temporary directory (see '-r'). No further actions will be
35 | done until all hosts will unpack the archive.
36 |
37 | If '-S' flag specified, then sync command tool will be launched after upload
38 | (see '--sync-cmd'). Sync command tool can send stdout and stderr back to the
39 | orgalorg, but it needs to be compatible with following procotol.
40 |
41 | First of all, sync command tool and orgalorg communicate through stdout/stdin.
42 | All lines, that are not match protocol will be printed untouched.
43 |
44 | orgalorg first send hello message to the each running node, where ''
45 | is an unique string
46 |
47 | HELLO
48 |
49 | All consequent communication must be prefixed by that prefix, followed by
50 | space.
51 |
52 | Then, orgalorg will pass nodes list to each running node by sending 'NODE'
53 | commands, where '' is unique node identifier:
54 |
55 | NODE [CURRENT]
56 |
57 | 'CURRENT' will present next to the node record which currently executes sync
58 | tool.
59 |
60 | After nodes list is exchausted, orgalorg will send 'START' marker, that means
61 | sync tool may proceed with execution.
62 |
63 | START
64 |
65 | Then, sync command tool can reply with 'SYNC' messages, that will be
66 | broadcasted to all connected nodes by orgalorg:
67 |
68 | SYNC
69 |
70 | Broadcasted sync message will contain source node:
71 |
72 | SYNC
73 |
74 | Each node can decide, when to wait synchronizations, based on amount of
75 | received sync messages.
76 |
77 | Usage:
78 | orgalorg -h | --help
79 | orgalorg [options] [-v]... (-o ...|-s) -r= -U ...
80 | orgalorg [options] [-v]... (-o ...|-s) [-r=] [-g=]... -S [...|--no-upload]
81 | orgalorg [options] [-v]... (-o ...|-s) [-r=] -C [--] ...
82 | orgalorg [options] [-v]... (-o ...|-s) -L
83 |
84 | Operation mode options:
85 | -S --sync Sync.
86 | Synchronizes files on the specified hosts via 3-stage
87 | process:
88 | * global cluster locking (use -L to stop here);
89 | * tar-ing files on local machine, transmitting and
90 | unpacking files to the intermediate directory
91 | (-U to stop here);
92 | * launching sync command tool such as gunter;
93 | -L --lock Will stop right after locking, e.g. will not try to
94 | do sync whatsoever. Will keep lock until interrupted.
95 | -U --upload Upload files to specified directory and exit.
96 | -C --command Run specified command on all hosts and exit.
97 |
98 | Required options:
99 | -o --host Target host in format [@][:].
100 | If value is started from '/' or from './', then it's
101 | considered file which should be used to read hosts
102 | from.
103 | -s --read-stdin Read hosts from stdin in addition to other flags.
104 |
105 | Options:
106 | -h --help Show this help.
107 | -k --key Identity file (private key), which will be used for
108 | authentication. This is default way of
109 | authentication.
110 | [default: $HOME/.ssh/id_rsa]
111 | -p --password Enable password authentication.
112 | Exclude '-k' option.
113 | Interactive TTY is required for reading password.
114 | -x --sudo Obtain root via 'sudo -n'.
115 | By default, orgalorg will not obtain root and do
116 | all actions from specified user. To change that
117 | behaviour, this option can be used.
118 | -y --no-lock Do not lock at all.
119 | -t --no-lock-fail Try to obtain global lock, but only print warning if
120 | it cannot be done, do not stop execution.
121 | -w --no-conn-fail Skip unreachable servers whatsoever.
122 | -r --root Specify root dir to extract files into.
123 | By default, orgalorg will create temporary directory
124 | inside of '$ROOT'.
125 | Removal of that directory is up to sync tool.
126 | -u --user Username used for connecting to all hosts by default.
127 | [default: $USER]
128 | -i --stdin Pass specified file as input for the command.
129 | -l --serial Run commands in serial mode, so they output will not
130 | interleave each other. Only one node is allowed to
131 | output, all other nodes will wait that node to
132 | finish.
133 | -q --quiet Be quiet, in command mode do not use prefixes.
134 | -v --verbose Print debug information on stderr.
135 | -V --version Print program version.
136 |
137 | Advanced options:
138 | --lock-file File to put lock onto. If not specified, value of '-r'
139 | will be used. If '-r' is not specified too, then
140 | use "$LOCK" as lock file.
141 | -e --relative Upload files by relative path. By default, all
142 | specified files will be uploaded on the target
143 | hosts by absolute paths, e.g. if you running
144 | orgalorg from '/tmp' dir with argument '-S x',
145 | then file will be uploaded into '/tmp/x' on the
146 | remote hosts. That option switches off that
147 | behavior.
148 | -n --sync-cmd Run specified sync command tool on each remote node.
149 | Orgalorg will communicate with sync command tool via
150 | stdin. See 'Protocol commands' below.
151 | [default: /usr/lib/orgalorg/sync "${@}"]
152 | -g --arg Arguments to pass untouched to the sync command tool.
153 | No modification will be done to the passed arg, so
154 | take care about escaping.
155 | -m --simple Treat sync command as simple tool, which is not
156 | support specified protocol messages. No sync
157 | is possible in that case and all stdout and stderr
158 | will be passed untouched back to the orgalorg.
159 | --shell Use following shell wrapper. '{}' will be replaced
160 | with properly escaped command. If empty, then no
161 | shell wrapper will be used. If any args are given
162 | using '-g', they will be appended to shell
163 | invocation.
164 | [default: ` + defaultRemoteExecutionShell + `]
165 | -d --threads Set threads count which will be used for connection,
166 | locking and execution commands.
167 | [default: 16].
168 | ` + helpNoPreserveUidGid + ` --no-upload Do not upload files while syncing.
169 |
170 | Output format and colors options:
171 | --json Output everything in line-by-line JSON format,
172 | printing objects with fields:
173 | * 'stream' = 'stdout' | 'stderr';
174 | * 'node' = | null (if internal output);
175 | * 'body' =
176 | --bar-format Format for the status bar.
177 | Full Go template syntax is available with delims
178 | of '{' and '}'.
179 | See https://github.com/reconquest/barely for more
180 | info.
181 | For example, run orgalorg with '-vv' flag.
182 | Two embedded themes are available by their names:
183 | ` + themeDark + ` and ` + themeLight + `
184 | [default: ` + themeDefault + `]
185 | --log-format Format for the logs.
186 | See https://github.com/reconquest/colorgful for more
187 | info.
188 | [default: ` + themeDefault + `]
189 | --colors-dark Set all available formats to predefined dark theme.
190 | --colors-light Set all available formats to predefined light theme.
191 | --color Specify, whether to use colors:
192 | * never - disable colors;
193 | * auto - use colors only when TTY presents.
194 | * always - always use colorized output.
195 | [default: auto]
196 |
197 | Timeout options:
198 | -c --conn-timeout Remote host connection timeout in milliseconds.
199 | [default: 10000]
200 | -j --send-timeout Remote host connection data sending timeout in
201 | milliseconds. [default: 60000]
202 | NOTE: send timeout will be also used for the
203 | heartbeat messages, that orgalorg and connected nodes
204 | exchanges through synchronization process.
205 | -z --recv-timeout Remote host connection data receiving timeout in
206 | milliseconds. [default: 60000]
207 | -a --keep-alive How long to keep connection keeped alive after session
208 | ends. [default: 10000]
209 | `
210 |
211 | const (
212 | defaultSSHPort = 22
213 |
214 | // heartbeatTimeoutCoefficient will be multiplied to send timeout and
215 | // resulting value will be used as time interval between heartbeats.
216 | heartbeatTimeoutCoefficient = 0.8
217 |
218 | runsDirectory = "/var/run/orgalorg/"
219 |
220 | defaultLockFile = "/"
221 |
222 | defaultRemoteExecutionShell = "bash -c '{}'"
223 | )
224 |
225 | var (
226 | sshPasswordPrompt = "Password: "
227 | sshPassphrasePrompt = "Key passphrase: "
228 | )
229 |
230 | var (
231 | logger = lorg.NewLog()
232 | verbose = verbosityNormal
233 | format = outputFormatText
234 |
235 | pool *threadPool
236 | statusbar *barely.StatusBar
237 | )
238 |
239 | var exit = os.Exit
240 |
241 | func main() {
242 | args := parseArgs()
243 |
244 | verbose = parseVerbosity(args)
245 |
246 | setLoggerVerbosity(verbose, logger)
247 |
248 | format = parseOutputFormat(args)
249 |
250 | setLoggerOutputFormat(logger, format)
251 |
252 | loreley.Colorize = parseColorMode(args)
253 |
254 | loggerStyle, err := getLoggerTheme(parseTheme("log", args))
255 | if err != nil {
256 | fatalln(hierr.Errorf(
257 | err,
258 | `can't use given logger style`,
259 | ))
260 | }
261 |
262 | setLoggerStyle(logger, loggerStyle)
263 |
264 | poolSize, err := parseThreadPoolSize(args)
265 | if err != nil {
266 | errorln(hierr.Errorf(
267 | err,
268 | `--threads given invalid value`,
269 | ))
270 | }
271 |
272 | pool = newThreadPool(poolSize)
273 |
274 | setupInteractiveMode(args)
275 |
276 | switch {
277 | case args["--upload"].(bool):
278 | fallthrough
279 | case args["--lock"].(bool):
280 | fallthrough
281 | case args["--sync"].(bool):
282 | err = handleSynchronize(args)
283 | case args["--command"].(bool):
284 | err = handleEvaluate(args)
285 | }
286 |
287 | if err != nil {
288 | fatalln(err)
289 | }
290 |
291 | clearStatus()
292 | }
293 |
294 | func parseArgs() map[string]interface{} {
295 | usage, err := formatUsage(string(usage))
296 | if err != nil {
297 | fatalln(hierr.Errorf(
298 | err,
299 | `can't format usage`,
300 | ))
301 | }
302 |
303 | args, err := docopt.Parse(usage, nil, true, version, true)
304 | if err != nil {
305 | panic(err)
306 | }
307 |
308 | return args
309 | }
310 |
311 | func formatUsage(template string) (string, error) {
312 | currentUser, err := user.Current()
313 | if err != nil {
314 | return "", hierr.Errorf(
315 | err,
316 | `can't get current user`,
317 | )
318 | }
319 |
320 | usage := template
321 |
322 | usage = strings.Replace(usage, "$USER", currentUser.Username, -1)
323 | usage = strings.Replace(usage, "$HOME", currentUser.HomeDir, -1)
324 | usage = strings.Replace(usage, "$ROOT", runsDirectory, -1)
325 | usage = strings.Replace(usage, "$LOCK", defaultLockFile, -1)
326 |
327 | return usage, nil
328 | }
329 |
330 | func handleEvaluate(args map[string]interface{}) error {
331 | var (
332 | stdin, hasStdin = args["--stdin"].(string)
333 | rootDir, _ = args["--root"].(string)
334 | sudo = args["--sudo"].(bool)
335 | shell = args["--shell"].(string)
336 | serial = args["--serial"].(bool)
337 |
338 | command = args[""].([]string)
339 | )
340 |
341 | canceler := sync.NewCond(&sync.Mutex{})
342 |
343 | cluster, err := connectAndLock(args, canceler)
344 | if err != nil {
345 | return err
346 | }
347 |
348 | runner := &remoteExecutionRunner{
349 | shell: shell,
350 | sudo: sudo,
351 | command: command,
352 | directory: rootDir,
353 | serial: serial,
354 | term: !hasStdin,
355 | }
356 |
357 | return run(cluster, runner, stdin)
358 | }
359 |
360 | func run(
361 | cluster *distributedLock,
362 | runner *remoteExecutionRunner,
363 | stdin string,
364 | ) error {
365 | execution, err := runner.run(cluster, nil)
366 | if err != nil {
367 | return hierr.Errorf(
368 | err,
369 | `can't run remote execution on %d nodes`,
370 | len(cluster.nodes),
371 | )
372 | }
373 |
374 | if stdin != "" {
375 | var inputFile *os.File
376 |
377 | inputFile, err = os.Open(stdin)
378 | if err != nil {
379 | return hierr.Errorf(
380 | err,
381 | `can't open file for passing as stdin: '%s'`,
382 | stdin,
383 | )
384 | }
385 |
386 | defer inputFile.Close()
387 |
388 | _, err = io.Copy(execution.stdin, inputFile)
389 | if err != nil {
390 | return hierr.Errorf(
391 | err,
392 | `can't copy input file to the execution processes`,
393 | )
394 | }
395 |
396 | }
397 |
398 | debugf(`commands are running, waiting for finish`)
399 |
400 | err = execution.stdin.Close()
401 | if err != nil {
402 | return hierr.Errorf(
403 | err,
404 | `can't close stdin`,
405 | )
406 | }
407 |
408 | err = execution.wait()
409 | if err != nil {
410 | return hierr.Errorf(
411 | err,
412 | `remote execution failed, because one of `+
413 | `command has been exited with non-zero exit `+
414 | `code (or timed out) at least on one node`,
415 | )
416 | }
417 |
418 | return nil
419 | }
420 |
421 | func handleSynchronize(args map[string]interface{}) error {
422 | var (
423 | stdin, _ = args["--stdin"].(string)
424 | rootDir, _ = args["--root"].(string)
425 | lockOnly = args["--lock"].(bool)
426 | uploadOnly = args["--upload"].(bool)
427 | relative = args["--relative"].(bool)
428 |
429 | isSimpleCommand = args["--simple"].(bool)
430 |
431 | commandString = args["--sync-cmd"].(string)
432 | commandArgs = args["--arg"].([]string)
433 |
434 | shell = args["--shell"].(string)
435 |
436 | sudo = args["--sudo"].(bool)
437 | serial = args["--serial"].(bool)
438 |
439 | fileSources = args[""].([]string)
440 |
441 | noUpload = args["--no-upload"].(bool)
442 | )
443 |
444 | var (
445 | filesList = []file{}
446 |
447 | err error
448 | )
449 |
450 | if !noUpload {
451 | if !lockOnly {
452 | debugf(`building files list from %d sources`, len(fileSources))
453 | filesList, err = getFilesList(relative, fileSources...)
454 | if err != nil {
455 | return hierr.Errorf(
456 | err,
457 | `can't build files list`,
458 | )
459 | }
460 |
461 | debugf(`file list contains %d files`, len(filesList))
462 | tracef(`files to upload: %+v`, filesList)
463 | }
464 | }
465 |
466 | canceler := sync.NewCond(&sync.Mutex{})
467 |
468 | cluster, err := connectAndLock(args, canceler)
469 | if err != nil {
470 | return err
471 | }
472 |
473 | if lockOnly {
474 | warningf("-L|--lock was passed, waiting for interrupt...")
475 |
476 | canceler.L.Lock()
477 | canceler.Wait()
478 | canceler.L.Unlock()
479 |
480 | return nil
481 | }
482 |
483 | if !noUpload {
484 | err = upload(args, cluster, filesList)
485 | if err != nil {
486 | return hierr.Errorf(
487 | err,
488 | `can't upload files on the remote nodes`,
489 | )
490 | }
491 |
492 | tracef(`upload done`)
493 | }
494 |
495 | if uploadOnly {
496 | return nil
497 | }
498 |
499 | tracef(`starting sync tool`)
500 |
501 | command, err := shellwords.NewParser().Parse(commandString)
502 | if err != nil {
503 | return hierr.Errorf(
504 | err,
505 | `can't parse sync tool command: '%s'`,
506 | commandString,
507 | )
508 | }
509 |
510 | runner := &remoteExecutionRunner{
511 | shell: shell,
512 | sudo: sudo,
513 | command: command,
514 | args: commandArgs,
515 | directory: rootDir,
516 | serial: serial,
517 | }
518 |
519 | if isSimpleCommand {
520 | return run(cluster, runner, stdin)
521 | }
522 |
523 | err = runSyncProtocol(cluster, runner)
524 | if err != nil {
525 | return hierr.Errorf(
526 | err,
527 | `failed to run sync command`,
528 | )
529 | }
530 |
531 | return nil
532 | }
533 |
534 | func upload(
535 | args map[string]interface{},
536 | cluster *distributedLock,
537 | filesList []file,
538 | ) error {
539 | var (
540 | rootDir, _ = args["--root"].(string)
541 | sudo = args["--sudo"].(bool)
542 |
543 | preserveUID = !args["--no-preserve-uid"].(bool)
544 | preserveGID = !args["--no-preserve-gid"].(bool)
545 |
546 | serial = args["--serial"].(bool)
547 | )
548 |
549 | if rootDir == "" {
550 | rootDir = filepath.Join(runsDirectory, generateRunID())
551 | }
552 |
553 | debugf(`file upload started into: '%s'`, rootDir)
554 |
555 | receivers, err := startArchiveReceivers(cluster, rootDir, sudo, serial)
556 | if err != nil {
557 | return hierr.Errorf(
558 | err,
559 | `can't start archive receivers on the cluster`,
560 | )
561 | }
562 |
563 | err = archiveFilesToWriter(
564 | receivers.stdin,
565 | filesList,
566 | preserveUID,
567 | preserveGID,
568 | )
569 | if err != nil {
570 | return hierr.Errorf(
571 | err,
572 | `can't archive files and send to the remote nodes`,
573 | )
574 | }
575 |
576 | tracef(`waiting file upload to finish`)
577 |
578 | err = receivers.stdin.Close()
579 | if err != nil {
580 | return hierr.Errorf(
581 | err,
582 | `can't close archive receiver stdin`,
583 | )
584 | }
585 |
586 | err = receivers.wait()
587 | if err != nil {
588 | return hierr.Errorf(
589 | err,
590 | `archive upload failed`,
591 | )
592 | }
593 |
594 | return nil
595 | }
596 |
597 | func connectAndLock(
598 | args map[string]interface{},
599 | canceler *sync.Cond,
600 | ) (*distributedLock, error) {
601 | var (
602 | hosts = args["--host"].([]string)
603 |
604 | sendTimeout = args["--send-timeout"].(string)
605 | defaultUser = args["--user"].(string)
606 |
607 | askPassword = args["--password"].(bool)
608 | fromStdin = args["--read-stdin"].(bool)
609 |
610 | rootDir, _ = args["--root"].(string)
611 | sshKeyPath, _ = args["--key"].(string)
612 | lockFile, _ = args["--lock-file"].(string)
613 |
614 | noConnFail = args["--no-conn-fail"].(bool)
615 | noLockFail = args["--no-lock-fail"].(bool)
616 |
617 | noLock = args["--no-lock"].(bool)
618 | )
619 |
620 | addresses, err := parseAddresses(hosts, defaultUser, fromStdin)
621 | if err != nil {
622 | return nil, hierr.Errorf(
623 | err,
624 | `can't parse all specified addresses`,
625 | )
626 | }
627 |
628 | timeouts, err := makeTimeouts(args)
629 | if err != nil {
630 | return nil, hierr.Errorf(
631 | err,
632 | `can't parse SSH connection timeouts`,
633 | )
634 | }
635 |
636 | runners, err := createRunnerFactory(timeouts, sshKeyPath, askPassword)
637 | if err != nil {
638 | return nil, hierr.Errorf(
639 | err,
640 | `can't create runner factory`,
641 | )
642 | }
643 |
644 | debugf(`using %d threads`, pool.size)
645 |
646 | debugf(`connecting to %d nodes`, len(addresses))
647 |
648 | if lockFile == "" {
649 | if rootDir == "" {
650 | lockFile = defaultLockFile
651 | } else {
652 | lockFile = rootDir
653 | }
654 | }
655 |
656 | heartbeatMillisecondsBase, err := strconv.Atoi(sendTimeout)
657 | if err != nil {
658 | return nil, hierr.Errorf(
659 | err,
660 | `can't use --send-timeout as heartbeat timeout`,
661 | )
662 | }
663 |
664 | heartbeatMilliseconds := time.Duration(
665 | float64(heartbeatMillisecondsBase)*heartbeatTimeoutCoefficient,
666 | ) * time.Millisecond
667 |
668 | cluster, err := connectToCluster(
669 | lockFile,
670 | runners,
671 | addresses,
672 | noLock,
673 | noLockFail,
674 | noConnFail,
675 | func(node *distributedLockNode) {
676 | heartbeat(heartbeatMilliseconds, node, canceler)
677 | },
678 | )
679 | if err != nil {
680 | return nil, hierr.Errorf(
681 | err,
682 | `connecting to cluster failed`,
683 | )
684 | }
685 |
686 | if noLock {
687 | debugf(`connection established to %d nodes`, len(cluster.nodes))
688 | } else {
689 | debugf(`global lock acquired on %d nodes`, len(cluster.nodes))
690 | }
691 |
692 | return cluster, nil
693 | }
694 |
695 | func createRunnerFactory(
696 | timeout *runcmd.Timeout,
697 | sshKeyPath string,
698 | askPassword bool,
699 | ) (runnerFactory, error) {
700 | if askPassword {
701 | password, err := readPassword(sshPasswordPrompt)
702 | if err != nil {
703 | return nil, hierr.Errorf(
704 | err,
705 | `can't read password`,
706 | )
707 | }
708 |
709 | return createRemoteRunnerFactoryWithPassword(
710 | string(password),
711 | timeout,
712 | ), nil
713 | }
714 |
715 | keyring, err := getSshAgent()
716 | if err != nil {
717 | debugf(`failed to connect to a ssh-agent: %s`, err)
718 | }
719 | keyListEmpty := true
720 | if keyring == nil {
721 | keyring = agent.NewKeyring()
722 | } else {
723 | li, err := keyring.List()
724 | if err != nil {
725 | debugf(` could not get agent keys list: %s`, err)
726 | }
727 | keyListEmpty = len(li) < 1
728 | if keyListEmpty {
729 | debugf(` agent keys list is empty`)
730 | }
731 | }
732 |
733 | if keyListEmpty && sshKeyPath != "" {
734 | err := readSSHKey(keyring, sshKeyPath)
735 | if err != nil {
736 | return nil, hierr.Errorf(
737 | err,
738 | `can't read SSH key: '%s'`,
739 | sshKeyPath,
740 | )
741 | }
742 | }
743 |
744 | return createRemoteRunnerFactoryWithAgent(
745 | keyring,
746 | timeout,
747 | ), nil
748 | }
749 |
750 | func parseAddresses(
751 | hosts []string,
752 | defaultUser string,
753 | fromStdin bool,
754 | ) ([]address, error) {
755 | hostsToParse := []string{}
756 |
757 | if fromStdin {
758 | scanner := bufio.NewScanner(os.Stdin)
759 | for scanner.Scan() {
760 | hostsToParse = append(hostsToParse, scanner.Text())
761 | }
762 | }
763 |
764 | for _, host := range hosts {
765 | if strings.HasPrefix(host, "/") || strings.HasPrefix(host, "./") {
766 | hostsFile, err := os.Open(host)
767 | if err != nil {
768 | return nil, hierr.Errorf(
769 | err,
770 | `can't open hosts file: '%s'`,
771 | host,
772 | )
773 | }
774 |
775 | scanner := bufio.NewScanner(hostsFile)
776 | for scanner.Scan() {
777 | hostsToParse = append(hostsToParse, scanner.Text())
778 | }
779 | } else {
780 | hostsToParse = append(hostsToParse, host)
781 | }
782 | }
783 |
784 | addresses := []address{}
785 |
786 | for _, host := range hostsToParse {
787 | parsedAddress, err := parseAddress(
788 | host, defaultUser, defaultSSHPort,
789 | )
790 | if err != nil {
791 | return nil, hierr.Errorf(
792 | err,
793 | `can't parse specified address '%s'`,
794 | host,
795 | )
796 | }
797 |
798 | addresses = append(addresses, parsedAddress)
799 | }
800 |
801 | return getUniqueAddresses(addresses), nil
802 | }
803 |
804 | func setupInteractiveMode(args map[string]interface{}) {
805 | var (
806 | _, hasStdin = args["--stdin"].(string)
807 |
808 | barLock = &sync.Mutex{}
809 | )
810 |
811 | barStyle, err := getStatusBarTheme(parseTheme("bar", args))
812 | if err != nil {
813 | errorln(hierr.Errorf(
814 | err,
815 | `can't use given status bar style`,
816 | ))
817 | }
818 |
819 | if loreley.HasTTY(int(os.Stderr.Fd())) {
820 | statusbar = barely.NewStatusBar(barStyle.Template)
821 | statusbar.SetLock(barLock)
822 | } else {
823 | statusbar = nil
824 |
825 | sshPasswordPrompt = ""
826 | sshPassphrasePrompt = ""
827 | }
828 |
829 | if hasStdin && loreley.HasTTY(int(os.Stdin.Fd())) {
830 | statusbar = nil
831 | }
832 | }
833 |
834 | func generateRunID() string {
835 | return time.Now().Format("20060102150405.999999")
836 | }
837 |
--------------------------------------------------------------------------------
/main_unix.go:
--------------------------------------------------------------------------------
1 | //go:build !windows
2 |
3 | package main
4 |
5 | const helpNoPreserveUidGid = ` --no-preserve-uid Do not preserve UIDs for transferred files.
6 | --no-preserve-gid Do not preserve GIDs for transferred files.
7 | `
8 |
--------------------------------------------------------------------------------
/main_windows.go:
--------------------------------------------------------------------------------
1 | //go:build windows
2 |
3 | package main
4 |
5 | const helpNoPreserveUidGid = ``
6 |
--------------------------------------------------------------------------------
/multiwrite_closer.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "io"
6 | "strings"
7 | )
8 |
9 | type multiWriteCloser struct {
10 | writers []io.WriteCloser
11 | }
12 |
13 | func (closer *multiWriteCloser) Write(data []byte) (int, error) {
14 | errs := []string{}
15 |
16 | for _, writer := range closer.writers {
17 | _, err := writer.Write(data)
18 | if err != nil && err != io.EOF {
19 | errs = append(errs, err.Error())
20 | }
21 | }
22 |
23 | if len(errs) > 0 {
24 | return 0, fmt.Errorf(
25 | "%d errors: %s",
26 | len(errs),
27 | strings.Join(errs, "; "),
28 | )
29 | }
30 |
31 | return len(data), nil
32 | }
33 |
34 | func (closer *multiWriteCloser) Close() error {
35 | errs := []string{}
36 |
37 | for _, closer := range closer.writers {
38 | err := closer.Close()
39 | if err != nil && err != io.EOF {
40 | errs = append(errs, err.Error())
41 | }
42 | }
43 |
44 | if len(errs) > 0 {
45 | return fmt.Errorf(
46 | "%d errors: %s",
47 | len(errs),
48 | strings.Join(errs, "; "),
49 | )
50 | }
51 |
52 | return nil
53 | }
54 |
--------------------------------------------------------------------------------
/nop_closer.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "io"
5 | )
6 |
7 | type nopCloser struct {
8 | io.Writer
9 | }
10 |
11 | func (closer nopCloser) Close() error {
12 | return nil
13 | }
14 |
--------------------------------------------------------------------------------
/protocol_node_writer.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bufio"
5 | "bytes"
6 | "io"
7 | )
8 |
9 | type protocolNodeWriter struct {
10 | node *remoteExecutionNode
11 | protocol *syncProtocol
12 |
13 | stdout io.Writer
14 |
15 | buffer *bytes.Buffer
16 | }
17 |
18 | func newProtocolNodeWriter(
19 | node *remoteExecutionNode,
20 | protocol *syncProtocol,
21 | ) *protocolNodeWriter {
22 | return &protocolNodeWriter{
23 | node: node,
24 | stdout: node.stdout,
25 | protocol: protocol,
26 | buffer: &bytes.Buffer{},
27 | }
28 | }
29 |
30 | func (writer *protocolNodeWriter) Write(data []byte) (int, error) {
31 | written, err := writer.buffer.Write(data)
32 | if err != nil {
33 | return written, err
34 | }
35 |
36 | reader := bufio.NewReader(writer.buffer)
37 |
38 | for {
39 | line, err := reader.ReadString('\n')
40 | if err != nil {
41 | if err == io.EOF {
42 | _, err := io.WriteString(writer.buffer, line)
43 | if err != nil {
44 | return 0, err
45 | }
46 |
47 | break
48 | }
49 | }
50 |
51 | switch {
52 | case writer.protocol.IsSyncCommand(line):
53 | tracef(
54 | "%s sent sync command: '%s'",
55 | writer.node.String(),
56 | line,
57 | )
58 |
59 | err := writer.protocol.SendSync(writer.node, line)
60 |
61 | if err != nil {
62 | return 0, err
63 | }
64 | default:
65 | _, err := io.WriteString(writer.stdout, line)
66 | if err != nil {
67 | return 0, err
68 | }
69 | }
70 | }
71 |
72 | return written, nil
73 | }
74 |
75 | func (writer *protocolNodeWriter) Close() error {
76 | return nil
77 | }
78 |
--------------------------------------------------------------------------------
/remote_execution.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "io"
6 | "reflect"
7 |
8 | "github.com/reconquest/hierr-go"
9 | )
10 |
11 | type remoteExecution struct {
12 | stdin io.WriteCloser
13 | nodes map[*distributedLockNode]*remoteExecutionNode
14 | }
15 |
16 | type remoteExecutionResult struct {
17 | node *remoteExecutionNode
18 |
19 | err error
20 | }
21 |
22 | func (execution *remoteExecution) wait() error {
23 | tracef(`waiting %d nodes to finish`, len(execution.nodes))
24 |
25 | results := make(chan *remoteExecutionResult, 0)
26 | for _, node := range execution.nodes {
27 | go func(node *remoteExecutionNode) {
28 | results <- &remoteExecutionResult{node, node.wait()}
29 | }(node)
30 | }
31 |
32 | executionErrors := fmt.Errorf(
33 | `commands are exited with non-zero code`,
34 | )
35 |
36 | var (
37 | status = &struct {
38 | Phase string
39 | Total int
40 | Fails int
41 | Success int
42 | }{
43 | Phase: `wait`,
44 | Total: len(execution.nodes),
45 | }
46 |
47 | exitCodes = map[int]int{}
48 | )
49 |
50 | setStatus(status)
51 |
52 | for range execution.nodes {
53 | result := <-results
54 | if result.err != nil {
55 | exitCodes[result.node.exitCode]++
56 |
57 | executionErrors = hierr.Push(
58 | executionErrors,
59 | hierr.Errorf(
60 | result.err,
61 | `%s has finished with error`,
62 | result.node.node.String(),
63 | ),
64 | )
65 |
66 | status.Fails++
67 | status.Total--
68 |
69 | tracef(
70 | `%s finished with exit code: '%d'`,
71 | result.node.node.String(),
72 | result.node.exitCode,
73 | )
74 |
75 | continue
76 | }
77 |
78 | status.Success++
79 |
80 | tracef(
81 | `%s has successfully finished execution`,
82 | result.node.node.String(),
83 | )
84 | }
85 |
86 | if status.Fails > 0 {
87 | if status.Fails == len(execution.nodes) {
88 | exitCodesValue := reflect.ValueOf(exitCodes)
89 |
90 | topError := fmt.Errorf(
91 | `commands are failed on all %d nodes`,
92 | len(execution.nodes),
93 | )
94 |
95 | for _, key := range exitCodesValue.MapKeys() {
96 | topError = hierr.Push(
97 | topError,
98 | fmt.Sprintf(
99 | `code %d (%d nodes)`,
100 | key.Int(),
101 | exitCodesValue.MapIndex(key).Int(),
102 | ),
103 | )
104 | }
105 |
106 | return topError
107 | }
108 |
109 | return hierr.Errorf(
110 | executionErrors,
111 | `commands are failed on %d out of %d nodes`,
112 | status.Fails,
113 | len(execution.nodes),
114 | )
115 | }
116 |
117 | return nil
118 | }
119 |
--------------------------------------------------------------------------------
/remote_execution_node.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "io"
6 |
7 | "golang.org/x/crypto/ssh"
8 |
9 | "github.com/reconquest/hierr-go"
10 | "github.com/reconquest/runcmd"
11 | )
12 |
13 | type remoteExecutionNode struct {
14 | node *distributedLockNode
15 | command runcmd.CmdWorker
16 |
17 | stdin io.WriteCloser
18 | stdout io.WriteCloser
19 | stderr io.WriteCloser
20 |
21 | exitCode int
22 | }
23 |
24 | func (node *remoteExecutionNode) wait() error {
25 | err := node.command.Wait()
26 | if err != nil {
27 | _ = node.stdout.Close()
28 | _ = node.stderr.Close()
29 | if sshErrors, ok := err.(*ssh.ExitError); ok {
30 | node.exitCode = sshErrors.Waitmsg.ExitStatus()
31 |
32 | return fmt.Errorf(
33 | `%s had failed to evaluate command, `+
34 | `remote command exited with non-zero code: %d`,
35 | node.node.String(),
36 | node.exitCode,
37 | )
38 | }
39 |
40 | return hierr.Errorf(
41 | err,
42 | `%s failed to finish execution, unexpected error`,
43 | node.node.String(),
44 | )
45 | }
46 |
47 | err = node.stdout.Close()
48 | if err != nil {
49 | return hierr.Errorf(
50 | err,
51 | `%s can't close stdout`,
52 | node.node.String(),
53 | )
54 | }
55 |
56 | err = node.stderr.Close()
57 | if err != nil {
58 | return hierr.Errorf(
59 | err,
60 | `%s can't close stderr`,
61 | node.node.String(),
62 | )
63 | }
64 |
65 | return nil
66 | }
67 |
68 | func (node *remoteExecutionNode) String() string {
69 | return node.node.String()
70 | }
71 |
--------------------------------------------------------------------------------
/remote_execution_runner.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "strings"
6 |
7 | "github.com/mattn/go-shellwords"
8 | "github.com/reconquest/hierr-go"
9 | )
10 |
11 | var sudoCommand = []string{"sudo", "-n", "-E", "-H"}
12 |
13 | type remoteExecutionRunner struct {
14 | command []string
15 | args []string
16 | shell string
17 | directory string
18 | sudo bool
19 | serial bool
20 | term bool
21 | }
22 |
23 | func (runner *remoteExecutionRunner) run(
24 | cluster *distributedLock,
25 | setupCallback func(*remoteExecutionNode),
26 | ) (*remoteExecution, error) {
27 | commandline := joinCommand(runner.command)
28 |
29 | if runner.directory != "" {
30 | commandline = fmt.Sprintf("cd %s && { %s; }",
31 | escapeCommandArgumentStrict(runner.directory),
32 | commandline,
33 | )
34 | }
35 |
36 | if len(runner.shell) != 0 {
37 | commandline = wrapCommandIntoShell(
38 | commandline,
39 | runner.shell,
40 | runner.args,
41 | )
42 | }
43 |
44 | if runner.sudo {
45 | commandline = joinCommand(sudoCommand) + " " + commandline
46 | }
47 |
48 | command, err := shellwords.Parse(commandline)
49 | if err != nil {
50 | return nil, hierr.Errorf(
51 | err, "unparsable command line: %s", commandline,
52 | )
53 | }
54 |
55 | return runRemoteExecution(cluster, command, setupCallback, runner.serial, runner.term)
56 | }
57 |
58 | func wrapCommandIntoShell(command, shell string, args []string) string {
59 | if shell == "" {
60 | return command
61 | }
62 |
63 | command = strings.Replace(shell, `{}`, command, -1)
64 |
65 | if len(args) == 0 {
66 | return command
67 | }
68 |
69 | escapedArgs := []string{}
70 | for _, arg := range args {
71 | escapedArgs = append(escapedArgs, escapeCommandArgumentStrict(arg))
72 | }
73 |
74 | return command + " _ " + strings.Join(escapedArgs, " ")
75 | }
76 |
77 | func joinCommand(command []string) string {
78 | escapedParts := []string{}
79 |
80 | for _, part := range command {
81 | escapedParts = append(escapedParts, escapeCommandArgument(part))
82 | }
83 |
84 | return strings.Join(escapedParts, ` `)
85 | }
86 |
87 | func escapeCommandArgument(argument string) string {
88 | argument = strings.Replace(argument, `'`, `'\''`, -1)
89 |
90 | return argument
91 | }
92 |
93 | func escapeCommandArgumentStrict(argument string) string {
94 | escaper := strings.NewReplacer(
95 | `\`, `\\`,
96 | "`", "\\`",
97 | `"`, `\"`,
98 | `'`, `'\''`,
99 | `$`, `\$`,
100 | )
101 |
102 | escaper.Replace(argument)
103 |
104 | return `"` + argument + `"`
105 | }
106 |
--------------------------------------------------------------------------------
/run_tests:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -euo pipefail
4 |
5 | cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
6 |
7 | source "vendor.bash/github.com/reconquest/import.bash/import.bash"
8 |
9 | import:use "github.com/reconquest/hastur.bash"
10 | import:use "github.com/reconquest/containers.bash"
11 | import:use "github.com/reconquest/progress.bash"
12 | import:use "github.com/reconquest/test-runner.bash"
13 | import:use "github.com/reconquest/tests.sh"
14 | import:use "github.com/reconquest/go-test.bash"
15 | import:use "github.com/reconquest/ssh-test.bash"
16 |
17 | import:include tests/build.sh
18 | import:include tests/orgalorg.sh
19 | import:include tests/deps.sh
20 |
21 | :deps:check
22 |
23 | test-runner:set-custom-opts \
24 | --keep-containers \
25 | --keep-images \
26 | --containers-count:
27 |
28 | test-runner:handle-custom-opt() {
29 | case "$1" in
30 | --keep-containers)
31 | containers:keep-containers
32 | hastur:keep-images
33 | ;;
34 |
35 | --keep-images)
36 | hastur:keep-images
37 | ;;
38 |
39 | --containers-count)
40 | containers:set-count "$2"
41 | ;;
42 | esac
43 | }
44 |
45 | progress:spinner:new _progress_spinner
46 |
47 | test-runner:progress() {
48 | if [ "${1:-}" = "stop" ]; then
49 | printf " ok."
50 | else
51 | progress:spinner:spin "$_progress_spinner" > /dev/null
52 | fi
53 | }
54 |
55 | :init() {
56 | go-test:set-output-dir "$(readlink -f .)"
57 | go-test:build orgalorg
58 |
59 | hastur:init openssh,pam,util-linux,tar,iproute2,sudo,sed,procps-ng,systemd
60 | }
61 |
62 | :cleanup() {
63 | containers:wipe
64 |
65 | hastur:destroy-root
66 |
67 | progress:spinner:stop "$_progress_spinner"
68 |
69 | go-test:merge-coverage
70 | }
71 |
72 | :init 2> >(progress:spinner:spin "$_progress_spinner" > /dev/null)
73 |
74 | trap :cleanup EXIT
75 |
76 | test-runner:run "${@}"
77 |
--------------------------------------------------------------------------------
/runner_factory.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 |
6 | "golang.org/x/crypto/ssh"
7 | "golang.org/x/crypto/ssh/agent"
8 |
9 | "github.com/reconquest/runcmd"
10 | )
11 |
12 | type (
13 | runnerFactory func(address address) (runcmd.Runner, error)
14 | )
15 |
16 | func createRemoteRunnerFactoryWithAgent(
17 | keyring agent.Agent,
18 | timeout *runcmd.Timeout,
19 | ) runnerFactory {
20 | return func(address address) (runcmd.Runner, error) {
21 | return runcmd.NewRemoteRunner(
22 | address.user,
23 | fmt.Sprintf("%s:%d", address.domain, address.port),
24 | []ssh.AuthMethod{
25 | ssh.PublicKeysCallback(keyring.Signers),
26 | },
27 | *timeout,
28 | )
29 | }
30 | }
31 |
32 | func createRemoteRunnerFactoryWithPassword(
33 | password string,
34 | timeout *runcmd.Timeout,
35 | ) runnerFactory {
36 | return func(address address) (runcmd.Runner, error) {
37 | return runcmd.NewRemotePasswordAuthRunner(
38 | address.user,
39 | fmt.Sprintf("%s:%d", address.domain, address.port),
40 | password,
41 | *timeout,
42 | )
43 | }
44 | }
45 |
--------------------------------------------------------------------------------
/sshagent_unix.go:
--------------------------------------------------------------------------------
1 | //go:build !windows
2 |
3 | package main
4 |
5 | import (
6 | "net"
7 | "os"
8 |
9 | "github.com/reconquest/hierr-go"
10 | "golang.org/x/crypto/ssh/agent"
11 | )
12 |
13 | func getSshAgent() (agent.Agent, error) {
14 | debugf(`trying unix ssh-agent pipe`)
15 | if os.Getenv("SSH_AUTH_SOCK") == "" {
16 | return nil, hierr.Errorf(
17 | nil,
18 | "SSH_AUTH_SOCK is not set",
19 | )
20 | }
21 | sock, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
22 | if err != nil {
23 | return nil, hierr.Errorf(
24 | err,
25 | "unable to dial to ssh agent socket: %s",
26 | os.Getenv("SSH_AUTH_SOCK"),
27 | )
28 | }
29 |
30 | return agent.NewClient(sock), nil
31 | }
32 |
--------------------------------------------------------------------------------
/sshagent_windows.go:
--------------------------------------------------------------------------------
1 | //go:build windows
2 |
3 | package main
4 |
5 | import (
6 | "github.com/davidmz/go-pageant"
7 | "github.com/reconquest/hierr-go"
8 |
9 | "github.com/Microsoft/go-winio"
10 | "golang.org/x/crypto/ssh/agent"
11 | )
12 |
13 | const (
14 | openSshAgentPipe = `\\.\pipe\openssh-ssh-agent`
15 | )
16 |
17 | func getSshAgent() (agent.Agent, error) {
18 | debugf(`trying windows pageant`)
19 | if pageant.Available() {
20 | return pageant.New(), nil
21 | } else {
22 | debugf(" pageant is not found")
23 | }
24 | debugf(`trying windows openssh-agent`)
25 | sock, err := winio.DialPipe(openSshAgentPipe, nil)
26 | if err != nil {
27 | return nil, hierr.Errorf(
28 | err,
29 | "unable to dial openssh-agent socket: %s",
30 | openSshAgentPipe,
31 | )
32 | }
33 | return agent.NewClient(sock), nil
34 | }
35 |
--------------------------------------------------------------------------------
/status_bar_update_writer.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import "io"
4 |
5 | type statusBarUpdateWriter struct {
6 | writer io.WriteCloser
7 | }
8 |
9 | func (writer *statusBarUpdateWriter) Write(data []byte) (int, error) {
10 | clearStatus()
11 |
12 | written, err := writer.writer.Write(data)
13 |
14 | drawStatus()
15 |
16 | return written, err
17 | }
18 |
19 | func (writer *statusBarUpdateWriter) Close() error {
20 | return writer.writer.Close()
21 | }
22 |
--------------------------------------------------------------------------------
/sync.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import "github.com/reconquest/hierr-go"
4 |
5 | func runSyncProtocol(
6 | cluster *distributedLock,
7 | runner *remoteExecutionRunner,
8 | ) error {
9 | protocol := newSyncProtocol()
10 |
11 | execution, err := runner.run(
12 | cluster,
13 | func(remoteNode *remoteExecutionNode) {
14 | remoteNode.stdout = newProtocolNodeWriter(remoteNode, protocol)
15 | },
16 | )
17 | if err != nil {
18 | return hierr.Errorf(
19 | err,
20 | `can't run sync tool command`,
21 | )
22 | }
23 |
24 | tracef(`starting sync protocol with %d nodes`, len(execution.nodes))
25 |
26 | err = protocol.Init(execution.stdin)
27 | if err != nil {
28 | return hierr.Errorf(
29 | err,
30 | `can't init protocol with sync tool`,
31 | )
32 | }
33 |
34 | tracef(`sending information about %d nodes to each`, len(execution.nodes))
35 |
36 | nodes := []*remoteExecutionNode{}
37 | for _, node := range execution.nodes {
38 | nodes = append(nodes, node)
39 | }
40 |
41 | for _, node := range execution.nodes {
42 | for _, neighbor := range nodes {
43 | err := protocol.SendNode(node, neighbor)
44 | if err != nil {
45 | return hierr.Errorf(
46 | err,
47 | `can't send node to sync tool: '%s'`,
48 | node.String(),
49 | )
50 | }
51 | }
52 | }
53 |
54 | tracef(`sending start message to sync tools`)
55 |
56 | err = protocol.SendStart()
57 | if err != nil {
58 | return hierr.Errorf(
59 | err,
60 | `can't start sync tool`,
61 | )
62 | }
63 |
64 | debugf(`waiting sync tool to finish`)
65 |
66 | err = execution.wait()
67 | if err != nil {
68 | return hierr.Errorf(
69 | err,
70 | `failed to finish sync tool command`,
71 | )
72 | }
73 |
74 | return nil
75 | }
76 |
--------------------------------------------------------------------------------
/sync_protocol.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "io"
6 | "strings"
7 | "time"
8 |
9 | "github.com/reconquest/prefixwriter-go"
10 | )
11 |
12 | var (
13 | syncProtocolPrefix = "ORGALORG"
14 | syncProtocolHello = "HELLO"
15 | syncProtocolNode = "NODE"
16 | syncProtocolNodeCurrent = "CURRENT"
17 | syncProtocolStart = "START"
18 | syncProtocolSync = "SYNC"
19 | )
20 |
21 | // syncProtocol handles SYNC protocol described in the main.go.
22 | //
23 | // It will handle protocol over all connected nodes.
24 | type syncProtocol struct {
25 | // output represents writer, that should be connected to stdins of
26 | // all connected nodes.
27 | output io.WriteCloser
28 |
29 | // prefix is a unique string which prefixes every protocol message.
30 | prefix string
31 | }
32 |
33 | // newSyncProtocol returns syncProtocol instantiated with unique prefix.
34 | func newSyncProtocol() *syncProtocol {
35 | return &syncProtocol{
36 | prefix: fmt.Sprintf(
37 | "%s:%d",
38 | syncProtocolPrefix,
39 | time.Now().UnixNano(),
40 | ),
41 | }
42 | }
43 |
44 | // Init starts protocol and sends HELLO message to the writer. Specified writer
45 | // will be used in all further communications.
46 | func (protocol *syncProtocol) Init(output io.WriteCloser) error {
47 | protocol.output = prefixwriter.New(output, protocol.prefix+" ")
48 |
49 | _, err := io.WriteString(
50 | protocol.output,
51 | syncProtocolHello+"\n",
52 | )
53 | if err != nil {
54 | return protocolSuspendEOF(err)
55 | }
56 |
57 | return nil
58 | }
59 |
60 | // SendNode sends to the writer serialized representation of specified node as
61 | // NODE message.
62 | func (protocol *syncProtocol) SendNode(
63 | current *remoteExecutionNode,
64 | neighbor *remoteExecutionNode,
65 | ) error {
66 | var line = syncProtocolNode + " " + neighbor.String()
67 |
68 | if current == neighbor {
69 | line += " " + syncProtocolNodeCurrent
70 | }
71 |
72 | _, err := io.WriteString(current.stdin, line+"\n")
73 | if err != nil {
74 | return protocolSuspendEOF(err)
75 | }
76 |
77 | return nil
78 | }
79 |
80 | // SendStart sends START message to the writer.
81 | func (protocol *syncProtocol) SendStart() error {
82 | _, err := io.WriteString(
83 | protocol.output,
84 | syncProtocolStart+"\n",
85 | )
86 | if err != nil {
87 | return protocolSuspendEOF(err)
88 | }
89 |
90 | return nil
91 | }
92 |
93 | // IsSyncCommand will return true, if specified line looks like incoming
94 | // SYNC message from the remote node.
95 | func (protocol *syncProtocol) IsSyncCommand(line string) bool {
96 | return strings.HasPrefix(line, protocol.prefix+" "+syncProtocolSync)
97 | }
98 |
99 | // SendSync sends SYNC message to the writer, tagging it as sent from node,
100 | // described by given source and adding optional description for the given
101 | // SYNC phase taken by extraction it from the original SYNC message, sent
102 | // by node.
103 | func (protocol *syncProtocol) SendSync(
104 | source fmt.Stringer,
105 | sync string,
106 | ) error {
107 | data := strings.TrimSpace(
108 | strings.TrimPrefix(sync, protocol.prefix+" "+syncProtocolSync),
109 | )
110 |
111 | _, err := io.WriteString(
112 | protocol.output,
113 | syncProtocolSync+" "+source.String()+" "+data+"\n",
114 | )
115 |
116 | if err != nil {
117 | return protocolSuspendEOF(err)
118 | }
119 |
120 | return nil
121 | }
122 |
123 | // Suspend EOF for be compatible with simple commands, that are not support
124 | // protocol, and therefore can close exit earlier, than protocol is initiated.
125 | func protocolSuspendEOF(err error) error {
126 | if err == io.EOF {
127 | return nil
128 | }
129 |
130 | return err
131 | }
132 |
--------------------------------------------------------------------------------
/tests/.gitignore:
--------------------------------------------------------------------------------
1 | /.last-testcase
2 |
--------------------------------------------------------------------------------
/tests/build.sh:
--------------------------------------------------------------------------------
1 | :build:init() {
2 | printf "[build] building go binary... "
3 |
4 | if build_out=$(go build -o orgalorg -v 2>&1 | tee /dev/stderr); then
5 | printf "ok.\n"
6 | else
7 | printf "fail.\n\n%s\n" "$build_out"
8 | return 1
9 | fi
10 | }
11 |
--------------------------------------------------------------------------------
/tests/deps.sh:
--------------------------------------------------------------------------------
1 | :deps:check() {
2 | if ! which brctl >/dev/null 2>&1; then
3 | echo "missing dependency: brctl (bridge-utils)"
4 | exit 1
5 | fi >&2
6 |
7 | if ! which expect >/dev/null 2>&1; then
8 | echo "missing dependency: expect"
9 | exit 1
10 | fi >&2
11 |
12 | if ! which hastur >/dev/null 2>&1; then
13 | echo "missing dependency: hastur (https://github.com/seletskiy/hastur)"
14 | exit 1
15 | fi >&2
16 | }
17 |
--------------------------------------------------------------------------------
/tests/orgalorg.sh:
--------------------------------------------------------------------------------
1 | # requires setup.sh to be sourced first!
2 |
3 | orgalorg_user="orgalorg"
4 |
5 | :orgalorg:with-key() {
6 | :orgalorg \
7 | -u $orgalorg_user ${ips[*]/#/-o} -k "$(ssh-test:print-key-path)" "${@}"
8 | }
9 |
10 | :orgalorg:with-password() {
11 | local password="$1"
12 | shift
13 |
14 | :expect() {
15 | expect -f <(cat) -- "${@}" /home/$(ssh-test:print-username)/.ssh/authorized_keys
33 |
34 | chown -R \\\\
35 | $(ssh-test:print-username): /home/$(ssh-test:print-username)"
36 | }
37 |
38 | :start-ssh-daemon() {
39 | local container_name="$1"
40 |
41 | tests:debug "[$container_name] starting sshd..."
42 |
43 | tests:run-background "pid" ssh-test:remote:run-daemon
44 |
45 | until containers:is-active "$container_name"; do
46 | tests:debug "[$container_name] is offline"
47 | done
48 |
49 | while :; do
50 | containers:get-ip ip "$container_name"
51 | if [[ "$ip" ]]; then
52 | break
53 | fi
54 |
55 | tests:debug "[$container_name] is online, but no ip assigned"
56 | done
57 |
58 | tests:debug "[$container_name] is online and has ip"
59 | }
60 |
61 | :wait-for-ssh-active() {
62 | local container_name="$1"
63 | local container_ip="$2"
64 |
65 | until ssh-test:connect:by-key "$container_ip" "true"; do
66 | sleep 0.1
67 | tests:debug "[$container_name] sshd is offline"
68 | done
69 |
70 | tests:debug "[$container_name] sshd is online"
71 | }
72 |
73 | :install-sync-command-into-container() {
74 | local file_name="$1"
75 | local container_name="$2"
76 |
77 | containers:get-rootfs rootfs "$container_name"
78 |
79 | tests:ensure sudo mkdir -p "$rootfs/usr/lib/orgalorg/"
80 | tests:ensure sudo cp "$file_name" "$rootfs/usr/lib/orgalorg/"
81 | tests:ensure sudo chmod +x "$rootfs/usr/lib/orgalorg/sync"
82 | }
83 |
84 | tests:debug "!!! setup"
85 |
86 | tests:clone "orgalorg" "bin/"
87 |
88 | tests:debug "!!! spawning $(containers:count) containers"
89 |
90 | containers:spawn "/bin/true"
91 |
92 | tests:debug "!!! generating local key pair"
93 |
94 | tests:ensure ssh-test:local:keygen
95 |
96 | tests:debug "!!! bootstrapping containers"
97 |
98 | containers:foreach :bootstrap-container
99 |
100 | tests:debug "!!! starting sshd instances"
101 |
102 | containers:foreach :start-ssh-daemon
103 |
104 | tests:debug "!!! waiting for sshd"
105 |
106 | containers:do :wait-for-ssh-active
107 |
108 | containers:get-list containers
109 | containers:get-ip-list ips
110 |
--------------------------------------------------------------------------------
/tests/ssh.sh:
--------------------------------------------------------------------------------
1 | :ssh:get-key() {
2 | printf "ssh-key"
3 | }
4 |
5 | :ssh:get-username() {
6 | printf "orgalorg"
7 | }
8 |
9 | :ssh:run-daemon() {
10 | local container_name=$1
11 | shift
12 |
13 | containers:run "$container_name" -- \
14 | /usr/bin/sshd "${@:--Dd}"
15 | }
16 |
17 | :ssh:keygen-local() {
18 | local output_file=$1
19 | shift
20 |
21 | ssh-keygen -P '' -f "$output_file"
22 | }
23 |
24 | :ssh:keygen-remote() {
25 | local container_name=$1
26 | shift
27 |
28 | containers:run "$container_name" -- \
29 | /usr/bin/ssh-keygen "${@:--A}"
30 | }
31 |
32 | :ssh:copy-id() {
33 | local container_name=$1
34 | local username=$2
35 | shift
36 |
37 | containers:run "$container_name" -- \
38 | /usr/bin/tee -a /home/$username/.ssh/authorized_keys > /dev/null
39 | }
40 |
41 | :ssh:run-with-key() {
42 | local ip=$1
43 | local user=$2
44 | local identity=$3
45 |
46 | shift 3
47 |
48 | ssh \
49 | -oStrictHostKeyChecking=no \
50 | -oPasswordAuthentication=no \
51 | -oControlPath=none \
52 | -i "$identity" \
53 | -l "$user" \
54 | "$ip" "${@}"
55 | }
56 |
57 | :ssh() {
58 | local ip=$1
59 | shift
60 |
61 | :ssh:run-with-key "$ip" \
62 | "$(:ssh:get-username)" "$(:ssh:get-key)" \
63 | "${@}"
64 | }
65 |
--------------------------------------------------------------------------------
/tests/teardown.sh:
--------------------------------------------------------------------------------
1 | containers:wipe
2 |
--------------------------------------------------------------------------------
/tests/testcases/auth/can-authenticate-via-key-with-passphrase.test.sh:
--------------------------------------------------------------------------------
1 | passphrase="theone"
2 |
3 | tests:ensure ssh-test:local:keygen -f "$(ssh-test:print-key-path)-encrypted" \
4 | -b 4096 -P "$passphrase"
5 |
6 | tests:ensure cat $(ssh-test:print-key-path)
7 |
8 | :copy-key() {
9 | local container_name="$1"
10 | local container_ip="$2"
11 |
12 | tests:ensure ssh-test:connect:by-key "$container_ip" \
13 | 'cat > ~/.ssh/authorized_keys' \
14 | < "$(ssh-test:print-key-path)-encrypted.pub"
15 | }
16 |
17 | containers:do :copy-key
18 |
19 | tests:ensure \
20 | mv "$(ssh-test:print-key-path)-encrypted" \
21 | "$(ssh-test:print-key-path)"
22 |
23 | tests:eval :orgalorg:with-key-passphrase "bla-$passphrase" -C -- \
24 | whoami
25 |
26 | tests:assert-stdout "invalid passphrase for private key specified"
27 |
28 | tests:ensure :orgalorg:with-key-passphrase "$passphrase" -C -- \
29 | whoami
30 |
31 | :check-output() {
32 | local container_name="$1"
33 | local container_ip="$2"
34 |
35 | tests:assert-stdout "$container_ip $orgalorg_user"
36 | }
37 |
38 | containers:do :check-output
39 |
--------------------------------------------------------------------------------
/tests/testcases/auth/can-authenticate-via-password.test.sh:
--------------------------------------------------------------------------------
1 | password="123456"
2 |
3 | :set-ssh-password() {
4 | local container_ip="$2"
5 |
6 | ssh-test:connect:by-key "$container_ip" sudo -n chpasswd \
7 | <<< "$orgalorg_user:$password"
8 | }
9 |
10 | containers:do :set-ssh-password
11 |
12 | tests:ensure :orgalorg:with-password "$password" -C -- whoami
13 |
14 | :check-output() {
15 | local container_name="$1"
16 | local container_ip="$2"
17 |
18 | tests:assert-stdout "$container_ip $orgalorg_user"
19 | }
20 |
21 | containers:do :check-output
22 |
--------------------------------------------------------------------------------
/tests/testcases/can-skip-unreachable-servers-if-flag-is-given.test.sh:
--------------------------------------------------------------------------------
1 | tests:not tests:ensure :orgalorg:with-key -o example.com -C whoami
2 |
3 | tests:ensure :orgalorg:with-key -o example.com -w -C whoami
4 |
5 | tests:assert-stderr-re "WARN.*can't connect to address.*example.com"
6 |
7 | :check-node-output() {
8 | local container_ip="$2"
9 |
10 | tests:assert-stdout "$container_ip $orgalorg_user"
11 | }
12 |
13 | containers:do :check-node-output
14 |
--------------------------------------------------------------------------------
/tests/testcases/commandline/can-handle-common-mistakes-in-arguments.test.sh:
--------------------------------------------------------------------------------
1 | tests:not tests:ensure :orgalorg -C echo blah
2 | tests:assert-stderr "Usage:"
3 |
4 | tests:not tests:ensure :orgalorg -o ./blah -C echo blah
5 | tests:assert-stderr-re "can't open.*blah"
6 | tests:assert-stderr-re "blah.*no such file or directory"
7 |
8 | tests:not tests:ensure :orgalorg -o blah --send-timeout=wazup -C echo dunno
9 | tests:assert-stderr-re "send timeout to number"
10 |
11 | tests:not tests:ensure :orgalorg -o blah --recv-timeout=wazup -C echo dunno
12 | tests:assert-stderr-re "receive timeout to number"
13 |
14 | tests:not tests:ensure :orgalorg -o blah --conn-timeout=wazup -C echo dunno
15 | tests:assert-stderr-re "connection timeout to number"
16 |
17 | tests:not tests:ensure :orgalorg -o blah --keep-alive=wazup -C echo dunno
18 | tests:assert-stderr-re "keep alive time to number"
19 |
--------------------------------------------------------------------------------
/tests/testcases/commandline/will-read-hosts-from-file-if-argument-starts-with-slash-or-dot-slash.test.sh:
--------------------------------------------------------------------------------
1 | xargs -n1 <<< "${ips[@]}" | tests:put hosts
2 |
3 | tests:ensure :orgalorg:with-key -o ./hosts -C echo hello '|' wc -l
4 |
5 | tests:assert-stdout "$(containers:count)"
6 |
--------------------------------------------------------------------------------
/tests/testcases/commands/can-aggregate-errors-on-the-end-of-execution.test.sh:
--------------------------------------------------------------------------------
1 | tests:not tests:ensure :orgalorg:with-key -e -C echo 1 '&&' exit 1
2 | tests:assert-stderr-re "exited with non-zero"
3 | tests:assert-stderr-re "all $(containers:count) nodes"
4 | tests:assert-stderr "code 1 ($(containers:count) nodes)"
5 |
--------------------------------------------------------------------------------
/tests/testcases/commands/can-escape-space-in-the-remote-command.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key -e -C echo '"two spaces"'
2 |
3 | tests:assert-stdout "two spaces"
4 |
--------------------------------------------------------------------------------
/tests/testcases/commands/can-run-remote-command-and-pass-stdin-to-it.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key -C -- wc -l
2 |
3 | containers:do tests:assert-stdout "0"
4 |
5 | tests:ensure :orgalorg:with-key -C -i <(echo 1) -- wc -l
6 |
7 | containers:do tests:assert-stdout "1"
8 |
--------------------------------------------------------------------------------
/tests/testcases/commands/can-run-remote-command.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key -e -C pwd
2 |
3 | tests:assert-stdout "/home/orgalorg"
4 |
--------------------------------------------------------------------------------
/tests/testcases/commands/can-run-shell-command.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key -C -- echo -n 1 '&&' echo 2
2 |
3 | tests:assert-stdout-re "${ips[0]} 12"
4 |
--------------------------------------------------------------------------------
/tests/testcases/commands/should-properly-escape-shell-arguments.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure echo "'1'"
2 | tests:ensure :orgalorg:with-key -e -C echo "'1'"
3 |
4 | tests:assert-stdout-re "1$"
5 |
6 | tests:ensure :orgalorg:with-key -e -C echo "\\'"
7 |
8 | tests:assert-stdout-re "'$"
9 |
--------------------------------------------------------------------------------
/tests/testcases/commands/should-return-non-zero-exit-code-if-command-fails.test.sh:
--------------------------------------------------------------------------------
1 | tests:not tests:ensure :orgalorg:with-key -e -C exit 17
2 |
3 | tests:assert-stderr "remote execution failed"
4 | tests:assert-stderr "code 17 ($(containers:count) nodes)"
5 |
--------------------------------------------------------------------------------
/tests/testcases/commands/should-use-default-user-directory-unless-specified.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key -C -- pwd
2 |
3 | tests:assert-stdout-re "${ips[0]} /home/$orgalorg_user"
4 |
--------------------------------------------------------------------------------
/tests/testcases/commands/should-use-specified-working-directory.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key -r /tmp -C -- pwd
2 |
3 | tests:assert-stdout-re "${ips[0]} /tmp"
4 |
--------------------------------------------------------------------------------
/tests/testcases/commands/will-flush-lines-so-they-do-not-interleave.test.sh:
--------------------------------------------------------------------------------
1 | # checking, that lines from different remote sources do not interleave
2 | # each other
3 | tests:ensure :orgalorg:with-key -C \
4 | seq 1 10000 '|' awk '{print $1}' '|' sort '|' uniq '|' wc -l
5 |
6 | tests:assert-no-diff "$(containers:count)" stdout
7 |
--------------------------------------------------------------------------------
/tests/testcases/locking/can-acquire-global-lock.test.sh:
--------------------------------------------------------------------------------
1 | tests:involve tests/testcases/locking/lock.sh
2 |
3 | :orgalorg:lock orgalorg_output orgalorg_pid
4 |
5 | tests:not tests:ensure :orgalorg:with-key --lock
6 | tests:assert-stderr "lock already"
7 |
8 | pkill -INT -P "$orgalorg_pid"
9 |
10 | _exited_with_ctrl_c=130
11 |
12 | wait "$orgalorg_pid" || tests:assert-equals "$_exited_with_ctrl_c" "$?"
13 |
--------------------------------------------------------------------------------
/tests/testcases/locking/can-detect-if-lock-process-is-aborted-by-remote-host-after-acquire.test.sh:
--------------------------------------------------------------------------------
1 | tests:involve tests/testcases/locking/lock.sh
2 |
3 | :orgalorg:lock orgalorg_output orgalorg_pid --send-timeout=2000
4 |
5 | tests:wait-file-changes "$orgalorg_output" 0.1 10 \
6 | ssh-test:connect:by-key "${ips[0]}" pkill -f flock
7 |
8 | tests:ensure grep -q "ERROR.*${ips[0]}.*heartbeat" "$orgalorg_output"
9 |
--------------------------------------------------------------------------------
/tests/testcases/locking/lock.sh:
--------------------------------------------------------------------------------
1 | :orgalorg:lock() {
2 | local _output_var="$1"
3 | local _pid_var="$2"
4 | shift 2
5 |
6 | local _orgalorg_output="$(tests:get-tmp-dir)/oralorg.stdout"
7 | local _orgalorg=""
8 |
9 | tests:run-background _orgalorg \
10 | tests:silence tests:pipe \
11 | :orgalorg:with-key --lock "${@}" '2>&1' \
12 | '|' tee "$_orgalorg_output"
13 |
14 | until grep -qF "waiting for interrupt" "$_orgalorg_output" 2>/dev/null
15 | do
16 | tests:debug "[orgalorg] waiting for global lock..."
17 | sleep 0.1
18 | done
19 |
20 | tests:debug "[orgalorg] global lock has been acquired"
21 |
22 | eval $_output_var=\$_orgalorg_output
23 | eval $_pid_var=\$\(tests:get-background-pid \$_orgalorg\)
24 | }
25 |
--------------------------------------------------------------------------------
/tests/testcases/locking/will-continue-execution-when-lock-failed-if-flag-is-specified.test.sh:
--------------------------------------------------------------------------------
1 | tests:involve tests/testcases/locking/lock.sh
2 |
3 | :orgalorg:lock orgalorg_output orgalorg_pid
4 |
5 | tests:ensure :orgalorg:with-key --no-lock-fail -C -- echo 1
6 | tests:assert-stdout "1"
7 |
8 | pkill -INT -P "$orgalorg_pid"
9 |
10 | _exited_with_ctrl_c=130
11 |
12 | wait "$orgalorg_pid" || tests:assert-equals "$_exited_with_ctrl_c" "$?"
13 |
--------------------------------------------------------------------------------
/tests/testcases/newlines/do-append-newlines-in-non-quite-mode-when-necessary.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key -C -- echo -n hello '|' wc -l
2 |
3 | tests:assert-stdout "$(containers:count)"
4 |
--------------------------------------------------------------------------------
/tests/testcases/newlines/do-not-append-newlines-if-they-are-not-present-in-remote-output-at-quiet-mode.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key -q -C -- echo -n hello
2 |
3 | tests:debug $(containers:count)
4 |
5 | tests:assert-stdout "$(printf "%0.shello" $(seq ${#containers[@]}))"
6 |
--------------------------------------------------------------------------------
/tests/testcases/sudo/can-run-command-under-sudo.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key -C 'whoami'
2 |
3 | containers:do tests:assert-stdout "$orgalorg_user"
4 |
5 | tests:ensure :orgalorg:with-key -x -C 'whoami'
6 |
7 | containers:do tests:assert-stdout "root"
8 |
--------------------------------------------------------------------------------
/tests/testcases/sudo/can-upload-files-under-sudo.test.sh:
--------------------------------------------------------------------------------
1 | tests:put test-file < output
25 |
26 | tests:ensure awk '$1 != $4 { exit 1 }' '<' output
27 | }
28 |
29 | containers:do :check-all-nodes-present-in-list
30 | :check-all-nodes-has-current-flag-set-correctly
31 |
--------------------------------------------------------------------------------
/tests/testcases/sync/will-send-stderr-from-sync-tool-back-to-master.test.sh:
--------------------------------------------------------------------------------
1 | tests:put test-file <&2
8 | EOF
9 |
10 | containers:do :install-sync-command-into-container "sync"
11 |
12 | tests:ensure :orgalorg:with-key -e -r /home/orgalorg/ -S test-file
13 |
14 | :check-stderr-returned-from-all-nodes() {
15 | local container_name="$1"
16 | local container_ip="$2"
17 |
18 | tests:assert-stderr-re "$container_ip XXX"
19 | }
20 |
21 | containers:do :check-stderr-returned-from-all-nodes
22 |
--------------------------------------------------------------------------------
/tests/testcases/verbosity/can-log-stderr-and-stdout-from-remote-at-verbose-level-to-stderr.test.sh:
--------------------------------------------------------------------------------
1 | # note! tests:ensure will eat '>&2' if it's passed without prefix
2 | tests:ensure :orgalorg:with-key -v -C -- echo 1 \; echo err\>\&2
3 |
4 | tests:assert-stderr-re ' \[.*\] 1'
5 | tests:assert-stderr-re ' \[.*\] err'
6 |
--------------------------------------------------------------------------------
/tests/testcases/verbosity/can-output-ip-in-response-from-remote-server-when-running-command.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key -C pwd
2 |
3 | containers:do tests:assert-stdout-re "${ips[0]} /home/orgalorg"
4 |
--------------------------------------------------------------------------------
/tests/testcases/verbosity/can-output-node-stdout-and-stderr-in-json-format.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key --json -C pwd
2 |
3 | tests:assert-stdout-re '"stream":"stdout"'
4 | tests:assert-stdout-re "\"node\":\".*${ips[0]}.*\""
5 | tests:assert-stdout-re "\"body\":\"/home/$orgalorg_user"
6 |
--------------------------------------------------------------------------------
/tests/testcases/verbosity/can-output-verbose-debug-info-in-json-format.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key --json -vv -C pwd
2 |
3 | tests:assert-stderr-re '"stream":"stderr"'
4 | tests:assert-stderr-re '"body":".*DEBUG.*connection established'
5 | tests:assert-stderr-re '"body":".*TRACE.*running lock command'
6 |
--------------------------------------------------------------------------------
/tests/testcases/verbosity/can-print-debug-output.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key -vv -C pwd
2 |
3 | tests:assert-stderr-re "DEBUG.*stdout.*/home/orgalorg"
4 |
--------------------------------------------------------------------------------
/tests/testcases/verbosity/can-report-internal-errors-in-json-format.test.sh:
--------------------------------------------------------------------------------
1 | tests:not tests:ensure :orgalorg:with-key --json -o example.com -C pwd
2 |
3 | tests:assert-stderr-re '"stream":"stderr"'
4 | tests:assert-stderr-re '"body":".*FATAL.*connect to address'
5 | tests:not tests:assert-stderr '└─'
6 |
--------------------------------------------------------------------------------
/tests/testcases/verbosity/do-not-output-ip-in-quiet-mode.test.sh:
--------------------------------------------------------------------------------
1 | tests:ensure :orgalorg:with-key -q -C pwd
2 |
3 | tests:assert-stdout-re "^/home/orgalorg$"
4 |
--------------------------------------------------------------------------------
/tests/testcases/verbosity/will-output-hierarchical-errors-by-default.test.sh:
--------------------------------------------------------------------------------
1 | tests:not tests:ensure :orgalorg:with-key -o example.com -C pwd
2 |
3 | tests:assert-stderr "└─ can't connect to address"
4 |
--------------------------------------------------------------------------------
/themes.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 |
6 | "github.com/kovetskiy/lorg"
7 | "github.com/reconquest/colorgful"
8 | "github.com/reconquest/loreley"
9 | )
10 |
11 | const (
12 | themeDefault = `default`
13 | themeDark = `dark`
14 | themeLight = `light`
15 | )
16 |
17 | var (
18 | statusBarThemeTemplate = `{bg %d}{fg %d}` +
19 | `{bold}` +
20 | `{if eq .Phase "lock"}{bg %d} LOCK{end}` +
21 | `{if eq .Phase "connect"}{bg %[3]d} CONNECT{end}` +
22 | `{if eq .Phase "exec"}{bg %d} EXEC{end}` +
23 | `{if eq .Phase "wait"}{bg %d} WAIT{end}` +
24 | `{if eq .Phase "upload"}{bg %d} UPLOAD{end}` +
25 | `{nobold} ` +
26 | `{from "" %d} ` +
27 | `{fg %d}{bold}{printf "%%4d" .Success}{nobold}{fg %d}` +
28 | `/{printf "%%4d" .Total} ` +
29 | `{if .Fails}{fg %d}✗ {.Fails}{end} ` +
30 | `{from "" %d}` +
31 | `{if eq .Phase "upload"}{fg %d} ` +
32 | `{printf "%%9s/%%s" .Written .Bytes} ` +
33 | `{end}`
34 |
35 | statusBarThemes = map[string]string{
36 | themeDark: fmt.Sprintf(
37 | statusBarThemeTemplate,
38 | 99, 7, 22, 1, 1, 25, 237, 46, 15, 214, -1, 140,
39 | ),
40 |
41 | themeLight: fmt.Sprintf(
42 | statusBarThemeTemplate,
43 | 99, 7, 22, 1, 1, 64, 254, 106, 16, 9, -1, 140,
44 | ),
45 |
46 | themeDefault: fmt.Sprintf(
47 | statusBarThemeTemplate,
48 | 234, 255, 22, 1, 1, 19, 245, 85, 255, 160, -1, 140,
49 | ),
50 | }
51 |
52 | logFormat = `${time} ${level:[%s]:right:true} %s`
53 | )
54 |
55 | func getLoggerTheme(theme string) (lorg.Formatter, error) {
56 | switch theme {
57 | case "default":
58 | return colorgful.ApplyDefaultTheme(
59 | logFormat,
60 | colorgful.Default,
61 | )
62 | case "dark":
63 | return colorgful.ApplyDefaultTheme(
64 | logFormat,
65 | colorgful.Dark,
66 | )
67 | case "light":
68 | return colorgful.ApplyDefaultTheme(
69 | logFormat,
70 | colorgful.Light,
71 | )
72 | default:
73 | return colorgful.Format(theme)
74 | }
75 | }
76 |
77 | func getStatusBarTheme(theme string) (*loreley.Style, error) {
78 | if format, ok := statusBarThemes[theme]; ok {
79 | theme = format
80 | }
81 |
82 | style, err := loreley.CompileWithReset(theme, nil)
83 | if err != nil {
84 | return nil, err
85 | }
86 |
87 | return style, nil
88 | }
89 |
90 | func parseTheme(target string, args map[string]interface{}) string {
91 | var (
92 | theme = args["--"+target+"-format"].(string)
93 | light = args["--colors-light"].(bool)
94 | dark = args["--colors-dark"].(bool)
95 | )
96 |
97 | switch {
98 | case light:
99 | return themeLight
100 |
101 | case dark:
102 | return themeDark
103 |
104 | default:
105 | return theme
106 | }
107 | }
108 |
--------------------------------------------------------------------------------
/thread_pool.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "strconv"
5 |
6 | "github.com/reconquest/hierr-go"
7 | )
8 |
9 | type threadPool struct {
10 | available chan struct{}
11 |
12 | size int
13 | }
14 |
15 | func newThreadPool(size int) *threadPool {
16 | available := make(chan struct{}, size)
17 | for i := 0; i < size; i++ {
18 | available <- struct{}{}
19 | }
20 |
21 | return &threadPool{
22 | available,
23 | size,
24 | }
25 | }
26 |
27 | func (pool *threadPool) run(task func()) {
28 | <-pool.available
29 | defer func() {
30 | pool.available <- struct{}{}
31 | }()
32 |
33 | task()
34 | }
35 |
36 | func parseThreadPoolSize(args map[string]interface{}) (int, error) {
37 | var (
38 | poolSizeRaw = args["--threads"].(string)
39 | )
40 |
41 | poolSize, err := strconv.Atoi(poolSizeRaw)
42 | if err != nil {
43 | return 0, hierr.Errorf(
44 | err,
45 | `can't parse threads count`,
46 | )
47 | }
48 |
49 | return poolSize, nil
50 | }
51 |
--------------------------------------------------------------------------------
/timeouts.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "strconv"
5 | "time"
6 |
7 | "github.com/reconquest/hierr-go"
8 | "github.com/reconquest/runcmd"
9 | )
10 |
11 | func makeTimeouts(args map[string]interface{}) (*runcmd.Timeout, error) {
12 | var (
13 | connectionTimeoutRaw = args["--conn-timeout"].(string)
14 | sendTimeoutRaw = args["--send-timeout"].(string)
15 | receiveTimeoutRaw = args["--recv-timeout"].(string)
16 | keepAliveRaw = args["--keep-alive"].(string)
17 | )
18 |
19 | connectionTimeout, err := strconv.Atoi(connectionTimeoutRaw)
20 | if err != nil {
21 | return nil, hierr.Errorf(
22 | err,
23 | `can't convert specified connection timeout to number: '%s'`,
24 | connectionTimeoutRaw,
25 | )
26 | }
27 |
28 | sendTimeout, err := strconv.Atoi(sendTimeoutRaw)
29 | if err != nil {
30 | return nil, hierr.Errorf(
31 | err,
32 | `can't convert specified send timeout to number: '%s'`,
33 | sendTimeoutRaw,
34 | )
35 | }
36 |
37 | receiveTimeout, err := strconv.Atoi(receiveTimeoutRaw)
38 | if err != nil {
39 | return nil, hierr.Errorf(
40 | err,
41 | `can't convert specified receive timeout to number: '%s'`,
42 | receiveTimeoutRaw,
43 | )
44 | }
45 |
46 | keepAlive, err := strconv.Atoi(keepAliveRaw)
47 | if err != nil {
48 | return nil, hierr.Errorf(
49 | err,
50 | `can't convert specified keep alive time to number: '%s'`,
51 | keepAliveRaw,
52 | )
53 | }
54 |
55 | return &runcmd.Timeout{
56 | Connection: time.Millisecond * time.Duration(connectionTimeout),
57 | Send: time.Millisecond * time.Duration(sendTimeout),
58 | Receive: time.Millisecond * time.Duration(receiveTimeout),
59 | KeepAlive: time.Millisecond * time.Duration(keepAlive),
60 | }, nil
61 | }
62 |
--------------------------------------------------------------------------------
/vendor.bash/.gitignore:
--------------------------------------------------------------------------------
1 | /github.com/reconquest/containers.bash
2 | /github.com/reconquest/coproc.bash
3 | /github.com/reconquest/go-test.bash
4 | /github.com/reconquest/hastur.bash
5 | /github.com/reconquest/opts.bash
6 | /github.com/reconquest/progress.bash
7 | /github.com/reconquest/ssh-test.bash
8 | /github.com/reconquest/sudo.bash
9 | /github.com/reconquest/test-runner.bash
10 | /github.com/reconquest/tests.sh
11 | /github.com/reconquest/types.bash
12 |
--------------------------------------------------------------------------------
/vendor.bash/Makefile:
--------------------------------------------------------------------------------
1 | .PHONY: .gitignore
2 |
3 | .gitignore:
4 | find . -type d -name .git -prune -printf '/%P\n' \
5 | | sed 's#/\.git$$##' \
6 | | sort \
7 | | tee .gitignore
8 |
--------------------------------------------------------------------------------
/verbosity.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | type (
4 | verbosity int
5 | )
6 |
7 | const (
8 | verbosityQuiet verbosity = iota
9 | verbosityNormal
10 | verbosityDebug
11 | verbosityTrace
12 | )
13 |
14 | func parseVerbosity(args map[string]interface{}) verbosity {
15 | var (
16 | quiet = args["--quiet"].(bool)
17 | level = args["--verbose"].(int)
18 | )
19 |
20 | if quiet {
21 | return verbosityQuiet
22 | }
23 |
24 | if level == 1 {
25 | return verbosityDebug
26 | }
27 |
28 | if level > 1 {
29 | return verbosityTrace
30 | }
31 |
32 | return verbosityNormal
33 | }
34 |
--------------------------------------------------------------------------------