├── .gitignore ├── .travis.yml ├── README.md ├── assets └── sasori.jpg ├── examples ├── .gitignore ├── README.md ├── project.clj ├── resources │ ├── elk │ │ ├── LICENSE │ │ ├── README.md │ │ ├── docker-compose.yml │ │ ├── elasticsearch │ │ │ ├── Dockerfile │ │ │ └── config │ │ │ │ └── elasticsearch.yml │ │ ├── extensions │ │ │ ├── README.md │ │ │ └── logspout │ │ │ │ ├── Dockerfile │ │ │ │ ├── README.md │ │ │ │ ├── build.sh │ │ │ │ ├── logspout-compose.yml │ │ │ │ └── modules.go │ │ ├── kibana │ │ │ ├── Dockerfile │ │ │ └── config │ │ │ │ └── kibana.yml │ │ └── logstash │ │ │ ├── Dockerfile │ │ │ ├── config │ │ │ └── logstash.yml │ │ │ └── pipeline │ │ │ └── logstash.conf │ └── ss │ │ └── ubuntu │ │ ├── config.json │ │ └── shadowsocks-server.service └── src │ └── examples │ ├── elk │ └── core.clj │ └── utils.clj ├── project.clj ├── src └── sasori │ ├── color.clj │ ├── core.clj │ ├── dsl.clj │ ├── exec.clj │ ├── log.clj │ ├── protocols.clj │ └── utils.clj └── test └── sasori ├── core_test.clj ├── dsl_test.clj └── utils_test.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .hgignore 11 | .hg/ 12 | /.idea 13 | *.iml 14 | .DS_Store 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: clojure 2 | script: lein test 3 | jdk: 4 | - oraclejdk8 5 | 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sasori/赤砂之蝎/サソリ 2 | [![Build Status](https://travis-ci.org/defclass/sasori.svg?branch=master)](https://travis-ci.org/defclass/sasori) 3 | 4 | Sasori is a lightweight shell command wrapper and composer. This is very early version, please feel free to submit bugs/thoughts/ideas/suggestions/patches etc. 5 | 6 | ## Features 7 | 8 | * Using function to compose shell commands. 9 | * REPL friendly. 10 | * Support customizable template engine. 11 | * Support executing commands sequentially and parallelly on different nodes. 12 | * Support colorful and real-time logging output. 13 | 14 | ## Usage 15 | 16 | Add sasori dependence. 17 | 18 | stable: 19 | 20 | [![Clojars Project](https://img.shields.io/clojars/v/defclass/sasori.svg)](https://clojars.org/defclass/sasori) 21 | 22 | Examples: 23 | 24 | ```clojure 25 | (require '[sasori.dsl :as dsl]) 26 | 27 | (let [cmd (dsl/cmd "mkdir /tmp/abc")] 28 | (dsl/emit cmd)) 29 | ;;=> "mkdir /tmp/abc" 30 | 31 | (let [cmd (dsl/cmds 32 | (dsl/cmd "mkdir /tmp/abc" :exit? false) 33 | (dsl/cmd "pwd"))] 34 | (dsl/emit cmd)) 35 | ;;=> "mkdir /tmp/abc ; pwd" 36 | 37 | (let [node (dsl/make-node {:host-info {:host "v1"}}) 38 | cmd (dsl/ssh 39 | (dsl/cmd "mkdir /tmp/abc") 40 | (dsl/cmd "ls -alh"))] 41 | (dsl/emit cmd node)) 42 | ;;=> "ssh v1 'mkdir /tmp/abc && ls -alh'" 43 | 44 | ;; Exec shell command 45 | 46 | (require '[sasori.core :as sasori]) 47 | 48 | (let [node (dsl/make-node {:host-info {:host "v1"}}) 49 | cmd (dsl/ssh 50 | (dsl/cmd "mkdir /tmp/a") 51 | (dsl/cmd "cd /tmp") 52 | (dsl/cmd "ls -alh"))] 53 | (-> (dsl/emit cmd node) 54 | (sasori/sh-string))) 55 | ;;=> 56 | ;#sasori.exec.Ok{:code 0, 57 | ; :out ["total 40K" 58 | ; "drwxrwxrwt 59 root root 12K Jul 4 17:56 ." 59 | ; "drwxr-xr-x 23 root root 4.0K Apr 15 23:10 .." 60 | ; "drwxrwxr-x 2 defclass defclass 4.0K Jul 4 17:56 a" 61 | ; ] 62 | ; :err nil} 63 | ``` 64 | 65 | A example to deploy a complete elk log service: [elk service deploy](https://github.com/defclass/sasori/blob/master/examples/src/examples/elk/core.clj) 66 | 67 | ## License 68 | 69 | Copyright © 2018 Michael Wong 70 | 71 | Distributed under the Eclipse Public License . 72 | -------------------------------------------------------------------------------- /assets/sasori.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defclass/sasori/86806d12db85598a08289902662e9b81b107871c/assets/sasori.jpg -------------------------------------------------------------------------------- /examples/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .hgignore 11 | .hg/ 12 | /.idea 13 | *.iml 14 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # examples 2 | 3 | A Clojure library designed to support ... 4 | 5 | 6 | ## Usage 7 | 8 | Add examples dependence: 9 | 10 | [![Clojars Project](https://img.shields.io/clojars/v/examples.svg)](https://clojars.org/examples) 11 | 12 | Examples: 13 | 14 | ```clojure 15 | 16 | (map prn ["This" "is" "example"]) 17 | 18 | ``` 19 | 20 | ## License 21 | 22 | Copyright © 2016 Michael Wong 23 | 24 | Distributed under the Eclipse Public License . 25 | -------------------------------------------------------------------------------- /examples/project.clj: -------------------------------------------------------------------------------- 1 | (defproject examples "0.1.0-SNAPSHOT" 2 | :description "A Clojure library designed to support..." 3 | :url "https://github.com/your-github-name/examples" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | :dependencies [[org.clojure/clojure "1.8.0"] 7 | [defclass/sasori "0.1.2-SNAPSHOT"] 8 | [selmer "1.11.8"]]) 9 | -------------------------------------------------------------------------------- /examples/resources/elk/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Anthony Lapenna 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /examples/resources/elk/README.md: -------------------------------------------------------------------------------- 1 | # Docker ELK stack 2 | 3 | [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 4 | [![Elastic Stack version](https://img.shields.io/badge/ELK-6.2.2-blue.svg?style=flat)](https://github.com/deviantony/docker-elk/issues/248) 5 | [![Build Status](https://api.travis-ci.org/deviantony/docker-elk.svg?branch=master)](https://travis-ci.org/deviantony/docker-elk) 6 | 7 | Run the latest version of the ELK (Elasticsearch, Logstash, Kibana) stack with Docker and Docker Compose. 8 | 9 | It will give you the ability to analyze any data set by using the searching/aggregation capabilities of Elasticsearch 10 | and the visualization power of Kibana. 11 | 12 | Based on the official Docker images: 13 | 14 | * [elasticsearch](https://github.com/elastic/elasticsearch-docker) 15 | * [logstash](https://github.com/elastic/logstash-docker) 16 | * [kibana](https://github.com/elastic/kibana-docker) 17 | 18 | **Note**: Other branches in this project are available: 19 | 20 | * ELK 6 with X-Pack support: https://github.com/deviantony/docker-elk/tree/x-pack 21 | * ELK 6 in Vagrant: https://github.com/deviantony/docker-elk/tree/vagrant 22 | * ELK 6 with Search Guard: https://github.com/deviantony/docker-elk/tree/searchguard 23 | 24 | ## Contents 25 | 26 | 1. [Requirements](#requirements) 27 | * [Host setup](#host-setup) 28 | * [SELinux](#selinux) 29 | * [DockerForWindows](#dockerforwindows) 30 | 2. [Getting started](#getting-started) 31 | * [Bringing up the stack](#bringing-up-the-stack) 32 | * [Initial setup](#initial-setup) 33 | 3. [Configuration](#configuration) 34 | * [How can I tune the Kibana configuration?](#how-can-i-tune-the-kibana-configuration) 35 | * [How can I tune the Logstash configuration?](#how-can-i-tune-the-logstash-configuration) 36 | * [How can I tune the Elasticsearch configuration?](#how-can-i-tune-the-elasticsearch-configuration) 37 | * [How can I scale out the Elasticsearch cluster?](#how-can-i-scale-up-the-elasticsearch-cluster) 38 | 4. [Storage](#storage) 39 | * [How can I persist Elasticsearch data?](#how-can-i-persist-elasticsearch-data) 40 | 5. [Extensibility](#extensibility) 41 | * [How can I add plugins?](#how-can-i-add-plugins) 42 | * [How can I enable the provided extensions?](#how-can-i-enable-the-provided-extensions) 43 | 6. [JVM tuning](#jvm-tuning) 44 | * [How can I specify the amount of memory used by a service?](#how-can-i-specify-the-amount-of-memory-used-by-a-service) 45 | * [How can I enable a remote JMX connection to a service?](#how-can-i-enable-a-remote-jmx-connection-to-a-service) 46 | 47 | ## Requirements 48 | 49 | ### Host setup 50 | 51 | 1. Install [Docker](https://www.docker.com/community-edition#/download) version **1.10.0+** 52 | 2. Install [Docker Compose](https://docs.docker.com/compose/install/) version **1.6.0+** 53 | 3. Clone this repository 54 | 55 | ### SELinux 56 | 57 | On distributions which have SELinux enabled out-of-the-box you will need to either re-context the files or set SELinux 58 | into Permissive mode in order for docker-elk to start properly. For example on Redhat and CentOS, the following will 59 | apply the proper context: 60 | 61 | ```console 62 | $ chcon -R system_u:object_r:admin_home_t:s0 docker-elk/ 63 | ``` 64 | 65 | ### DockerForWindows 66 | 67 | If you're using Docker for Windows, ensure the 'Shared Drives' feature is enabled for the C: drive (Docker for Windows > Settings > Shared Drives). [MSDN article detailing Shared Drives config](https://blogs.msdn.microsoft.com/stevelasker/2016/06/14/configuring-docker-for-windows-volumes/). 68 | 69 | ## Usage 70 | 71 | ### Bringing up the stack 72 | 73 | **Note**: In case you switched branch or updated a base image - you may need to run `docker-compose build` first 74 | 75 | Start the ELK stack using `docker-compose`: 76 | 77 | ```console 78 | $ docker-compose up 79 | ``` 80 | 81 | You can also choose to run it in background (detached mode): 82 | 83 | ```console 84 | $ docker-compose up -d 85 | ``` 86 | 87 | Give Kibana a few seconds to initialize, then access the Kibana web UI by hitting 88 | [http://localhost:5601](http://localhost:5601) with a web browser. 89 | 90 | By default, the stack exposes the following ports: 91 | * 5000: Logstash TCP input. 92 | * 9200: Elasticsearch HTTP 93 | * 9300: Elasticsearch TCP transport 94 | * 5601: Kibana 95 | 96 | **WARNING**: If you're using `boot2docker`, you must access it via the `boot2docker` IP address instead of `localhost`. 97 | 98 | **WARNING**: If you're using *Docker Toolbox*, you must access it via the `docker-machine` IP address instead of 99 | `localhost`. 100 | 101 | Now that the stack is running, you will want to inject some log entries. The shipped Logstash configuration allows you 102 | to send content via TCP: 103 | 104 | ```console 105 | $ nc localhost 5000 < /path/to/logfile.log 106 | ``` 107 | 108 | ## Initial setup 109 | 110 | ### Default Kibana index pattern creation 111 | 112 | When Kibana launches for the first time, it is not configured with any index pattern. 113 | 114 | #### Via the Kibana web UI 115 | 116 | **NOTE**: You need to inject data into Logstash before being able to configure a Logstash index pattern via the Kibana web 117 | UI. Then all you have to do is hit the *Create* button. 118 | 119 | Refer to [Connect Kibana with 120 | Elasticsearch](https://www.elastic.co/guide/en/kibana/current/connect-to-elasticsearch.html) for detailed instructions 121 | about the index pattern configuration. 122 | 123 | #### On the command line 124 | 125 | Create an index pattern via the Kibana API: 126 | 127 | ```console 128 | $ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ 129 | -H 'Content-Type: application/json' \ 130 | -H 'kbn-version: 6.2.2' \ 131 | -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' 132 | ``` 133 | 134 | The created pattern will automatically be marked as the default index pattern as soon as the Kibana UI is opened for the first time. 135 | 136 | ## Configuration 137 | 138 | **NOTE**: Configuration is not dynamically reloaded, you will need to restart the stack after any change in the 139 | configuration of a component. 140 | 141 | ### How can I tune the Kibana configuration? 142 | 143 | The Kibana default configuration is stored in `kibana/config/kibana.yml`. 144 | 145 | It is also possible to map the entire `config` directory instead of a single file. 146 | 147 | ### How can I tune the Logstash configuration? 148 | 149 | The Logstash configuration is stored in `logstash/config/logstash.yml`. 150 | 151 | It is also possible to map the entire `config` directory instead of a single file, however you must be aware that 152 | Logstash will be expecting a 153 | [`log4j2.properties`](https://github.com/elastic/logstash-docker/tree/master/build/logstash/config) file for its own 154 | logging. 155 | 156 | ### How can I tune the Elasticsearch configuration? 157 | 158 | The Elasticsearch configuration is stored in `elasticsearch/config/elasticsearch.yml`. 159 | 160 | You can also specify the options you want to override directly via environment variables: 161 | 162 | ```yml 163 | elasticsearch: 164 | 165 | environment: 166 | network.host: "_non_loopback_" 167 | cluster.name: "my-cluster" 168 | ``` 169 | 170 | ### How can I scale out the Elasticsearch cluster? 171 | 172 | Follow the instructions from the Wiki: [Scaling out 173 | Elasticsearch](https://github.com/deviantony/docker-elk/wiki/Elasticsearch-cluster) 174 | 175 | ## Storage 176 | 177 | ### How can I persist Elasticsearch data? 178 | 179 | The data stored in Elasticsearch will be persisted after container reboot but not after container removal. 180 | 181 | In order to persist Elasticsearch data even after removing the Elasticsearch container, you'll have to mount a volume on 182 | your Docker host. Update the `elasticsearch` service declaration to: 183 | 184 | ```yml 185 | elasticsearch: 186 | 187 | volumes: 188 | - /path/to/storage:/usr/share/elasticsearch/data 189 | ``` 190 | 191 | This will store Elasticsearch data inside `/path/to/storage`. 192 | 193 | **NOTE:** beware of these OS-specific considerations: 194 | * **Linux:** the [unprivileged `elasticsearch` user][esuser] is used within the Elasticsearch image, therefore the 195 | mounted data directory must be owned by the uid `1000`. 196 | * **macOS:** the default Docker for Mac configuration allows mounting files from `/Users/`, `/Volumes/`, `/private/`, 197 | and `/tmp` exclusively. Follow the instructions from the [documentation][macmounts] to add more locations. 198 | 199 | [esuser]: https://github.com/elastic/elasticsearch-docker/blob/016bcc9db1dd97ecd0ff60c1290e7fa9142f8ddd/templates/Dockerfile.j2#L22 200 | [macmounts]: https://docs.docker.com/docker-for-mac/osxfs/ 201 | 202 | ## Extensibility 203 | 204 | ### How can I add plugins? 205 | 206 | To add plugins to any ELK component you have to: 207 | 208 | 1. Add a `RUN` statement to the corresponding `Dockerfile` (eg. `RUN logstash-plugin install logstash-filter-json`) 209 | 2. Add the associated plugin code configuration to the service configuration (eg. Logstash input/output) 210 | 3. Rebuild the images using the `docker-compose build` command 211 | 212 | ### How can I enable the provided extensions? 213 | 214 | A few extensions are available inside the [`extensions`](extensions) directory. These extensions provide features which 215 | are not part of the standard Elastic stack, but can be used to enrich it with extra integrations. 216 | 217 | The documentation for these extensions is provided inside each individual subdirectory, on a per-extension basis. Some 218 | of them require manual changes to the default ELK configuration. 219 | 220 | ## JVM tuning 221 | 222 | ### How can I specify the amount of memory used by a service? 223 | 224 | By default, both Elasticsearch and Logstash start with [1/4 of the total host 225 | memory](https://docs.oracle.com/javase/8/docs/technotes/guides/vm/gctuning/parallel.html#default_heap_size) allocated to 226 | the JVM Heap Size. 227 | 228 | The startup scripts for Elasticsearch and Logstash can append extra JVM options from the value of an environment 229 | variable, allowing the user to adjust the amount of memory that can be used by each component: 230 | 231 | | Service | Environment variable | 232 | |---------------|----------------------| 233 | | Elasticsearch | ES_JAVA_OPTS | 234 | | Logstash | LS_JAVA_OPTS | 235 | 236 | To accomodate environments where memory is scarce (Docker for Mac has only 2 GB available by default), the Heap Size 237 | allocation is capped by default to 256MB per service in the `docker-compose.yml` file. If you want to override the 238 | default JVM configuration, edit the matching environment variable(s) in the `docker-compose.yml` file. 239 | 240 | For example, to increase the maximum JVM Heap Size for Logstash: 241 | 242 | ```yml 243 | logstash: 244 | 245 | environment: 246 | LS_JAVA_OPTS: "-Xmx1g -Xms1g" 247 | ``` 248 | 249 | ### How can I enable a remote JMX connection to a service? 250 | 251 | As for the Java Heap memory (see above), you can specify JVM options to enable JMX and map the JMX port on the docker 252 | host. 253 | 254 | Update the `{ES,LS}_JAVA_OPTS` environment variable with the following content (I've mapped the JMX service on the port 255 | 18080, you can change that). Do not forget to update the `-Djava.rmi.server.hostname` option with the IP address of your 256 | Docker host (replace **DOCKER_HOST_IP**): 257 | 258 | ```yml 259 | logstash: 260 | 261 | environment: 262 | LS_JAVA_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=18080 -Dcom.sun.management.jmxremote.rmi.port=18080 -Djava.rmi.server.hostname=DOCKER_HOST_IP -Dcom.sun.management.jmxremote.local.only=false" 263 | ``` 264 | -------------------------------------------------------------------------------- /examples/resources/elk/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | elasticsearch: 6 | build: 7 | context: elasticsearch/ 8 | volumes: 9 | - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro 10 | - {{volumes}}/path/to/volume{{/volumes}}/volumes/elasticsearch/data:/usr/share/elasticsearch/data 11 | ports: 12 | - "9200:9200" 13 | - "9300:9300" 14 | environment: 15 | ES_JAVA_OPTS: "-Xmx256m -Xms256m" 16 | networks: 17 | - elk 18 | 19 | logstash: 20 | build: 21 | context: logstash/ 22 | volumes: 23 | - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro 24 | - ./logstash/pipeline:/usr/share/logstash/pipeline:ro 25 | ports: 26 | - "5000:5000" 27 | environment: 28 | LS_JAVA_OPTS: "-Xmx256m -Xms256m" 29 | networks: 30 | - elk 31 | depends_on: 32 | - elasticsearch 33 | 34 | kibana: 35 | build: 36 | context: kibana/ 37 | volumes: 38 | - ./kibana/config/:/usr/share/kibana/config:ro 39 | ports: 40 | - "5601:5601" 41 | networks: 42 | - elk 43 | depends_on: 44 | - elasticsearch 45 | 46 | networks: 47 | 48 | elk: 49 | driver: bridge -------------------------------------------------------------------------------- /examples/resources/elk/elasticsearch/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/elastic/elasticsearch-docker 2 | FROM docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.2 3 | 4 | # Add your elasticsearch plugins setup here 5 | # Example: RUN elasticsearch-plugin install analysis-icu 6 | -------------------------------------------------------------------------------- /examples/resources/elk/elasticsearch/config/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Elasticsearch configuration from elasticsearch-docker. 3 | ## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml 4 | # 5 | cluster.name: "docker-cluster" 6 | network.host: 0.0.0.0 7 | 8 | # minimum_master_nodes need to be explicitly set when bound on a public IP 9 | # set to 1 to allow single node clusters 10 | # Details: https://github.com/elastic/elasticsearch/pull/17288 11 | discovery.zen.minimum_master_nodes: 1 12 | 13 | ## Use single node discovery in order to disable production mode and avoid bootstrap checks 14 | ## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html 15 | # 16 | discovery.type: single-node 17 | -------------------------------------------------------------------------------- /examples/resources/elk/extensions/README.md: -------------------------------------------------------------------------------- 1 | Third-party extensions that enable extra integrations with the ELK stack. 2 | -------------------------------------------------------------------------------- /examples/resources/elk/extensions/logspout/Dockerfile: -------------------------------------------------------------------------------- 1 | # uses ONBUILD instructions described here: 2 | # https://github.com/gliderlabs/logspout/tree/master/custom 3 | 4 | FROM gliderlabs/logspout:master 5 | ENV SYSLOG_FORMAT rfc3164 6 | -------------------------------------------------------------------------------- /examples/resources/elk/extensions/logspout/README.md: -------------------------------------------------------------------------------- 1 | # Logspout extension 2 | 3 | Logspout collects all Docker logs using the Docker logs API, and forwards them to Logstash without any additional 4 | configuration. 5 | 6 | ## Usage 7 | 8 | If you want to include the Logspout extension, run Docker Compose from the root of the repository with an additional 9 | command line argument referencing the `logspout-compose.yml` file: 10 | 11 | ```bash 12 | $ docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml up 13 | ``` 14 | 15 | In your Logstash pipeline configuration, enable the `udp` input and set the input codec to `json`: 16 | 17 | ``` 18 | input { 19 | udp { 20 | port => 5000 21 | codec => json 22 | } 23 | } 24 | ``` 25 | 26 | ## Documentation 27 | 28 | https://github.com/looplab/logspout-logstash 29 | -------------------------------------------------------------------------------- /examples/resources/elk/extensions/logspout/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # unmodified from: 4 | # https://github.com/gliderlabs/logspout/blob/67ee3831cbd0594361bb3381380c65bdbeb3c20f/custom/build.sh 5 | 6 | set -e 7 | apk add --update go git mercurial build-base 8 | mkdir -p /go/src/github.com/gliderlabs 9 | cp -r /src /go/src/github.com/gliderlabs/logspout 10 | cd /go/src/github.com/gliderlabs/logspout 11 | export GOPATH=/go 12 | go get 13 | go build -ldflags "-X main.Version=$1" -o /bin/logspout 14 | apk del go git mercurial build-base 15 | rm -rf /go /var/cache/apk/* /root/.glide 16 | 17 | # backwards compatibility 18 | ln -fs /tmp/docker.sock /var/run/docker.sock 19 | -------------------------------------------------------------------------------- /examples/resources/elk/extensions/logspout/logspout-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | logspout: 5 | build: 6 | context: extensions/logspout 7 | volumes: 8 | - /var/run/docker.sock:/var/run/docker.sock:ro 9 | environment: 10 | ROUTE_URIS: logstash://logstash:5000 11 | LOGSTASH_TAGS: docker-elk 12 | networks: 13 | - elk 14 | depends_on: 15 | - logstash 16 | restart: on-failure 17 | -------------------------------------------------------------------------------- /examples/resources/elk/extensions/logspout/modules.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // installs the Logstash adapter for Logspout, and required dependencies 4 | // https://github.com/looplab/logspout-logstash 5 | import ( 6 | _ "github.com/looplab/logspout-logstash" 7 | _ "github.com/gliderlabs/logspout/transports/udp" 8 | _ "github.com/gliderlabs/logspout/transports/tcp" 9 | ) 10 | -------------------------------------------------------------------------------- /examples/resources/elk/kibana/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/elastic/kibana-docker 2 | FROM docker.elastic.co/kibana/kibana-oss:6.2.2 3 | 4 | # Add your kibana plugins setup here 5 | # Example: RUN kibana-plugin install 6 | -------------------------------------------------------------------------------- /examples/resources/elk/kibana/config/kibana.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Kibana configuration from kibana-docker. 3 | ## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml 4 | # 5 | server.name: kibana 6 | server.host: "0" 7 | elasticsearch.url: http://elasticsearch:9200 8 | -------------------------------------------------------------------------------- /examples/resources/elk/logstash/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/elastic/logstash-docker 2 | FROM docker.elastic.co/logstash/logstash-oss:6.2.2 3 | 4 | # Add your logstash plugins setup here 5 | # Example: RUN logstash-plugin install logstash-filter-json 6 | -------------------------------------------------------------------------------- /examples/resources/elk/logstash/config/logstash.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Logstash configuration from logstash-docker. 3 | ## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash-oss.yml 4 | # 5 | http.host: "0.0.0.0" 6 | path.config: /usr/share/logstash/pipeline 7 | -------------------------------------------------------------------------------- /examples/resources/elk/logstash/pipeline/logstash.conf: -------------------------------------------------------------------------------- 1 | input { 2 | tcp { 3 | port => 5000 4 | } 5 | } 6 | 7 | ## Add your filters / logstash plugins configuration here 8 | 9 | filter{ 10 | json{ 11 | source => "message" 12 | remove_field => ["message"] 13 | } 14 | } 15 | 16 | output { 17 | elasticsearch { 18 | hosts => "elasticsearch:9200" 19 | index => "api-request-%{+YYYY.MM.dd}" 20 | } 21 | } -------------------------------------------------------------------------------- /examples/resources/ss/ubuntu/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "server": "0.0.0.0", 3 | "server_port": 9600, 4 | "password": "aw42a49AiXiJ56p6", 5 | "method": "aes-256-cfb", 6 | "mode": "tcp_and_udp" 7 | } -------------------------------------------------------------------------------- /examples/resources/ss/ubuntu/shadowsocks-server.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Shadowsocks Server 3 | After=network.target 4 | 5 | [Service] 6 | ExecStart=/usr/bin/ssserver -c /etc/shadowsocks/config.json 7 | Restart=on-abort 8 | 9 | [Install] 10 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /examples/src/examples/elk/core.clj: -------------------------------------------------------------------------------- 1 | (ns examples.elk.core 2 | "To deploy out of the box elk logging service." 3 | (:require [sasori 4 | [core :as sasori] 5 | [dsl :as dsl]]) 6 | (:require [examples.utils :as utils])) 7 | 8 | (defn remote-base-path 9 | "Ensure remote base path." 10 | [node _] 11 | (let [remote-base-path (str (sasori/get-home node) "/elk")] 12 | (sasori/make-context (utils/create-kw-map remote-base-path)))) 13 | 14 | (defn sync-docker-elk 15 | [node {:keys [remote-base-path]}] 16 | (let [opts (merge {:recursive true} node)] 17 | (utils/rsync-resource-to "./elk" remote-base-path opts))) 18 | 19 | (defn template-docker-compose 20 | "Replace docker-compose template" 21 | [node {:keys [remote-base-path]}] 22 | (let [local-tpl "./elk/docker-compose.yml" 23 | remote-path (str remote-base-path "/docker-compose.yml") 24 | tpl-params {:volumes (format "%s/%s" remote-base-path "volumes")}] 25 | (utils/template node local-tpl remote-path tpl-params))) 26 | 27 | (defn docker-build 28 | "Build images" 29 | [node {:keys [remote-base-path]}] 30 | (let [dsl (dsl/ssh 31 | (dsl/cmd "cd" remote-base-path) 32 | (dsl/cmd "docker-compose build"))] 33 | (sasori/sh-dsl dsl node))) 34 | 35 | (defn docker-up 36 | [node {:keys [remote-base-path]}] 37 | (let [dsl (dsl/ssh 38 | (dsl/cmd "cd" remote-base-path) 39 | (dsl/cmd "docker-compose up"))] 40 | (sasori/sh-dsl dsl node))) 41 | 42 | ;; host is for the nickname of the host, represent a entry in ~/.ssh/config. 43 | ;; eg: 44 | ;; host v1 45 | ;; HostName ip-or-domain 46 | ;; user username 47 | 48 | (def host-info {:host "v1"}) 49 | (def global-opts {:verbose false :color true}) 50 | 51 | (defn play 52 | [& [opts]] 53 | (let [parallel? (:parallel? opts) 54 | task-vars (sasori/task-vars 55 | remote-base-path 56 | sync-docker-elk 57 | template-docker-compose 58 | docker-build)] 59 | (sasori/play task-vars 60 | {:parallel? parallel? 61 | :hosts-info host-info 62 | :global-opts global-opts 63 | :context nil}))) 64 | 65 | (defn -main 66 | " Usage: 67 | 68 | lein run -m examples.elk.core '{:parallel? true}' 69 | " 70 | [& args] 71 | (let [args-m (sasori/parse-from-clj args)] 72 | (play args-m) 73 | ;; JVM will wait for `clojure-agent-send-off-pool-x` exit, so need exit 74 | ;; manually. 75 | (System/exit 0))) 76 | 77 | 78 | ; output: 79 | ; 80 | ;$ lein run -m examples.elk.core '{:parallel? true}' 81 | ;Exec: Ensure remote base path. 82 | ;v1: Success. 83 | ; 84 | ;Exec: sync-docker-elk 85 | ;v1: Success. 86 | ; 87 | ;Exec: Replace docker-compose template 88 | ;v1: Success. 89 | ; 90 | ;Exec: Build images 91 | ;v1: Success. 92 | ; 93 | -------------------------------------------------------------------------------- /examples/src/examples/utils.clj: -------------------------------------------------------------------------------- 1 | (ns examples.utils 2 | (:require [clojure.java.io :as io] 3 | [sasori 4 | [core :as sasori] 5 | [utils :as u] 6 | [dsl :as dsl]] 7 | [selmer.parser :as tpl]) 8 | (:import (java.io File))) 9 | 10 | (defmacro create-kw-map 11 | [& symbols] 12 | `(zipmap (map keyword '~symbols) (list ~@symbols))) 13 | 14 | (defn rsync-resource-to [resource-path remote-path & [opts]] 15 | (let [resource-file (io/as-file (io/resource resource-path)) 16 | _ (when-not (instance? File resource-file) 17 | (u/error! (format "resource-path not exist: %s" resource-path))) 18 | local-path (.getAbsolutePath resource-file) 19 | dir-default-opts {:recursive true} 20 | [local-path opts] (if (.isFile resource-file) 21 | [local-path opts] 22 | [(str local-path "/") 23 | (merge dir-default-opts opts)]) 24 | rsync-opts (select-keys opts [:sudo :compress :progress :recursive :delete :human]) 25 | dsl (dsl/rsync local-path remote-path rsync-opts)] 26 | (sasori/sh-dsl dsl opts))) 27 | 28 | 29 | (defn template [node source destination & [transform-label-m]] 30 | (let [s (tpl/render-file source transform-label-m)] 31 | (sasori/scp-str-to-file s destination node))) -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject defclass/sasori "0.1.2" 2 | :description "A Clojure library designed to wrap and compose shell commands." 3 | :url "https://github.com/defclass/sasori" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | :dependencies [[org.clojure/clojure "1.8.0"]]) 7 | -------------------------------------------------------------------------------- /src/sasori/color.clj: -------------------------------------------------------------------------------- 1 | (ns sasori.color) 2 | 3 | (defn- gen-mark [n] 4 | (str "\033[38;5;" n "m")) 5 | 6 | (def ^:private clear-mark "\033[0m") 7 | 8 | (defn- assemble-color [start-mark s] 9 | (str start-mark s clear-mark)) 10 | 11 | (defn- wrap-mark [color-index s opts] 12 | (let [opts (merge {:color true} opts)] 13 | (if-not (:color opts) 14 | s 15 | (-> (gen-mark color-index) 16 | (assemble-color s))))) 17 | 18 | (defn get-cached-color-generator [] 19 | (let [color-count (atom 0) 20 | start-marks (atom {})] 21 | (fn [k opts] 22 | (let [opts (merge {:color true} opts)] 23 | (if-not (:color opts) 24 | k 25 | (let [start-mark 26 | (if-let [exist-mark (get @start-marks k)] 27 | exist-mark 28 | (let [n (swap! color-count inc) 29 | new-color-mark (gen-mark n)] 30 | (swap! start-marks assoc k new-color-mark) 31 | new-color-mark))] 32 | (assemble-color start-mark k))))))) 33 | 34 | (def wrap-host (get-cached-color-generator)) 35 | 36 | (def wrap-red (partial wrap-mark 1)) 37 | (def wrap-green (partial wrap-mark 2)) 38 | (def wrap-yellow (partial wrap-mark 3)) 39 | (def wrap-blue (partial wrap-mark 4)) 40 | (def wrap-magenta (partial wrap-mark 5)) 41 | (def wrap-cyan (partial wrap-mark 6)) 42 | (def wrap-gray (partial wrap-mark 7)) 43 | (def wrap-dark-grey (partial wrap-mark 8)) 44 | (def wrap-light-red (partial wrap-mark 9)) 45 | (def wrap-light-green (partial wrap-mark 10)) 46 | (def wrap-light-yellow (partial wrap-mark 11)) 47 | (def wrap-light-blue (partial wrap-mark 12)) 48 | (def wrap-light-magenta (partial wrap-mark 13)) 49 | (def wrap-light-cyan (partial wrap-mark 14)) 50 | (def wrap-light-gray (partial wrap-mark 15)) 51 | -------------------------------------------------------------------------------- /src/sasori/core.clj: -------------------------------------------------------------------------------- 1 | (ns sasori.core 2 | (:require [clojure.string :as str] 3 | [clojure.java.io :as io] 4 | [clojure.walk :as walk] 5 | [clojure.set :as set]) 6 | (:require [sasori 7 | [utils :as u] 8 | [exec :as exec] 9 | [dsl :as dsl] 10 | [log :as log] 11 | [color :as color] 12 | [protocols :as protocols]])) 13 | 14 | (def make-node dsl/make-node) 15 | (def make-nodes dsl/make-nodes) 16 | 17 | (defn local? [node] 18 | (dsl/check-node-type! node) 19 | (get-in node [:host-info :local?])) 20 | 21 | ;;;; Exec 22 | 23 | (defn sh-string 24 | [cmd & [args]] 25 | {:pre [(or (nil? args) (map? args))]} 26 | (let [args-in-seq (flatten (seq args))] 27 | (when (:verbose args) 28 | (log/info cmd args)) 29 | (->> (into ["/bin/sh" "-c" cmd] args-in-seq) 30 | (apply exec/sh)))) 31 | 32 | (defn sh-dsl 33 | [dsl node] 34 | (let [cmd-str (dsl/emit dsl node)] 35 | (->> (u/node->opts-m node) 36 | (sh-string cmd-str)))) 37 | 38 | (defn safe-sh-dsl 39 | "Same to sh-dsl exception throw exception when return isn't ok." 40 | [dsl node] 41 | (let [cmd-str (dsl/emit dsl node) 42 | result (sh-string cmd-str node)] 43 | (if (protocols/ok? result) 44 | result 45 | (throw (IllegalStateException. 46 | (format "Process exec failed: %s" 47 | cmd-str)))))) 48 | 49 | ;;;; Tools 50 | 51 | (defn get-home [node] 52 | (-> (dsl/ssh (dsl/cmd "pwd")) 53 | (sh-dsl node) 54 | :out 55 | (first))) 56 | 57 | (defn get-host [node] 58 | (-> (dsl/ssh (dsl/cmd "hostname -s")) 59 | (sh-dsl node) 60 | :out 61 | (first))) 62 | 63 | (defn get-ip [node] 64 | (-> (dsl/ssh (dsl/cmd "ip route get 1 | awk '{print $NF;exit}'")) 65 | (sh-dsl node) 66 | :out 67 | (first))) 68 | 69 | (defn scp-str-to-file [s path node] 70 | (let [f (io/as-file path) 71 | directory (.getParent f) 72 | filename (.getAbsolutePath f) 73 | main-cmd (format "mkdir -p %s ; cat - > %s" directory filename) 74 | dsl (dsl/ssh (dsl/cmd main-cmd)) 75 | node (update node :global-opts assoc :in s)] 76 | (sh-dsl dsl node))) 77 | 78 | ;;;; Compose tasks 79 | 80 | (defn- gen-asserts 81 | [m ks] 82 | (let [f (fn [k] 83 | `(when-not (contains? ~m ~k) 84 | (u/error! (format "Safe-let: Key %s is not exist in %s" ~k ~m))))] 85 | `(do ~@(map f ks)))) 86 | 87 | (defn- insert-validator 88 | [bindings] 89 | (let [f (fn [[left right]] 90 | (if (and (map? left) (some? (:safe-keys left))) 91 | (let [syms (:safe-keys left) 92 | ks (mapv keyword syms)] 93 | (vector '_ (gen-asserts right ks) 94 | (set/rename-keys left {:safe-keys :keys}) right)) 95 | [left right]))] 96 | (vec (mapcat f bindings)))) 97 | 98 | (defmacro safe-let 99 | [bindings & body] 100 | (assert (vector? bindings) "Binding is not a vector.") 101 | (assert (even? (count bindings)) "An even number of forms in binding vector.") 102 | (let [binding-entries (partition 2 bindings) 103 | new-bindings (insert-validator binding-entries)] 104 | `(let ~new-bindings 105 | ~@body))) 106 | 107 | ;; Context are keys/values (map) that shared between tasks. 108 | 109 | (defrecord Context [value] 110 | protocols/ITaskReturn 111 | (merge-to-msg [this msg] 112 | (let [ks (keys (:value this))] 113 | (doseq [k ks] 114 | (when (contains? (:context msg) k) 115 | (u/error! (format "Key %s is exists in context." k)))) 116 | (update msg :context merge (:value this))))) 117 | 118 | (defn make-context [& [m]] 119 | (assert ((some-fn map? nil?) m)) 120 | (->Context m)) 121 | 122 | ;; A msg is everything needed by one task, including share context , node 123 | ;; information, and error if ever produced. 124 | 125 | (defrecord Msg [node context error]) 126 | 127 | (defn make-msg [node & [init-content]] 128 | (->Msg node init-content nil)) 129 | 130 | (defn msg? [x] 131 | (instance? Msg x)) 132 | 133 | (defn failed-msg? [^Msg msg] 134 | (when-not (msg? msg) 135 | (u/error! "Msg is not Msg instance." {:msg msg})) 136 | (some? (:error msg))) 137 | 138 | (defn success-msg? [^Msg msg] 139 | (not (failed-msg? msg))) 140 | 141 | (defn- gen-doc-logger 142 | "Return a log fn which receive a var. This fn will println var's doc or name." 143 | [fmt] 144 | (fn [v node] 145 | (when-not (dsl/node? node)) 146 | (let [m (meta v) 147 | description (or (:doc m) (:name m)) 148 | msg (format fmt description)] 149 | (->> (u/node->opts-m node) 150 | (log/info msg))))) 151 | 152 | (def start-logger (gen-doc-logger "Starting: %s")) 153 | (def done-logger (gen-doc-logger "Done: %s")) 154 | 155 | (defn- do-task [task-var node context] 156 | (let [verbose (get-in node [:global-opts :verbose])] 157 | (when verbose 158 | (start-logger task-var node)) 159 | (let [resp (task-var node context)] 160 | (when verbose 161 | (done-logger task-var node)) 162 | resp))) 163 | 164 | (defn- task [task-var {:keys [node context] :as msg}] 165 | (if (failed-msg? msg) 166 | msg 167 | (try 168 | (let [return (do-task task-var node context)] 169 | (cond 170 | (satisfies? protocols/ITaskReturn return) 171 | (protocols/merge-to-msg return msg) 172 | 173 | :else 174 | (throw 175 | (RuntimeException. 176 | "Return didn't satisfied sasori.protocols/ITaskReturn")))) 177 | (catch Throwable t 178 | (assoc msg :error (into [t] (map str (.getStackTrace t)))))))) 179 | 180 | (defn- print-stats 181 | [task-var msgs] 182 | (let [{:keys [doc name]} (meta task-var) 183 | title (format "Exec: %s" (or doc name))] 184 | (->> (color/wrap-cyan title (:node (first msgs))) 185 | (println)) 186 | (doseq [msg msgs] 187 | (let [node (:node msg) 188 | colored-host (log/build-host-info (u/node->opts-m node)) 189 | success-or-failed 190 | (if (failed-msg? msg) 191 | (color/wrap-red "Failed." (:global-opts node)) 192 | (color/wrap-green "Success." (:global-opts node)))] 193 | (println (format "%s: %s" colored-host success-or-failed)))) 194 | (println))) 195 | 196 | (defn- do-tasks [map-f init-msgs tasks] 197 | {:pre [(every? msg? init-msgs)]} 198 | (let [results 199 | (reduce (fn [pre-msgs task-var] 200 | (let [task-f (partial task task-var) 201 | new-msgs (map-f task-f pre-msgs)] 202 | (print-stats task-var new-msgs) 203 | new-msgs)) 204 | init-msgs 205 | tasks)] 206 | (doseq [r results] 207 | (when (failed-msg? r) 208 | (let [host-info (-> (u/node->opts-m (:node r)) 209 | (log/build-host-info))] 210 | (println (format "[%s] Found Error: " host-info))) 211 | (println (str/join "\n" (:error r))))) 212 | results)) 213 | 214 | (defmacro task-vars 215 | "Generate task vars from sequence of symbols." 216 | [& syms] 217 | {:pre [(every? symbol? syms)]} 218 | `(vector 219 | ~@(map (fn [task-sym#] `(var ~task-sym#)) 220 | syms))) 221 | 222 | (defn build-init-msgs 223 | [hosts-info global-opts & [context-m]] 224 | (let [nodes (dsl/make-nodes {:hosts-info hosts-info 225 | :global-opts global-opts})] 226 | (map #(make-msg % context-m) nodes))) 227 | 228 | (defn- exec-task 229 | [map-f task-vars hosts-info global-opts context] 230 | {:pre [(and (sequential? task-vars) 231 | (every? var? task-vars) 232 | (sequential? hosts-info) 233 | (u/maybe-map? global-opts) 234 | (u/maybe-map? context))]} 235 | (let [init-msgs (build-init-msgs hosts-info global-opts context)] 236 | (do-tasks map-f init-msgs task-vars))) 237 | 238 | (defn play 239 | [task-vars {:keys [parallel? hosts-info global-opts context] 240 | :or {parallel? false}}] 241 | (when-not (or (map? hosts-info) (sequential? hosts-info)) 242 | (u/error! "hosts-info is not set or format is wrong.")) 243 | (let [map-f (if parallel? pmap map) 244 | hosts-info (if (sequential? hosts-info) hosts-info [hosts-info])] 245 | (exec-task map-f task-vars hosts-info global-opts context))) 246 | 247 | ;;;; Read from cli 248 | 249 | (defn parse-from-clj 250 | "From cmd line: 251 | lein run -m ns/fn '{:hostname \"hostname\" :username \"some-username\"}' 252 | 253 | Return: 254 | 255 | {:hostname \"hostname\" :username \"some-username\"}" 256 | [args] 257 | (when (seq args) 258 | (read-string (first args)))) 259 | 260 | (defn parse-from-seq 261 | "From cmd line: 262 | lein run -m ns/fn hostname hostname username some-username 263 | 264 | Return: 265 | {:hostname \"hostname\" :username \"some-username\"}" 266 | [args] 267 | (-> (apply array-map args) 268 | (walk/keywordize-keys))) 269 | -------------------------------------------------------------------------------- /src/sasori/dsl.clj: -------------------------------------------------------------------------------- 1 | (ns sasori.dsl 2 | "About host and hostname, follow ~/.ssh/config: 3 | host: host is for the nickname of the host, represent a entry in ~/.ssh/config. 4 | hostname: hostname is for the actual hostname." 5 | (:require [clojure.string :as str]) 6 | (:require [sasori 7 | [utils :as u] 8 | [protocols :as protocols]])) 9 | 10 | (defrecord HostInfo [local? host hostname port username]) 11 | 12 | (defn make-host-info [{:keys [local? host hostname port username]}] 13 | (when-not (u/boolean? local?) 14 | (u/error! "Should set local? a boolean value.")) 15 | (when-not local? 16 | (when-not (or (every? string? [hostname username]) 17 | (some? host)) 18 | (u/error! "Host or `hostname && username` should be set."))) 19 | (->HostInfo local? host hostname port username)) 20 | 21 | (defrecord GlobalOpts [verbose in env dir clear-env color]) 22 | 23 | (defn make-global-opts [m] 24 | (assert (map? m) "Opts should be map.") 25 | (map->GlobalOpts m)) 26 | 27 | (defrecord Node [host-info global-opts]) 28 | 29 | (defn make-node 30 | [{:keys [host-info global-opts] :as m}] 31 | (when (contains? m :local?) 32 | (u/error! ":local key should in host-info map.")) 33 | (let [local? (true? (:local? host-info)) 34 | host (make-host-info (assoc host-info :local? local?)) 35 | global-opts (when global-opts (make-global-opts global-opts))] 36 | (->Node host global-opts))) 37 | 38 | (defn make-nodes 39 | [{:keys [hosts-info global-opts]}] 40 | {:pre (sequential? hosts-info)} 41 | (let [make-node-fn 42 | (fn [host-info] (make-node {:host-info host-info :global-opts global-opts}))] 43 | (map make-node-fn hosts-info))) 44 | 45 | (defn node? [node] 46 | (instance? Node node)) 47 | 48 | (defn check-node-type! [node] 49 | (when-not (node? node) 50 | (u/error! "Should be node." {:node node}))) 51 | 52 | (defn cmd? [x] 53 | (satisfies? protocols/ICmd x)) 54 | 55 | (defrecord Cmd [cmd-seq opts] 56 | protocols/ICmd 57 | (plain [_ _] (str/join " " cmd-seq)) 58 | (exit? [_] (:exit? opts))) 59 | 60 | (defn- parse-opts [opts] 61 | (assert (even? (count opts)) "Opts should be satisfy even?") 62 | (let [opts (apply hash-map opts)] 63 | (assert (every? keyword? (keys opts)) "Opts keys should be keyword.") 64 | opts)) 65 | 66 | (defn cmd [& args] 67 | (let [[str-seq local-opts] (split-with string? args) 68 | local-opts (parse-opts local-opts) 69 | local-opts (merge {:exit? true} local-opts)] 70 | (->Cmd str-seq local-opts))) 71 | 72 | (defrecord Cmds [seq-cmds local-opts] 73 | protocols/ICmd 74 | (plain [_ node] 75 | (check-node-type! node) 76 | (loop [acc "" [f & r] seq-cmds] 77 | (if (nil? f) 78 | acc 79 | (let [next-fragment 80 | (cond 81 | (nil? r) 82 | (protocols/plain f node) 83 | 84 | (protocols/exit? f) 85 | (str (protocols/plain f node) " &&") 86 | 87 | (not (protocols/exit? f)) 88 | (str (protocols/plain f node) " ;") 89 | 90 | :else 91 | (u/error! "Unknown error: " 92 | {:seq-cmds seq-cmds 93 | :acc acc})) 94 | acc (u/join-not-blank acc next-fragment)] 95 | (recur acc r))))) 96 | (exit? [_] (:exit? local-opts))) 97 | 98 | (defn cmds [& args] 99 | (let [[cmds local-opts] (split-with cmd? args) 100 | local-opts (parse-opts local-opts)] 101 | (->Cmds cmds local-opts))) 102 | 103 | (defn escape-cmd 104 | "https://stackoverflow.com/questions/1250079/how-to-escape-single-quotes-within-single-quoted-strings" 105 | [s] 106 | (str/escape s {\' "'\"'\"'"})) 107 | 108 | (defn build-ssh-conn 109 | [host] 110 | (instance? HostInfo host) 111 | (let [{:keys [host hostname port username]} host] 112 | (if host 113 | host 114 | (let [conn-info (format "%s@%s" username hostname) 115 | port (when port (str "-p " port))] 116 | (u/join-not-blank port conn-info))))) 117 | 118 | (defrecord Ssh [seq-cmds local-opts] 119 | protocols/ICmd 120 | (plain [_ node] 121 | (check-node-type! node) 122 | (let [conn-info (build-ssh-conn (:host-info node))] 123 | (->> (protocols/plain seq-cmds node) 124 | (escape-cmd) 125 | (format "ssh %s '%s'" conn-info)))) 126 | (exit? [_] (:exit? local-opts))) 127 | 128 | (defn ssh [& args] 129 | (let [[seq-cmds local-opts] (split-with cmd? args) 130 | local-opts (parse-opts local-opts) 131 | seq-cmd-obj (if (and (= 1 (count seq-cmds)) 132 | (instance? Cmds (first seq-cmds))) 133 | (first seq-cmds) 134 | (apply cmds seq-cmds))] 135 | (->Ssh seq-cmd-obj local-opts))) 136 | 137 | (defrecord Sudo [cmd local-opts] 138 | protocols/ICmd 139 | (plain [_ node] 140 | (check-node-type! node) 141 | ;; sudo sh -c would fit `sudo cmd > some-file-need-privilege` 142 | (format "sudo sh -c '%s'" (protocols/plain cmd node))) 143 | (exit? [_] (:exit? local-opts))) 144 | 145 | (defn sudo 146 | "Not work on macos at present." 147 | [cmd & [local-opts]] 148 | (->Sudo cmd local-opts)) 149 | 150 | (defn- assemble-excludes [excludes] 151 | (let [complete-param (fn [path] (str "--exclude=" (str/trim path)))] 152 | (->> (map complete-param excludes) 153 | (str/join " ")))) 154 | 155 | (defn- build-rsync-dest 156 | [host-info dest-path] 157 | (assert (instance? HostInfo host-info) "Should be HostInfo instance.") 158 | (when-not (string? dest-path) (u/error! "Dest-path should be string.")) 159 | (let [{:keys [host hostname username]} host-info] 160 | (if host 161 | (str host ":" dest-path) 162 | (str username "@" hostname ":" dest-path)))) 163 | 164 | (defrecord Rsync [src dest local-opts] 165 | protocols/ICmd 166 | (plain [this node] 167 | (let [{:keys [verbose]} (:global-opts node) 168 | {:keys [compress progress recursive delete human excludes 169 | archive sudo]} local-opts 170 | dest (build-rsync-dest (:host-info node) dest) 171 | rsync-path (if sudo 172 | "sudo `which rsync`" 173 | "`which rsync`") 174 | params-in-s (u/cond-join 175 | delete "--delete" 176 | 177 | (u/not-blank? rsync-path) 178 | (format "--rsync-path='%s'" rsync-path) 179 | 180 | archive "-a" 181 | verbose "-v" 182 | compress "-z" 183 | excludes (assemble-excludes excludes) 184 | recursive "-r" 185 | progress "--progress" 186 | human "-h" 187 | true src 188 | true dest)] 189 | (u/join-not-blank "rsync" params-in-s))) 190 | (exit? [_] (:exit? local-opts))) 191 | 192 | (defn rsync 193 | [src dest & [opts]] 194 | (let [opts opts] 195 | (let [{:keys [excludes]} opts 196 | default-opts {:verbose true :human true :progress true}] 197 | (when excludes 198 | (assert (sequential? excludes) "Excludes should match sequential?")) 199 | (->Rsync src dest (merge default-opts opts))))) 200 | 201 | (defn- build-scp-dest 202 | [host dest-path] 203 | (assert (instance? HostInfo host) "Should be HostInfo instance.") 204 | (when-not (string? dest-path) (u/error! "dest-path should be string.")) 205 | (let [{:keys [host hostname username]} host] 206 | (if host 207 | (str host ":" dest-path) 208 | (format "%s@%s:%s" username hostname dest-path)))) 209 | 210 | (defrecord Scp [src dest local-opts] 211 | protocols/ICmd 212 | (protocols/plain [this node] 213 | (let [{:keys [recursive]} local-opts 214 | {:keys [verbose]} (:global-opts node) 215 | {:keys [port] :as host} (:host-info node) 216 | dest (build-scp-dest host dest) 217 | params-in-s (u/cond-join 218 | verbose "-vv" 219 | recursive "-r" 220 | port (u/join "-P" port) 221 | true src 222 | true dest)] 223 | (u/join-not-blank "scp" params-in-s))) 224 | (exit? [_] (:exit? local-opts))) 225 | 226 | (defn scp 227 | [src dest & [local-opts]] 228 | (->Scp src dest local-opts)) 229 | 230 | (def local-node (make-node {:host-info {:local? true}})) 231 | 232 | (defn emit [cmd & [node]] 233 | (assert (cmd? cmd)) 234 | (let [node (if (some? node) node local-node)] 235 | (protocols/plain cmd node))) 236 | -------------------------------------------------------------------------------- /src/sasori/exec.clj: -------------------------------------------------------------------------------- 1 | (ns sasori.exec 2 | (:require [clojure.java.io :as io]) 3 | (:require [sasori 4 | [utils :as u] 5 | [log :as log] 6 | [protocols :as protocols]])) 7 | 8 | (defrecord Ok [code out err] 9 | protocols/ICmdStatus 10 | (ok? [_] true) 11 | 12 | protocols/ITaskReturn 13 | (merge-to-msg [_ msg] msg)) 14 | 15 | (defrecord Failed [code out err] 16 | protocols/ICmdStatus 17 | (ok? [_] false) 18 | 19 | protocols/ITaskReturn 20 | (merge-to-msg [this msg] 21 | (assoc msg :error (:out this)))) 22 | 23 | (defn sh 24 | [& args] 25 | (let [[cmd opts] (split-with (complement keyword?) args) 26 | opts (apply hash-map opts) 27 | opts (merge {:verbose true} opts) 28 | builder (-> (ProcessBuilder. (into-array String cmd)) 29 | (.redirectErrorStream true)) 30 | env (.environment builder)] 31 | (-> (apply u/join cmd) 32 | (log/debug opts)) 33 | (when (:clear-env opts) 34 | (.clear env)) 35 | (doseq [[k v] (:env opts)] 36 | (.put env k v)) 37 | (when-let [dir (:dir opts)] 38 | (.directory builder (io/file dir))) 39 | (when (= :very (:verbose opts)) 40 | (when-let [env (:env opts)] (log/info {:env env} opts)) 41 | (when-let [dir (:dir opts)] (log/info {:dir dir} opts))) 42 | (let [proc (.start builder) 43 | in (:in opts)] 44 | (if in 45 | (future 46 | (with-open [os (.getOutputStream proc)] 47 | (io/copy in os))) 48 | (.close (.getOutputStream proc))) 49 | (let [out 50 | (with-open 51 | [out-reader (io/reader (.getInputStream proc))] 52 | (loop [out []] 53 | (if-let [line (.readLine out-reader)] 54 | (do (when (:verbose opts) 55 | (log/info line opts)) 56 | (recur (conj out line))) 57 | out))) 58 | exit-code (.waitFor proc)] 59 | (if (= 0 exit-code) 60 | (->Ok exit-code out nil) 61 | (->Failed exit-code out nil)))))) -------------------------------------------------------------------------------- /src/sasori/log.clj: -------------------------------------------------------------------------------- 1 | (ns sasori.log 2 | (:require [clojure.string :as str]) 3 | (:require [sasori.color :as color] 4 | [sasori.utils :as u]) 5 | (:import (java.text SimpleDateFormat))) 6 | 7 | (def time-formatter (SimpleDateFormat. "HH:mm:ss")) 8 | 9 | (defn get-now [] 10 | (let [date (new java.util.Date)] 11 | (.format time-formatter date))) 12 | 13 | (defn build-host-info [m] 14 | (let [{:keys [local? host hostname]} m 15 | host (if local? "local" (or host hostname))] 16 | (when-not (string? host) 17 | (u/error! "Unknown host" {:m m})) 18 | (color/wrap-host host m))) 19 | 20 | (def levels (atom #{:info :error})) 21 | 22 | (defn set-level! [level] 23 | (let [level-seq [:debug :info :error]] 24 | (reset! levels 25 | (-> (drop-while #(not= % level) level-seq) 26 | (set))))) 27 | 28 | (defn- log [level msg & [opts]] 29 | (let [hostname (build-host-info opts) 30 | now (get-now)] 31 | (when (contains? @levels level) 32 | (let [level (str/upper-case (name level)) 33 | msg (if (string? msg) 34 | msg 35 | (with-out-str (pr msg)))] 36 | (locking *out* 37 | (println (format "%s [%s] %s - %s" now hostname level msg))))))) 38 | 39 | (def debug (partial log :debug)) 40 | (def info (partial log :info)) 41 | (def error (partial log :error)) 42 | -------------------------------------------------------------------------------- /src/sasori/protocols.clj: -------------------------------------------------------------------------------- 1 | (ns sasori.protocols) 2 | 3 | (defprotocol ICmd 4 | (plain [_ opts] "Transform to string form.") 5 | (exit? [_] "Whether exit when failed")) 6 | 7 | (defprotocol ICmdStatus 8 | (ok? [_] "Success or failed.")) 9 | 10 | (defprotocol ITaskReturn 11 | (merge-to-msg [this msg] "Merge new state from task return to msg.")) 12 | -------------------------------------------------------------------------------- /src/sasori/utils.clj: -------------------------------------------------------------------------------- 1 | (ns sasori.utils 2 | (:require [clojure.string :as str]) 3 | (:refer-clojure :exclude [boolean?])) 4 | 5 | (defn error! [msg & [map]] 6 | (let [map (or map {})] 7 | (throw (ex-info msg map)))) 8 | 9 | (defn not-blank? 10 | [x] 11 | (and (string? x) 12 | (not (str/blank? x)))) 13 | 14 | (defn join [& strings] 15 | (str/join " " strings)) 16 | 17 | (defn join-not-blank [& seq-of-s] 18 | (->> (filter not-blank? seq-of-s) 19 | (apply join))) 20 | 21 | (defmacro cond-join 22 | [& clauses] 23 | (assert (even? (count clauses))) 24 | (let [steps (map (fn [[test result]] `(when ~test ~result)) 25 | (partition 2 clauses))] 26 | `(join-not-blank 27 | ~@steps))) 28 | 29 | (defn maybe-map? [m] 30 | (let [maybe-map-f (some-fn nil? map?)] 31 | (maybe-map-f m))) 32 | 33 | (defn node->opts-m [node] 34 | (let [{:keys [global-opts host-info]} node] 35 | (merge host-info global-opts))) 36 | 37 | (defn boolean? 38 | "Return true if x is a Boolean" 39 | [x] 40 | (instance? Boolean x)) 41 | 42 | -------------------------------------------------------------------------------- /test/sasori/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns sasori.core-test 2 | (:require [clojure.test :refer :all] 3 | [sasori 4 | [core :as sasori] 5 | [dsl :as dsl]] 6 | [sasori.protocols :as protocols])) 7 | 8 | (defn test-local-ls 9 | [node context] 10 | (let [dsl (dsl/cmd "ls")] 11 | (sasori/sh-dsl dsl node))) 12 | 13 | (deftest test-do-parallel 14 | (let [task-vars (sasori/task-vars test-local-ls) 15 | result (sasori/play task-vars 16 | {:parallel? true 17 | :global-opts {:verbose false} 18 | :hosts-info {:host "local"}}) 19 | stub :stub 20 | error-result (sasori/play task-vars 21 | {:parallel? true 22 | :global-opts {:verbose false} 23 | :hosts-info {:host "local"} 24 | :context {:key-to-test-context stub}})] 25 | (is (not (sasori/failed-msg? (first result)))) 26 | (is (= stub (-> error-result first :context :key-to-test-context))))) 27 | 28 | (deftest test-do-sequence 29 | (let [task-vars (sasori/task-vars test-local-ls) 30 | host-info {:host "local"} 31 | result (sasori/play task-vars 32 | {:hosts-info {:host "local"} 33 | :global-opts {:verbose false}}) 34 | stub :stub 35 | error-result (sasori/play task-vars 36 | {:hosts-info host-info 37 | :global-opts {:verbose false} 38 | :context {:key-to-test-context stub}})] 39 | (is (not (sasori/failed-msg? (first result)))) 40 | (is (= stub (-> error-result first :context :key-to-test-context))))) 41 | 42 | (deftest test-context-merge-msg 43 | (let [m {:a 1 :b 2} 44 | context (sasori/make-context m) 45 | msg (sasori/make-msg (sasori/make-node {:host-info {:local? true}}))] 46 | (is (= m (-> (protocols/merge-to-msg context msg) :context))))) 47 | 48 | (deftest test-local 49 | (testing "local" 50 | (is (true? (-> (sasori/make-node {:host-info {:local? true}}) 51 | (sasori/local?))))) 52 | (testing "remote" 53 | (is (false? (-> (dsl/make-node {:host-info {:host "v1"}}) 54 | (sasori/local?)))))) -------------------------------------------------------------------------------- /test/sasori/dsl_test.clj: -------------------------------------------------------------------------------- 1 | (ns sasori.dsl-test 2 | (:require [clojure.test :refer :all]) 3 | (:require [sasori.dsl :as dsl])) 4 | 5 | (deftest test-cmd 6 | (is (= "mkdir /tmp/abc" 7 | (let [cmd (dsl/cmd "mkdir /tmp/abc" :exit? false)] 8 | (dsl/emit cmd)))) 9 | 10 | (is (= "mkdir /tmp/abc" 11 | (let [cmd (dsl/cmd "mkdir /tmp/abc" :exit? true)] 12 | (dsl/emit cmd))))) 13 | 14 | (deftest test-cmds 15 | (is (= "mkdir /tmp/abc && ls -alh && touch /tmp/abc/touch-file && pwd" 16 | (let [cmd (dsl/cmds 17 | (dsl/cmd "mkdir /tmp/abc" :exit? true) 18 | (dsl/cmd "ls -alh") 19 | (dsl/cmd "touch /tmp/abc/touch-file") 20 | (dsl/cmd "pwd"))] 21 | (dsl/emit cmd)))) 22 | 23 | (is (= "mkdir /tmp/abc && ls -alh && touch /tmp/abc/touch-file && pwd" 24 | (let [cmd (dsl/cmds 25 | (dsl/cmd "mkdir /tmp/abc") 26 | (dsl/cmd "ls -alh") 27 | (dsl/cmd "touch /tmp/abc/touch-file") 28 | (dsl/cmd "pwd" :exit? false))] 29 | (dsl/emit cmd)))) 30 | 31 | (is (= "mkdir /tmp/abc ; ls -alh && touch /tmp/abc/touch-file && pwd" 32 | (let [cmd (dsl/cmds 33 | (dsl/cmd "mkdir /tmp/abc" :exit? false) 34 | (dsl/cmd "ls -alh") 35 | (dsl/cmd "touch /tmp/abc/touch-file") 36 | (dsl/cmd "pwd"))] 37 | (dsl/emit cmd))))) 38 | 39 | (deftest test-ssh 40 | (is (= "ssh v1 'mkdir /tmp/abc && ls -alh && touch /tmp/abc/touch-file && pwd'" 41 | (let [node (dsl/make-node {:host-info {:host "v1"}}) 42 | cmd (dsl/ssh 43 | (dsl/cmd "mkdir /tmp/abc") 44 | (dsl/cmd "ls -alh") 45 | (dsl/cmd "touch /tmp/abc/touch-file") 46 | (dsl/cmd "pwd"))] 47 | (dsl/emit cmd node)))) 48 | 49 | (is (= "ssh v1 'mkdir /tmp/abc && ls -alh && touch /tmp/abc/touch-file && pwd'" 50 | (let [node (dsl/make-node {:host-info {:host "v1"}}) 51 | cmd (dsl/ssh 52 | (dsl/cmds 53 | (dsl/cmd "mkdir /tmp/abc") 54 | (dsl/cmd "ls -alh") 55 | (dsl/cmd "touch /tmp/abc/touch-file") 56 | (dsl/cmd "pwd")))] 57 | (dsl/emit cmd node))))) 58 | 59 | (deftest test-sudo 60 | (is (= "ssh v1 'sudo sh -c '\"'\"'mkdir /tmp/abc'\"'\"''" 61 | (let [node (dsl/make-node {:host-info {:host "v1"}}) 62 | cmd (dsl/ssh 63 | (dsl/sudo (dsl/cmd "mkdir /tmp/abc")))] 64 | (dsl/emit cmd node))))) 65 | 66 | (deftest test-sync 67 | (is (= "rsync --rsync-path='`which rsync`' -v --progress -h /local/path v1:/remote/path" 68 | (let [node (dsl/make-node {:host-info {:host "v1"} 69 | :global-opts {:verbose true}}) 70 | cmd (dsl/rsync "/local/path" "/remote/path")] 71 | (dsl/emit cmd node)))) 72 | 73 | (is (= "rsync --rsync-path='`which rsync`' -v -z --progress -h /local/path v1:/remote/path" 74 | (let [node (dsl/make-node {:host-info {:host "v1"} 75 | :global-opts {:verbose true}}) 76 | cmd (dsl/rsync "/local/path" "/remote/path" 77 | {:verbose true 78 | :compress true 79 | :progress true})] 80 | (dsl/emit cmd node)))) 81 | 82 | (is (= "rsync --rsync-path='`which rsync`' -v --progress -h /local/path v1:/remote/path" 83 | (let [node (dsl/make-node {:host-info {:host "v1"} 84 | :global-opts {:verbose true}}) 85 | cmd (dsl/rsync "/local/path" "/remote/path")] 86 | (dsl/emit cmd node))))) 87 | 88 | (deftest test-scp 89 | (is (= "scp -vv /local/path/file v1:/remote/path" 90 | (let [node (dsl/make-node {:host-info {:host "v1"} 91 | :global-opts {:verbose true}}) 92 | cmd (dsl/scp "/local/path/file" "/remote/path")] 93 | (dsl/emit cmd node)))) 94 | 95 | (is (= "scp -r /local/path/file v1:/remote/path" 96 | (let [node (dsl/make-node {:host-info {:host "v1"}}) 97 | cmd (dsl/scp "/local/path/file" "/remote/path" 98 | {:recursive true})] 99 | (dsl/emit cmd node)))) 100 | 101 | (is (= "scp -vv -r /local/path/file v1:/remote/path" 102 | (let [node (dsl/make-node {:host-info {:host "v1"} 103 | :global-opts {:verbose true}}) 104 | cmd (dsl/scp "/local/path/file" "/remote/path" 105 | {:recursive true})] 106 | (dsl/emit cmd node)))) 107 | 108 | (is (= "scp -r -P 22 /local/path/file user@host:/remote/path" 109 | (let [node (dsl/make-node {:host-info {:hostname "host" 110 | :username "user" 111 | :port 22}}) 112 | cmd (dsl/scp "/local/path/file" "/remote/path" 113 | {:verbose true 114 | :recursive true})] 115 | (dsl/emit cmd node))))) -------------------------------------------------------------------------------- /test/sasori/utils_test.clj: -------------------------------------------------------------------------------- 1 | (ns sasori.utils-test 2 | (:require [clojure.test :refer :all] 3 | [sasori.utils :as u])) 4 | 5 | (deftest test-cond->join 6 | (is (= "rsync -v -z -r" 7 | (let [params-in-s 8 | (u/cond-join 9 | true "-v" 10 | true "-z" 11 | true "-r")] 12 | (u/join-not-blank "rsync" params-in-s)))) 13 | 14 | (is (= "rsync -v -r" 15 | (let [params-in-s 16 | (u/cond-join 17 | true "-v" 18 | false "-z" 19 | true "-r")] 20 | (u/join-not-blank "rsync" params-in-s))))) 21 | --------------------------------------------------------------------------------