├── CODEOWNERS ├── Makefile ├── dev-resources ├── puppetlabs │ └── puppetserver │ │ └── shell_utils_test │ │ ├── cat │ │ ├── echo │ │ ├── false │ │ ├── true │ │ ├── num-args │ │ ├── warn │ │ ├── echo_cwd │ │ ├── echo_foo_env_var │ │ ├── gen-output │ │ ├── echo_and_warn │ │ └── list_env_var_names └── Makefile.i18n ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── src ├── java │ └── com │ │ └── puppetlabs │ │ └── puppetserver │ │ ├── ExecutionResult.java │ │ └── ShellUtils.java └── clj │ └── puppetlabs │ └── puppetserver │ └── shell_utils.clj ├── README.md ├── project.clj ├── .github └── workflows │ └── snyk.yml ├── test └── unit │ └── puppetlabs │ └── puppetserver │ └── shell_utils_test.clj └── LICENSE /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @puppetlabs/dumpling 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include dev-resources/Makefile.i18n 2 | -------------------------------------------------------------------------------- /dev-resources/puppetlabs/puppetserver/shell_utils_test/cat: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | cat - -------------------------------------------------------------------------------- /dev-resources/puppetlabs/puppetserver/shell_utils_test/echo: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | echo $@ 4 | -------------------------------------------------------------------------------- /dev-resources/puppetlabs/puppetserver/shell_utils_test/false: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | exit 1 4 | -------------------------------------------------------------------------------- /dev-resources/puppetlabs/puppetserver/shell_utils_test/true: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | exit 0 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .lein* 2 | pom.xml 3 | /target/ 4 | /resources/locales.clj 5 | /dev-resources/i18n/bin 6 | -------------------------------------------------------------------------------- /dev-resources/puppetlabs/puppetserver/shell_utils_test/num-args: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | exit $# 4 | -------------------------------------------------------------------------------- /dev-resources/puppetlabs/puppetserver/shell_utils_test/warn: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | echo $@ 1>&2 4 | -------------------------------------------------------------------------------- /dev-resources/puppetlabs/puppetserver/shell_utils_test/echo_cwd: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | echo $PWD 4 | -------------------------------------------------------------------------------- /dev-resources/puppetlabs/puppetserver/shell_utils_test/echo_foo_env_var: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | echo $FOO 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: clojure 2 | lein: 2.9.1 3 | jdk: 4 | - openjdk8 5 | - openjdk11 6 | notifications: 7 | email: false 8 | -------------------------------------------------------------------------------- /dev-resources/puppetlabs/puppetserver/shell_utils_test/gen-output: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | dd if=/dev/zero bs=$1 count=1 2>/dev/null 4 | -------------------------------------------------------------------------------- /dev-resources/puppetlabs/puppetserver/shell_utils_test/echo_and_warn: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | echo "to out:" $@ 4 | echo "to err:" $@ 1>&2 5 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 2.0.1 release version of 2.0.0 2 | ## 2.0.0 - not released 3 | * update to clj parent 7.3.15 4 | * add in the i18n plugin, and Makefiles 5 | 6 | ## 1.0.2 7 | 8 | * Built with the same code as 1.0.0, but with java8 instead of java11 9 | 10 | ## 1.0.0 11 | 12 | * Initial release 13 | -------------------------------------------------------------------------------- /dev-resources/puppetlabs/puppetserver/shell_utils_test/list_env_var_names: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # -s supresses lines that don't have an equals sign in them (this prevents bar in FOO=foo\nbar from being interpretted as a environment key) 4 | # -f selects which part of the split line to select 5 | # -d specifies the delimiter 6 | printenv | cut -s -f 1 -d "=" 7 | 8 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | 3 | Third-party patches are essential for keeping puppet open-source projects 4 | great. We want to keep it as easy as possible to contribute changes that 5 | allow you to get the most out of our projects. There are a few guidelines 6 | that we need contributors to follow so that we can have a chance of keeping on 7 | top of things. For more info, see our canonical guide to contributing: 8 | 9 | [https://github.com/puppetlabs/puppet/blob/master/CONTRIBUTING.md](https://github.com/puppetlabs/puppet/blob/master/CONTRIBUTING.md) 10 | -------------------------------------------------------------------------------- /src/java/com/puppetlabs/puppetserver/ExecutionResult.java: -------------------------------------------------------------------------------- 1 | package com.puppetlabs.puppetserver; 2 | 3 | import java.io.InputStream; 4 | import java.io.IOException; 5 | import org.apache.commons.io.IOUtils; 6 | 7 | public class ExecutionResult { 8 | private final InputStream output; 9 | private final String error; 10 | private final int exitCode; 11 | 12 | public ExecutionResult(InputStream output, String error, int exitCode) { 13 | this.output = output; 14 | this.error = error; 15 | this.exitCode = exitCode; 16 | } 17 | 18 | public int getExitCode() { 19 | return exitCode; 20 | } 21 | 22 | public String getOutput() throws IOException { 23 | return IOUtils.toString(output, "UTF-8"); 24 | } 25 | 26 | public InputStream getOutputAsStream() { 27 | return output; 28 | } 29 | 30 | public String getError() { 31 | return error; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # puppetlabs/clj-shell-utils 2 | 3 | A library for shell execution common to Puppet clojure projects. 4 | 5 | ## Installation 6 | 7 | Add the following dependency to your `project.clj` file: 8 | 9 | [![Clojars Project](http://clojars.org/puppetlabs/clj-shell-utils/latest-version.svg)](http://clojars.org/puppetlabs/clj-shell-utils) 10 | 11 | ## License 12 | 13 | Copyright © 2019 Puppet Labs 14 | 15 | See [LICENSE](LICENSE) file. 16 | 17 | ## Support 18 | 19 | Please log tickets and issues at our [JIRA tracker](https://tickets.puppetlabs.com/). 20 | 21 | We use semantic version numbers for our releases, and recommend that users stay 22 | as up-to-date as possible by upgrading to patch releases and minor releases as 23 | they become available. 24 | 25 | Bugfixes and ongoing development will occur in minor releases for the current 26 | major version. Security fixes will be backported to a previous major version on 27 | a best-effort basis, until the previous major version is no longer maintained. 28 | 29 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject puppetlabs/clj-shell-utils "2.0.2-SNAPSHOT" 2 | :description "Clojure shell execution utilities" 3 | 4 | :min-lein-version "2.9.0" 5 | 6 | :parent-project {:coords [puppetlabs/clj-parent "7.3.15"] 7 | :inherit [:managed-dependencies]} 8 | :license {:name "Apache-2.0" 9 | :url "https://www.apache.org/licenses/LICENSE-2.0.txt"} 10 | 11 | :pedantic? :abort 12 | 13 | :test-paths ["test/unit"] 14 | 15 | :plugins [[lein-project-version "0.1.0"] 16 | [lein-parent "0.3.6"] 17 | [puppetlabs/i18n "0.9.0"]] 18 | 19 | :source-paths ["src/clj"] 20 | :java-source-paths ["src/java"] 21 | 22 | :dependencies [[org.clojure/clojure] 23 | [prismatic/schema] 24 | [org.apache.commons/commons-exec] 25 | [commons-io] 26 | [org.slf4j/log4j-over-slf4j] 27 | [org.slf4j/slf4j-api] 28 | [puppetlabs/trapperkeeper] 29 | [puppetlabs/kitchensink] 30 | [puppetlabs/i18n]] 31 | 32 | 33 | 34 | :profiles { :test { :dependencies [[puppetlabs/trapperkeeper nil :classifier "test" :scope "test"]]}} 35 | 36 | :deploy-repositories [["releases" {:url "https://clojars.org/repo" 37 | :username :env/clojars_jenkins_username 38 | :password :env/clojars_jenkins_password 39 | :sign-releases false}] 40 | ["snapshots" "https://artifactory.delivery.puppetlabs.net/artifactory/list/clojure-snapshots__local/"]]) 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /.github/workflows/snyk.yml: -------------------------------------------------------------------------------- 1 | name: mend_scan 2 | on: 3 | workflow_dispatch: 4 | push: 5 | branches: 6 | - main 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: connect_twingate 12 | uses: twingate/github-action@v1 13 | with: 14 | service-key: ${{ secrets.TWINGATE_PUBLIC_REPO_KEY }} 15 | - name: checkout repo content 16 | uses: actions/checkout@v2 # checkout the repository content to github runner. 17 | with: 18 | fetch-depth: 1 19 | # install java which is required for mend and clojure 20 | - name: setup java 21 | uses: actions/setup-java@v3 22 | with: 23 | distribution: temurin 24 | java-version: 17 25 | # install clojure tools 26 | - name: Install Clojure tools 27 | uses: DeLaGuardo/setup-clojure@10.1 28 | with: 29 | # Install just one or all simultaneously 30 | # The value must indicate a particular version of the tool, or use 'latest' 31 | # to always provision the latest version 32 | cli: latest # Clojure CLI based on tools.deps 33 | lein: latest # Leiningen 34 | boot: latest # Boot.clj 35 | bb: latest # Babashka 36 | clj-kondo: latest # Clj-kondo 37 | cljstyle: latest # cljstyle 38 | zprint: latest # zprint 39 | # run lein gen 40 | - name: create pom.xml 41 | run: lein pom 42 | # download mend 43 | - name: download_mend 44 | run: curl -o wss-unified-agent.jar https://unified-agent.s3.amazonaws.com/wss-unified-agent.jar 45 | - name: run mend 46 | run: env WS_INCLUDES=pom.xml java -jar wss-unified-agent.jar 47 | env: 48 | WS_APIKEY: ${{ secrets.MEND_API_KEY }} 49 | WS_WSS_URL: https://saas-eu.whitesourcesoftware.com/agent 50 | WS_USERKEY: ${{ secrets.MEND_TOKEN }} 51 | WS_PRODUCTNAME: Puppet Enterprise 52 | WS_PROJECTNAME: ${{ github.event.repository.name }} 53 | -------------------------------------------------------------------------------- /src/clj/puppetlabs/puppetserver/shell_utils.clj: -------------------------------------------------------------------------------- 1 | (ns puppetlabs.puppetserver.shell-utils 2 | (:require [schema.core :as schema] 3 | [clojure.java.io :as io] 4 | [puppetlabs.kitchensink.core :as ks] 5 | [clojure.string :as string] 6 | [puppetlabs.i18n.core :as i18n]) 7 | (:import (com.puppetlabs.puppetserver ShellUtils ShellUtils$ExecutionOptions) 8 | (java.io InputStream) 9 | (org.apache.commons.io IOUtils))) 10 | 11 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 12 | ;;; Schemas 13 | 14 | (def ExecutionResult 15 | "A map that contains the details of the result of executing a command." 16 | {:exit-code schema/Int 17 | :stderr schema/Str 18 | :stdout schema/Str}) 19 | 20 | (def ExecutionResultStreamed 21 | "A map that contains the details of the result of executing a command with 22 | stdout as a stream." 23 | {:exit-code schema/Int 24 | :stderr schema/Str 25 | :stdout InputStream}) 26 | 27 | (def ExecutionOptions 28 | {(schema/optional-key :args) [schema/Str] 29 | (schema/optional-key :env) (schema/maybe {schema/Str schema/Str}) 30 | (schema/optional-key :in) (schema/maybe InputStream) 31 | (schema/optional-key :cwd) (schema/maybe schema/Str)}) 32 | 33 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 34 | ;;; Internal 35 | 36 | (def default-execution-options 37 | {:args [] 38 | :env nil 39 | :in nil 40 | :cwd nil}) 41 | 42 | (schema/defn ^:always-validate java-exe-options :- ShellUtils$ExecutionOptions 43 | [{:keys [env in cwd]} :- ExecutionOptions] 44 | (let [exe-options (ShellUtils$ExecutionOptions.)] 45 | (.setStdin exe-options in) 46 | (when env 47 | (.setEnv exe-options (ks/mapkeys name env))) 48 | (when cwd 49 | (.setWorkingDirectory exe-options cwd)) 50 | exe-options)) 51 | 52 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 53 | ;;; Public 54 | 55 | (schema/defn ^:always-validate 56 | validate-command! 57 | "Checks the command string to ensure that it is an absolute path, executable 58 | and that the file exists. An exception is thrown if any of those are not the 59 | case." 60 | [command :- schema/Str] 61 | (let [command-file (io/as-file command)] 62 | (cond 63 | (not (.isAbsolute command-file)) 64 | (throw (IllegalArgumentException. 65 | (i18n/trs "An absolute path is required, but ''{0}'' is not an absolute path" command))) 66 | (not (.exists command-file)) 67 | (let [cmds (string/split command #" ")] 68 | (if (and (> (count cmds) 1) (.exists (io/as-file (first cmds)))) 69 | (throw (IllegalArgumentException. 70 | (i18n/trs "Command ''{0}'' appears to use command-line arguments, but this is not allowed." command))) 71 | (throw (IllegalArgumentException. 72 | (i18n/trs "The referenced command ''{0}'' does not exist" command))))) 73 | (not (.canExecute command-file)) 74 | (throw (IllegalArgumentException. 75 | (i18n/trs "The referenced command ''{0}'' is not executable" command)))))) 76 | 77 | (schema/defn ^:always-validate 78 | execute-command-streamed :- ExecutionResultStreamed 79 | "Execute the specified fully qualified command (string) and any included 80 | command-arguments (vector of strings) and return the exit-code (integer), 81 | and the contents of the stdout (stream) and stderr (string) for the command." 82 | ([command :- schema/Str] 83 | (execute-command-streamed command {})) 84 | ([command :- schema/Str 85 | opts :- ExecutionOptions] 86 | (let [{:keys [args] :as opts} (merge default-execution-options opts)] 87 | (validate-command! command) 88 | (let [process (ShellUtils/executeCommand 89 | command 90 | (into-array String args) 91 | (java-exe-options opts))] 92 | {:exit-code (.getExitCode process) 93 | :stderr (.getError process) 94 | :stdout (.getOutputAsStream process)})))) 95 | 96 | (schema/defn ^:always-validate 97 | execute-command :- ExecutionResult 98 | "Execute the specified fully qualified command (string) and any included 99 | command-arguments (vector of strings) and return the exit-code (integer), 100 | and the contents of the stdout (string) and stderr (string) for the command." 101 | ([command :- schema/Str] 102 | (execute-command command {})) 103 | ([command :- schema/Str 104 | opts :- ExecutionOptions] 105 | (let [result (execute-command-streamed command opts)] 106 | (update-in result [:stdout] 107 | (fn [stdout] (IOUtils/toString stdout "UTF-8")))))) 108 | -------------------------------------------------------------------------------- /test/unit/puppetlabs/puppetserver/shell_utils_test.clj: -------------------------------------------------------------------------------- 1 | (ns puppetlabs.puppetserver.shell-utils-test 2 | (:require [clojure.test :refer :all] 3 | [puppetlabs.puppetserver.shell-utils :as sh-utils] 4 | [puppetlabs.kitchensink.core :as ks] 5 | [puppetlabs.trapperkeeper.testutils.logging :as logging] 6 | [clojure.string :as str] 7 | [clojure.set :as set]) 8 | (:import (java.io ByteArrayInputStream) 9 | (com.puppetlabs.puppetserver ShellUtils ShellUtils$ExecutionOptions))) 10 | 11 | (def test-resources 12 | (ks/absolute-path 13 | "./dev-resources/puppetlabs/puppetserver/shell_utils_test")) 14 | 15 | (defn script-path 16 | [script-name] 17 | (str test-resources "/" script-name)) 18 | 19 | (defn parse-env-output 20 | [env-output] 21 | (set (str/split-lines env-output))) 22 | 23 | (deftest returns-the-exit-code 24 | (testing "true should return 0" 25 | (is (zero? (:exit-code (sh-utils/execute-command (script-path "true")))))) 26 | (testing "false should return 1" 27 | (is (= 1 (:exit-code (sh-utils/execute-command (script-path "false"))))))) 28 | 29 | (deftest returns-stdout-correctly 30 | (testing "echo should add content to stdout" 31 | (is (= "foo\n" (:stdout (sh-utils/execute-command 32 | (script-path "echo") 33 | {:args ["foo"]})))))) 34 | 35 | (deftest returns-stderr-correctly 36 | (testing "echo can add content to stderr as well" 37 | (logging/with-test-logging 38 | (is (= "bar\n" (:stderr (sh-utils/execute-command 39 | (script-path "warn") 40 | {:args ["bar"]}))))))) 41 | 42 | (deftest combines-stderr-and-stdout-correctly 43 | (logging/with-test-logging 44 | (let [options (ShellUtils$ExecutionOptions.) 45 | _ (.setCombineStdoutStderr options true) 46 | results (ShellUtils/executeCommand (str (script-path "echo_and_warn") 47 | " baz") 48 | options)] 49 | (testing "combined info echoed to stdout and stderr captured as output" 50 | (let [output (.getOutput results)] 51 | ;; Allow stdout and stderr messages to come in either order since 52 | ;; the order in which they are read from the different stream 53 | ;; consuming threads is not reliable. 54 | (is (or (= "to out: baz\nto err: baz\n" output) 55 | (= "to err: baz\nto out: baz\n" output)) 56 | (format "Output produced, '%s', did not match expected output" 57 | output)))) 58 | (testing "only info echoed to stderr captured as error" 59 | (is (= "to err: baz\n" (.getError results)))) 60 | (testing "only stderr info (and not stdout info) is logged" 61 | (is (logged? 62 | "Executed an external process which logged to STDERR: to err: baz\n" 63 | :warn)))))) 64 | 65 | (deftest pass-args-correctly 66 | (testing "passes the expected number of args to cmd" 67 | (is (= 5 (:exit-code (sh-utils/execute-command 68 | (script-path "num-args") 69 | {:args ["a" "b" "c" "d" "e"]})))))) 70 | 71 | (deftest inherits-env-correctly 72 | (testing "inherits environment variables if not specified" 73 | (let [env-output (:stdout (sh-utils/execute-command 74 | (script-path "list_env_var_names"))) 75 | env (parse-env-output env-output)] 76 | (is (< 3 (count env)) 77 | (str "Expected at least 3 environment variables, got:\n" env-output)) 78 | (is (contains? env "PATH")) 79 | (is (contains? env "PWD")) 80 | (is (contains? env "HOME"))))) 81 | 82 | (deftest sets-env-correctly 83 | (testing "sets environment variables correctly" 84 | (is (= "foo\n" (:stdout (sh-utils/execute-command 85 | (script-path "echo_foo_env_var") 86 | {:env {"FOO" "foo"}})))) 87 | 88 | (let [env-output (:stdout (sh-utils/execute-command 89 | (script-path "list_env_var_names") 90 | {:env {"FOO" "foo\nbar"}})) 91 | env (parse-env-output env-output) 92 | ;; it seems that the JVM always includes a PWD env var, no 93 | ;; matter what, and in certain terminals it may also include a few 94 | ;; other vars, so we are writing the test to be tolerant of that. 95 | expected-keys #{"FOO" "PWD" "_" "SHLVL"} 96 | extra-keys (set/difference env expected-keys)] 97 | (is (empty? extra-keys) 98 | (str "Found unexpected environment variables:" extra-keys))))) 99 | 100 | (deftest pass-stdin-correctly 101 | (testing "passes stdin stream to command" 102 | (is (= "foo" (:stdout (sh-utils/execute-command 103 | (script-path "cat") 104 | {:in (ByteArrayInputStream. 105 | (.getBytes "foo" "UTF-8"))})))))) 106 | 107 | (deftest sets-cwd-correctly 108 | (testing "sets current working directory correctly" 109 | (let [tmpdir (System/getProperty "java.io.tmpdir")] 110 | (is (= (str tmpdir "\n") (:stdout (sh-utils/execute-command 111 | (script-path "echo_cwd") 112 | {:cwd tmpdir}))))))) 113 | 114 | (deftest throws-exception-for-non-absolute-path 115 | (testing "Commands must be given using absolute paths" 116 | (is (thrown? IllegalArgumentException 117 | (sh-utils/execute-command "echo"))))) 118 | 119 | (deftest throws-exception-for-non-existent-file 120 | (testing "The given command must exist" 121 | (is (thrown-with-msg? IllegalArgumentException 122 | #"command '/usr/bin/footest' does not exist" 123 | (sh-utils/execute-command "/usr/bin/footest"))))) 124 | 125 | (deftest throws-reasonable-error-for-arguments-in-command 126 | (testing "A meaningful error is raised if arguments are added to the command" 127 | (is (thrown-with-msg? IllegalArgumentException 128 | #"appears to use command-line arguments, but this is not allowed" 129 | (sh-utils/execute-command 130 | (str (script-path "echo") " foo")))))) 131 | 132 | (deftest can-read-more-than-the-pipe-buffer 133 | (testing "Doesn't deadlock when reading more than the pipe can hold" 134 | (is (= 128000 (count (:stdout (sh-utils/execute-command 135 | (script-path "gen-output") 136 | {:args ["128000"]}))))))) 137 | -------------------------------------------------------------------------------- /dev-resources/Makefile.i18n: -------------------------------------------------------------------------------- 1 | # -*- Makefile -*- 2 | # This file was generated by the i18n leiningen plugin 3 | # Do not edit this file; it will be overwritten the next time you run 4 | # lein i18n init 5 | # 6 | 7 | # The name of the package into which the translations bundle will be placed 8 | BUNDLE=puppetlabs.clj_shell_utils 9 | 10 | # The name of the POT file into which the gettext code strings (msgid) will be placed 11 | POT_NAME=clj-shell-utils.pot 12 | 13 | # The list of names of packages covered by the translation bundle; 14 | # by default it contains a single package - the same where the translations 15 | # bundle itself is placed - but this can be overridden - preferably in 16 | # the top level Makefile 17 | PACKAGES?=$(BUNDLE) 18 | LOCALES=$(basename $(notdir $(wildcard locales/*.po))) 19 | BUNDLE_DIR=$(subst .,/,$(BUNDLE)) 20 | BUNDLE_FILES=$(patsubst %,resources/$(BUNDLE_DIR)/Messages_%.class,$(LOCALES)) 21 | FIND_SOURCES=find src -name \*.clj 22 | # xgettext before 0.19 does not understand --add-location=file. Even CentOS 23 | # 7 ships with an older gettext. We will therefore generate full location 24 | # info on those systems, and only file names where xgettext supports it 25 | LOC_OPT=$(shell xgettext --add-location=file -f - /dev/null 2>&1 && echo --add-location=file || echo --add-location) 26 | 27 | LOCALES_CLJ=resources/locales.clj 28 | define LOCALES_CLJ_CONTENTS 29 | { 30 | :locales #{$(patsubst %,"%",$(LOCALES))} 31 | :packages [$(patsubst %,"%",$(PACKAGES))] 32 | :bundle $(patsubst %,"%",$(BUNDLE).Messages) 33 | } 34 | endef 35 | export LOCALES_CLJ_CONTENTS 36 | 37 | 38 | i18n: msgfmt 39 | 40 | # Update locales/.pot 41 | update-pot: locales/$(POT_NAME) 42 | 43 | locales/$(POT_NAME): $(shell $(FIND_SOURCES)) | locales 44 | @tmp=$$(mktemp $@.tmp.XXXX); \ 45 | $(FIND_SOURCES) \ 46 | | xgettext --from-code=UTF-8 --language=lisp \ 47 | --copyright-holder='Puppet ' \ 48 | --package-name="$(BUNDLE)" \ 49 | --package-version="$(BUNDLE_VERSION)" \ 50 | --msgid-bugs-address="docs@puppet.com" \ 51 | -k \ 52 | -kmark:1 -ki18n/mark:1 \ 53 | -ktrs:1 -ki18n/trs:1 \ 54 | -ktru:1 -ki18n/tru:1 \ 55 | -ktrun:1,2 -ki18n/trun:1,2 \ 56 | -ktrsn:1,2 -ki18n/trsn:1,2 \ 57 | $(LOC_OPT) \ 58 | --add-comments --sort-by-file \ 59 | -o $$tmp -f -; \ 60 | sed -i.bak -e 's/charset=CHARSET/charset=UTF-8/' $$tmp; \ 61 | sed -i.bak -e 's/POT-Creation-Date: [^\\]*/POT-Creation-Date: /' $$tmp; \ 62 | rm -f $$tmp.bak; \ 63 | if ! diff -q -I POT-Creation-Date $$tmp $@ >/dev/null 2>&1; then \ 64 | mv $$tmp $@; \ 65 | else \ 66 | rm $$tmp; touch $@; \ 67 | fi 68 | 69 | # Run msgfmt over all .po files to generate Java resource bundles 70 | # and create the locales.clj file 71 | msgfmt: $(BUNDLE_FILES) $(LOCALES_CLJ) clean-orphaned-bundles 72 | 73 | # Force rebuild of locales.clj if its contents is not the the desired one. The 74 | # shell echo is used to add a trailing newline to match the one from `cat` 75 | ifneq ($(shell cat $(LOCALES_CLJ) 2> /dev/null),$(shell echo '$(LOCALES_CLJ_CONTENTS)')) 76 | .PHONY: $(LOCALES_CLJ) 77 | endif 78 | $(LOCALES_CLJ): | resources 79 | @echo "Writing $@" 80 | @echo "$$LOCALES_CLJ_CONTENTS" > $@ 81 | 82 | # Remove every resource bundle that wasn't generated from a PO file. 83 | # We do this because we used to generate the english bundle directly from the POT. 84 | .PHONY: clean-orphaned-bundles 85 | clean-orphaned-bundles: 86 | @for bundle in resources/$(BUNDLE_DIR)/Messages_*.class; do \ 87 | locale=$$(basename "$$bundle" | sed -E -e 's/\$$?1?\.class$$/_class/' | cut -d '_' -f 2;); \ 88 | if [ ! -f "locales/$$locale.po" -a -f "$$bundle" ]; then \ 89 | rm "$$bundle"; \ 90 | fi \ 91 | done 92 | 93 | resources/$(BUNDLE_DIR)/Messages_%.class: locales/%.po | resources 94 | msgfmt --java2 -d resources -r $(BUNDLE).Messages -l $(*F) $< 95 | 96 | # Use this to initialize translations. Updating the PO files is done 97 | # automatically through a CI job that utilizes the scripts in the project's 98 | # `bin` file, which themselves come from the `clj-i18n` project. 99 | locales/%.po: | locales 100 | @if [ ! -f $@ ]; then \ 101 | touch $@ && msginit --no-translator -l $(*F) -o $@ -i locales/$(POT_NAME); \ 102 | fi 103 | 104 | resources locales: 105 | @mkdir $@ 106 | 107 | help: 108 | $(info $(HELP)) 109 | @echo 110 | 111 | .PHONY: help 112 | 113 | define HELP 114 | This Makefile assists in handling i18n related tasks during development. Files 115 | that need to be checked into source control are put into the locales/ directory. 116 | They are 117 | 118 | locales/$(POT_NAME) - the POT file generated by 'make update-pot' 119 | locales/$$LANG.po - the translations for $$LANG 120 | 121 | Only the $$LANG.po files should be edited manually; this is usually done by 122 | translators. 123 | 124 | You can use the following targets: 125 | 126 | i18n: refresh all the files in locales/ and recompile resources 127 | update-pot: extract strings and update locales/$(POT_NAME) 128 | locales/LANG.po: create translations for LANG 129 | msgfmt: compile the translations into Java classes; this step is 130 | needed to make translations available to the Clojure code 131 | and produces Java class files in resources/ 132 | endef 133 | # @todo lutter 2015-04-20: for projects that use libraries with their own 134 | # translation, we need to combine all their translations into one big po 135 | # file and then run msgfmt over that so that we only have to deal with one 136 | # resource bundle 137 | -------------------------------------------------------------------------------- /src/java/com/puppetlabs/puppetserver/ShellUtils.java: -------------------------------------------------------------------------------- 1 | package com.puppetlabs.puppetserver; 2 | 3 | import org.apache.commons.exec.DefaultExecutor; 4 | import org.apache.commons.exec.PumpStreamHandler; 5 | import org.apache.commons.exec.CommandLine; 6 | import org.apache.commons.io.output.TeeOutputStream; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.io.ByteArrayOutputStream; 11 | import java.io.ByteArrayInputStream; 12 | import java.io.IOException; 13 | import java.io.InputStream; 14 | import java.io.File; 15 | import java.util.Map; 16 | 17 | public class ShellUtils { 18 | 19 | public static class ExecutionOptions { 20 | private boolean combineStdoutStderr = false; 21 | private Map env = null; 22 | private InputStream stdin = null; 23 | private String cwd = null; 24 | 25 | public boolean getCombineStdoutStderr() { 26 | return combineStdoutStderr; 27 | } 28 | 29 | public void setCombineStdoutStderr(boolean combineStdoutStderr) { 30 | this.combineStdoutStderr = combineStdoutStderr; 31 | } 32 | 33 | public InputStream getStdin() { 34 | return stdin; 35 | } 36 | 37 | public void setStdin(InputStream stdin) { 38 | this.stdin = stdin; 39 | } 40 | 41 | public Map getEnv() { 42 | return env; 43 | } 44 | 45 | public void setEnv(Map env) { 46 | this.env = env; 47 | } 48 | 49 | public String getWorkingDirectory() { 50 | return cwd; 51 | } 52 | 53 | public void setWorkingDirectory(String cwd) { 54 | this.cwd = cwd; 55 | } 56 | } 57 | 58 | private static final Logger log = LoggerFactory.getLogger(ShellUtils.class); 59 | 60 | /** 61 | * Takes a prepared CommandLine instance and executes it using some sane 62 | * defaults and a DefaultExecutor. Also makes a delicious pancake. 63 | * 64 | * @param commandLine CommandLine instance to execute 65 | * @param options optional object [ExecutionOptions] to control behavior; may be null. 66 | * @return An ExecutionResult with output[String], error[String], and 67 | * the exit code[Integer] of the process 68 | * 69 | * @throws InterruptedException 70 | * @throws IOException 71 | */ 72 | private static ExecutionResult executeExecutor(CommandLine commandLine, 73 | ExecutionOptions options) 74 | throws InterruptedException, IOException { 75 | if (options == null) { 76 | options = new ExecutionOptions(); 77 | } 78 | DefaultExecutor executor = new DefaultExecutor(); 79 | ByteArrayOutputStream errStream = new ByteArrayOutputStream(); 80 | // TODO the nice thing here would be to set up a piped stream 81 | // arrangement like: 82 | // PipedOutputStream stdoutOutputStream = new PipedOutputStream(); 83 | // PipedInputStream stdoutInputStream = new PipedInputStream(stdoutOutputStream); 84 | // but this requires that the input stream be read on a different thread 85 | // than this one. this is currently out of scope. 86 | ByteArrayOutputStream stdoutOutputStream = new ByteArrayOutputStream(); 87 | PumpStreamHandler streamHandler; 88 | if (options.getCombineStdoutStderr()) { 89 | log.debug("Combining STDOUT/STDERR for external command '" + commandLine.toString() + "'"); 90 | streamHandler = new PumpStreamHandler( 91 | stdoutOutputStream, 92 | new TeeOutputStream(stdoutOutputStream, errStream), 93 | options.getStdin()); 94 | } else { 95 | streamHandler = new PumpStreamHandler(stdoutOutputStream, errStream, 96 | options.getStdin()); 97 | } 98 | 99 | // Don't throw exception on non-zero exit code 100 | executor.setExitValues(null); 101 | 102 | // Set up the handlers 103 | executor.setStreamHandler(streamHandler); 104 | 105 | // Set up cwd 106 | String cwd = options.getWorkingDirectory(); 107 | if (cwd != null) { 108 | executor.setWorkingDirectory(new File(cwd)); 109 | } 110 | 111 | Integer exitCode = executor.execute(commandLine, options.getEnv()); 112 | 113 | ByteArrayInputStream stdoutInputStream = new ByteArrayInputStream(stdoutOutputStream.toByteArray()); 114 | String stdErr = errStream.toString(); 115 | 116 | if ( ! stdErr.isEmpty() ) { 117 | log.warn("Executed an external process which logged to STDERR: " + stdErr); 118 | } 119 | 120 | return new ExecutionResult(stdoutInputStream, stdErr, exitCode); 121 | } 122 | 123 | /** 124 | * Executes the given command in a separate process. 125 | * 126 | * @param commandLine the command to execute. 127 | * @param options optional object [ExecutionOptions] to control behavior; may be null. 128 | * 129 | * @return An ExecutionResult with output[String], error[String], and 130 | * the exit code[Integer] of the process 131 | * 132 | * @throws InterruptedException 133 | */ 134 | protected static ExecutionResult executeCommand(CommandLine commandLine, 135 | ExecutionOptions options) 136 | throws InterruptedException { 137 | try { 138 | return executeExecutor(commandLine, options); 139 | } catch (IOException e) { 140 | // this nonsense is due to a weird edge-case incompatibility between JDK8 141 | // and apache commons-exec. See SERVER-1116; hopefully we can remove this 142 | // conditional once that is resolved. 143 | if (e.getMessage() == "Stream closed") { 144 | log.warn("An error occurred while executing the command '" + commandLine.getExecutable() + 145 | ". The most likely culprit is that you are on JDK8, " + 146 | "and we executed an external process with data on its STDIN that was not " + 147 | "consumed by the process. Please make sure the command above processes STDIN " + 148 | "correctly. For more information, see " + 149 | "https://tickets.puppetlabs.com/browse/SERVER-1116 . If you do not believe " + 150 | "that this is the root cause of this error message, please file a bug at " + 151 | "https://tickets.puppetlabs.com/browse/SERVER."); 152 | } 153 | throw new IllegalStateException( 154 | "Exception while executing '" + commandLine.getExecutable() + "': " + e.getMessage(), 155 | e); 156 | } 157 | } 158 | 159 | /** 160 | * Executes the given command in a separate process. 161 | * 162 | * @param command the command [String] to execute. arguments can be 163 | * included in the string. 164 | * @return An ExecutionResult with output[String], error[String], and 165 | * the exit code[Integer] of the process 166 | * 167 | * @throws InterruptedException 168 | * @throws IOException 169 | */ 170 | public static ExecutionResult executeCommand(String command) 171 | throws InterruptedException, IOException { 172 | CommandLine commandLine = CommandLine.parse(command); 173 | 174 | return executeCommand(commandLine, null); 175 | } 176 | 177 | /** 178 | * Executes the given command in a separate process. 179 | * 180 | * 181 | * @param command the command [String] to execute. arguments can be 182 | * included in the string. 183 | * @param options optional object [ExecutionOptions] to control behavior; may be null. 184 | * @return An ExecutionResult with output[String], error[String], and 185 | * the exit code[Integer] of the process 186 | * 187 | * @throws InterruptedException 188 | * @throws IOException 189 | */ 190 | public static ExecutionResult executeCommand(String command, ExecutionOptions options) 191 | throws InterruptedException, IOException { 192 | CommandLine commandLine = CommandLine.parse(command); 193 | 194 | return executeCommand(commandLine, options); 195 | } 196 | 197 | /** 198 | * Executes the given command in a separate process. 199 | * 200 | * @param command the command [String] to execute. 201 | * @param arguments arguments [Array of Strings] to add to the command being executed 202 | * @return An ExecutionResult with output[String], error[String], and 203 | * the exit code[Integer] of the process 204 | * 205 | * @throws InterruptedException 206 | * @throws IOException 207 | */ 208 | public static ExecutionResult executeCommand(String command, String[] arguments) 209 | throws InterruptedException, IOException { 210 | return executeCommand(command, arguments, null); 211 | } 212 | 213 | /** 214 | * Executes the given command in a separate process. 215 | * 216 | * @param command the command [String] to execute. 217 | * @param arguments arguments [Array of Strings] to add to the command being executed 218 | * @param options optional object [ExecutionOptions] to control behavior; may be null. 219 | * 220 | * @return An ExecutionResult with output[String], error[String], and 221 | * the exit code[Integer] of the process 222 | * 223 | * @throws InterruptedException 224 | * @throws IOException 225 | */ 226 | public static ExecutionResult executeCommand(String command, String[] arguments, 227 | ExecutionOptions options) 228 | throws InterruptedException, IOException { 229 | CommandLine commandLine = new CommandLine(command); 230 | commandLine.addArguments(arguments, false); 231 | 232 | return executeCommand(commandLine, options); 233 | } 234 | 235 | 236 | } 237 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------