├── testdata ├── basic │ ├── expected │ │ ├── stderr │ │ ├── exitcode │ │ └── stdout │ ├── context1.yaml │ ├── context2.yaml │ ├── template.yaml │ └── run.sh ├── complex │ ├── expected │ │ ├── stderr │ │ ├── exitcode │ │ └── stdout │ ├── input.txt │ ├── named.yaml │ ├── context2.yaml │ ├── context1.yaml │ ├── run.sh │ └── template.yaml ├── format │ ├── expected │ │ ├── stderr │ │ ├── exitcode │ │ └── stdout │ ├── context.txt │ ├── context.kv │ ├── context.yaml │ ├── context.json │ ├── run.sh │ └── template.yaml └── merge │ ├── expected │ ├── stderr │ ├── exitcode │ └── stdout │ ├── run.sh │ ├── sub.yaml │ ├── template.yaml │ └── root.yaml ├── _config.yml ├── .gitignore ├── Gopkg.toml ├── Gopkg.lock ├── LICENSE ├── .travis.yml ├── test.sh ├── README.md ├── main.go └── context.go /testdata/basic/expected/stderr: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /testdata/complex/expected/stderr: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /testdata/format/expected/stderr: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /testdata/merge/expected/stderr: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman -------------------------------------------------------------------------------- /testdata/basic/expected/exitcode: -------------------------------------------------------------------------------- 1 | 0 2 | -------------------------------------------------------------------------------- /testdata/complex/input.txt: -------------------------------------------------------------------------------- 1 | foo 2 | -------------------------------------------------------------------------------- /testdata/merge/expected/exitcode: -------------------------------------------------------------------------------- 1 | 0 2 | -------------------------------------------------------------------------------- /testdata/complex/expected/exitcode: -------------------------------------------------------------------------------- 1 | 0 2 | -------------------------------------------------------------------------------- /testdata/complex/named.yaml: -------------------------------------------------------------------------------- 1 | everything 2 | -------------------------------------------------------------------------------- /testdata/format/context.txt: -------------------------------------------------------------------------------- 1 | text value 2 | -------------------------------------------------------------------------------- /testdata/format/expected/exitcode: -------------------------------------------------------------------------------- 1 | 0 2 | -------------------------------------------------------------------------------- /testdata/basic/context1.yaml: -------------------------------------------------------------------------------- 1 | foo: something 2 | -------------------------------------------------------------------------------- /testdata/basic/context2.yaml: -------------------------------------------------------------------------------- 1 | bar: nothing 2 | -------------------------------------------------------------------------------- /testdata/complex/context2.yaml: -------------------------------------------------------------------------------- 1 | bar: nothing 2 | -------------------------------------------------------------------------------- /testdata/complex/context1.yaml: -------------------------------------------------------------------------------- 1 | foo: something 2 | -------------------------------------------------------------------------------- /testdata/format/context.kv: -------------------------------------------------------------------------------- 1 | fookv something kv 2 | -------------------------------------------------------------------------------- /testdata/basic/template.yaml: -------------------------------------------------------------------------------- 1 | a: ${foo} 2 | b: ${bar} 3 | -------------------------------------------------------------------------------- /testdata/format/context.yaml: -------------------------------------------------------------------------------- 1 | fooyaml: something yaml 2 | -------------------------------------------------------------------------------- /testdata/basic/expected/stdout: -------------------------------------------------------------------------------- 1 | a: something 2 | b: nothing 3 | -------------------------------------------------------------------------------- /testdata/format/context.json: -------------------------------------------------------------------------------- 1 | {"foojson": "something json"} 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /rjsone 2 | /.vscode 3 | 4 | # dep stuff 5 | /vendor/ 6 | -------------------------------------------------------------------------------- /testdata/basic/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | exec rjsone -y -t template.yaml context1.yaml context2.yaml 4 | -------------------------------------------------------------------------------- /testdata/merge/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | rjsone -d -y -t template.yaml root.yaml root:sub.yaml root_list::+"string override" 4 | -------------------------------------------------------------------------------- /testdata/merge/sub.yaml: -------------------------------------------------------------------------------- 1 | subkey1: 'overridden subkey1 value' 2 | subkey3: 3 | nested2: 'overridden subkey3.nested2 value' 4 | -------------------------------------------------------------------------------- /testdata/format/expected/stdout: -------------------------------------------------------------------------------- 1 | textinsert: | 2 | text value 3 | toplevel: 4 | - something json 5 | - something yaml 6 | - something kv 7 | -------------------------------------------------------------------------------- /testdata/format/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | exec rjsone -y -t template.yaml :yaml:context.yaml :kv:context.kv :json:context.json bar:text:context.txt 4 | -------------------------------------------------------------------------------- /testdata/format/template.yaml: -------------------------------------------------------------------------------- 1 | toplevel: 2 | - ${foojson} 3 | - ${fooyaml} 4 | - ${fookv} 5 | # Can't have text at top level 6 | textinsert: ${bar} 7 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | [[constraint]] 2 | branch = "master" 3 | name = "github.com/wryun/yaml-1" 4 | 5 | [prune] 6 | go-tests = true 7 | unused-packages = true 8 | -------------------------------------------------------------------------------- /testdata/complex/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | rjsone -y -t template.yaml encode::--'tr a-zA-Z' decode::--'tr b-zaB-ZA' rawparam::+hello context1.yaml context2.yaml foobar:named.yaml text::input.txt list:.. context1.yaml context2.yaml list2::.. input.txt withfilename::... input.txt 4 | -------------------------------------------------------------------------------- /testdata/merge/template.yaml: -------------------------------------------------------------------------------- 1 | should_be_overridden: {$eval: root.subkey1} 2 | should_be_original: {$eval: root.subkey2} 3 | nested: 4 | should_be_original: {$eval: root.subkey3.nested1} 5 | should_be_overridden: {$eval: root.subkey3.nested2} 6 | should_be_a_string: {$eval: root_list} 7 | -------------------------------------------------------------------------------- /testdata/merge/expected/stdout: -------------------------------------------------------------------------------- 1 | nested: 2 | should_be_original: original subkey3.nested1 value 3 | should_be_overridden: overridden subkey3.nested2 value 4 | should_be_a_string: string override 5 | should_be_original: original subkey2 value 6 | should_be_overridden: overridden subkey1 value 7 | -------------------------------------------------------------------------------- /testdata/merge/root.yaml: -------------------------------------------------------------------------------- 1 | root: 2 | subkey1: 'original subkey1 value' 3 | subkey2: 'original subkey2 value' 4 | subkey3: 5 | nested1: 'original subkey3.nested1 value' 6 | nested2: 7 | complex_nested_structure_will_be_overridden: 'original subkey3.nested2 value' 8 | root_list: 9 | - this 10 | - was 11 | - a 12 | - list 13 | -------------------------------------------------------------------------------- /testdata/complex/template.yaml: -------------------------------------------------------------------------------- 1 | encoded: 2 | $eval: encode(["n-za-mN-ZA-M"], "foo") 3 | decoded: 4 | $eval: decode(["a-zA-Z"], encode(["b-zaB-ZA"], "foooooo")) 5 | raw: ${rawparam} 6 | a: ${foo} 7 | b: ${bar} 8 | c: ${foobar} 9 | d: ${text} 10 | stuff: 11 | somelist: 12 | $eval: list 13 | someotherlist: 14 | $eval: list2 15 | fileinfo: 16 | $eval: withfilename 17 | -------------------------------------------------------------------------------- /testdata/complex/expected/stdout: -------------------------------------------------------------------------------- 1 | a: something 2 | b: nothing 3 | c: everything 4 | d: | 5 | foo 6 | decoded: foooooo 7 | encoded: sbb 8 | fileinfo: 9 | - basename: input.txt 10 | content: | 11 | foo 12 | filename: input.txt 13 | name: input 14 | raw: hello 15 | stuff: 16 | somelist: 17 | - foo: something 18 | - bar: nothing 19 | someotherlist: 20 | - | 21 | foo 22 | -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | name = "github.com/imdario/mergo" 6 | packages = ["."] 7 | revision = "9316a62528ac99aaecb4e47eadd6dc8aa6533d58" 8 | version = "v0.3.5" 9 | 10 | [[projects]] 11 | name = "github.com/taskcluster/json-e" 12 | packages = [ 13 | ".", 14 | "interpreter", 15 | "interpreter/prattparser" 16 | ] 17 | revision = "e7c5057d292797e63f41e90ae469a1007e1f69bc" 18 | version = "v2.5.0" 19 | 20 | [[projects]] 21 | branch = "master" 22 | name = "github.com/wryun/yaml-1" 23 | packages = ["."] 24 | revision = "e5213689ab3ec721209263e51f9edf8615d93085" 25 | 26 | [[projects]] 27 | name = "gopkg.in/yaml.v2" 28 | packages = ["."] 29 | revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" 30 | version = "v2.2.1" 31 | 32 | [solve-meta] 33 | analyzer-name = "dep" 34 | analyzer-version = 1 35 | inputs-digest = "1a46a12c324553e6998d528e22e809ea3063d913f83f240110b996f7d8c066c9" 36 | solver-name = "gps-cdcl" 37 | solver-version = 1 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.10.x 5 | 6 | env: 7 | - DEP_VERSION="0.4.1" 8 | 9 | before_install: 10 | - curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 11 | -o $GOPATH/bin/dep 12 | - chmod +x $GOPATH/bin/dep 13 | - go get github.com/mitchellh/gox 14 | 15 | install: 16 | - dep ensure 17 | 18 | script: 19 | - go build 20 | - ./test.sh testdata/* 21 | - if [ -n "$TRAVIS_TAG" ]; then gox; fi 22 | 23 | deploy: 24 | provider: releases 25 | api_key: 26 | secure: FhdaRLKhLICahJR1QHt9KCD12ukAP6uWdxreSFAv3+i95tFXJmATDggmP1OXEW5LmD7W/MJ0PD4UtlljoDZwsOrhdEBDdiJPNkwug8fdfcvLOmIOgjsZVzNf3ZZGue9isEcgAxm63xCmTpAIWAj+UC4WlUJfGAFTIDCRRsMJd202uzXuRmDwk3fUF+HNoOctxVAObmG0GrUstbVM+d/pfrT8v1epyN2MNM4KxoSk+yEtYs8ejtrB+ShPQHT8+AvFLBvVBBqI3l+SlEHg9cerFSuAGFyGgUMHqGJDuIVFSzXLh+RHZhd6JeY85Wep8YS7EPM/3BPwZpl2GB58z39AVLC8FxXoVjDo1mM97VdmV27EyXfGkRaVpz4cJyBgN+wKTntbwuxH05Hu2hvM8n4Oj6uP6FSd02xZ6sMr/iFlPwqRXlSNPwiTO0zFA2z3CRqpjZ48e3GDBvB1e7/eV5RLN2vkLvMF/MiW+F7tG4I93RYjR33GheyK20BNYf8qrIBv6LNtIOfEO6pYKItiU2gII6igchRDcaXlWoKuTHXyzQyzrR3jjI//8UanAfHBk1e1DtghBgBFQmr3h1S5osvjvIafkinrz5tWFjqbp788Sf29qCHnJbje7X+MW+QJfws3cHXLxjknlCbYnt9i7fPfnnvoowULfdUTPSFIfcCsXDY= 27 | file_glob: true 28 | file: rjsone_* 29 | skip_cleanup: true 30 | on: 31 | repo: wryun/rjsone 32 | tags: true 33 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -eu 4 | 5 | xreadlink() { 6 | # This sort of implements readlink -f except it works on OSX too 7 | 8 | node=$1 9 | 10 | cd `dirname $node` 11 | node=`basename $node` 12 | 13 | while [ -L "$node" ] 14 | do 15 | node=`readlink $node` 16 | 17 | cd `dirname $node` 18 | node=`basename $node` 19 | done 20 | 21 | echo `pwd`/$node 22 | } 23 | 24 | TEMPDIR="$(mktemp -d)" 25 | trap "rm -rf $TEMPDIR" EXIT 26 | 27 | go build 28 | export PATH="$(xreadlink .):$PATH" 29 | 30 | run() { 31 | local outputdir 32 | set +e 33 | outputdir="$(xreadlink "$2")" 34 | cd "$1" 35 | "./run.sh" > "$outputdir/stdout" 2> "$outputdir/stderr" 36 | echo "$?" > "$outputdir/exitcode" 37 | cd - > /dev/null 38 | set -e 39 | } 40 | 41 | PASS=0 42 | FAIL=0 43 | 44 | if [ "${1-}" = "-g" ]; then 45 | # generate 'golden' files (i.e. test output) 46 | shift 47 | 48 | echo "Generating new test outputs - be careful..." 49 | for f in "$@"; do 50 | OUTPUTDIR="$f/expected" 51 | if [ -e $OUTPUTDIR ]; then 52 | echo "Skipping $OUTPUTDIR since it already exists." 53 | continue 54 | fi 55 | mkdir -p "$OUTPUTDIR" 56 | run "$f" "$OUTPUTDIR" 57 | echo "$f exitcode=$(cat "$OUTPUTDIR/exitcode") stderr=$(cat "$OUTPUTDIR/stderr")" 58 | cat "$OUTPUTDIR/stdout" 59 | done 60 | else 61 | for f in "$@"; do 62 | echo "$f" 63 | run "$f" "$TEMPDIR" 64 | if diff "$f"/expected "$TEMPDIR"; then 65 | PASS="$(expr $PASS + 1)" 66 | else 67 | FAIL="$(expr $FAIL + 1)" 68 | fi 69 | done 70 | 71 | echo "--- $FAIL failed, $PASS passed ---" 72 | test "$FAIL" -eq 0 73 | fi 74 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/wryun/rjsone.svg?branch=master)](https://travis-ci.org/wryun/rjsone) 2 | 3 | `rjsone` (Render JSON-e) is a simple wrapper around the 4 | [JSON-e templating language](https://taskcluster.github.io/json-e/). 5 | 6 | It's a safe and easy way to have templates of moderate complexity 7 | for configuration as code 'languages' like Kubernetes and CloudFormation. 8 | 9 | Usage: rjsone [options] [context ...] 10 | -d performs a deep merge of contexts 11 | -i int 12 | indentation of JSON output; 0 means no pretty-printing (default 2) 13 | -o string 14 | output to a file (default is -, which is stdout) (default "-") 15 | -t string 16 | file to use for template (- is stdin) (default "-") 17 | -v show information about processing on stderr 18 | -y output YAML rather than JSON (always reads YAML/JSON) 19 | 20 | Context is usually provided by a list of arguments. By default, 21 | these are interpreted as files. Data is loaded as YAML/JSON by default 22 | and merged into the main context. If the 'filename' begins with a `+`, 23 | the rest of the argument is interpreted as a raw string rather than 24 | reading the file. For example: 25 | 26 | rjsone -t template.yaml context.yaml '+{"foo": 1}' 27 | 28 | When duplicate keys are found, later entries replace earlier at the 29 | top level only unless the `-d` flag is passed to perform deep merging. 30 | 31 | You can specify a particular context key to load a YAML/JSON file into 32 | using `keyname:filename.yaml`. You can also use `keyname:..` to indicate 33 | that subsequent entries without keys should be loaded as a list element 34 | into that key. If you instead use `keyname:...`, metadata information is 35 | loaded as well and each list element is an object containing `{filename, 36 | basename, content}`. 37 | 38 | When loading the context, the default input format is YAML but you can 39 | also use JSON, plain text, and `kv` (key value pairs, space separated, 40 | as used by bazel and many unix tools). To specify the format, rather 41 | than using a `:` you use `:format:`. For example: 42 | 43 | :yaml:ctx.yaml :kv:ctx.kv :json:ctx.json mykey:text:ctx.txt 44 | 45 | Note that you must specify a key name under which to load the plain text 46 | file, since it cannot define keys (i.e. is a plain text string). Also, 47 | although the default format is yaml, the default format with `::` is 48 | text. So the following equivalencies hold: 49 | 50 | mykey::context.txt == mykey:text:context.txt 51 | context.yaml == :context.yaml == :yaml:context.yaml 52 | 53 | A common pattern, therefore, is to provide plain text arguments to 54 | the template: 55 | 56 | rjsone -t template.yaml env::+production context.yaml 57 | 58 | For complex applications, single argument functions can be added by 59 | prefixing the filename with a `-` (or a `--` for raw string input). For 60 | example: 61 | 62 | b64decode::--'base64 -d' 63 | 64 | This adds a base64 decode function to the context which accepts two 65 | arguments as input, an array (command line arguments) and string (stdin), 66 | and outputs a string. In your template, you would use this function by 67 | like `b64decode([], 'Zm9vCg==')`. As with before, you can use format 68 | specifiers (:- is yaml on both sides for the default behaviour, and 69 | you can explicitly specify kv/json/text/yaml between both `::` and 70 | `--`). 71 | 72 | # Getting it 73 | 74 | [Grab the latest binary](https://github.com/wryun/rjsone/releases) or 75 | build it yourself: 76 | 77 | go get github.com/wryun/rjsone 78 | 79 | # Rationale 80 | 81 | I often want to template JSON/YAML for declarative infrastructure as code 82 | things (e.g. Kubernetes, CloudFormation, ...), and JSON-e is one of the 83 | few languages that is also valid YAML/JSON, unlike the common option of 84 | hijacking languages designed for plain text (or HTML) templating. If your 85 | template is valid YAML/JSON, your editor can help you out with syntax 86 | highlighting, and after you apply the template you will always have 87 | valid YAML/JSON. 88 | 89 | I also want to be 'declarative configuration language' agnostic 90 | (i.e. avoiding Kubernetes specific templating tools...). 91 | 92 | Before I discovered JSON-e, I wrote 93 | [o-stache](https://github.com/wryun/ostache/). There are a 94 | list of other structured templating options there, the most 95 | prominent of which is [Jsonnet](http://jsonnet.org/). 96 | 97 | # Basic usage example 98 | 99 | Please see the JSON-e documentation for how to really use it. 100 | 101 | `template.yaml` 102 | ```yaml 103 | a: ${foo} 104 | b: ${bar} 105 | c: ${foobar} 106 | ``` 107 | 108 | `context1.yaml` 109 | ```yaml 110 | foo: something 111 | ``` 112 | 113 | `context2.yaml` 114 | ```yaml 115 | bar: nothing 116 | ``` 117 | 118 | `named.yaml` 119 | ```yaml 120 | everything 121 | ``` 122 | 123 | Use YAML files for context: 124 | 125 | ```sh 126 | $ rjsone -y -t template.yaml context1.yaml context2.yaml foobar:named.yaml 127 | a: something 128 | b: nothing 129 | c: everything 130 | ``` 131 | 132 | Use context on command line (:: rather than : means interpret as raw string 133 | not as JSON/YAML, and + means treat it as a string rather than a filename): 134 | ```sh 135 | $ rjsone -y -t template.yaml foo::+something bar::+nothing foobar::+everything 136 | a: something 137 | b: nothing 138 | c: everything 139 | ``` 140 | 141 | *Warning*: if you need to construct anything that's not a pure string from templated 142 | input, you'll probably need to understand JSON-e's '$eval' operator. 143 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "log" 9 | "os" 10 | "strings" 11 | 12 | "github.com/imdario/mergo" 13 | jsone "github.com/taskcluster/json-e" 14 | // Quick hack of ghodss YAML to expose a new method 15 | yaml_ghodss "github.com/wryun/yaml-1" 16 | yaml_v2 "gopkg.in/yaml.v2" 17 | ) 18 | 19 | const description = `rjsone is a simple wrapper around the JSON-e templating language. 20 | 21 | See: https://taskcluster.github.io/json-e/ 22 | 23 | Context is usually provided by a list of arguments. By default, 24 | these are interpreted as files. Data is loaded as YAML/JSON by default 25 | and merged into the main context. If the 'filename' begins with a +, 26 | the rest of the argument is interpreted as a raw string rather than 27 | reading the file. For example: 28 | 29 | rjsone -t template.yaml context.yaml '+{"foo": 1}' 30 | 31 | When duplicate keys are found, later entries replace earlier at the 32 | top level only unless the -d flag is passed to perform deep merging. 33 | 34 | You can specify a particular context key to load a YAML/JSON file into 35 | using keyname:filename.yaml. You can also use keyname:.. to indicate 36 | that subsequent entries without keys should be loaded as a list element 37 | into that key. If you instead use keyname:..., metadata information is 38 | loaded as well and each list element is an object containing {filename, 39 | basename, content}. 40 | 41 | When loading the context, the default input format is YAML but you can 42 | also use JSON, plain text, and kv (key value pairs, space separated, 43 | as used by bazel and many unix tools). To specify the format, rather 44 | than using a : you use :format:. For example: 45 | 46 | :yaml:ctx.yaml :kv:ctx.kv :json:ctx.json mykey:text:ctx.txt 47 | 48 | Note that you must specify a key name under which to load the plain text 49 | file, since it cannot define keys (i.e. is a plain text string). Also, 50 | although the default format is yaml, the default format with :: is 51 | text. So the following equivalencies hold: 52 | 53 | mykey::context.txt == mykey:text:context.txt 54 | context.yaml == :context.yaml == :yaml:context.yaml 55 | 56 | A common pattern, therefore, is to provide plain text arguments to 57 | the template: 58 | 59 | rjsone -t template.yaml env::+production context.yaml 60 | 61 | For complex applications, single argument functions can be added by 62 | prefixing the filename with a - (or a -- for raw string input). For 63 | example: 64 | 65 | b64decode::--'base64 -d' 66 | 67 | This adds a base64 decode function to the context which accepts two 68 | arguments as input, an array (command line arguments) and string (stdin), 69 | and outputs a string. In your template, you would use this function by 70 | like b64decode([], 'Zm9vCg=='). As with before, you can use format 71 | specifiers (:- is yaml on both sides for the default behaviour, and 72 | you can explicitly specify kv/json/text/yaml between both :: and 73 | --). 74 | ` 75 | 76 | type arguments struct { 77 | yaml bool 78 | indentation int 79 | templateFile string 80 | verbose bool 81 | deepMerge bool 82 | outputFile string 83 | contexts []context 84 | } 85 | 86 | type content interface { 87 | load() (interface{}, error) 88 | metadata() map[string]interface{} 89 | } 90 | 91 | func main() { 92 | var args arguments 93 | flag.Usage = func() { 94 | fmt.Fprint(flag.CommandLine.Output(), description) 95 | fmt.Fprintf(flag.CommandLine.Output(), "\nUsage: %s [options] [context ...]\n", os.Args[0]) 96 | flag.PrintDefaults() 97 | fmt.Fprint(flag.CommandLine.Output(), "\n") 98 | } 99 | flag.StringVar(&args.templateFile, "t", "-", "file to use for template (- is stdin)") 100 | flag.BoolVar(&args.yaml, "y", false, "output YAML rather than JSON (always reads YAML/JSON)") 101 | flag.BoolVar(&args.verbose, "v", false, "show information about processing on stderr") 102 | flag.BoolVar(&args.deepMerge, "d", false, "performs a deep merge of contexts") 103 | flag.StringVar(&args.outputFile, "o", "-", "output to a file (default is -, which is stdout)") 104 | flag.IntVar(&args.indentation, "i", 2, "indentation of JSON output; 0 means no pretty-printing") 105 | flag.Parse() 106 | 107 | args.contexts = parseContexts(flag.Args()) 108 | logger := log.New(os.Stderr, "", 0) 109 | 110 | if err := run(logger, args); err != nil { 111 | fmt.Fprintf(flag.CommandLine.Output(), "Fatal error: %s\n", err) 112 | os.Exit(2) 113 | } 114 | } 115 | 116 | func run(l *log.Logger, args arguments) (finalError error) { 117 | closeWithError := func(c io.Closer) { 118 | if err := c.Close(); err != nil && finalError == nil { 119 | finalError = err 120 | } 121 | } 122 | 123 | context, err := loadContext(args.contexts, args.deepMerge) 124 | if err != nil { 125 | return err 126 | } 127 | 128 | if args.verbose { 129 | l.Println("Calculated context:") 130 | output, err := yaml_ghodss.Marshal(context) 131 | if err != nil { 132 | return err 133 | } 134 | l.Println(string(output)) 135 | } 136 | 137 | var input io.ReadCloser 138 | if args.templateFile == "-" { 139 | input = os.Stdin 140 | } else { 141 | input, err = os.Open(args.templateFile) 142 | if err != nil { 143 | return err 144 | } 145 | defer closeWithError(input) 146 | } 147 | 148 | var out io.WriteCloser 149 | if args.outputFile == "-" { 150 | out = os.Stdout 151 | } else { 152 | out, err = os.Create(args.outputFile) 153 | if err != nil { 154 | return err 155 | } 156 | defer closeWithError(out) 157 | } 158 | 159 | var encoder *yaml_v2.Encoder 160 | if args.yaml { 161 | encoder = yaml_v2.NewEncoder(out) 162 | defer closeWithError(encoder) 163 | } 164 | 165 | decoder := yaml_v2.NewDecoder(input) 166 | for { 167 | // json-e wants types as output by json, so we have to reach 168 | // into the annoying ghodss/yaml code to do the type conversion. 169 | // We can't use it directly (trivially), because it doesn't have 170 | // multi-document support. 171 | var passthroughTemplate interface{} 172 | err := decoder.Decode(&passthroughTemplate) 173 | if err == io.EOF { 174 | return nil 175 | } 176 | if err != nil { 177 | return err 178 | } 179 | var template interface{} 180 | err = yaml_ghodss.YAMLTypesToJSONTypes(passthroughTemplate, &template) 181 | if err != nil { 182 | return err 183 | } 184 | 185 | output, err := jsone.Render(template, context) 186 | if err != nil { 187 | return err 188 | } 189 | 190 | if args.yaml { 191 | err = encoder.Encode(output) 192 | if err != nil { 193 | return err 194 | } 195 | } else { 196 | var byteOutput []byte 197 | if args.indentation == 0 { 198 | byteOutput, err = json.Marshal(output) 199 | } else { 200 | byteOutput, err = json.MarshalIndent(output, "", strings.Repeat(" ", args.indentation)) 201 | // MarshalIndent, sadly, doesn't add a newline at the end. Which I think it should. 202 | byteOutput = append(byteOutput, 0x0a) 203 | } 204 | 205 | if err != nil { 206 | return err 207 | } 208 | 209 | _, err = out.Write(byteOutput) 210 | if err != nil { 211 | return err 212 | } 213 | } 214 | } 215 | } 216 | 217 | func loadContext(contexts []context, deepMerge bool) (map[string]interface{}, error) { 218 | finalContext := make(map[string]interface{}) 219 | 220 | for _, context := range contexts { 221 | untypedNewContext, err := context.eval() 222 | if err != nil { 223 | return nil, err 224 | } 225 | 226 | newContext, ok := untypedNewContext.(map[string]interface{}) 227 | if !ok { 228 | return nil, fmt.Errorf("context %s had no top level keys: %q", context.original, untypedNewContext) 229 | } 230 | 231 | if deepMerge { 232 | err = mergo.Merge(&finalContext, newContext, mergo.WithOverride) 233 | if err != nil { 234 | return nil, err 235 | } 236 | } else { 237 | for k, v := range newContext { 238 | finalContext[k] = v 239 | } 240 | } 241 | } 242 | 243 | return finalContext, nil 244 | } 245 | -------------------------------------------------------------------------------- /context.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io/ioutil" 9 | "os" 10 | "os/exec" 11 | "path" 12 | "path/filepath" 13 | "strings" 14 | 15 | "github.com/imdario/mergo" 16 | 17 | jsone_interpreter "github.com/taskcluster/json-e/interpreter" 18 | // Quick hack of ghodss YAML to expose a new method 19 | yaml_ghodss "github.com/wryun/yaml-1" 20 | ) 21 | 22 | func parseContexts(rawContexts []string) []context { 23 | contexts := make([]context, 0) 24 | 25 | var lc *listContent 26 | 27 | for _, rawContext := range rawContexts { 28 | key := "" 29 | var rawContent string 30 | if strings.HasPrefix(rawContext, "+") { 31 | // if it starts with a '+', we know it's raw and we shouldn't 32 | // try to find keys in it (otherwise we can't easily pass raw 33 | // JSON/YAML as an argument) 34 | rawContent = rawContext 35 | } else { 36 | splitContext := strings.SplitN(rawContext, ":", 2) 37 | if len(splitContext) < 2 { 38 | rawContent = splitContext[0] 39 | } else { 40 | key = splitContext[0] 41 | rawContent = ":" + splitContext[1] 42 | } 43 | } 44 | 45 | if key != "" { 46 | // If we have a new key, we should jump out of any list we're in 47 | lc = nil 48 | } 49 | 50 | parsedContext := context{rawContext, key, parseContent(rawContent, lc)} 51 | if newLc, ok := parsedContext.content.(*listContent); ok { 52 | lc = newLc 53 | contexts = append(contexts, parsedContext) 54 | } else if lc != nil { 55 | lc.contexts = append(lc.contexts, parsedContext) 56 | } else { 57 | contexts = append(contexts, parsedContext) 58 | } 59 | 60 | } 61 | 62 | return contexts 63 | } 64 | 65 | func parseContent(content string, lc *listContent) content { 66 | fmtPointer, data := parseFormat(content) 67 | 68 | var format inputFormat 69 | if fmtPointer == nil { 70 | if lc == nil { 71 | format = yamlFormat 72 | } else { 73 | // we inherit rawness from the listContent. Quirk of the syntax... 74 | // (NB - there will not be a colon, because if there were it would 75 | // have seen a key and aborted the list) 76 | format = lc.childFormat 77 | } 78 | } else if *fmtPointer == "" { 79 | format = textFormat 80 | } else { 81 | format = *fmtPointer 82 | } 83 | 84 | // TODO: this currently allows a bunch of stupid things 85 | // (e.g. embedded listContents...). Should write a proper grammar. 86 | switch { 87 | case data == "..": 88 | return &listContent{childFormat: format, showMetadata: false} 89 | case data == "...": 90 | return &listContent{childFormat: format, showMetadata: true} 91 | case strings.HasPrefix(data, "+"): 92 | return &textContent{format: format, text: data[1:]} 93 | case data == "-": 94 | return &stdinContent{format: format} 95 | case strings.HasPrefix(data, "--"): 96 | return &functionContent{rawInput: format == textFormat, rawOutput: true, function: data[2:]} 97 | case strings.HasPrefix(data, "-"): 98 | return &functionContent{rawInput: format == textFormat, rawOutput: false, function: data[1:]} 99 | default: 100 | return &fileContent{format: format, filename: data} 101 | } 102 | } 103 | 104 | type inputFormat string 105 | 106 | const ( 107 | yamlFormat = inputFormat("yaml") 108 | jsonFormat = inputFormat("json") 109 | kvFormat = inputFormat("kv") 110 | textFormat = inputFormat("text") 111 | ) 112 | 113 | // parseFormat part of content (content = :format:data) 114 | // nothing there -> nil 115 | func parseFormat(content string) (*inputFormat /* format */, string /* data */) { 116 | if !strings.HasPrefix(content, ":") { 117 | // it's just a text string/filename - no format specifier 118 | return nil, content 119 | } 120 | 121 | content = content[1:] 122 | 123 | // Hack: if the first thing after the ':' is a '+', it must be 124 | // raw yaml/json (and we don't want to split on ':', as that might 125 | // be a valid char...) 126 | if strings.HasPrefix(content, "+") { 127 | f := yamlFormat 128 | return &f, content 129 | } 130 | 131 | splitContent := strings.SplitN(content, ":", 2) 132 | 133 | if len(splitContent) == 1 { 134 | return nil, splitContent[0] 135 | } 136 | 137 | // we don't validate here to avoid errors - will fail on eval. 138 | f := inputFormat(splitContent[0]) 139 | return &f, splitContent[1] 140 | } 141 | 142 | type context struct { 143 | original string 144 | key string 145 | 146 | content content 147 | } 148 | 149 | func (c *context) eval() (interface{}, error) { 150 | result, err := c.content.load() 151 | if err != nil { 152 | return nil, err 153 | } 154 | 155 | if c.key != "" { 156 | return map[string]interface{}{c.key: result}, nil 157 | } 158 | 159 | return result, nil 160 | } 161 | 162 | func loadBytes(format inputFormat, data []byte) (interface{}, error) { 163 | switch format { 164 | case jsonFormat: 165 | var result interface{} 166 | if err := json.Unmarshal(data, &result); err != nil { 167 | return nil, err 168 | } 169 | return result, nil 170 | case yamlFormat: 171 | var result interface{} 172 | if err := yaml_ghodss.Unmarshal(data, &result); err != nil { 173 | return nil, err 174 | } 175 | return result, nil 176 | case textFormat: 177 | return string(data), nil 178 | case kvFormat: 179 | // TODO unicode? 180 | lines := strings.Split(string(data), "\n") 181 | result := make(map[string]interface{}, len(lines)) 182 | for _, line := range lines { 183 | if line == "" { 184 | continue 185 | } 186 | 187 | splitLine := strings.SplitN(line, " ", 2) 188 | if len(splitLine) != 2 { 189 | return nil, fmt.Errorf("line not in kv format: %q", line) 190 | } 191 | result[splitLine[0]] = splitLine[1] 192 | } 193 | return result, nil 194 | default: 195 | return nil, fmt.Errorf("format %q not supported", format) 196 | } 197 | } 198 | 199 | type fileContent struct { 200 | format inputFormat 201 | filename string 202 | } 203 | 204 | func (fc *fileContent) load() (interface{}, error) { 205 | resultBytes, err := ioutil.ReadFile(fc.filename) 206 | if err != nil { 207 | return nil, err 208 | } 209 | return loadBytes(fc.format, resultBytes) 210 | } 211 | 212 | func (fc *fileContent) metadata() map[string]interface{} { 213 | basename := path.Base(fc.filename) 214 | return map[string]interface{}{ 215 | "filename": fc.filename, 216 | "basename": basename, 217 | "name": strings.TrimSuffix(basename, filepath.Ext(basename)), 218 | } 219 | } 220 | 221 | type stdinContent struct { 222 | format inputFormat 223 | } 224 | 225 | func (sc *stdinContent) load() (interface{}, error) { 226 | resultBytes, err := ioutil.ReadAll(os.Stdin) 227 | if err != nil { 228 | return nil, err 229 | } 230 | return loadBytes(sc.format, resultBytes) 231 | } 232 | 233 | func (sc *stdinContent) metadata() map[string]interface{} { 234 | return map[string]interface{}{} 235 | } 236 | 237 | type functionContent struct { 238 | function string 239 | rawOutput bool 240 | rawInput bool 241 | } 242 | 243 | type textContent struct { 244 | format inputFormat 245 | text string 246 | } 247 | 248 | func (tc *textContent) load() (interface{}, error) { 249 | return loadBytes(tc.format, []byte(tc.text)) 250 | } 251 | 252 | func (tc *textContent) metadata() map[string]interface{} { 253 | return map[string]interface{}{} 254 | } 255 | 256 | func (fc *functionContent) load() (interface{}, error) { 257 | var f interface{} 258 | commandArray := strings.Split(fc.function, " ") 259 | 260 | if fc.rawInput && fc.rawOutput { 261 | f = func(args []interface{}, stdin string) (string, error) { 262 | stringArgs, err := castToStrings(args) 263 | if err != nil { 264 | return "", err 265 | } 266 | extendedCommandArray := append(commandArray, stringArgs...) 267 | command := exec.Command(extendedCommandArray[0], extendedCommandArray[1:]...) 268 | command.Stderr = os.Stderr 269 | command.Stdin = bytes.NewReader([]byte(stdin)) 270 | stdoutBytes, err := command.Output() 271 | if err != nil { 272 | return "", err 273 | } 274 | return string(stdoutBytes), nil 275 | } 276 | } else if fc.rawInput { 277 | f = func(args []interface{}, stdin string) (interface{}, error) { 278 | stringArgs, err := castToStrings(args) 279 | if err != nil { 280 | return "", err 281 | } 282 | extendedCommandArray := append(commandArray, stringArgs...) 283 | command := exec.Command(extendedCommandArray[0], extendedCommandArray[1:]...) 284 | command.Stderr = os.Stderr 285 | command.Stdin = bytes.NewReader([]byte(stdin)) 286 | stdoutBytes, err := command.Output() 287 | if err != nil { 288 | return nil, err 289 | } 290 | 291 | var o interface{} 292 | err = yaml_ghodss.Unmarshal(stdoutBytes, &o) 293 | if err != nil { 294 | return nil, err 295 | } 296 | return o, nil 297 | } 298 | } else if fc.rawOutput { 299 | f = func(args []interface{}, stdin interface{}) (string, error) { 300 | jsonBytes, err := json.Marshal(stdin) 301 | if err != nil { 302 | return "", err 303 | } 304 | 305 | stringArgs, err := castToStrings(args) 306 | if err != nil { 307 | return "", err 308 | } 309 | extendedCommandArray := append(commandArray, stringArgs...) 310 | command := exec.Command(extendedCommandArray[0], extendedCommandArray[1:]...) 311 | command.Stderr = os.Stderr 312 | command.Stdin = bytes.NewReader(jsonBytes) 313 | stdoutBytes, err := command.Output() 314 | return string(stdoutBytes), err 315 | } 316 | } else { 317 | f = func(args []interface{}, stdin interface{}) (interface{}, error) { 318 | jsonBytes, err := json.Marshal(stdin) 319 | if err != nil { 320 | return "", err 321 | } 322 | 323 | stringArgs, err := castToStrings(args) 324 | if err != nil { 325 | return "", err 326 | } 327 | extendedCommandArray := append(commandArray, stringArgs...) 328 | command := exec.Command(extendedCommandArray[0], extendedCommandArray[1:]...) 329 | command.Stderr = os.Stderr 330 | command.Stdin = bytes.NewReader(jsonBytes) 331 | stdoutBytes, err := command.Output() 332 | if err != nil { 333 | return nil, err 334 | } 335 | 336 | var o interface{} 337 | err = yaml_ghodss.Unmarshal(stdoutBytes, &o) 338 | if err != nil { 339 | return nil, err 340 | } 341 | return stdin, nil 342 | } 343 | } 344 | 345 | return jsone_interpreter.WrapFunction(f), nil 346 | } 347 | 348 | func (fc *functionContent) metadata() map[string]interface{} { 349 | return map[string]interface{}{} 350 | } 351 | 352 | type listContent struct { 353 | contexts []context 354 | showMetadata bool 355 | childFormat inputFormat 356 | } 357 | 358 | func (lc *listContent) load() (interface{}, error) { 359 | outputList := make([]interface{}, 0, len(lc.contexts)) 360 | 361 | for _, context := range lc.contexts { 362 | result, err := context.eval() 363 | if err != nil { 364 | return nil, err 365 | } 366 | 367 | if !lc.showMetadata { 368 | outputList = append(outputList, result) 369 | continue 370 | } 371 | 372 | metadataResult := map[string]interface{}{ 373 | "content": result, 374 | } 375 | err = mergo.Merge(&metadataResult, context.content.metadata()) 376 | if err != nil { 377 | return nil, err 378 | } 379 | outputList = append(outputList, metadataResult) 380 | } 381 | 382 | return outputList, nil 383 | } 384 | 385 | func (lc *listContent) metadata() map[string]interface{} { 386 | return map[string]interface{}{} 387 | } 388 | 389 | func castToStrings(slice []interface{}) ([]string, error) { 390 | result := make([]string, len(slice)) 391 | for i, v := range slice { 392 | s, ok := v.(string) 393 | if !ok { 394 | return nil, errors.New("function command line arguments must be strings (use stdin or $json)") 395 | } 396 | result[i] = s 397 | } 398 | return result, nil 399 | } 400 | --------------------------------------------------------------------------------