├── .gitignore ├── LICENSE ├── README.md ├── dist.sh ├── go.mod ├── go.sum ├── gq ├── gq.go └── gq_test.go ├── main.go └── program.go.tmpl /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | .DS_Store -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Hunter Herman 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![GoDoc](https://godoc.org/github.com/hherman1/gq?status.svg)](https://pkg.go.dev/mod/github.com/hherman1/gq/gq) 2 | 3 | # gq 4 | jq but using go instead. 5 | 6 | 7 | ## Install 8 | 9 | ``` 10 | go install github.com/hherman1/gq@latest 11 | ``` 12 | 13 | Or download the binaries [here](https://github.com/hherman1/gq/releases/latest) 14 | 15 | # Usage 16 | 17 | Pipe some input into `gq` and pass in a block of go code as the argument. The variable `j` is set to the passed in JSON and printed at the end of the function, so manipulate it (e.g below) in order to change the output. The `gq` library is inlined in the program, and described in detail in the API reference above. 18 | 19 | # Examples 20 | 21 | Accesses 22 | 23 | ``` 24 | $ echo '{"weird": ["nested", {"complex": "structure"}]}' | gq 'j.G("weird").I(1).G("complex")' 25 | "structure" 26 | ``` 27 | 28 | Filters 29 | 30 | ``` 31 | $ echo '["list", "of", "mostly", "uninterseting", "good/strings", "good/welp"]' | gq 'j.Filter(func(n *Node) bool { return strings.Contains(n.String(), "good/") })' 32 | [ 33 | "good/strings", 34 | "good/welp" 35 | ] 36 | ``` 37 | 38 | Maps 39 | 40 | ``` 41 | $ echo '[1, 2, 3, 4]' | gq 'j.Map(func(n *Node) *Node {n.val = n.Int() + 50 / 2; return n})' 42 | [ 43 | 26, 44 | 27, 45 | 28, 46 | 29 47 | ] 48 | ``` 49 | 50 | # Why? 51 | 52 | `jq` is hard to use. There are alternatives like `fq` and `zq`, but they still make you learn a new programming language. Im tired of learning new programming languages. 53 | 54 | `gq` is not optimized for speed, flexibility or beauty. `gq` is optimized for minimal learning/quick usage. `gq` understands that you don't use it constantly, you use it once a month and then forget about it. So when you come back to it, `gq` will be easy to relearn. Just use the builtin library just like you would any other go project and you're done. No unfamiliar syntax or operations, or surprising limits. Thats it. 55 | 56 | 57 | # How's it work? 58 | 59 | Speaking of surprising limits, `gq` runs your code in the [yaegi](https://github.com/traefik/yaegi) go interpreter. This means that it runs quickly for small inputs/programs (the alternative was `go run`, which is... not quite as quick). However, it also means its not the fastest `*q` out there, and further it means that you might run into quirks with `yaegi` interpretation limitations. Seems to work pretty well so far though. 60 | 61 | # This tool sucks. 62 | 63 | Yea, well, I built it last night, so, yea it kinda sucks. But please file issues! Or submit PRs! Maybe this will turn into something useful? At least maybe it will inspire some conversation. 64 | 65 | I think it needs a lot of refinement still, in short. 66 | -------------------------------------------------------------------------------- /dist.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | rm -rf build/* 6 | 7 | GOOS=darwin GOARCH=amd64 go build -o build/gq . 8 | zip -j build/darwin-amd64.zip build/gq 9 | GOOS=darwin GOARCH=arm64 go build -o build/gq . 10 | zip -j build/darwin-arm64.zip build/gq 11 | GOOS=linux GOARCH=amd64 go build -o build/gq . 12 | zip -j build/linux-amd64.zip build/gq 13 | GOOS=windows GOARCH=amd64 go build -o build/gq . 14 | zip -j build/windows-amd64.zip build/gq 15 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/hherman1/gq 2 | 3 | go 1.18 4 | 5 | require github.com/traefik/yaegi v0.11.3 // indirect 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/traefik/yaegi v0.11.3 h1:TuuIc0TC4oaWkVngjVAKkFd4fH35B0B95DmbS76uqs8= 2 | github.com/traefik/yaegi v0.11.3/go.mod h1:RuCwD8/wsX7b6KoQHOaIFUfuH3gQIK4KWnFFmJMw5VA= 3 | -------------------------------------------------------------------------------- /gq/gq.go: -------------------------------------------------------------------------------- 1 | // Package gq provides functionality for use in gq's generated programs, and serves as the prelude to all of those programs. 2 | package gq 3 | 4 | import ( 5 | "bytes" 6 | "encoding/json" 7 | "fmt" 8 | "strings" 9 | ) 10 | 11 | // The central datastructure. If a Node is returned by a gq program it is pretty printed as JSON. 12 | type Node struct { 13 | // Parsed JSON as a Go structure 14 | val interface{} 15 | // Any errors generated during construction or manipulation. 16 | err error 17 | // The command that generated this node. 18 | origin string 19 | } 20 | 21 | // Pretty prints the contained json. If something is wrong with the node, returns an error string instead. 22 | func (n Node) String() string { 23 | if n.err != nil { 24 | return fmt.Sprintf("Error: %v: %v", n.origin, n.err) 25 | } 26 | var bs bytes.Buffer 27 | enc := json.NewEncoder(&bs) 28 | enc.SetIndent("", "\t") 29 | err := enc.Encode(n.val) 30 | if err != nil { 31 | return fmt.Sprintf("Error: %v: %v\n%v", n.origin, err, n.val) 32 | } 33 | return bs.String() 34 | } 35 | 36 | func (n *Node) UnmarshalJSON(bs []byte) error { 37 | err := json.Unmarshal(bs, &n.val) 38 | if err != nil { 39 | return fmt.Errorf("gq.Node: %w", err) 40 | } 41 | return nil 42 | } 43 | 44 | // Fetches the current value of the node as an integer, if possible. Otherwise, sets the error for the node. 45 | func (n *Node) Int() int { 46 | if f, ok := n.val.(float64); ok { 47 | return int(f) 48 | } 49 | n.trace("Int") 50 | n.err = fmt.Errorf("expected numeric, was %T", n.val) 51 | return 0 52 | } 53 | 54 | // Fetches the current value of the node as float, if possible. Otherwise, sets the error for the node. 55 | func (n *Node) Float() float64 { 56 | if f, ok := n.val.(float64); ok { 57 | return f 58 | } 59 | n.trace("Float") 60 | n.err = fmt.Errorf("expected numeric, was %T", n.val) 61 | return 0 62 | } 63 | 64 | // Fetches the current value of the node as a map, if possible. Otherwise, sets the error for the node. 65 | func (n *Node) MapValue() map[string]interface{} { 66 | if f, ok := n.val.(map[string]interface{}); ok { 67 | return f 68 | } 69 | n.trace("MapValue") 70 | n.err = fmt.Errorf("expected map, was %T", n.val) 71 | return map[string]interface{}{} 72 | } 73 | 74 | // Fetches the current value of the node as an array, if possible. Otherwise, sets the error for the node. 75 | func (n *Node) Array() []interface{} { 76 | if f, ok := n.val.([]interface{}); ok { 77 | return f 78 | } 79 | n.trace("Array") 80 | n.err = fmt.Errorf("expected array, was %T", n.val) 81 | return []interface{}{} 82 | } 83 | 84 | // Fetches the current value of the node as string, if possible. Otherwise, sets the error for the node. 85 | func (n *Node) Str() string { 86 | if s, ok := n.val.(string); ok { 87 | return s 88 | } 89 | n.trace("String") 90 | n.err = fmt.Errorf("expected string, was %T", n.val) 91 | return "error" 92 | } 93 | 94 | // G fetches the values at the given keys in the map node. If there is only one key, returns that key's value. If there are many keys, returns 95 | // an array of their non-null values. If this is not a map node, returns an error node. If none of the keys are not found, returns null. 96 | func (n *Node) G(keys ...string) *Node { 97 | if n.err != nil { 98 | return n 99 | } 100 | n.trace("G", keys) 101 | m, ok := n.val.(map[string]interface{}) 102 | if !ok { 103 | n.err = fmt.Errorf("expected JSON map, found: %T", n.val) 104 | return n 105 | } 106 | if len(keys) == 1 { 107 | n.val = m[keys[0]] 108 | return n 109 | } 110 | vals := []interface{}{} 111 | for _, k := range keys { 112 | val := m[k] 113 | if val == nil { 114 | continue 115 | } 116 | vals = append(vals, val) 117 | } 118 | if len(vals) == 0 { 119 | n.val = nil 120 | return n 121 | } 122 | n.val = vals 123 | return n 124 | } 125 | 126 | // I fetches the value at the given array indices. If this is not an array node, returns an error node. If none of the indices are found, returns null. 127 | // If there is only one index given, returns just that value. Otherwise returns an array of values. 128 | func (n *Node) I(is ...int) *Node { 129 | if n.err != nil { 130 | return n 131 | } 132 | n.trace("I", is) 133 | a, ok := n.val.([]interface{}) 134 | if !ok { 135 | n.err = fmt.Errorf("expected JSON array, found: %T", n.val) 136 | return n 137 | } 138 | if len(is) == 1 { 139 | i := is[0] 140 | if i < 0 || i >= len(a) { 141 | n.val = nil 142 | return n 143 | } 144 | n.val = a[i] 145 | return n 146 | } 147 | var vals []interface{} 148 | for _, i := range is { 149 | if i < 0 || i >= len(a) { 150 | continue 151 | } 152 | vals = append(vals, a[i]) 153 | } 154 | if len(vals) == 0 { 155 | n.val = nil 156 | return n 157 | } 158 | n.val = vals 159 | return n 160 | } 161 | 162 | // Filter removes nodes from the interior of the given map or array node if they fail the filter function. 163 | func (n *Node) Filter(f func(*Node) bool) *Node { 164 | if n.err != nil { 165 | return n 166 | } 167 | cn := *n // operate on a copy in case we need to revert. 168 | cn.trace("Filter", "func") 169 | // map implementation 170 | if m, ok := cn.val.(map[string]interface{}); ok { 171 | filtered := make(map[string]interface{}) 172 | for k, v := range m { 173 | subN := cn 174 | subN.trace("G", k) 175 | subN.val = v 176 | success := f(&subN) 177 | if subN.err != nil { 178 | // uh oh, error. 179 | *n = subN 180 | return n 181 | } 182 | if success { 183 | filtered[k] = v 184 | } 185 | } 186 | cn.val = filtered 187 | *n = cn 188 | return n 189 | } 190 | 191 | // array implementation 192 | if a, ok := cn.val.([]interface{}); ok { 193 | filtered := make([]interface{}, 0) 194 | for i, v := range a { 195 | subN := cn 196 | subN.trace("I", i) 197 | subN.val = v 198 | success := f(&subN) 199 | if subN.err != nil { 200 | // uh oh, error. 201 | *n = subN 202 | return n 203 | } 204 | if success { 205 | filtered = append(filtered, v) 206 | } 207 | } 208 | cn.val = filtered 209 | *n = cn 210 | return n 211 | } 212 | 213 | // whoops 214 | cn.err = fmt.Errorf("expected map or array, was %T", n.val) 215 | *n = cn 216 | return n 217 | } 218 | 219 | // Map replaces nodes from the interior of the given map or array node with the output of the function. 220 | func (n *Node) Map(f func(*Node) *Node) *Node { 221 | if n.err != nil { 222 | return n 223 | } 224 | cn := *n // operate on a copy in case we need to revert. 225 | cn.trace("Map", "func") 226 | // map implementation 227 | if m, ok := cn.val.(map[string]interface{}); ok { 228 | filtered := make(map[string]interface{}) 229 | for k, v := range m { 230 | subN := cn 231 | subN.trace("G", k) 232 | subN.val = v 233 | replace := f(&subN) 234 | if subN.err != nil { 235 | // uh oh, error. 236 | *n = subN 237 | return n 238 | } 239 | filtered[k] = replace.val 240 | } 241 | cn.val = filtered 242 | *n = cn 243 | return n 244 | } 245 | 246 | // array implementation 247 | if a, ok := cn.val.([]interface{}); ok { 248 | filtered := make([]interface{}, 0) 249 | for i, v := range a { 250 | subN := cn 251 | subN.trace("I", i) 252 | subN.val = v 253 | replace := f(&subN) 254 | if subN.err != nil { 255 | // uh oh, error. 256 | *n = subN 257 | return n 258 | } 259 | filtered = append(filtered, replace.val) 260 | } 261 | cn.val = filtered 262 | *n = cn 263 | return n 264 | } 265 | 266 | // whoops 267 | cn.err = fmt.Errorf("expected map or array, was %T", n.val) 268 | *n = cn 269 | return n 270 | } 271 | 272 | // Checks if this is a map node 273 | func (n Node) IsMap() bool { 274 | _, ok := n.val.(map[string]interface{}) 275 | return ok 276 | } 277 | 278 | // Adds the given trace information to the node. 279 | func (n *Node) trace(method string, args ...interface{}) { 280 | var sargs []string 281 | for _, a := range args { 282 | sargs = append(sargs, fmt.Sprint(a)) 283 | } 284 | origin := fmt.Sprintf("%v(%v)", method, strings.Join(sargs, ", ")) 285 | if n.origin != "" { 286 | origin = fmt.Sprintf("%v: %v", n.origin, origin) 287 | } 288 | n.origin = origin 289 | } 290 | -------------------------------------------------------------------------------- /gq/gq_test.go: -------------------------------------------------------------------------------- 1 | package gq 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "testing" 7 | ) 8 | 9 | func TestNode(t *testing.T) { 10 | sample := `{ 11 | "test": [1, 2, 3, {"1": "b"}] 12 | }` 13 | var n Node 14 | err := json.Unmarshal([]byte(sample), &n) 15 | if err != nil { 16 | t.Fatalf("unmarshal sample: %v", err) 17 | } 18 | fmt.Println(n) 19 | } 20 | 21 | func TestNodeGet(t *testing.T) { 22 | sample := `{ 23 | "test": [1, 2, 3, {"1": "b"}] 24 | }` 25 | var n Node 26 | err := json.Unmarshal([]byte(sample), &n) 27 | if err != nil { 28 | t.Fatalf("unmarshal sample: %v", err) 29 | } 30 | fmt.Println(n.G("test")) 31 | fmt.Println(n.G("test").G("1")) 32 | } 33 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | _ "embed" 6 | "fmt" 7 | "os" 8 | "strings" 9 | "text/template" 10 | 11 | "github.com/traefik/yaegi/interp" 12 | "github.com/traefik/yaegi/stdlib" 13 | ) 14 | 15 | //go:embed program.go.tmpl 16 | var programTmpl string 17 | 18 | //go:embed gq/gq.go 19 | var prelude string 20 | 21 | func main() { 22 | if err := run(); err != nil { 23 | fmt.Println("Error: ", err) 24 | fmt.Println(help) 25 | os.Exit(1) 26 | } 27 | } 28 | 29 | func run() error { 30 | // Check for help invocation. 31 | for _, a := range os.Args { 32 | if a == "-h" || a == "--help" { 33 | fmt.Println(help) 34 | return nil 35 | } 36 | } 37 | 38 | // prepare program 39 | tmpl, err := template.New("tmpl").Parse(programTmpl) 40 | if err != nil { 41 | return fmt.Errorf("parse template: %w", err) 42 | } 43 | program := strings.Join(os.Args[1:], " ") 44 | type Dot struct { 45 | Program string 46 | Prelude string 47 | } 48 | var runnable bytes.Buffer 49 | err = tmpl.Execute(&runnable, Dot{Program: program, Prelude: strings.ReplaceAll(prelude, "package gq", "")}) 50 | if err != nil { 51 | return fmt.Errorf("generating program: %w", err) 52 | } 53 | 54 | // execute 55 | i := interp.New(interp.Options{}) 56 | i.Use(stdlib.Symbols) 57 | p, err := i.Compile(runnable.String()) 58 | if err != nil { 59 | return fmt.Errorf("compile generated program: %w", err) 60 | } 61 | _, err = i.Execute(p) 62 | if err != nil { 63 | return fmt.Errorf("execute program: %w", err) 64 | } 65 | /* 66 | // save to disk 67 | f, err := os.CreateTemp(os.TempDir(), "*.go") 68 | if err != nil { 69 | return fmt.Errorf("create temp file: %w", err) 70 | } 71 | defer func() { 72 | f.Close() 73 | os.Remove(f.Name()) 74 | }() 75 | _, err = io.Copy(f, bytes.NewReader(runnable.Bytes())) 76 | if err != nil { 77 | return fmt.Errorf("write to temp file: %w", err) 78 | } 79 | err = f.Close() 80 | if err != nil { 81 | return fmt.Errorf("closing temp file: %w", err) 82 | } 83 | 84 | // execute new program 85 | cmd := exec.Command("go", "run", f.Name()) 86 | cmd.Stdin = os.Stdin 87 | cmd.Stdout = os.Stdout 88 | cmd.Stderr = os.Stderr 89 | err = cmd.Run() 90 | if err != nil { 91 | if cmd.ProcessState.ExitCode() == 1 { 92 | // the generated program compiled successfully, but had some error output. We should just exit immediately. 93 | os.Exit(1) 94 | } 95 | return fmt.Errorf("running generated program: %w", err) 96 | } 97 | */ 98 | return nil 99 | } 100 | 101 | const help = `gq executes go scripts against json. 102 | 103 | Usage: cat f.json | gq 'j.G("hello").G("world") 104 | 105 | The script you pass into 'gq' has access to a variable called 'j' which contains the parsed JSON. After your script is run, whatever remains in 'j' is printed. 'j' is of type *Node. The following functions are available to the script: 106 | 107 | func (n *Node) Array() []interface{} 108 | Fetches the current value of the node as an array, if possible. Otherwise, 109 | sets the error for the node. 110 | 111 | func (n *Node) Filter(f func(*Node) bool) *Node 112 | Filter removes nodes from the interior of the given map or array node if 113 | they fail the filter function. 114 | 115 | func (n *Node) Float() float64 116 | Fetches the current value of the node as float, if possible. Otherwise, sets 117 | the error for the node. 118 | 119 | func (n *Node) G(keys ...string) *Node 120 | G fetches the values at the given keys in the map node. If there is only one 121 | key, returns that key's value. If there are many keys, returns an array of 122 | their non-null values. If this is not a map node, returns an error node. If 123 | none of the keys are not found, returns null. 124 | 125 | func (n *Node) I(is ...int) *Node 126 | I fetches the value at the given array indices. If this is not an array 127 | node, returns an error node. If none of the indices are found, returns null. 128 | If there is only one index given, returns just that value. Otherwise returns 129 | an array of values. 130 | 131 | func (n *Node) Int() int 132 | Fetches the current value of the node as an integer, if possible. Otherwise, 133 | sets the error for the node. 134 | 135 | func (n Node) IsMap() bool 136 | Checks if this is a map node 137 | 138 | func (n *Node) Map(f func(*Node) *Node) *Node 139 | Map replaces nodes from the interior of the given map or array node with the 140 | output of the function. 141 | 142 | func (n *Node) MapValue() map[string]interface{} 143 | Fetches the current value of the node as a map, if possible. Otherwise, sets 144 | the error for the node. 145 | 146 | func (n *Node) Str() string 147 | Fetches the current value of the node as string, if possible. Otherwise, 148 | sets the error for the node. 149 | ` 150 | -------------------------------------------------------------------------------- /program.go.tmpl: -------------------------------------------------------------------------------- 1 | package main 2 | import "os" 3 | import "io" 4 | {{.Prelude}} 5 | 6 | func main() { 7 | if err := run(); err != nil { 8 | fmt.Println("Error: ", err) 9 | os.Exit(1) 10 | } 11 | } 12 | 13 | func run() error { 14 | j := new(Node) 15 | var val interface{} 16 | err := json.NewDecoder(os.Stdin).Decode(&val) 17 | if err != nil { 18 | return fmt.Errorf("unmarshal: %w", err) 19 | } 20 | j.val = val 21 | 22 | {{.Program}} 23 | 24 | fmt.Println(j.String()) 25 | return nil 26 | } --------------------------------------------------------------------------------