├── .gitignore ├── LICENSE ├── README.md ├── jqpipe.go └── jqpipe_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, threatGRID 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | jqpipe-go 2 | ========= 3 | 4 | This package is a wrapper of [JQ](http://stedolan.github.io/jq/) for the [Go](http://golang.org) programming language, designed to make it easy for Go programs to filter JSON input using JQ. This is used internally at [ThreatGRID](http://threatgrid.com) to perform unit testing using [expectjq](http://github.com/threatgrid/expectjq) and as a map/reduce VM for ThreatGRID's internal data storage. 5 | 6 | API information is available at [godoc.org](http://godoc.org/github.com/threatgrid/jqpipe-go). -------------------------------------------------------------------------------- /jqpipe.go: -------------------------------------------------------------------------------- 1 | /* 2 | Wraps the "jq" utility as a pipe. 3 | 4 | This package makes it easy for Go programs to filter JSON data using 5 | stedolan's "jq". This is used internally at ThreatGRID as a sort of 6 | expedient map/reduce in its distributed data store and in its "expectjq" 7 | test utility. 8 | */ 9 | package jq 10 | 11 | import ( 12 | "bytes" 13 | "encoding/json" 14 | "errors" 15 | "io" 16 | "os/exec" 17 | ) 18 | 19 | // Eval starts a new Jq process to evaluate an expression with json input 20 | func Eval(js string, expr string, opts ...string) ([]json.RawMessage, error) { 21 | jq, err := New(bytes.NewReader([]byte(js)), expr, opts...) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | ret := make([]json.RawMessage, 0, 16) 27 | for { 28 | next, err := jq.Next() 29 | switch err { 30 | case nil: 31 | ret = append(ret, next) 32 | case io.EOF: 33 | return ret, nil 34 | default: 35 | return ret, err 36 | } 37 | } 38 | panic("unreachable") // for go 1.0 39 | } 40 | 41 | // New wraps a jq.Pipe around an existing io.Reader, applying a JQ expression 42 | func New(r io.Reader, expr string, opts ...string) (*Pipe, error) { 43 | var err error 44 | 45 | proc := new(Pipe) 46 | opts = append(opts, expr) 47 | proc.jq = exec.Command("jq", opts...) 48 | proc.jq.Stdin = r 49 | 50 | proc.stdout, err = proc.jq.StdoutPipe() 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | proc.jq.Stderr = &proc.stderr 56 | err = proc.jq.Start() 57 | if err != nil { 58 | proc.stdout.Close() 59 | return nil, err 60 | } 61 | 62 | proc.dec = json.NewDecoder(proc.stdout) 63 | return proc, nil 64 | } 65 | 66 | // Pipe encapsulates a child "jq" process with a fixed expression, returning each JSON output from jq. 67 | type Pipe struct { 68 | jq *exec.Cmd 69 | dec *json.Decoder 70 | stdout io.ReadCloser 71 | stderr bytes.Buffer 72 | } 73 | 74 | // Next provides the next JSON result from JQ. If there are no more results, io.EOF is returned. 75 | func (p *Pipe) Next() (json.RawMessage, error) { 76 | var msg json.RawMessage 77 | err := p.dec.Decode(&msg) 78 | 79 | //TODO: guard against a Next() after we have terminated. 80 | if err == nil { 81 | return msg, nil 82 | } 83 | p.stdout.Close() 84 | 85 | // if we have a decoding error, jq is sick and we need to kill it with fire.. 86 | if err != io.EOF { 87 | p.Close() 88 | return nil, err 89 | } 90 | 91 | // terminate jq (if it hasn't died already) 92 | p.jq.Process.Kill() 93 | p.jq.Wait() 94 | 95 | // if jq complained, that's our error 96 | if p.stderr.Len() != 0 { 97 | return nil, errors.New(p.stderr.String()) 98 | } 99 | 100 | if p.jq.ProcessState.Success() { 101 | return nil, io.EOF 102 | } 103 | 104 | return nil, errors.New("unexplained jq failure") 105 | } 106 | 107 | // Close attempts to halt the jq process if it has not already exited. This is only necessary if Next has not returned io.EOF. 108 | func (p *Pipe) Close() error { 109 | if p.stdout != nil { 110 | p.stdout.Close() 111 | } 112 | if p.jq == nil { 113 | return nil 114 | } 115 | if p.jq.ProcessState != nil && p.jq.ProcessState.Exited() { 116 | return nil 117 | } 118 | if p.jq.Process != nil { 119 | p.jq.Process.Kill() 120 | go p.jq.Process.Wait() 121 | } 122 | return nil 123 | } 124 | -------------------------------------------------------------------------------- /jqpipe_test.go: -------------------------------------------------------------------------------- 1 | package jq 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | ) 7 | 8 | func TestJqEval(t *testing.T) { 9 | exSucc(t, "", ".[0] + .[1]") 10 | exSucc(t, "[1,2]", ".[0] + .[1]", "3") 11 | exSucc(t, "[1,2][11,22]", ".[0] + .[1]", "3", "33") 12 | exFail(t, "[", ".") 13 | exFail(t, "[]", "]") 14 | exFail(t, "[1,2] 3", ".[0] + .[1]", "3") 15 | } 16 | 17 | func exFail(t *testing.T, inp string, expr string, out ...string) bool { 18 | ok := true 19 | seq, err := Eval(inp, expr) 20 | 21 | if len(seq) != len(out) { 22 | return failEval(t, inp, expr, seq, "expected %v results and failure", len(out)) 23 | } 24 | for i := range seq { 25 | if string(out[i]) != string(seq[i]) { 26 | ok = failEval(t, inp, expr, seq, "expected [%v]: %v", i, out[i]) 27 | } 28 | } 29 | if err == nil { 30 | return failEval(t, inp, expr, seq, "expected failure") 31 | } 32 | return ok 33 | } 34 | 35 | func exSucc(t *testing.T, inp string, expr string, out ...string) bool { 36 | ok := true 37 | seq, err := Eval(inp, expr) 38 | if err != nil { 39 | return failEval(t, inp, expr, seq, "jq-exit: %v", err) 40 | } 41 | 42 | if len(seq) != len(out) { 43 | return failEval(t, inp, expr, seq, "expected %v results", len(out)) 44 | } 45 | 46 | for i := range seq { 47 | if string(out[i]) != string(seq[i]) { 48 | ok = failEval(t, inp, expr, seq, "expected [%v]: %v", i, out[i]) 49 | } 50 | } 51 | return ok 52 | } 53 | 54 | func failEval(t *testing.T, inp, expr string, results []json.RawMessage, layout string, info ...interface{}) bool { 55 | t.Logf("json: %v", inp) 56 | t.Logf("expr: %v", expr) 57 | for i, item := range results { 58 | t.Logf(" [%v]: %v", i, string(item)) 59 | } 60 | t.Errorf(layout, info...) 61 | return false 62 | } 63 | --------------------------------------------------------------------------------