├── .circleci └── config.yml ├── .gitignore ├── Gopkg.lock ├── Gopkg.toml ├── LICENSE ├── README.md ├── cmd └── jqrepl │ └── main.go ├── jq ├── cgo_callback.c ├── jq.go ├── jq_test.go ├── jv.go └── jv_test.go ├── jqrepl.go └── jqrepl_unix.go /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | workflows: 3 | version: 2 4 | build: 5 | jobs: 6 | - build 7 | 8 | commands: 9 | checkout-merge: 10 | description: "checkout merge branch" 11 | steps: 12 | - checkout 13 | - run: 14 | name: Checkout merge if PR 15 | command: | 16 | set -exu 17 | 18 | if [[ -n "${CIRCLE_PULL_REQUEST:-}" ]]; then 19 | pr_num="${CIRCLE_PULL_REQUEST##*/}" 20 | git fetch origin "+refs/pull/${pr_num}/*:refs/remotes/origin/pull/${pr_num}/*" 21 | git checkout "pull/${pr_num}/merge" 22 | fi 23 | 24 | jobs: 25 | build: 26 | working_directory: /go/src/github.com/ashb/jqrepl/ 27 | environment: 28 | CGO_CFLAGS: -I/go/src/github.com/ashb/jqrepl/jq-1.5/BUILD/include 29 | CGO_LDFLAGS: -L/go/src/github.com/ashb/jqrepl/jq-1.5/BUILD/lib 30 | TEST_RESULTS: /tmp/test-results 31 | 32 | docker: 33 | - image: circleci/golang:1.15 34 | 35 | steps: 36 | - checkout-merge 37 | - run: mkdir -p $TEST_RESULTS 38 | - run: go get github.com/jstemmer/go-junit-report 39 | 40 | - run: 41 | name: Install dep 42 | command: | 43 | set -x 44 | GOBIN=${GOPATH%%:*}/bin 45 | mkdir -p $GOBIN 46 | curl -LfsS https://github.com/golang/dep/releases/download/v0.3.2/dep-linux-amd64 > $GOBIN/dep 47 | chmod +x $GOBIN/dep 48 | 49 | - restore_cache: 50 | name: jq-1.5 build cached 51 | key: v1-jq-1.5 52 | 53 | - run: 54 | name: Build JQ 1.5 55 | command: | 56 | if [[ ! -d jq-1.5/BUILD ]]; then 57 | curl -L -fsS https://github.com/stedolan/jq/releases/download/jq-1.5/jq-1.5.tar.gz | tar -xz 58 | cd jq-1.5 59 | ./configure --disable-maintainer-mode --prefix=$PWD/BUILD 60 | make install-libLTLIBRARIES install-includeHEADERS 61 | # Remove the .so, we want to use the .a for static builds. 62 | rm -v BUILD/lib/*.so 63 | fi 64 | - save_cache: 65 | name: Save JQ 1.5 cache 66 | paths: 67 | - jq-1.5/BUILD 68 | key: v1-jq-1.5 69 | 70 | # Download and cache dependencies 71 | - restore_cache: 72 | name: Restore Go vendor cache 73 | keys: 74 | - v1-go-dependencies-{{ checksum "Gopkg.lock" }} 75 | # fallback to using the latest cache if no exact match is found 76 | - v1-go-dependencies- 77 | 78 | - run: 79 | name: Install Go dependencies 80 | command: 81 | dep ensure --vendor-only -v 82 | 83 | - save_cache: 84 | name: Save Go vendor cache 85 | paths: 86 | - vendor 87 | - /go/pkg/dep 88 | key: v1-go-dependencies-{{ checksum "Gopkg.lock" }} 89 | 90 | - run: 91 | name: go test 92 | command: | 93 | trap "go-junit-report <${TEST_RESULTS}/go-test.out > ${TEST_RESULTS}/go-test-report.xml" EXIT 94 | go test -v --race ./... | tee ${TEST_RESULTS}/go-test.out 95 | 96 | - run: 97 | name: Build Linux CLI 98 | command: go build -v -o /tmp/jqrepl-linux-amd64 ./cmd/jqrepl 99 | 100 | - store_artifacts: 101 | path: /tmp/jqrepl-linux-amd64 102 | destination: binaries/jqrepl-linux-amd64 103 | 104 | - store_test_results: 105 | path: /tmp/test-results 106 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | /jq-*/ 27 | /jq-*.tar.gz 28 | -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | branch = "master" 6 | name = "github.com/cheekybits/is" 7 | packages = ["."] 8 | revision = "68e9c0620927fb5427fda3708222d0edee89eae9" 9 | 10 | [[projects]] 11 | name = "gopkg.in/chzyer/readline.v1" 12 | packages = ["."] 13 | revision = "62c6fe6193755f722b8b8788aa7357be55a50ff1" 14 | version = "v1.4" 15 | 16 | [solve-meta] 17 | analyzer-name = "dep" 18 | analyzer-version = 1 19 | inputs-digest = "fdc1ea383e3865c050485683d8f80ade121f335dc892a123ec5d561fedb27985" 20 | solver-name = "gps-cdcl" 21 | solver-version = 1 22 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | [[constraint]] 2 | branch = "master" 3 | name = "github.com/cheekybits/is" 4 | 5 | [[constraint]] 6 | name = "gopkg.in/chzyer/readline.v1" 7 | version = "1.4.0" 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Ash Berlin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # jq-repl 2 | 3 | ## What is it? 4 | 5 | A REPL to make exploring data with JQ easier. 6 | 7 | I'm a huge fan of [JQ][jq] and use it in a lot of small utilities or to 8 | explore JSON APIs from the command line, and I often found myself doing things 9 | like this: 10 | 11 | ```bash 12 | aws ec2 describe-images | jq '.Images' 13 | 14 | # Hmm, that's still large 15 | aws ec2 describe-images | jq '.Images | keys' 16 | 17 | aws ec2 describe-images | jq '.Images | .Tags' 18 | ``` 19 | 20 | i.e. I was using `jq` as a tool to explore a complex JSON data structure -- 21 | but each invokation of `aws ec2 describe-images` took 5 to 15 seconds which 22 | which made the process of building a jq filter quite jaring. 23 | 24 | Now, I could have just piped the result of the `aws` command to a file and then 25 | invoked JQ on that many times, and to start with that's what I did. But it 26 | turned out that each of the `Images` above has differente keys, so finding the 27 | error with jq alone was painful, so in another terminal I fired up ipython, 28 | loaded that JSON file into a python dictionary and started exploring the data 29 | that way. Somehow it got suggested that a REPL for JQ would be the right tool 30 | for this job - and thus the seed for this tool was planted. (P.S. Samir and 31 | James: this is all your fault for egging me on) 32 | 33 | ## Does it work? 34 | 35 | **Sort of**. I'm working on it slowly. 36 | 37 | I am using this project as excuse and reason to learn Go so it will take me a 38 | while to get it functional and bug free. And then even longer so that other 39 | pepole can use it. 40 | 41 | 42 | ## What's the current state? 43 | 44 | ``` 45 | $ aws ec2 describe-images --owner self | jq-repl 46 | 0 » type 47 | $out[0] = "object" 48 | 49 | 1 » keys 50 | $out[1] = [ 51 | "Images" 52 | ] 53 | 54 | 2 » .Images[0] 55 | $out[2] = { 56 | "VirtualizationType": "hvm", 57 | "Name": "leader 2015-11-05T16-50-35Z", 58 | "Tags": [ 59 | { 60 | "Value": "2015-11-05T16:50:35Z", 61 | "Key": "build_date" 62 | } 63 | ], 64 | "Hypervisor": "xen", 65 | "SriovNetSupport": "simple", 66 | "ImageId": "ami-abc01234", 67 | "State": "available", 68 | "BlockDeviceMappings": [ 69 | { 70 | "DeviceName": "/dev/sda1", 71 | "Ebs": { 72 | "DeleteOnTermination": true, 73 | "SnapshotId": "snap-01234fed", 74 | "VolumeSize": 16, 75 | "VolumeType": "gp2", 76 | "Encrypted": false 77 | } 78 | }, 79 | ], 80 | "Architecture": "x86_64", 81 | "RootDeviceType": "ebs", 82 | "RootDeviceName": "/dev/sda1", 83 | "CreationDate": "2015-11-05T16:55:15.000Z", 84 | "Public": false, 85 | "ImageType": "machine", 86 | "Description": "My AMI" 87 | } 88 | ``` 89 | 90 | So far all fairly mundane. This is where I think things start to get 91 | interesting - you will be able to refer back to previous results. 92 | 93 | ``` 94 | 3 » $_ | .Name 95 | $out[3] = "leader 2015-11-05T16-50-35Z" 96 | ``` 97 | 98 | ### Special variables 99 | 100 | We define some special variables. 101 | 102 | 103 | - `$_` 104 | 105 | The previous result. Equivalent to `$out[-1]` 106 | 107 | - `$__` 108 | 109 | The result before last. Equivalent to `$out[-2]` 110 | 111 | - `$out` 112 | 113 | An array of all previous results. You can use negative indcies to count 114 | backwards from the end. 115 | 116 | ## Current status and future plans 117 | 118 | 119 | - [x] Accept JSON input on stdin 120 | - [ ] Accept JSON input from a file 121 | - [ ] Sepcify a command to run to get input 122 | - [ ] Be able to re-execute command to refresh input 123 | - [ ] Autocompletion 124 | - [x] Refer to previous results 125 | - [ ] Build up complex filters by refering to previous results. 126 | - [ ] Automated tests for the JqRepl methods 127 | 128 | # Building it 129 | 130 | 131 | ## Prerequisites 132 | 133 | * [JQ souce code][JQ src] and anything it needs to compile 134 | 135 | [JQ]: https://stedolan.github.io/jq/ 136 | [JQ src]: https://stedolan.github.io/jq/download/ 137 | 138 | ## Build it 139 | 140 | It doesn't do much of anything yet. But to build it you will need to do 141 | something like this: 142 | 143 | ```bash 144 | curl -fL https://github.com/stedolan/jq/releases/download/jq-1.5/jq-1.5.tar.gz | tar -zx 145 | cd jq-1.5 146 | ./configure --disable-maintainer-mode --prefix=$PWD/BUILD 147 | # We could run `make install` but we only actually need these components. 148 | make install-libLTLIBRARIES install-includeHEADERS 149 | go test ./... 150 | ``` 151 | 152 | I have no idea if this will work on platforms other than OSX right now. I will 153 | work on that later once I have some basic functionality 154 | -------------------------------------------------------------------------------- /cmd/jqrepl/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | 7 | "github.com/ashb/jqrepl" 8 | "github.com/ashb/jqrepl/jq" 9 | ) 10 | 11 | func main() { 12 | 13 | var ( 14 | jv *jq.Jv 15 | err error 16 | ) 17 | 18 | repl, err := jqrepl.New() 19 | 20 | if err != nil { 21 | // TODO: don't use panic 22 | panic(err) 23 | } 24 | 25 | defer repl.Close() 26 | 27 | if err != nil { 28 | // TODO: don't use panic 29 | panic(err) 30 | } 31 | 32 | if jqrepl.StdinIsTTY() { 33 | // TODO: Get input from a file, or exec a command! 34 | jv, err = jq.JvFromJSONString(` 35 | { "simple": 123, 36 | "nested": { 37 | "a": [1,2,"a"], 38 | "b": true, 39 | "c": null 40 | }, 41 | "non_printable": "\ud83c\uddec\ud83c\udde7" 42 | }`) 43 | if err != nil { 44 | // TODO: don't use panic 45 | panic(err) 46 | } 47 | } else { 48 | 49 | input, err := ioutil.ReadAll(os.Stdin) 50 | if err != nil { 51 | // TODO: don't use panic 52 | panic(err) 53 | } 54 | 55 | jv, err = jq.JvFromJSONBytes(input) 56 | 57 | if err != nil { 58 | // TODO: don't use panic 59 | panic(err) 60 | } 61 | } 62 | 63 | repl.SetJvInput(jv) 64 | 65 | repl.Loop() 66 | } 67 | -------------------------------------------------------------------------------- /jq/cgo_callback.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | 6 | #include "_cgo_export.h" 7 | 8 | static inline void callGoErrorHandler(void *data, jv it) { 9 | goLibjqErrorHandler((GoUint64)data, it); 10 | } 11 | 12 | void install_jq_error_cb(jq_state *jq, GoUint64 id) { 13 | jq_set_error_cb(jq, callGoErrorHandler, (void*)id); 14 | } 15 | -------------------------------------------------------------------------------- /jq/jq.go: -------------------------------------------------------------------------------- 1 | // Package jq provides go bindings for libjq providing a streaming filter of 2 | // JSON documents. 3 | // 4 | // This package provides a thin layer on top of stedolan's libjq -- it would 5 | // likely be helpful to read through the wiki pages about it: 6 | // 7 | // jv: the JSON value type https://github.com/stedolan/jq/wiki/C-API:-jv 8 | // 9 | // libjq: https://github.com/stedolan/jq/wiki/C-API:-libjq 10 | package jq 11 | 12 | /* 13 | To install 14 | $ ./configure --disable-maintainer-mode --prefix=$PWD/BUILD 15 | $ make install-libLTLIBRARIES install-includeHEADERS 16 | */ 17 | 18 | /* 19 | #cgo LDFLAGS: -ljq 20 | #cgo linux LDFLAGS: -lm 21 | 22 | 23 | #include 24 | #include 25 | 26 | #include 27 | 28 | void install_jq_error_cb(jq_state *jq, unsigned long long id); 29 | */ 30 | import "C" 31 | import ( 32 | "errors" 33 | "fmt" 34 | "sync" 35 | "sync/atomic" 36 | "unsafe" 37 | ) 38 | 39 | // Jq encapsulates the state needed to interface with the libjq C library 40 | type Jq struct { 41 | _state *C.struct_jq_state 42 | errorStoreId uint64 43 | running sync.WaitGroup 44 | } 45 | 46 | // New initializes a new JQ object and the underlying C library. 47 | func New() (*Jq, error) { 48 | jq := new(Jq) 49 | 50 | var err error 51 | jq._state, err = C.jq_init() 52 | 53 | if err != nil { 54 | return nil, err 55 | } else if jq == nil { 56 | return nil, errors.New("jq_init returned nil -- out of memory?") 57 | } 58 | 59 | return jq, nil 60 | } 61 | 62 | // Close the handle to libjq and free C resources. 63 | // 64 | // If Start() has been called this will block until the input Channel it 65 | // returns has been closed. 66 | func (jq *Jq) Close() { 67 | // If the goroutine from Start() is running we need to make sure it finished cleanly 68 | // Wait until we aren't running before freeing C things. 69 | // 70 | jq.running.Wait() 71 | if jq._state != nil { 72 | C.jq_teardown(&jq._state) 73 | jq._state = nil 74 | } 75 | if jq.errorStoreId != 0 { 76 | globalErrorChannels.Delete(jq.errorStoreId) 77 | jq.errorStoreId = 0 78 | } 79 | } 80 | 81 | // We cant pass many things over the Go/C boundary, so instead of passing the error channel we pass an opaque indentifier (a 64bit int as it turns out) and use that to look up in a global variable 82 | type errorLookupState struct { 83 | sync.RWMutex 84 | idCounter uint64 85 | channels map[uint64]chan<- error 86 | } 87 | 88 | func (e *errorLookupState) Add(c chan<- error) uint64 { 89 | newID := atomic.AddUint64(&e.idCounter, 1) 90 | e.RWMutex.Lock() 91 | defer e.RWMutex.Unlock() 92 | e.channels[newID] = c 93 | return newID 94 | } 95 | 96 | func (e *errorLookupState) Get(id uint64) chan<- error { 97 | e.RWMutex.RLock() 98 | defer e.RWMutex.RUnlock() 99 | c, ok := e.channels[id] 100 | if !ok { 101 | panic(fmt.Sprintf("Tried to get error channel #%d out of store but it wasn't there!", id)) 102 | } 103 | return c 104 | } 105 | 106 | func (e *errorLookupState) Delete(id uint64) { 107 | e.RWMutex.Lock() 108 | defer e.RWMutex.Unlock() 109 | delete(e.channels, id) 110 | } 111 | 112 | // The global state - this also serves to keep the channel in scope by keeping 113 | // a reference to it that the GC can see 114 | var globalErrorChannels = errorLookupState{ 115 | channels: make(map[uint64]chan<- error), 116 | } 117 | 118 | //export goLibjqErrorHandler 119 | func goLibjqErrorHandler(id uint64, value C.jv) { 120 | ch := globalErrorChannels.Get(id) 121 | 122 | err := _ConvertError(value) 123 | ch <- err 124 | } 125 | 126 | // Start will compile `program` and return a three channels: input, output and 127 | // error. Sending a jq.Jv* to input cause the program to be run to it and 128 | // one-or-more results returned as jq.Jv* on the output channel, or one or more 129 | // error values sent to the error channel. When you are done sending values 130 | // close the input channel. 131 | // 132 | // args is a list of key/value pairs to bind as variables into the program, and 133 | // must be an array type even if empty. Each element of the array should be an 134 | // object with a "name" and "value" properties. Name should exclude the "$" 135 | // sign. For example this is `[ {"name": "n", "value": 1 } ]` would then be 136 | // `$n` in the programm. 137 | // 138 | // This function is not reentereant -- in that you cannot and should not call 139 | // Start again until you have closed the previous input channel. 140 | // 141 | // If there is a problem compiling the JQ program then the errors will be 142 | // reported on error channel before any input is read so makle sure you account 143 | // for this case. 144 | // 145 | // Any jq.Jv* values passed to the input channel will be owned by the channel. 146 | // If you want to keep them afterwards ensure you Copy() them before passing to 147 | // the channel 148 | func (jq *Jq) Start(program string, args *Jv) (in chan<- *Jv, out <-chan *Jv, errs <-chan error) { 149 | // Create out two way copy of the channels. We need to be able to recv from 150 | // input, so need to store the original channel 151 | cIn := make(chan *Jv) 152 | cOut := make(chan *Jv) 153 | cErr := make(chan error) 154 | 155 | // And assign the read/write only versions to the output fars 156 | in = cIn 157 | out = cOut 158 | errs = cErr 159 | 160 | // Before setting up any of the global error handling state, lets check that 161 | // args is of the right type! 162 | if args.Kind() != JV_KIND_ARRAY { 163 | go func() { 164 | // Take ownership of the inputs 165 | for jv := range cIn { 166 | jv.Free() 167 | } 168 | cErr <- fmt.Errorf("`args` parameter is of type %s not array!", args.Kind().String()) 169 | args.Free() 170 | close(cOut) 171 | close(cErr) 172 | }() 173 | return 174 | } 175 | 176 | if jq.errorStoreId != 0 { 177 | // We might have called Compile 178 | globalErrorChannels.Delete(jq.errorStoreId) 179 | } 180 | jq.errorStoreId = globalErrorChannels.Add(cErr) 181 | 182 | // Because we can't pass a function pointer to an exported Go func we have to 183 | // call a C function which uses the exported fund for us. 184 | // https://github.com/golang/go/wiki/cgo#function-variables 185 | C.install_jq_error_cb(jq._state, C.ulonglong(jq.errorStoreId)) 186 | 187 | jq.running.Add(1) 188 | go func() { 189 | 190 | if jq._Compile(program, args) == false { 191 | // Even if compile failed follow the contract. Read any inputs and take 192 | // ownership of them (aka free them) 193 | // 194 | // Errors from compile will be sent to the error channel 195 | for jv := range cIn { 196 | jv.Free() 197 | } 198 | } else { 199 | for jv := range cIn { 200 | results, err := jq.Execute(jv) 201 | for _, result := range results { 202 | cOut <- result 203 | } 204 | if err != nil { 205 | cErr <- err 206 | } 207 | } 208 | } 209 | // Once we've read all the inputs close the output to signal to caller that 210 | // we are done. 211 | close(cOut) 212 | close(cErr) 213 | C.install_jq_error_cb(jq._state, 0) 214 | jq.running.Done() 215 | }() 216 | 217 | return 218 | } 219 | 220 | // Execute will run the Compiled() program against a single input and return 221 | // the results. 222 | // 223 | // Using this interface directly is not thread-safe -- it is up to the caller to 224 | // ensure that this is not called from two goroutines concurrently. 225 | func (jq *Jq) Execute(input *Jv) (results []*Jv, err error) { 226 | flags := C.int(0) 227 | results = make([]*Jv, 0) 228 | 229 | C.jq_start(jq._state, input.jv, flags) 230 | result := &Jv{C.jq_next(jq._state)} 231 | for result.IsValid() { 232 | results = append(results, result) 233 | result = &Jv{C.jq_next(jq._state)} 234 | } 235 | msg, ok := result.GetInvalidMessageAsString() 236 | if ok { 237 | // Uncaught jq exception 238 | // TODO: get file:line position in input somehow. 239 | err = errors.New(msg) 240 | } 241 | 242 | return 243 | } 244 | 245 | // Compile the program and make it ready to Execute() 246 | // 247 | // Only a single program can be compiled on a Jq object at once. Calling this 248 | // again a second time will replace the current program. 249 | // 250 | // args is a list of key/value pairs to bind as variables into the program, and 251 | // must be an array type even if empty. Each element of the array should be an 252 | // object with a "name" and "value" properties. Name should exclude the "$" 253 | // sign. For example this is `[ {"name": "n", "value": 1 } ]` would then be 254 | // `$n` in the program. 255 | func (jq *Jq) Compile(prog string, args *Jv) (errs []error) { 256 | 257 | // Before setting up any of the global error handling state, lets check that 258 | // args is of the right type! 259 | if args.Kind() != JV_KIND_ARRAY { 260 | args.Free() 261 | return []error{fmt.Errorf("`args` parameter is of type %s not array", args.Kind().String())} 262 | } 263 | 264 | cErr := make(chan error) 265 | 266 | if jq.errorStoreId != 0 { 267 | // We might have called Compile 268 | globalErrorChannels.Delete(jq.errorStoreId) 269 | } 270 | jq.errorStoreId = globalErrorChannels.Add(cErr) 271 | 272 | C.install_jq_error_cb(jq._state, C.ulonglong(jq.errorStoreId)) 273 | defer C.install_jq_error_cb(jq._state, 0) 274 | var wg sync.WaitGroup 275 | 276 | wg.Add(1) 277 | go func() { 278 | for err := range cErr { 279 | if err == nil { 280 | break 281 | } 282 | errs = append(errs, err) 283 | } 284 | wg.Done() 285 | }() 286 | 287 | compiled := jq._Compile(prog, args) 288 | cErr <- nil // Sentinel to break the loop above 289 | 290 | wg.Wait() 291 | globalErrorChannels.Delete(jq.errorStoreId) 292 | jq.errorStoreId = 0 293 | 294 | if !compiled && len(errs) == 0 { 295 | return []error{fmt.Errorf("jq_compile returned error, but no errors were reported. Oops")} 296 | } 297 | return errs 298 | } 299 | 300 | func (jq *Jq) _Compile(prog string, args *Jv) bool { 301 | cs := C.CString(prog) 302 | defer C.free(unsafe.Pointer(cs)) 303 | 304 | // If there was an error it will have been sent to errorChannel via the 305 | // installed error handler 306 | return C.jq_compile_args(jq._state, cs, args.jv) != 0 307 | } 308 | -------------------------------------------------------------------------------- /jq/jq_test.go: -------------------------------------------------------------------------------- 1 | package jq_test 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/ashb/jqrepl/jq" 8 | ) 9 | 10 | func TestJqNewClose(t *testing.T) { 11 | jq, err := jq.New() 12 | 13 | if err != nil { 14 | t.Errorf("Error initializing jq_state: %v", err) 15 | } 16 | 17 | jq.Close() 18 | 19 | // We should be able to safely close multiple times. 20 | jq.Close() 21 | 22 | } 23 | 24 | func TestJqCloseRace(t *testing.T) { 25 | state, err := jq.New() 26 | 27 | if err != nil { 28 | t.Errorf("Error initializing jq_state: %v", err) 29 | } 30 | 31 | cIn, _, _ := state.Start(".", jq.JvArray()) 32 | go state.Close() 33 | go close(cIn) 34 | } 35 | 36 | func feedJq(val *jq.Jv, in chan<- *jq.Jv, out <-chan *jq.Jv, errs <-chan error) ([]*jq.Jv, []error) { 37 | if val == nil { 38 | close(in) 39 | in = nil 40 | } 41 | outputs := make([]*jq.Jv, 0) 42 | errors := make([]error, 0) 43 | for errs != nil && out != nil { 44 | select { 45 | case e, ok := <-errs: 46 | if !ok { 47 | errs = nil 48 | } else { 49 | errors = append(errors, e) 50 | } 51 | case o, ok := <-out: 52 | if !ok { 53 | out = nil 54 | } else { 55 | outputs = append(outputs, o) 56 | } 57 | case in <- val: 58 | // We've sent our input, close the channel to tell Jq we're done 59 | close(in) 60 | in = nil 61 | } 62 | } 63 | return outputs, errors 64 | } 65 | 66 | func TestStartCompileError(t *testing.T) { 67 | state, err := jq.New() 68 | 69 | if err != nil { 70 | t.Errorf("Error initializing jq_state: %v", err) 71 | } 72 | defer state.Close() 73 | 74 | const program = "a b" 75 | cIn, cOut, cErr := state.Start(program, jq.JvArray()) 76 | _, errors := feedJq(nil, cIn, cOut, cErr) 77 | 78 | // JQ might (and currently does) report multiple errors. One of them will 79 | // contain our input program. Check for that but don't be overly-specific 80 | // about the string or order of errors 81 | 82 | gotErrors := false 83 | for _, err := range errors { 84 | gotErrors = true 85 | if strings.Contains(err.Error(), program) { 86 | // t.Pass("Found the error we expected: %#v\n", 87 | return 88 | } 89 | } 90 | 91 | if !gotErrors { 92 | t.Fatal("Errors were expected but none seen") 93 | } 94 | t.Fatal("No error containing the program source found") 95 | } 96 | 97 | func TestCompileError(t *testing.T) { 98 | state, err := jq.New() 99 | 100 | if err != nil { 101 | t.Errorf("Error initializing jq_state: %v", err) 102 | } 103 | defer state.Close() 104 | 105 | const program = "a b" 106 | errors := state.Compile(program, jq.JvArray()) 107 | 108 | // JQ might (and currently does) report multiple errors. One of them will 109 | // contain our input program. Check for that but don't be overly-specific 110 | // about the string or order of errors 111 | 112 | gotErrors := false 113 | for _, err := range errors { 114 | gotErrors = true 115 | if strings.Contains(err.Error(), program) { 116 | // t.Pass("Found the error we expected: %#v\n", 117 | return 118 | } 119 | } 120 | 121 | if !gotErrors { 122 | t.Fatal("Errors were expected but none seen") 123 | } 124 | t.Fatal("No error containing the program source found") 125 | } 126 | 127 | func TestCompileGood(t *testing.T) { 128 | state, err := jq.New() 129 | 130 | if err != nil { 131 | t.Errorf("Error initializing jq_state: %v", err) 132 | } 133 | defer state.Close() 134 | 135 | const program = "." 136 | errors := state.Compile(program, jq.JvArray()) 137 | 138 | // JQ might (and currently does) report multiple errors. One of them will 139 | // contain our input program. Check for that but don't be overly-specific 140 | // about the string or order of errors 141 | 142 | if len(errors) != 0 { 143 | t.Fatal("Expected no errors, got", errors) 144 | } 145 | } 146 | 147 | func TestJqSimpleProgram(t *testing.T) { 148 | state, err := jq.New() 149 | 150 | if err != nil { 151 | t.Errorf("Error initializing state_state: %v", err) 152 | } 153 | defer state.Close() 154 | 155 | input, err := jq.JvFromJSONString("{\"a\": 123}") 156 | if err != nil { 157 | t.Error(err) 158 | } 159 | 160 | cIn, cOut, cErrs := state.Start(".a", jq.JvArray()) 161 | outputs, errs := feedJq(input, cIn, cOut, cErrs) 162 | 163 | if len(errs) > 0 { 164 | t.Errorf("Expected no errors, but got %#v", errs) 165 | } 166 | 167 | if l := len(outputs); l != 1 { 168 | t.Errorf("Got %d outputs (%#v), expected %d", l, outputs, 1) 169 | } else if val := outputs[0].ToGoVal(); val != 123 { 170 | t.Errorf("Got %#v, expected %#v", val, 123) 171 | } 172 | } 173 | 174 | func TestJqNonChannelInterface(t *testing.T) { 175 | state, err := jq.New() 176 | 177 | if err != nil { 178 | t.Errorf("Error initializing state_state: %v", err) 179 | } 180 | defer state.Close() 181 | 182 | input, err := jq.JvFromJSONString("{\"a\": 123}") 183 | if err != nil { 184 | t.Error(err) 185 | } 186 | 187 | errs := state.Compile(".a", jq.JvArray()) 188 | if errs != nil { 189 | t.Errorf("Expected no errors, but got %#v", errs) 190 | } 191 | 192 | outputs, err := state.Execute(input.Copy()) 193 | if err != nil { 194 | t.Errorf("Expected no error, but got %#v", err) 195 | } 196 | 197 | if l := len(outputs); l != 1 { 198 | t.Errorf("Got %d outputs (%#v), expected %d", l, outputs, 1) 199 | } else if val := outputs[0].ToGoVal(); val != 123 { 200 | t.Errorf("Got %#v, expected %#v", val, 123) 201 | } 202 | } 203 | 204 | func TestJqRuntimeError(t *testing.T) { 205 | state, err := jq.New() 206 | 207 | if err != nil { 208 | t.Errorf("Error initializing state_state: %v", err) 209 | } 210 | defer state.Close() 211 | 212 | input, err := jq.JvFromJSONString(`{"a": 123}`) 213 | if err != nil { 214 | t.Error(err) 215 | } 216 | 217 | cIn, cOut, cErrs := state.Start(".[0]", jq.JvArray()) 218 | _, errors := feedJq(input, cIn, cOut, cErrs) 219 | 220 | if l := len(errors); l != 1 { 221 | t.Errorf("Got %d errors (%#v), expected %d", l, errors, 1) 222 | } 223 | } 224 | -------------------------------------------------------------------------------- /jq/jv.go: -------------------------------------------------------------------------------- 1 | package jq 2 | 3 | /* 4 | #cgo LDFLAGS: -ljq 5 | 6 | #include 7 | 8 | #include 9 | #include 10 | */ 11 | import "C" 12 | import ( 13 | "encoding/json" 14 | "errors" 15 | "fmt" 16 | "reflect" 17 | "unsafe" 18 | ) 19 | 20 | // Helper functions for dealing with JV objects. You can't use this from 21 | // another go package as the cgo types are 'unique' per go package 22 | 23 | // JvKind represents the type of value that a `Jv` contains. 24 | type JvKind int 25 | 26 | // Jv represents a JSON value from libjq. 27 | // 28 | // The go wrapper uses the same memory management semantics as the underlying C 29 | // library, so you should familiarize yourself with 30 | // https://github.com/stedolan/jq/wiki/C-API:-jv#memory-management. In summary 31 | // this package and all JQ functions operate on the assumption that any jv value 32 | // you pass to a function is then owned by that function -- if you do not wish 33 | // this to be the case call Copy() on it first. 34 | type Jv struct { 35 | jv C.jv 36 | } 37 | 38 | const ( 39 | JV_KIND_INVALID JvKind = C.JV_KIND_INVALID 40 | JV_KIND_NULL JvKind = C.JV_KIND_NULL 41 | JV_KIND_FALSE JvKind = C.JV_KIND_FALSE 42 | JV_KIND_TRUE JvKind = C.JV_KIND_TRUE 43 | JV_KIND_NUMBER JvKind = C.JV_KIND_NUMBER 44 | JV_KIND_STRING JvKind = C.JV_KIND_STRING 45 | JV_KIND_ARRAY JvKind = C.JV_KIND_ARRAY 46 | JV_KIND_OBJECT JvKind = C.JV_KIND_OBJECT 47 | ) 48 | 49 | // String returns a string representation of what type this Jv contains 50 | func (kind JvKind) String() string { 51 | // Rather than rely on converting from a C string to go every time, store our 52 | // own list 53 | switch kind { 54 | case JV_KIND_INVALID: 55 | return "" 56 | case JV_KIND_NULL: 57 | return "null" 58 | case JV_KIND_FALSE: 59 | return "boolean" 60 | case JV_KIND_TRUE: 61 | return "boolean" 62 | case JV_KIND_NUMBER: 63 | return "number" 64 | case JV_KIND_STRING: 65 | return "string" 66 | case JV_KIND_ARRAY: 67 | return "array" 68 | case JV_KIND_OBJECT: 69 | return "object" 70 | default: 71 | return "" 72 | } 73 | } 74 | 75 | // JvNull returns a value representing a JSON null 76 | func JvNull() *Jv { 77 | return &Jv{C.jv_null()} 78 | } 79 | 80 | // JvInvalid returns an invalid jv object without an error property 81 | func JvInvalid() *Jv { 82 | return &Jv{C.jv_invalid()} 83 | } 84 | 85 | // JvInvalidWithMessage creates an "invalid" jv with the given error message. 86 | // 87 | // msg can be a string or an object 88 | // 89 | // Consumes `msg` 90 | func JvInvalidWithMessage(msg *Jv) *Jv { 91 | return &Jv{C.jv_invalid_with_msg(msg.jv)} 92 | } 93 | 94 | // JvFromString returns a new jv string-typed value containing the given go 95 | // string. 96 | func JvFromString(str string) *Jv { 97 | cs := C.CString(str) 98 | defer C.free(unsafe.Pointer(cs)) 99 | return &Jv{C.jv_string_sized(cs, C.int(len(str)))} 100 | } 101 | 102 | // JvFromFloat returns a new jv number-typed value containing the given float 103 | // value. 104 | func JvFromFloat(n float64) *Jv { 105 | return &Jv{C.jv_number(C.double(n))} 106 | } 107 | 108 | // JvFromBool returns a new jv of "true" or "false" kind depending on the given 109 | // boolean value 110 | func JvFromBool(b bool) *Jv { 111 | if b { 112 | return &Jv{C.jv_true()} 113 | } else { 114 | return &Jv{C.jv_false()} 115 | } 116 | } 117 | 118 | func jvFromArray(val reflect.Value) (*Jv, error) { 119 | len := val.Len() 120 | ret := &Jv{C.jv_array_sized(C.int(len))} 121 | for i := 0; i < len; i++ { 122 | newjv, err := JvFromInterface( 123 | val.Index(i).Interface(), 124 | ) 125 | if err != nil { 126 | // TODO: error context 127 | ret.Free() 128 | return nil, err 129 | } 130 | ret = &Jv{C.jv_array_set(ret.jv, C.int(i), newjv.jv)} 131 | } 132 | return ret, nil 133 | } 134 | 135 | func jvFromMap(val reflect.Value) (*Jv, error) { 136 | keys := val.MapKeys() 137 | ret := JvObject() 138 | 139 | for _, key := range keys { 140 | keyjv := JvFromString(key.String()) 141 | valjv, err := JvFromInterface(val.MapIndex(key).Interface()) 142 | if err != nil { 143 | // TODO: error context 144 | keyjv.Free() 145 | ret.Free() 146 | return nil, err 147 | } 148 | ret = ret.ObjectSet(keyjv, valjv) 149 | } 150 | 151 | return ret, nil 152 | } 153 | 154 | func JvFromInterface(intf interface{}) (*Jv, error) { 155 | if intf == nil { 156 | return JvNull(), nil 157 | } 158 | 159 | switch x := intf.(type) { 160 | case float32: 161 | return JvFromFloat(float64(x)), nil 162 | case float64: 163 | return JvFromFloat(x), nil 164 | case uint: 165 | return JvFromFloat(float64(x)), nil 166 | case int: 167 | return JvFromFloat(float64(x)), nil 168 | case int8: 169 | return JvFromFloat(float64(x)), nil 170 | case uint8: 171 | return JvFromFloat(float64(x)), nil 172 | case int16: 173 | return JvFromFloat(float64(x)), nil 174 | case uint16: 175 | return JvFromFloat(float64(x)), nil 176 | case int32: 177 | return JvFromFloat(float64(x)), nil 178 | case uint32: 179 | return JvFromFloat(float64(x)), nil 180 | case int64: 181 | return JvFromFloat(float64(x)), nil 182 | case uint64: 183 | return JvFromFloat(float64(x)), nil 184 | case string: 185 | return JvFromString(x), nil 186 | case []byte: 187 | return JvFromString(string(x)), nil 188 | case bool: 189 | return JvFromBool(x), nil 190 | } 191 | 192 | val := reflect.ValueOf(intf) 193 | switch val.Kind() { 194 | case reflect.Array, reflect.Slice: 195 | return jvFromArray(val) 196 | case reflect.Map: 197 | return jvFromMap(val) 198 | case reflect.Struct: 199 | marshalled, err := json.Marshal(intf) 200 | if err != nil { 201 | return nil, err 202 | } 203 | return JvFromJSONString(string(marshalled)) 204 | default: 205 | return nil, errors.New("JvFromInterface can't handle " + val.Kind().String()) 206 | } 207 | } 208 | 209 | func _ConvertError(inv C.jv) error { 210 | // We might want to not call this as it prefixes things with "jq: " 211 | jv := &Jv{C.jq_format_error(inv)} 212 | defer jv.Free() 213 | 214 | return errors.New(jv._string()) 215 | } 216 | 217 | // JvFromJSONString takes a JSON string and returns the jv representation of 218 | // it. 219 | func JvFromJSONString(str string) (*Jv, error) { 220 | cs := C.CString(str) 221 | defer C.free(unsafe.Pointer(cs)) 222 | jv := C.jv_parse_sized(cs, C.int(len(str))) 223 | 224 | if C.jv_is_valid(jv) == 0 { 225 | return nil, _ConvertError(jv) 226 | } 227 | return &Jv{jv}, nil 228 | } 229 | 230 | // JvFromJSONBytes takes a utf-8 byte sequence containing JSON and returns the 231 | // jv representation of it. 232 | func JvFromJSONBytes(b []byte) (*Jv, error) { 233 | jv := C.jv_parse_sized((*C.char)(unsafe.Pointer(&b[0])), C.int(len(b))) 234 | 235 | if C.jv_is_valid(jv) == 0 { 236 | return nil, _ConvertError(jv) 237 | } 238 | return &Jv{jv}, nil 239 | } 240 | 241 | // Free this reference to a Jv value. 242 | // 243 | // Don't call this more than once per jv - might not actually free the memory 244 | // as libjq uses reference counting. To make this more like the libjq interface 245 | // we return a nil pointer. 246 | func (jv *Jv) Free() *Jv { 247 | C.jv_free(jv.jv) 248 | return nil 249 | } 250 | 251 | // Kind returns a JvKind saying what type this jv contains. 252 | // 253 | // Does not consume the invocant. 254 | func (jv *Jv) Kind() JvKind { 255 | return JvKind(C.jv_get_kind(jv.jv)) 256 | } 257 | 258 | // Copy returns a *Jv so that the original won't get freed. 259 | // 260 | // Does not consume the invocant. 261 | func (jv *Jv) Copy() *Jv { 262 | C.jv_copy(jv.jv) 263 | // Becasue jv uses ref counting under the hood we can return the same value 264 | return jv 265 | } 266 | 267 | // IsValid returns true if this Jv represents a valid JSON type, or false if it 268 | // is unitiaizlied or if it represents an error type 269 | // 270 | // Does not consume the invocant. 271 | func (jv *Jv) IsValid() bool { 272 | return C.jv_is_valid(jv.jv) != 0 273 | } 274 | 275 | // GetInvalidMessageAsString gets the error message for this Jv. If there is none it 276 | // will return ("", false). Otherwise it will return the message as a string and true, 277 | // converting non-string values if necessary. If you want the message in it's 278 | // native Jv type use `GetInvalidMessage()` 279 | // 280 | // Consumes the invocant. 281 | func (jv *Jv) GetInvalidMessageAsString() (string, bool) { 282 | msg := C.jv_invalid_get_msg(jv.jv) 283 | defer C.jv_free(msg) 284 | 285 | if C.jv_get_kind(msg) == C.JV_KIND_NULL { 286 | return "", false 287 | } else if C.jv_get_kind(msg) != C.JV_KIND_STRING { 288 | msg = C.jv_dump_string(msg, 0) 289 | } 290 | return C.GoString(C.jv_string_value(msg)), true 291 | } 292 | 293 | // GetInvalidMessage returns the message associcated 294 | func (jv *Jv) GetInvalidMessage() *Jv { 295 | return &Jv{C.jv_invalid_get_msg(jv.jv)} 296 | } 297 | 298 | func (jv *Jv) _string() string { 299 | // Raw string value. If called on 300 | cs := C.jv_string_value(jv.jv) 301 | // Don't free cs - freed when the jv is 302 | return C.GoString(cs) 303 | } 304 | 305 | // If jv is a string, return its value. Will not stringify other types 306 | // 307 | // Does not consume the invocant. 308 | func (jv *Jv) String() (string, error) { 309 | // Doing this might be a bad idea as it means we almost implement the Stringer 310 | // interface but not quite (cos the error type) 311 | 312 | // If we don't do this check JV will assert 313 | if C.jv_get_kind(jv.jv) != C.JV_KIND_STRING { 314 | return "", fmt.Errorf("Cannot return String for jv of type %s", jv.Kind()) 315 | } 316 | 317 | return jv._string(), nil 318 | } 319 | 320 | // ToGoVal converts a jv into it's closest Go approximation 321 | // 322 | // Does not consume the invocant. 323 | func (jv *Jv) ToGoVal() interface{} { 324 | switch kind := C.jv_get_kind(jv.jv); kind { 325 | case C.JV_KIND_NULL: 326 | return nil 327 | case C.JV_KIND_FALSE: 328 | return false 329 | case C.JV_KIND_TRUE: 330 | return true 331 | case C.JV_KIND_NUMBER: 332 | dbl := C.jv_number_value(jv.jv) 333 | 334 | if C.jv_is_integer(jv.jv) == 0 { 335 | return float64(dbl) 336 | } 337 | return int(dbl) 338 | case C.JV_KIND_STRING: 339 | return jv._string() 340 | case C.JV_KIND_ARRAY: 341 | len := jv.Copy().ArrayLength() 342 | ary := make([]interface{}, len) 343 | for i := 0; i < len; i++ { 344 | v := jv.Copy().ArrayGet(i) 345 | ary[i] = v.ToGoVal() 346 | v.Free() 347 | } 348 | return ary 349 | case C.JV_KIND_OBJECT: 350 | obj := make(map[string]interface{}) 351 | for iter := C.jv_object_iter(jv.jv); C.jv_object_iter_valid(jv.jv, iter) != 0; iter = C.jv_object_iter_next(jv.jv, iter) { 352 | k := Jv{C.jv_object_iter_key(jv.jv, iter)} 353 | v := Jv{C.jv_object_iter_value(jv.jv, iter)} 354 | // jv_object_iter_key already asserts that the kind is string, so using _string is OK here 355 | obj[k._string()] = v.ToGoVal() 356 | k.Free() 357 | v.Free() 358 | } 359 | return obj 360 | default: 361 | panic(fmt.Sprintf("Unknown JV kind %d", kind)) 362 | } 363 | } 364 | 365 | type JvPrintFlags int 366 | 367 | const ( 368 | JvPrintNone JvPrintFlags = 0 // None of the below 369 | JvPrintPretty JvPrintFlags = C.JV_PRINT_PRETTY // Print across multiple lines 370 | JvPrintAscii JvPrintFlags = C.JV_PRINT_ASCII // Escape non-ascii printable characters 371 | JvPrintColour JvPrintFlags = C.JV_PRINT_COLOUR // Include ANSI color escapes based on data types 372 | JvPrintSorted JvPrintFlags = C.JV_PRINT_SORTED // Sort output keys 373 | JvPrintInvalid JvPrintFlags = C.JV_PRINT_INVALID // Print invalid as "" 374 | JvPrintRefCount JvPrintFlags = C.JV_PRINT_REFCOUNT // Display refcount of objects in in parenthesis 375 | JvPrintTab JvPrintFlags = C.JV_PRINT_TAB // Indent with tabs, 376 | JvPrintIsATty JvPrintFlags = C.JV_PRINT_ISATTY // 377 | JvPrintSpace0 JvPrintFlags = C.JV_PRINT_SPACE0 // Indent with zero extra chars beyond the parent bracket 378 | JvPrintSpace1 JvPrintFlags = C.JV_PRINT_SPACE1 // Indent with zero extra chars beyond the parent bracket 379 | JvPrintSpace2 JvPrintFlags = C.JV_PRINT_SPACE2 // Indent with zero extra chars beyond the parent bracket 380 | ) 381 | 382 | // Dump produces a human readable version of the string with the requested formatting. 383 | // 384 | // Consumes the invocant 385 | func (jv *Jv) Dump(flags JvPrintFlags) string { 386 | jv_str := Jv{C.jv_dump_string(jv.jv, C.int(flags))} 387 | defer jv_str.Free() 388 | return jv_str._string() 389 | } 390 | 391 | // JvArray creates a new, empty array-typed JV 392 | func JvArray() *Jv { 393 | return &Jv{C.jv_array()} 394 | } 395 | 396 | // ArrayAppend appends a single value to the end of the array. 397 | // 398 | // If jv is not an array this will cause an assertion. 399 | // 400 | // Consumes the invocant 401 | func (jv *Jv) ArrayAppend(val *Jv) *Jv { 402 | return &Jv{C.jv_array_append(jv.jv, val.jv)} 403 | } 404 | 405 | // ArrayLength returns the number of elements in the array. 406 | // 407 | // Consumes the invocant 408 | func (jv *Jv) ArrayLength() int { 409 | return int(C.jv_array_length(jv.jv)) 410 | } 411 | 412 | // ArrayGet returns the element at the given array index. 413 | // 414 | // If the index is out of bounds it will return an Invalid Jv object (with no 415 | // error message set). 416 | // 417 | // `idx` cannot be negative. 418 | // 419 | // Consumes the invocant 420 | func (jv *Jv) ArrayGet(idx int) *Jv { 421 | return &Jv{C.jv_array_get(jv.jv, C.int(idx))} 422 | } 423 | 424 | func JvObject() *Jv { 425 | return &Jv{C.jv_object()} 426 | } 427 | 428 | // ObjectSet will add val to the object under the given key. 429 | // 430 | // This is the equivalent of `jv[key] = val`. 431 | // 432 | // Consumes invocant and both key and val 433 | func (jv *Jv) ObjectSet(key *Jv, val *Jv) *Jv { 434 | return &Jv{C.jv_object_set(jv.jv, key.jv, val.jv)} 435 | } 436 | -------------------------------------------------------------------------------- /jq/jv_test.go: -------------------------------------------------------------------------------- 1 | package jq_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ashb/jqrepl/jq" 7 | "github.com/cheekybits/is" 8 | ) 9 | 10 | func TestJvKind(t *testing.T) { 11 | is := is.New(t) 12 | 13 | cases := []struct { 14 | *jq.Jv 15 | jq.JvKind 16 | string 17 | }{ 18 | {jq.JvNull(), jq.JV_KIND_NULL, "null"}, 19 | {jq.JvFromString("a"), jq.JV_KIND_STRING, "string"}, 20 | } 21 | 22 | for _, c := range cases { 23 | defer c.Free() 24 | is.Equal(c.Kind(), c.JvKind) 25 | is.Equal(c.Kind().String(), c.string) 26 | } 27 | } 28 | 29 | func TestJvString(t *testing.T) { 30 | is := is.New(t) 31 | 32 | jv := jq.JvFromString("test") 33 | defer jv.Free() 34 | 35 | str, err := jv.String() 36 | 37 | is.Equal(str, "test") 38 | is.NoErr(err) 39 | 40 | i := jv.ToGoVal() 41 | 42 | is.Equal(i, "test") 43 | } 44 | 45 | func TestJvStringOnNonStringType(t *testing.T) { 46 | is := is.New(t) 47 | 48 | // Test that on a non-string value we get a go error, not a C assert 49 | jv := jq.JvNull() 50 | defer jv.Free() 51 | 52 | _, err := jv.String() 53 | is.Err(err) 54 | } 55 | 56 | func TestJvFromJSONString(t *testing.T) { 57 | is := is.New(t) 58 | 59 | jv, err := jq.JvFromJSONString("[]") 60 | is.NoErr(err) 61 | is.OK(jv) 62 | is.Equal(jv.Kind(), jq.JV_KIND_ARRAY) 63 | 64 | jv, err = jq.JvFromJSONString("not valid") 65 | is.Err(err) 66 | is.Nil(jv) 67 | } 68 | 69 | func TestJvFromFloat(t *testing.T) { 70 | is := is.New(t) 71 | 72 | jv := jq.JvFromFloat(1.23) 73 | is.OK(jv) 74 | is.Equal(jv.Kind(), jq.JV_KIND_NUMBER) 75 | gv := jv.ToGoVal() 76 | n, ok := gv.(float64) 77 | is.True(ok) 78 | is.Equal(n, float64(1.23)) 79 | } 80 | 81 | func TestJvFromInterface(t *testing.T) { 82 | is := is.New(t) 83 | 84 | // Null 85 | jv, err := jq.JvFromInterface(nil) 86 | is.NoErr(err) 87 | is.OK(jv) 88 | is.Equal(jv.Kind(), jq.JV_KIND_NULL) 89 | 90 | // Boolean 91 | jv, err = jq.JvFromInterface(true) 92 | is.NoErr(err) 93 | is.OK(jv) 94 | is.Equal(jv.Kind(), jq.JV_KIND_TRUE) 95 | 96 | jv, err = jq.JvFromInterface(false) 97 | is.NoErr(err) 98 | is.OK(jv) 99 | is.Equal(jv.Kind(), jq.JV_KIND_FALSE) 100 | 101 | // Float 102 | jv, err = jq.JvFromInterface(1.23) 103 | is.NoErr(err) 104 | is.OK(jv) 105 | is.Equal(jv.Kind(), jq.JV_KIND_NUMBER) 106 | gv := jv.ToGoVal() 107 | n, ok := gv.(float64) 108 | is.True(ok) 109 | is.Equal(n, float64(1.23)) 110 | 111 | // Integer 112 | jv, err = jq.JvFromInterface(456) 113 | is.NoErr(err) 114 | is.OK(jv) 115 | is.Equal(jv.Kind(), jq.JV_KIND_NUMBER) 116 | gv = jv.ToGoVal() 117 | n2, ok := gv.(int) 118 | is.True(ok) 119 | is.Equal(n2, 456) 120 | 121 | // String 122 | jv, err = jq.JvFromInterface("test") 123 | is.NoErr(err) 124 | is.OK(jv) 125 | is.Equal(jv.Kind(), jq.JV_KIND_STRING) 126 | gv = jv.ToGoVal() 127 | s, ok := gv.(string) 128 | is.True(ok) 129 | is.Equal(s, "test") 130 | 131 | jv, err = jq.JvFromInterface([]string{"test", "one", "two"}) 132 | is.NoErr(err) 133 | is.OK(jv) 134 | is.Equal(jv.Kind(), jq.JV_KIND_ARRAY) 135 | gv = jv.ToGoVal() 136 | is.Equal(gv.([]interface{})[2], "two") 137 | 138 | jv, err = jq.JvFromInterface(map[string]int{"one": 1, "two": 2}) 139 | is.NoErr(err) 140 | is.OK(jv) 141 | is.Equal(jv.Kind(), jq.JV_KIND_OBJECT) 142 | gv = jv.ToGoVal() 143 | is.Equal(gv.(map[string]interface{})["two"], 2) 144 | 145 | aStruct := struct { 146 | One int `json:"one"` 147 | Two int `json:"two"` 148 | }{ 149 | One: 1, 150 | Two: 2, 151 | } 152 | jv, err = jq.JvFromInterface(aStruct) 153 | is.NoErr(err) 154 | is.OK(jv) 155 | is.Equal(jv.Kind(), jq.JV_KIND_OBJECT) 156 | gv = jv.ToGoVal() 157 | is.Equal(gv.(map[string]interface{})["two"], 2) 158 | } 159 | 160 | func TestJvDump(t *testing.T) { 161 | is := is.New(t) 162 | 163 | jv := jq.JvFromString("test") 164 | defer jv.Free() 165 | 166 | dump := jv.Copy().Dump(jq.JvPrintNone) 167 | 168 | is.Equal(`"test"`, dump) 169 | dump = jv.Copy().Dump(jq.JvPrintColour) 170 | 171 | is.Equal([]byte("\x1b[0;32m"+`"test"`+"\x1b[0m"), []byte(dump)) 172 | } 173 | 174 | func TestJvInvalid(t *testing.T) { 175 | is := is.New(t) 176 | 177 | jv := jq.JvInvalid() 178 | 179 | is.False(jv.IsValid()) 180 | 181 | _, ok := jv.Copy().GetInvalidMessageAsString() 182 | is.False(ok) // "Expected no Invalid message" 183 | 184 | jv = jv.GetInvalidMessage() 185 | is.Equal(jv.Kind(), jq.JV_KIND_NULL) 186 | } 187 | 188 | func TestJvInvalidWithMessage_string(t *testing.T) { 189 | is := is.New(t) 190 | 191 | jv := jq.JvInvalidWithMessage(jq.JvFromString("Error message 1")) 192 | 193 | is.False(jv.IsValid()) 194 | 195 | msg := jv.Copy().GetInvalidMessage() 196 | is.Equal(msg.Kind(), jq.JV_KIND_STRING) 197 | msg.Free() 198 | 199 | str, ok := jv.GetInvalidMessageAsString() 200 | is.True(ok) 201 | is.Equal("Error message 1", str) 202 | } 203 | 204 | func TestJvInvalidWithMessage_object(t *testing.T) { 205 | is := is.New(t) 206 | 207 | jv := jq.JvInvalidWithMessage(jq.JvObject()) 208 | 209 | is.False(jv.IsValid()) 210 | 211 | msg := jv.Copy().GetInvalidMessage() 212 | is.Equal(msg.Kind(), jq.JV_KIND_OBJECT) 213 | msg.Free() 214 | 215 | str, ok := jv.GetInvalidMessageAsString() 216 | is.True(ok) 217 | is.Equal("{}", str) 218 | 219 | } 220 | -------------------------------------------------------------------------------- /jqrepl.go: -------------------------------------------------------------------------------- 1 | package jqrepl 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | 8 | "github.com/ashb/jqrepl/jq" 9 | "gopkg.in/chzyer/readline.v1" 10 | ) 11 | 12 | const promptTemplate = "\033[0;36m%3d »\033[0m " 13 | const outputTemplate = "\033[0;34m$out[%d]\033[0m = %s\n\n" 14 | 15 | var ( 16 | jvStringName, jvStringValue, jvStringOut, jvStringUnderscore, jvStringDunderscore *jq.Jv 17 | ) 18 | 19 | func init() { 20 | // Pre-create thre jv_string objects that we will use over and over again. 21 | jvStringName = jq.JvFromString("name") 22 | jvStringValue = jq.JvFromString("value") 23 | jvStringOut = jq.JvFromString("out") 24 | jvStringUnderscore = jq.JvFromString("_") 25 | jvStringDunderscore = jq.JvFromString("__") 26 | } 27 | 28 | type JqRepl struct { 29 | programCounter int 30 | promptTemplate string 31 | reader *readline.Instance 32 | libJq *jq.Jq 33 | input *jq.Jv 34 | 35 | // a jv array of previous outputs 36 | results *jq.Jv 37 | } 38 | 39 | func StdinIsTTY() bool { 40 | return readline.IsTerminal(int(os.Stdin.Fd())) 41 | } 42 | 43 | // New creates a nwq JqRepl 44 | // 45 | // If stdin is not a tty then it will re-open the controlling tty ("/dev/tty" 46 | // on unix) to be able to run in interactive mode 47 | func New() (*JqRepl, error) { 48 | repl := JqRepl{ 49 | promptTemplate: promptTemplate, 50 | } 51 | 52 | cfg, err := repl.readlineReplConfig() 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | repl.reader, err = readline.NewEx(cfg) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | repl.libJq, err = jq.New() 63 | if err != nil { 64 | repl.reader.Close() 65 | return nil, err 66 | } 67 | 68 | repl.results = jq.JvArray() 69 | 70 | return &repl, nil 71 | } 72 | 73 | func (repl *JqRepl) readlineReplConfig() (*readline.Config, error) { 74 | cfg := readline.Config{ 75 | Prompt: repl.currentPrompt(), 76 | Stdin: os.Stdin, 77 | } 78 | 79 | // If stdin is not a tty (i.e. cos we've had input redirected) then we need 80 | // to re-open the controlling TTY to get interactive input from the user. 81 | if !StdinIsTTY() { 82 | tty, err := ReopenTTY() 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | fd := int(tty.Fd()) 88 | cfg.Stdin = tty 89 | cfg.ForceUseInteractive = true 90 | 91 | // The default impl of Make/ExitRaw operate on os.Stdin, which is not 92 | // re-settable 93 | var previousState *readline.State 94 | cfg.FuncMakeRaw = func() error { 95 | var err error 96 | previousState, err = readline.MakeRaw(fd) 97 | return err 98 | } 99 | cfg.FuncExitRaw = func() error { 100 | return readline.Restore(fd, previousState) 101 | } 102 | } 103 | 104 | cfg.Init() 105 | return &cfg, nil 106 | } 107 | 108 | func (repl *JqRepl) Close() { 109 | repl.reader.Close() 110 | repl.libJq.Close() 111 | if repl.input != nil { 112 | repl.input.Free() 113 | } 114 | } 115 | 116 | func (repl *JqRepl) currentPrompt() string { 117 | return fmt.Sprintf(repl.promptTemplate, repl.programCounter) 118 | } 119 | 120 | // JvInput returns the current input the JQ program will operate on 121 | func (repl *JqRepl) JvInput() *jq.Jv { 122 | return repl.input 123 | } 124 | 125 | func (repl *JqRepl) SetJvInput(input *jq.Jv) { 126 | if repl.input != nil { 127 | repl.input.Free() 128 | } 129 | repl.input = input 130 | } 131 | 132 | func (repl *JqRepl) Loop() { 133 | for { 134 | repl.reader.SetPrompt(repl.currentPrompt()) 135 | 136 | line, err := repl.reader.Readline() 137 | if err == io.EOF { 138 | break 139 | } else if err == readline.ErrInterrupt { 140 | // Stop the streaming of any results - if we were 141 | continue 142 | } else if err != nil { 143 | panic(fmt.Errorf("%#v", err)) 144 | } 145 | 146 | repl.RunProgram(line) 147 | } 148 | } 149 | 150 | func (repl *JqRepl) Error(err error) { 151 | fmt.Fprintf(repl.reader.Stderr(), "\033[0;31m%s\033[0m\n", err) 152 | } 153 | 154 | func (repl *JqRepl) Output(o *jq.Jv) { 155 | fmt.Fprintf(repl.reader.Stdout(), outputTemplate, repl.programCounter, o.Dump(jq.JvPrintPretty|jq.JvPrintSpace1|jq.JvPrintColour)) 156 | } 157 | 158 | func (repl *JqRepl) RunProgram(program string) { 159 | args := makeProgramArgs(repl.results.Copy()) 160 | chanIn, chanOut, chanErr := repl.libJq.Start(program, args) 161 | inCopy := repl.JvInput().Copy() 162 | 163 | // Run until the channels are closed 164 | for chanErr != nil && chanOut != nil { 165 | select { 166 | case e, ok := <-chanErr: 167 | if !ok { 168 | chanErr = nil 169 | } else { 170 | repl.Error(e) 171 | 172 | } 173 | case o, ok := <-chanOut: 174 | if !ok { 175 | chanOut = nil 176 | } else { 177 | // Store the result in the history 178 | repl.results = repl.results.ArrayAppend(o.Copy()) 179 | 180 | repl.Output(o) 181 | repl.programCounter++ 182 | } 183 | case chanIn <- inCopy: 184 | // We've sent our input, close the channel to tell Jq we're done 185 | close(chanIn) 186 | chanIn = nil 187 | } 188 | } 189 | } 190 | 191 | func makeProgramArgs(history *jq.Jv) *jq.Jv { 192 | // Create this structure: 193 | // programArgs = [ 194 | // {"name": "out", "value": history }, 195 | // {"name": "_", "value": history[-1] }, 196 | // {"name": "__", "value": history[-2] }, 197 | // ] 198 | 199 | arg := jq.JvObject().ObjectSet(jvStringName.Copy(), jvStringOut.Copy()).ObjectSet(jvStringValue.Copy(), history.Copy()) 200 | res := jq.JvArray().ArrayAppend(arg) 201 | 202 | len := history.Copy().ArrayLength() 203 | 204 | if len >= 1 { 205 | arg = jq.JvObject().ObjectSet(jvStringName.Copy(), jvStringUnderscore.Copy()).ObjectSet(jvStringValue.Copy(), history.Copy().ArrayGet(len-1)) 206 | res = res.ArrayAppend(arg) 207 | } 208 | if len >= 2 { 209 | arg = jq.JvObject().ObjectSet(jvStringName.Copy(), jvStringDunderscore.Copy()).ObjectSet(jvStringValue.Copy(), history.Copy().ArrayGet(len-2)) 210 | res = res.ArrayAppend(arg) 211 | } 212 | 213 | return res 214 | } 215 | -------------------------------------------------------------------------------- /jqrepl_unix.go: -------------------------------------------------------------------------------- 1 | // +build darwin dragonfly freebsd linux,!appengine netbsd openbsd 2 | 3 | package jqrepl 4 | 5 | import "os" 6 | 7 | // ReopenTTY will open a new filehandle to the controlling terminal. 8 | // 9 | // Used when stdin has been redirected so that we can still get an interactive 10 | // prompt to the user. 11 | func ReopenTTY() (*os.File, error) { 12 | return os.Open("/dev/tty") 13 | } 14 | --------------------------------------------------------------------------------