├── .github └── workflows │ ├── clippy.yaml │ ├── lint.yaml │ └── test.yaml ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── Cargo.toml ├── README.md ├── benches └── simple.rs ├── examples └── simple-cli.rs ├── src ├── errors.rs ├── jq.rs └── lib.rs └── tests └── error-chain-compat.rs /.github/workflows/clippy.yaml: -------------------------------------------------------------------------------- 1 | name: clippy 2 | on: push 3 | env: 4 | JQ_LIB_DIR: $(eval which jq) 5 | jobs: 6 | clippy_stable: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - uses: actions-rs/toolchain@v1 11 | with: 12 | toolchain: stable 13 | components: clippy 14 | override: true 15 | - uses: actions-rs/clippy-check@v1 16 | with: 17 | args: "--all-features" 18 | token: ${{ secrets.GITHUB_TOKEN }} 19 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | name: lint 2 | on: push 3 | 4 | jobs: 5 | cargo-deny: 6 | name: cargo deny 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | checks: 11 | - advisories 12 | 13 | # Prevent sudden announcement of a new advisory from failing ci: 14 | continue-on-error: ${{ matrix.checks == 'advisories' }} 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - uses: EmbarkStudios/cargo-deny-action@v1 19 | with: 20 | command: check ${{ matrix.checks }} 21 | 22 | rustfmt: 23 | name: rustfmt 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: actions/checkout@v2 27 | - uses: actions-rs/toolchain@v1.0.7 28 | with: 29 | profile: minimal 30 | toolchain: stable 31 | components: rustfmt 32 | override: true 33 | - name: Run rustfmt 34 | id: rustfmt 35 | run: rustfmt --edition 2018 --check $(find . -type f -iname *.rs) 36 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: test 2 | on: push 3 | env: 4 | RUST_BACKTRACE: 1 5 | 6 | jobs: 7 | unit: 8 | strategy: 9 | fail-fast: false 10 | matrix: 11 | os: [ubuntu-latest, macos-latest] 12 | include: 13 | - os: ubuntu-latest 14 | # n.b. libjq is pinned at 1.6 for now, with 1.7 support planned for the future (ref: #37) 15 | apt-deps: libjq-dev=1.6-2.1ubuntu3 libonig-dev 16 | jq-lib-dir: /usr/lib/x86_64-linux-gnu/ 17 | onig-lib-dir: /usr/lib/x86_64-linux-gnu/ 18 | - os: macos-latest 19 | use-install-jq-action: true 20 | runs-on: ${{ matrix.os }} 21 | steps: 22 | - uses: actions/checkout@v2 23 | - uses: actions-rs/toolchain@v1 24 | with: 25 | override: true 26 | toolchain: stable 27 | profile: minimal 28 | - uses: Swatinem/rust-cache@v2 29 | 30 | - name: Install System Deps (Linux) 31 | if: ${{ matrix.apt-deps }} 32 | run: sudo apt install -y ${{ matrix.apt-deps }} 33 | 34 | - name: Install jq (macOS) 35 | uses: dcarbone/install-jq-action@v1.0.1 36 | with: 37 | version: 1.6 38 | if: ${{ matrix.use-install-jq-action }} 39 | 40 | - name: Build workspace 41 | # FIXME: config for linking is a real mess - figure out something tidier now that both linux/macOS are green 42 | run: | 43 | [[ ! -z "${{ matrix.use-install-jq-action }}" ]] && export JQ_LIB_DIR="$(eval which jq)" 44 | [[ ! -z "${{ matrix.jq-lib-dir }}" ]] && export JQ_LIB_DIR="${{ matrix.jq-lib-dir }}" 45 | [[ ! -z "${{ matrix.onig-lib-dir }}" ]] && export ONIG_LIB_DIR="${{ matrix.onig-lib-dir }}" 46 | cargo build 47 | 48 | - name: Test workspace 49 | # FIXME: config for linking is a real mess - figure out something tidier now that both linux/macOS are green 50 | run: | 51 | [[ ! -z "${{ matrix.use-install-jq-action }}" ]] && export JQ_LIB_DIR="$(eval which jq)" 52 | [[ ! -z "${{ matrix.jq-lib-dir }}" ]] && export JQ_LIB_DIR="${{ matrix.jq-lib-dir }}" 53 | [[ ! -z "${{ matrix.onig-lib-dir }}" ]] && export ONIG_LIB_DIR="${{ matrix.onig-lib-dir }}" 54 | cargo test 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | Cargo.lock 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | dist: xenial 3 | rust: 4 | - stable 5 | # - beta 6 | - nightly 7 | cache: cargo 8 | matrix: 9 | allow_failures: 10 | - rust: nightly 11 | fast_finish: true 12 | before_script: 13 | - echo "$TRAVIS_RUST_VERSION" > rust-toolchain 14 | - rustup component add rustfmt 15 | - rustup show 16 | script: 17 | - cargo fmt -- --check 18 | - cargo test --features bundled 19 | notifications: 20 | email: false 21 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v0.4.1 ([2019-08-17](https://github.com/onelson/jq-rs/compare/v0.4.0..v0.4.1 "diff")) 4 | 5 | Additions 6 | 7 | - Implements `std::error::Error + Send + 'static` for `jq_rs::Error` to better 8 | integrate with popular error handling crate [error-chain] and others ([#22]). 9 | 10 | ## v0.4.0 ([2019-07-06](https://github.com/onelson/jq-rs/compare/v0.3.1..v0.4.0 "diff")) 11 | 12 | Breaking Changes 13 | 14 | - Renamed crate from `json-query` to `jq-rs` ([#12]). 15 | - Adopted 2018 edition. The minimum supported rust version is now **1.32** ([#14]). 16 | - Output from jq programs now includes a trailing newline, just like the output 17 | from the `jq` binary ([#6]). 18 | - Added custom `Error` and `Result` types, returned from fallible 19 | functions/methods in this crate ([#8]). 20 | 21 | ## v0.3.1 ([2019-07-04](https://github.com/onelson/json-query/compare/v0.3.0..v0.3.1 "diff")) 22 | 23 | **Note: This is final release with the name [json-query]. 24 | Future releases will be published as [jq-rs].** 25 | 26 | Bugfixes 27 | 28 | - Fixed issue where newlines in output were not being preserved correctly ([#3]). 29 | - Resolved a memory error which could cause a crash when running a jq program 30 | which could attempt to access missing fields on an object ([#4]). 31 | - Fixed some memory leaks which could occur during processing ([#10]). 32 | 33 | ## v0.3.0 ([2019-06-01](https://github.com/onelson/json-query/compare/v0.2.1..v0.3.0 "diff")) 34 | 35 | - Added `json_query::compile()`. Compile a jq program, then reuse it, running 36 | it against several inputs. 37 | 38 | ## v0.2.1 ([2019-06-01](https://github.com/onelson/json-query/compare/v0.2.0..v0.2.1 "diff")) 39 | 40 | - [#1] Enabled `bundled` feature when building on docs.rs. 41 | 42 | ## v0.2.0 ([2019-02-18](https://github.com/onelson/json-query/compare/v0.1.1..v0.2.0 "diff")) 43 | 44 | - Updates [jq-sys] dep to v0.2.0 for better build controls. 45 | - Settles on 2015 edition style imports (for now). 46 | 47 | Breaking Changes: 48 | 49 | - `bundled` feature is no longer enabled by default. 50 | 51 | 52 | ## v0.1.1 ([2019-01-14](https://github.com/onelson/json-query/compare/v0.1.0..v0.1.1 "diff")) 53 | 54 | - Added extra links to cargo manifest. 55 | - Added some basic docs. 56 | - Added a `bundled` feature to opt in or out of using the bundled source. 57 | 58 | ## v0.1.0 (2019-01-13) 59 | 60 | Initial release. 61 | 62 | [jq]: https://github.com/stedolan/jq 63 | [serde_json]: https://github.com/serde-rs/json 64 | [jq-rs]: https://crates.io/crates/jq-rs 65 | [json-query]: https://crates.io/crates/json-query 66 | [jq-sys]: https://github.com/onelson/jq-sys 67 | [jq-sys-building]: https://github.com/onelson/jq-sys#building 68 | [jq-src]: https://github.com/onelson/jq-src 69 | [error-chain]: https://crates.io/crates/error-chain 70 | 71 | [#1]: https://github.com/onelson/json-query/issues/1 72 | [#3]: https://github.com/onelson/json-query/issues/3 73 | [#4]: https://github.com/onelson/json-query/issues/4 74 | [#6]: https://github.com/onelson/jq-rs/pull/6 75 | [#8]: https://github.com/onelson/jq-rs/pull/8 76 | [#10]: https://github.com/onelson/json-query/issues/10 77 | [#12]: https://github.com/onelson/jq-rs/issues/12 78 | [#14]: https://github.com/onelson/jq-rs/issues/14 79 | [#22]: https://github.com/onelson/jq-rs/pull/22 80 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "jq-rs" 3 | version = "0.4.1" 4 | authors = ["Owen Nelson "] 5 | description = "Run jq programs to extract data from json strings." 6 | repository = "https://github.com/onelson/jq-rs" 7 | homepage = "https://github.com/onelson/jq-rs" 8 | license = "Apache-2.0/MIT" 9 | categories = ["api-bindings"] 10 | keywords = ["json", "jq", "query"] 11 | readme = "README.md" 12 | edition = "2018" 13 | 14 | [features] 15 | default = [] 16 | bundled = ["jq-sys/bundled"] 17 | 18 | [dependencies] 19 | jq-sys = "0.2.*" 20 | 21 | [dev-dependencies] 22 | criterion = "0.2" 23 | serde_json = "1.0" 24 | matches = "0.1.8" 25 | error-chain = "0.12.*" 26 | 27 | [package.metadata.docs.rs] 28 | features = ["bundled"] 29 | 30 | [[bench]] 31 | name = "simple" 32 | harness = false 33 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # jq-rs 2 | 3 | [![crates.io](https://img.shields.io/crates/v/jq-rs.svg)](https://crates.io/crates/jq-rs) 4 | [![crates.io](https://img.shields.io/crates/d/jq-rs.svg)](https://crates.io/crates/jq-rs) 5 | [![docs.rs](https://docs.rs/jq-rs/badge.svg)](https://docs.rs/jq-rs) 6 | [![Build Status](https://github.com/onelson/jq-rs/actions/workflows/test.yaml/badge.svg)](https://github.com/onelson/jq-rs/actions/workflows/test.yaml) 7 | 8 | ## Overview 9 | 10 | > Prior to v0.4.0 this crate was named [json-query]. 11 | 12 | This rust crate provides access to [jq] 1.6 via the `libjq` C API (rather than 13 | "shelling out"). 14 | 15 | By leveraging [jq] we can extract data from json strings using `jq`'s dsl. 16 | 17 | This crate requires Rust **1.32** or above. 18 | 19 | ## Usage 20 | 21 | The interface provided by this crate is very basic. You supply a jq program 22 | string and a string to run the program over. 23 | 24 | ```rust 25 | use jq_rs; 26 | // ... 27 | 28 | let res = jq_rs::run(".name", r#"{"name": "test"}"#); 29 | assert_eq!(res.unwrap(), "\"test\"\n".to_string()); 30 | ``` 31 | 32 | In addition to running one-off programs with `jq_rs::run()`, you can also 33 | use `jq_rs::compile()` to compile a jq program and reuse it with 34 | different inputs. 35 | 36 | ```rust 37 | use jq_rs; 38 | 39 | let tv_shows = r#"[ 40 | {"title": "Twilight Zone"}, 41 | {"title": "X-Files"}, 42 | {"title": "The Outer Limits"} 43 | ]"#; 44 | 45 | let movies = r#"[ 46 | {"title": "The Omen"}, 47 | {"title": "Amityville Horror"}, 48 | {"title": "The Thing"} 49 | ]"#; 50 | 51 | let mut program = jq_rs::compile("[.[].title] | sort").unwrap(); 52 | 53 | assert_eq!( 54 | &program.run(tv_shows).unwrap(), 55 | "[\"The Outer Limits\",\"Twilight Zone\",\"X-Files\"]\n" 56 | ); 57 | 58 | assert_eq!( 59 | &program.run(movies).unwrap(), 60 | "[\"Amityville Horror\",\"The Omen\",\"The Thing\"]\n", 61 | ); 62 | ``` 63 | 64 | ## A Note on Performance 65 | 66 | While the benchmarks are far from exhaustive, they indicate that much of the 67 | runtime of a simple jq program goes to the compilation. In fact, the compilation 68 | is _quite expensive_. 69 | 70 | ```text 71 | run one off time: [48.594 ms 48.689 ms 48.800 ms] 72 | Found 6 outliers among 100 measurements (6.00%) 73 | 3 (3.00%) high mild 74 | 3 (3.00%) high severe 75 | 76 | run pre-compiled time: [4.0351 us 4.0708 us 4.1223 us] 77 | Found 15 outliers among 100 measurements (15.00%) 78 | 6 (6.00%) high mild 79 | 9 (9.00%) high severe 80 | ``` 81 | 82 | If you have a need to run the same jq program multiple times it is 83 | _highly recommended_ to retain a pre-compiled `JqProgram` and reuse it. 84 | 85 | ## Handling Output 86 | 87 | The return values from jq are _strings_ since there is no certainty that the 88 | output will be valid json. As such the output will need to be parsed if you want 89 | to work with the actual data types being represented. 90 | 91 | In such cases you may want to pair this crate with [serde_json] or similar. 92 | 93 | For example, here we want to extract the numbers from a set of objects: 94 | 95 | ```rust 96 | use jq_rs; 97 | use serde_json::{self, json}; 98 | 99 | // ... 100 | 101 | let data = json!({ 102 | "movies": [ 103 | { "title": "Coraline", "year": 2009 }, 104 | { "title": "ParaNorman", "year": 2012 }, 105 | { "title": "Boxtrolls", "year": 2014 }, 106 | { "title": "Kubo and the Two Strings", "year": 2016 }, 107 | { "title": "Missing Link", "year": 2019 } 108 | ] 109 | }); 110 | 111 | let query = "[.movies[].year]"; 112 | // program output as a json string... 113 | let output = jq_rs::run(query, &data.to_string()).unwrap(); 114 | // ... parse via serde 115 | let parsed: Vec = serde_json::from_str(&output).unwrap(); 116 | 117 | assert_eq!(vec![2009, 2012, 2014, 2016, 2019], parsed); 118 | ``` 119 | 120 | Barely any of the options or flags available from the [jq] cli are exposed 121 | currently. 122 | Literally all that is provided is the ability to execute a _jq program_ on a blob 123 | of json. 124 | Please pardon my dust as I sort out the details. 125 | 126 | ## Linking to libjq 127 | 128 | This crate requires access to `libjq` at build and/or runtime depending on the 129 | your choice. 130 | 131 | When the `bundled` feature is enabled (**off by default**) `libjq` is provided 132 | and linked statically to your crate by [jq-sys] and [jq-src]. Using this feature 133 | requires having autotools and gcc in `PATH` in order for the to build to work. 134 | 135 | Without the `bundled` feature, _you_ will need to ensure your crate 136 | can link to `libjq` in order for the bindings to work. 137 | 138 | You can choose to compile `libjq` yourself, or perhaps install it via your 139 | system's package manager. 140 | See the [jq-sys building docs][jq-sys-building] for details on how to share 141 | hints with the [jq-sys] crate on how to link. 142 | 143 | 144 | [jq]: https://github.com/stedolan/jq 145 | [serde_json]: https://github.com/serde-rs/json 146 | [json-query]: https://crates.io/crates/json-query 147 | [jq-sys]: https://github.com/onelson/jq-sys 148 | [jq-sys-building]: https://github.com/onelson/jq-sys#building 149 | [jq-src]: https://github.com/onelson/jq-src 150 | -------------------------------------------------------------------------------- /benches/simple.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate criterion; 3 | extern crate jq_rs; 4 | 5 | use criterion::black_box; 6 | use criterion::Criterion; 7 | use jq_rs::{JqProgram, Result}; 8 | 9 | fn run_one_off(prog: &str, input: &str) -> Result { 10 | jq_rs::run(prog, input) 11 | } 12 | 13 | fn run_pre_compiled(prog: &mut JqProgram, input: &str) -> Result { 14 | prog.run(input) 15 | } 16 | 17 | fn criterion_benchmark(c: &mut Criterion) { 18 | c.bench_function("run one off", |b| { 19 | b.iter(|| run_one_off(black_box(".name"), black_box(r#"{"name": "John Wick"}"#))) 20 | }); 21 | 22 | c.bench_function("run pre-compiled", |b| { 23 | let mut prog = jq_rs::compile(".name").unwrap(); 24 | b.iter(|| run_pre_compiled(black_box(&mut prog), black_box(r#"{"name": "John Wick"}"#))) 25 | }); 26 | } 27 | 28 | criterion_group!(benches, criterion_benchmark); 29 | criterion_main!(benches); 30 | -------------------------------------------------------------------------------- /examples/simple-cli.rs: -------------------------------------------------------------------------------- 1 | extern crate jq_rs; 2 | use std::env; 3 | 4 | fn main() { 5 | let mut args = env::args().skip(1); 6 | 7 | let program = args.next().expect("jq program"); 8 | let input = args.next().expect("data input"); 9 | match jq_rs::run(&program, &input) { 10 | Ok(s) => print!("{}", s), // The output will include a trailing newline 11 | Err(e) => eprintln!("{}", e), 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/errors.rs: -------------------------------------------------------------------------------- 1 | use std::error; 2 | use std::fmt; 3 | use std::result; 4 | 5 | const ERR_UNKNOWN: &str = "JQ: Unknown error"; 6 | const ERR_COMPILE: &str = "JQ: Program failed to compile"; 7 | const ERR_STRING_CONV: &str = "JQ: Failed to convert string"; 8 | 9 | /// This is the common Result type for the crate. Fallible operations will 10 | /// return this. 11 | pub type Result = result::Result; 12 | 13 | /// There are many potential causes for failure when running jq programs. 14 | /// This enum attempts to unify them all under a single type. 15 | #[derive(Debug)] 16 | pub enum Error { 17 | /// The jq program failed to compile. 18 | InvalidProgram { 19 | /// JQ's explanation of the compilation error 20 | reason: String, 21 | }, 22 | /// System errors are raised by the internal jq state machine. These can 23 | /// indicate problems parsing input, or even failures while initializing 24 | /// the state machine itself. 25 | System { 26 | /// Feedback from jq about what went wrong, when available. 27 | reason: Option, 28 | }, 29 | /// Errors encountered during conversion between CString/String or vice 30 | /// versa. 31 | StringConvert { 32 | /// The original error which lead to this. 33 | err: Box, 34 | }, 35 | /// Something bad happened, but it was unexpected. 36 | Unknown, 37 | } 38 | 39 | unsafe impl Send for Error {} 40 | 41 | impl error::Error for Error { 42 | fn description(&self) -> &str { 43 | match self { 44 | Error::StringConvert { .. } => ERR_STRING_CONV, 45 | Error::InvalidProgram { reason } => reason, 46 | Error::System { reason } => reason 47 | .as_ref() 48 | .map(|x| x.as_str()) 49 | .unwrap_or_else(|| ERR_UNKNOWN), 50 | Error::Unknown => ERR_UNKNOWN, 51 | } 52 | } 53 | 54 | fn source(&self) -> Option<&(dyn error::Error + 'static)> { 55 | match self { 56 | Error::StringConvert { err } => { 57 | if let Some(err) = err.downcast_ref::() { 58 | Some(err) 59 | } else if let Some(err) = err.downcast_ref::() { 60 | Some(err) 61 | } else { 62 | None 63 | } 64 | } 65 | _ => None, 66 | } 67 | } 68 | } 69 | 70 | impl From for Error { 71 | fn from(err: std::ffi::NulError) -> Self { 72 | Error::StringConvert { err: Box::new(err) } 73 | } 74 | } 75 | 76 | impl From for Error { 77 | fn from(err: std::str::Utf8Error) -> Self { 78 | Error::StringConvert { err: Box::new(err) } 79 | } 80 | } 81 | 82 | impl fmt::Display for Error { 83 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 84 | let detail: String = match self { 85 | Error::InvalidProgram { reason } => format!("{}: {}", ERR_COMPILE, reason), 86 | Error::System { reason } => reason 87 | .as_ref() 88 | .cloned() 89 | .unwrap_or_else(|| ERR_UNKNOWN.into()), 90 | Error::StringConvert { err } => format!("{} - `{}`", ERR_STRING_CONV, err), 91 | Error::Unknown => ERR_UNKNOWN.into(), 92 | }; 93 | write!(f, "{}", detail) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/jq.rs: -------------------------------------------------------------------------------- 1 | //! This module takes the unsafe bindings from `jq-sys` then (hopefully) 2 | //! wrapping to present a slightly safer API to use. 3 | //! 4 | //! These are building blocks and not intended for use from the public API. 5 | 6 | use crate::errors::{Error, Result}; 7 | use jq_sys::{ 8 | jq_compile, jq_format_error, jq_get_exit_code, jq_halted, jq_init, jq_next, jq_set_error_cb, 9 | jq_start, jq_state, jq_teardown, jv, jv_copy, jv_dump_string, jv_free, jv_get_kind, 10 | jv_invalid_get_msg, jv_invalid_has_msg, jv_kind_JV_KIND_INVALID, jv_kind_JV_KIND_NUMBER, 11 | jv_kind_JV_KIND_STRING, jv_number_value, jv_parser, jv_parser_free, jv_parser_new, 12 | jv_parser_next, jv_parser_set_buf, jv_string_value, 13 | }; 14 | use std::ffi::{CStr, CString}; 15 | use std::os::raw::{c_char, c_void}; 16 | 17 | pub struct Jq { 18 | state: *mut jq_state, 19 | err_buf: String, 20 | } 21 | 22 | impl Jq { 23 | pub fn compile_program(program: CString) -> Result { 24 | let mut jq = Jq { 25 | state: { 26 | // jq's master branch shows this can be a null pointer, in 27 | // which case the binary will exit with a `Error::System`. 28 | let ptr = unsafe { jq_init() }; 29 | if ptr.is_null() { 30 | return Err(Error::System { 31 | reason: Some("Failed to init".into()), 32 | }); 33 | } else { 34 | ptr 35 | } 36 | }, 37 | err_buf: "".to_string(), 38 | }; 39 | 40 | extern "C" fn err_cb(data: *mut c_void, msg: jv) { 41 | unsafe { 42 | let formatted = jq_format_error(msg); 43 | let jq = &mut *(data as *mut Jq); 44 | jq.err_buf += &(CStr::from_ptr(jv_string_value(formatted)) 45 | .to_str() 46 | .unwrap_or("") 47 | .to_string() 48 | + "\n"); 49 | jv_free(formatted); 50 | } 51 | } 52 | unsafe { 53 | jq_set_error_cb(jq.state, Some(err_cb), &mut jq as *mut Jq as *mut c_void); 54 | } 55 | 56 | if unsafe { jq_compile(jq.state, program.as_ptr()) } == 0 { 57 | Err(Error::InvalidProgram { 58 | reason: jq.err_buf.clone(), 59 | }) 60 | } else { 61 | Ok(jq) 62 | } 63 | } 64 | 65 | fn is_halted(&self) -> bool { 66 | unsafe { jq_halted(self.state) != 0 } 67 | } 68 | 69 | fn get_exit_code(&self) -> ExitCode { 70 | let exit_code = JV { 71 | ptr: unsafe { jq_get_exit_code(self.state) }, 72 | }; 73 | 74 | // The rules for this seem odd, but I'm trying to model this after the 75 | // similar block in the jq `main.c`s `process()` function. 76 | 77 | if exit_code.is_valid() { 78 | ExitCode::JQ_OK 79 | } else { 80 | exit_code 81 | .as_number() 82 | .map(|i| (i as isize).into()) 83 | .unwrap_or(ExitCode::JQ_ERROR_UNKNOWN) 84 | } 85 | } 86 | 87 | /// Run the jq program against an input. 88 | pub fn execute(&mut self, input: CString) -> Result { 89 | let mut parser = Parser::new(); 90 | self.process(parser.parse(input)?) 91 | } 92 | 93 | /// Unwind the parser and return the rendered result. 94 | /// 95 | /// When this results in `Err`, the String value should contain a message about 96 | /// what failed. 97 | fn process(&mut self, initial_value: JV) -> Result { 98 | let mut buf = String::new(); 99 | 100 | unsafe { 101 | // `jq_start` seems to be a consuming call. 102 | // In order to avoid a double-free, when `initial_value` is dropped, 103 | // we have to use `jv_copy` on the inner `jv`. 104 | jq_start(self.state, jv_copy(initial_value.ptr), 0); 105 | // After, we can manually free the `initial_value` with `drop` since 106 | // it is no longer needed. 107 | drop(initial_value); 108 | 109 | dump(self, &mut buf)?; 110 | } 111 | 112 | Ok(buf) 113 | } 114 | } 115 | 116 | impl Drop for Jq { 117 | fn drop(&mut self) { 118 | unsafe { jq_teardown(&mut self.state) } 119 | } 120 | } 121 | 122 | struct JV { 123 | ptr: jv, 124 | } 125 | 126 | impl JV { 127 | /// Convert the current `JV` into the "dump string" rendering of itself. 128 | pub fn as_dump_string(&self) -> Result { 129 | let dump = JV { 130 | ptr: unsafe { jv_dump_string(jv_copy(self.ptr), 0) }, 131 | }; 132 | unsafe { get_string_value(jv_string_value(dump.ptr)) } 133 | } 134 | 135 | /// Attempts to extract feedback from jq if the JV is invalid. 136 | pub fn get_msg(&self) -> Option { 137 | if self.invalid_has_msg() { 138 | let reason = { 139 | let msg = JV { 140 | ptr: unsafe { 141 | // This call is gross since we're dipping outside of the 142 | // safe/drop-enabled wrapper to get a copy which will be freed 143 | // by jq. If we wrap it in a `JV`, we'll run into a double-free 144 | // situation. 145 | jv_invalid_get_msg(jv_copy(self.ptr)) 146 | }, 147 | }; 148 | 149 | format!( 150 | "JQ: Parse error: {}", 151 | msg.as_string().unwrap_or_else(|_| "unknown".into()) 152 | ) 153 | }; 154 | Some(reason) 155 | } else { 156 | None 157 | } 158 | } 159 | 160 | pub fn as_number(&self) -> Option { 161 | unsafe { 162 | if jv_get_kind(self.ptr) == jv_kind_JV_KIND_NUMBER { 163 | Some(jv_number_value(self.ptr)) 164 | } else { 165 | None 166 | } 167 | } 168 | } 169 | 170 | pub fn as_string(&self) -> Result { 171 | unsafe { 172 | if jv_get_kind(self.ptr) == jv_kind_JV_KIND_STRING { 173 | get_string_value(jv_string_value(self.ptr)) 174 | } else { 175 | Err(Error::Unknown) 176 | } 177 | } 178 | } 179 | 180 | pub fn is_valid(&self) -> bool { 181 | unsafe { jv_get_kind(self.ptr) != jv_kind_JV_KIND_INVALID } 182 | } 183 | 184 | pub fn invalid_has_msg(&self) -> bool { 185 | unsafe { jv_invalid_has_msg(jv_copy(self.ptr)) == 1 } 186 | } 187 | } 188 | 189 | impl Drop for JV { 190 | fn drop(&mut self) { 191 | unsafe { jv_free(self.ptr) }; 192 | } 193 | } 194 | 195 | struct Parser { 196 | ptr: *mut jv_parser, 197 | } 198 | 199 | impl Parser { 200 | pub fn new() -> Self { 201 | Self { 202 | ptr: unsafe { jv_parser_new(0) }, 203 | } 204 | } 205 | 206 | pub fn parse(&mut self, input: CString) -> Result { 207 | // For a single run, we could set this to `1` (aka `true`) but this will 208 | // break the repeated `JqProgram` usage. 209 | // It may be worth exposing this to the caller so they can set it for each 210 | // use case, but for now we'll just "leave it open." 211 | let is_last = 0; 212 | 213 | // Originally I planned to have a separate "set_buf" method, but it looks like 214 | // the C api really wants you to set the buffer, then call `jv_parser_next()` in 215 | // the same logical block. 216 | // Mainly I think the important thing is to ensure the `input` outlives both the 217 | // set_buf and next calls. 218 | unsafe { 219 | jv_parser_set_buf( 220 | self.ptr, 221 | input.as_ptr(), 222 | input.as_bytes().len() as i32, 223 | is_last, 224 | ) 225 | }; 226 | 227 | let value = JV { 228 | ptr: unsafe { jv_parser_next(self.ptr) }, 229 | }; 230 | if value.is_valid() { 231 | Ok(value) 232 | } else { 233 | Err(Error::System { 234 | reason: Some( 235 | value 236 | .get_msg() 237 | .unwrap_or_else(|| "JQ: Parser error".to_string()), 238 | ), 239 | }) 240 | } 241 | } 242 | } 243 | 244 | impl Drop for Parser { 245 | fn drop(&mut self) { 246 | unsafe { 247 | jv_parser_free(self.ptr); 248 | } 249 | } 250 | } 251 | 252 | /// Takes a pointer to a nul term string, and attempts to convert it to a String. 253 | unsafe fn get_string_value(value: *const c_char) -> Result { 254 | let s = CStr::from_ptr(value).to_str()?; 255 | Ok(s.to_owned()) 256 | } 257 | 258 | /// Renders the data from the parser and pushes it into the buffer. 259 | unsafe fn dump(jq: &Jq, buf: &mut String) -> Result<()> { 260 | // Looks a lot like an iterator... 261 | 262 | let mut value = JV { 263 | ptr: jq_next(jq.state), 264 | }; 265 | 266 | while value.is_valid() { 267 | let s = value.as_dump_string()?; 268 | buf.push_str(&s); 269 | buf.push('\n'); 270 | 271 | value = JV { 272 | ptr: jq_next(jq.state), 273 | }; 274 | } 275 | 276 | if jq.is_halted() { 277 | use ExitCode::*; 278 | match jq.get_exit_code() { 279 | JQ_ERROR_SYSTEM => Err(Error::System { 280 | reason: value.get_msg(), 281 | }), 282 | // As far as I know, we should not be able to see a compile error 283 | // this deep into the execution of a jq program (it would need to be 284 | // compiled already, right?) 285 | // Still, compile failure is represented by an exit code, so in 286 | // order to be exhaustive we have to check for it. 287 | JQ_ERROR_COMPILE => Err(Error::InvalidProgram { 288 | reason: jq.err_buf.clone(), 289 | }), 290 | // Any of these `OK_` variants are "success" cases. 291 | // I suppose the jq program can halt successfully, or not, or not at 292 | // all and still terminate some other way? 293 | JQ_OK | JQ_OK_NULL_KIND | JQ_OK_NO_OUTPUT => Ok(()), 294 | JQ_ERROR_UNKNOWN => Err(Error::Unknown), 295 | } 296 | } else if let Some(reason) = value.get_msg() { 297 | Err(Error::System { 298 | reason: Some(reason), 299 | }) 300 | } else { 301 | Ok(()) 302 | } 303 | } 304 | 305 | /// Various exit codes jq checks for during the `if (jq_halted(jq))` branch of 306 | /// their processing loop. 307 | /// 308 | /// Adapted from the enum seen in jq's master branch right now. 309 | /// The numbers seem to line up with the magic numbers seen in 310 | /// the 1.6 release, though there's no enum that I saw at that point in the git 311 | /// history. 312 | #[allow(non_camel_case_types, dead_code)] 313 | #[rustfmt::skip] 314 | enum ExitCode { 315 | JQ_OK = 0, 316 | JQ_OK_NULL_KIND = -1, 317 | JQ_ERROR_SYSTEM = 2, 318 | JQ_ERROR_COMPILE = 3, 319 | JQ_OK_NO_OUTPUT = -4, 320 | JQ_ERROR_UNKNOWN = 5, 321 | } 322 | 323 | impl From for ExitCode { 324 | #[rustfmt::skip] 325 | fn from(number: isize) -> Self { 326 | use ExitCode::*; 327 | match number { 328 | 0 => JQ_OK, 329 | -1 => JQ_OK_NULL_KIND, 330 | 2 => JQ_ERROR_SYSTEM, 331 | 3 => JQ_ERROR_COMPILE, 332 | -4 => JQ_OK_NO_OUTPUT, 333 | // `5` is called out explicitly in the jq source, but also "unknown" 334 | // seems to make good sense for other unexpected number. 335 | 5 | _ => JQ_ERROR_UNKNOWN, 336 | } 337 | } 338 | } 339 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! ## Overview 2 | //! 3 | //! > Prior to v0.4.0 this crate was named [json-query]. 4 | //! 5 | //! This rust crate provides access to [jq] 1.6 via the `libjq` C API (rather than 6 | //! "shelling out"). 7 | //! 8 | //! By leveraging [jq] we can extract data from json strings using `jq`'s dsl. 9 | //! 10 | //! This crate requires Rust **1.32** or above. 11 | //! 12 | //! ## Usage 13 | //! 14 | //! The interface provided by this crate is very basic. You supply a jq program 15 | //! string and a string to run the program over. 16 | //! 17 | //! ```rust 18 | //! use jq_rs; 19 | //! // ... 20 | //! 21 | //! let res = jq_rs::run(".name", r#"{"name": "test"}"#); 22 | //! assert_eq!(res.unwrap(), "\"test\"\n".to_string()); 23 | //! ``` 24 | //! 25 | //! In addition to running one-off programs with `jq_rs::run()`, you can also 26 | //! use `jq_rs::compile()` to compile a jq program and reuse it with 27 | //! different inputs. 28 | //! 29 | //! ```rust 30 | //! use jq_rs; 31 | //! 32 | //! let tv_shows = r#"[ 33 | //! {"title": "Twilight Zone"}, 34 | //! {"title": "X-Files"}, 35 | //! {"title": "The Outer Limits"} 36 | //! ]"#; 37 | //! 38 | //! let movies = r#"[ 39 | //! {"title": "The Omen"}, 40 | //! {"title": "Amityville Horror"}, 41 | //! {"title": "The Thing"} 42 | //! ]"#; 43 | //! 44 | //! let mut program = jq_rs::compile("[.[].title] | sort").unwrap(); 45 | //! 46 | //! assert_eq!( 47 | //! &program.run(tv_shows).unwrap(), 48 | //! "[\"The Outer Limits\",\"Twilight Zone\",\"X-Files\"]\n" 49 | //! ); 50 | //! 51 | //! assert_eq!( 52 | //! &program.run(movies).unwrap(), 53 | //! "[\"Amityville Horror\",\"The Omen\",\"The Thing\"]\n", 54 | //! ); 55 | //! ``` 56 | //! 57 | //! ## A Note on Performance 58 | //! 59 | //! While the benchmarks are far from exhaustive, they indicate that much of the 60 | //! runtime of a simple jq program goes to the compilation. In fact, the compilation 61 | //! is _quite expensive_. 62 | //! 63 | //! ```text 64 | //! run one off time: [48.594 ms 48.689 ms 48.800 ms] 65 | //! Found 6 outliers among 100 measurements (6.00%) 66 | //! 3 (3.00%) high mild 67 | //! 3 (3.00%) high severe 68 | //! 69 | //! run pre-compiled time: [4.0351 us 4.0708 us 4.1223 us] 70 | //! Found 15 outliers among 100 measurements (15.00%) 71 | //! 6 (6.00%) high mild 72 | //! 9 (9.00%) high severe 73 | //! ``` 74 | //! 75 | //! If you have a need to run the same jq program multiple times it is 76 | //! _highly recommended_ to retain a pre-compiled `JqProgram` and reuse it. 77 | //! 78 | //! ## Handling Output 79 | //! 80 | //! The return values from jq are _strings_ since there is no certainty that the 81 | //! output will be valid json. As such the output will need to be parsed if you want 82 | //! to work with the actual data types being represented. 83 | //! 84 | //! In such cases you may want to pair this crate with [serde_json] or similar. 85 | //! 86 | //! For example, here we want to extract the numbers from a set of objects: 87 | //! 88 | //! ```rust 89 | //! use jq_rs; 90 | //! use serde_json::{self, json}; 91 | //! 92 | //! // ... 93 | //! 94 | //! let data = json!({ 95 | //! "movies": [ 96 | //! { "title": "Coraline", "year": 2009 }, 97 | //! { "title": "ParaNorman", "year": 2012 }, 98 | //! { "title": "Boxtrolls", "year": 2014 }, 99 | //! { "title": "Kubo and the Two Strings", "year": 2016 }, 100 | //! { "title": "Missing Link", "year": 2019 } 101 | //! ] 102 | //! }); 103 | //! 104 | //! let query = "[.movies[].year]"; 105 | //! // program output as a json string... 106 | //! let output = jq_rs::run(query, &data.to_string()).unwrap(); 107 | //! // ... parse via serde 108 | //! let parsed: Vec = serde_json::from_str(&output).unwrap(); 109 | //! 110 | //! assert_eq!(vec![2009, 2012, 2014, 2016, 2019], parsed); 111 | //! ``` 112 | //! 113 | //! Barely any of the options or flags available from the [jq] cli are exposed 114 | //! currently. 115 | //! Literally all that is provided is the ability to execute a _jq program_ on a blob 116 | //! of json. 117 | //! Please pardon my dust as I sort out the details. 118 | //! 119 | //! ## Linking to libjq 120 | //! 121 | //! This crate requires access to `libjq` at build and/or runtime depending on the 122 | //! your choice. 123 | //! 124 | //! When the `bundled` feature is enabled (**off by default**) `libjq` is provided 125 | //! and linked statically to your crate by [jq-sys] and [jq-src]. Using this feature 126 | //! requires having autotools and gcc in `PATH` in order for the to build to work. 127 | //! 128 | //! Without the `bundled` feature, _you_ will need to ensure your crate 129 | //! can link to `libjq` in order for the bindings to work. 130 | //! 131 | //! You can choose to compile `libjq` yourself, or perhaps install it via your 132 | //! system's package manager. 133 | //! See the [jq-sys building docs][jq-sys-building] for details on how to share 134 | //! hints with the [jq-sys] crate on how to link. 135 | //! 136 | //! [jq]: https://github.com/stedolan/jq 137 | //! [serde_json]: https://github.com/serde-rs/json 138 | //! [jq-rs]: https://crates.io/crates/jq-rs 139 | //! [json-query]: https://crates.io/crates/json-query 140 | //! [jq-sys]: https://github.com/onelson/jq-sys 141 | //! [jq-sys-building]: https://github.com/onelson/jq-sys#building 142 | //! [jq-src]: https://github.com/onelson/jq-src 143 | 144 | #![deny(missing_docs)] 145 | 146 | extern crate jq_sys; 147 | #[cfg(test)] 148 | #[macro_use] 149 | extern crate serde_json; 150 | 151 | mod errors; 152 | mod jq; 153 | 154 | use std::ffi::CString; 155 | 156 | pub use errors::{Error, Result}; 157 | 158 | /// Run a jq program on a blob of json data. 159 | /// 160 | /// In the case of failure to run the program, feedback from the jq api will be 161 | /// available in the supplied `String` value. 162 | /// Failures can occur for a variety of reasons, but mostly you'll see them as 163 | /// a result of bad jq program syntax, or invalid json data. 164 | pub fn run(program: &str, data: &str) -> Result { 165 | compile(program)?.run(data) 166 | } 167 | 168 | /// A pre-compiled jq program which can be run against different inputs. 169 | pub struct JqProgram { 170 | jq: jq::Jq, 171 | } 172 | 173 | impl JqProgram { 174 | /// Runs a json string input against a pre-compiled jq program. 175 | pub fn run(&mut self, data: &str) -> Result { 176 | if data.trim().is_empty() { 177 | // During work on #4, #7, the parser test which allows us to avoid a memory 178 | // error shows that an empty input just yields an empty response BUT our 179 | // implementation would yield a parse error. 180 | return Ok("".into()); 181 | } 182 | let input = CString::new(data)?; 183 | self.jq.execute(input) 184 | } 185 | } 186 | 187 | /// Compile a jq program then reuse it, running several inputs against it. 188 | pub fn compile(program: &str) -> Result { 189 | let prog = CString::new(program)?; 190 | Ok(JqProgram { 191 | jq: jq::Jq::compile_program(prog)?, 192 | }) 193 | } 194 | 195 | #[cfg(test)] 196 | mod test { 197 | 198 | use super::{compile, run, Error}; 199 | use matches::assert_matches; 200 | use serde_json; 201 | 202 | #[test] 203 | fn reuse_compiled_program() { 204 | let query = r#"if . == 0 then "zero" elif . == 1 then "one" else "many" end"#; 205 | let mut prog = compile(&query).unwrap(); 206 | assert_eq!(prog.run("2").unwrap(), "\"many\"\n"); 207 | assert_eq!(prog.run("1").unwrap(), "\"one\"\n"); 208 | assert_eq!(prog.run("0").unwrap(), "\"zero\"\n"); 209 | } 210 | 211 | #[test] 212 | fn jq_state_is_not_global() { 213 | let input = r#"{"id": 123, "name": "foo"}"#; 214 | let query1 = r#".name"#; 215 | let query2 = r#".id"#; 216 | 217 | // Basically this test is just to check that the state pointers returned by 218 | // `jq::init()` are completely independent and don't share any global state. 219 | let mut prog1 = compile(&query1).unwrap(); 220 | let mut prog2 = compile(&query2).unwrap(); 221 | 222 | assert_eq!(prog1.run(input).unwrap(), "\"foo\"\n"); 223 | assert_eq!(prog2.run(input).unwrap(), "123\n"); 224 | assert_eq!(prog1.run(input).unwrap(), "\"foo\"\n"); 225 | assert_eq!(prog2.run(input).unwrap(), "123\n"); 226 | } 227 | 228 | fn get_movies() -> serde_json::Value { 229 | json!({ 230 | "movies": [ 231 | { "title": "Coraline", "year": 2009 }, 232 | { "title": "ParaNorman", "year": 2012 }, 233 | { "title": "Boxtrolls", "year": 2014 }, 234 | { "title": "Kubo and the Two Strings", "year": 2016 }, 235 | { "title": "Missing Link", "year": 2019 } 236 | ] 237 | }) 238 | } 239 | 240 | #[test] 241 | fn identity_nothing() { 242 | assert_eq!(run(".", "").unwrap(), "".to_string()); 243 | } 244 | 245 | #[test] 246 | fn identity_empty() { 247 | assert_eq!(run(".", "{}").unwrap(), "{}\n".to_string()); 248 | } 249 | 250 | #[test] 251 | fn extract_dates() { 252 | let data = get_movies(); 253 | let query = "[.movies[].year]"; 254 | let output = run(query, &data.to_string()).unwrap(); 255 | let parsed: Vec = serde_json::from_str(&output).unwrap(); 256 | assert_eq!(vec![2009, 2012, 2014, 2016, 2019], parsed); 257 | } 258 | 259 | #[test] 260 | fn extract_name() { 261 | let res = run(".name", r#"{"name": "test"}"#); 262 | assert_eq!(res.unwrap(), "\"test\"\n".to_string()); 263 | } 264 | 265 | #[test] 266 | fn unpack_array() { 267 | let res = run(".[]", "[1,2,3]"); 268 | assert_eq!(res.unwrap(), "1\n2\n3\n".to_string()); 269 | } 270 | 271 | #[test] 272 | fn compile_error() { 273 | let res = run(". aa12312me dsaafsdfsd", "{\"name\": \"test\"}"); 274 | assert_matches!(res, Err(Error::InvalidProgram { .. })); 275 | } 276 | 277 | #[test] 278 | fn parse_error() { 279 | let res = run(".", "{1233 invalid json ahoy : est\"}"); 280 | assert_matches!(res, Err(Error::System { .. })); 281 | } 282 | 283 | #[test] 284 | fn just_open_brace() { 285 | let res = run(".", "{"); 286 | assert_matches!(res, Err(Error::System { .. })); 287 | } 288 | 289 | #[test] 290 | fn just_close_brace() { 291 | let res = run(".", "}"); 292 | assert_matches!(res, Err(Error::System { .. })); 293 | } 294 | 295 | #[test] 296 | fn total_garbage() { 297 | let data = r#" 298 | { 299 | moreLike: "an object literal but also bad" 300 | loveToDangleComma: true, 301 | }"#; 302 | 303 | let res = run(".", data); 304 | assert_matches!(res, Err(Error::System { .. })); 305 | } 306 | 307 | pub mod mem_errors { 308 | //! Attempting run a program resulting in bad field access has been 309 | //! shown to sometimes trigger a use after free or double free memory 310 | //! error. 311 | //! 312 | //! Technically the program and inputs are both valid, but the 313 | //! evaluation of the program causes bad memory access to happen. 314 | //! 315 | //! https://github.com/onelson/json-query/issues/4 316 | 317 | use super::*; 318 | 319 | #[test] 320 | fn missing_field_access() { 321 | let prog = ".[] | .hello"; 322 | let data = "[1,2,3]"; 323 | let res = run(prog, data); 324 | assert_matches!(res, Err(Error::System { .. })); 325 | } 326 | 327 | #[test] 328 | fn missing_field_access_compiled() { 329 | let mut prog = compile(".[] | .hello").unwrap(); 330 | let data = "[1,2,3]"; 331 | let res = prog.run(data); 332 | assert_matches!(res, Err(Error::System { .. })); 333 | } 334 | } 335 | } 336 | -------------------------------------------------------------------------------- /tests/error-chain-compat.rs: -------------------------------------------------------------------------------- 1 | extern crate jq_rs; 2 | #[macro_use] 3 | extern crate error_chain; 4 | use error_chain::ChainedError; 5 | 6 | mod errors { 7 | error_chain! { 8 | foreign_links { 9 | Jq(jq_rs::Error); 10 | } 11 | } 12 | } 13 | 14 | use self::errors::{Error, ErrorKind, ResultExt}; 15 | 16 | #[test] 17 | fn test_match_errorkind() { 18 | match jq_rs::run(".", "[[[{}}").unwrap_err().into() { 19 | Error(ErrorKind::Jq(e), _s) => { 20 | // Proving that jq_rs::Error does in fact implement `std::error::Error`. 21 | // Sort of redundant considering error_chain requires this, but it 22 | // can't hurt to say it explicitly here. 23 | use std::error::Error as _; 24 | let _ = e.source(); 25 | } 26 | _ => unreachable!("error-chain should be converting."), 27 | } 28 | } 29 | 30 | #[test] 31 | fn test_chain_err() { 32 | let chain = jq_rs::run(".", "[[[{}}") 33 | .chain_err(|| "custom message") 34 | .unwrap_err() 35 | .display_chain() 36 | .to_string(); 37 | 38 | // the chain is a multi-line string mentioning each error in the chain. 39 | assert!(chain.contains("custom message")); 40 | assert!(chain.contains("Parse error")) 41 | } 42 | --------------------------------------------------------------------------------