├── .dir-locals.el ├── .editorconfig ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── appveyor.yml ├── build.rs ├── ci ├── before_deploy.ps1 ├── before_deploy.sh ├── install.sh └── script.sh ├── doc ├── image │ └── highlighting.png ├── installation.md ├── protobuf.md └── tutorial.md ├── install.sh └── src ├── bin └── rq.rs ├── config.rs ├── error.rs ├── lib.rs ├── proto_index.rs └── value ├── avro.rs ├── cbor.rs ├── csv.rs ├── json.rs ├── messagepack.rs ├── mod.rs ├── protobuf.rs ├── raw.rs ├── toml.rs └── yaml.rs /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((rust-mode . ((fill-column . 100)))) 2 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | indent_style = space 6 | insert_final_newline = true 7 | trim_trailing_whitespace = true 8 | 9 | [*.rs] 10 | indent_size = 4 11 | 12 | [*.js] 13 | indent_size = 2 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | *.bk 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Based on the "trust" template v0.1.2 2 | # https://github.com/japaric/trust/tree/v0.1.2 3 | 4 | dist: trusty 5 | language: rust 6 | services: docker 7 | sudo: required 8 | 9 | env: 10 | global: 11 | - CRATE_NAME=rq 12 | 13 | matrix: 14 | include: 15 | # Android 16 | - env: TARGET=aarch64-linux-android DISABLE_TESTS=1 17 | - env: TARGET=arm-linux-androideabi DISABLE_TESTS=1 18 | - env: TARGET=armv7-linux-androideabi DISABLE_TESTS=1 19 | - env: TARGET=i686-linux-android DISABLE_TESTS=1 20 | - env: TARGET=x86_64-linux-android DISABLE_TESTS=1 21 | 22 | # iOS 23 | - env: TARGET=aarch64-apple-ios DISABLE_TESTS=1 24 | os: osx 25 | - env: TARGET=armv7-apple-ios DISABLE_TESTS=1 26 | os: osx 27 | - env: TARGET=armv7s-apple-ios DISABLE_TESTS=1 28 | os: osx 29 | - env: TARGET=i386-apple-ios DISABLE_TESTS=1 30 | os: osx 31 | - env: TARGET=x86_64-apple-ios DISABLE_TESTS=1 32 | os: osx 33 | 34 | # Linux 35 | - env: TARGET=aarch64-unknown-linux-gnu 36 | - env: TARGET=arm-unknown-linux-gnueabi 37 | - env: TARGET=armv7-unknown-linux-gnueabihf 38 | - env: TARGET=i686-unknown-linux-gnu 39 | - env: TARGET=i686-unknown-linux-musl 40 | - env: TARGET=mips-unknown-linux-gnu 41 | - env: TARGET=mips64-unknown-linux-gnuabi64 42 | - env: TARGET=mips64el-unknown-linux-gnuabi64 43 | - env: TARGET=mipsel-unknown-linux-gnu 44 | - env: TARGET=powerpc-unknown-linux-gnu 45 | - env: TARGET=powerpc64-unknown-linux-gnu 46 | - env: TARGET=powerpc64le-unknown-linux-gnu 47 | - env: TARGET=s390x-unknown-linux-gnu DISABLE_TESTS=1 48 | - env: TARGET=x86_64-unknown-linux-gnu 49 | - env: TARGET=x86_64-unknown-linux-musl 50 | 51 | # OSX 52 | - env: TARGET=i686-apple-darwin 53 | os: osx 54 | - env: TARGET=x86_64-apple-darwin 55 | os: osx 56 | 57 | # *BSD 58 | #- env: TARGET=i686-unknown-freebsd DISABLE_TESTS=1 59 | #- env: TARGET=x86_64-unknown-freebsd DISABLE_TESTS=1 60 | - env: TARGET=x86_64-unknown-netbsd DISABLE_TESTS=1 61 | 62 | # Windows 63 | #- env: TARGET=x86_64-pc-windows-gnu 64 | 65 | # Bare metal 66 | # These targets don't support std and as such are likely not suitable for 67 | # most crates. 68 | # - env: TARGET=thumbv6m-none-eabi 69 | # - env: TARGET=thumbv7em-none-eabi 70 | # - env: TARGET=thumbv7em-none-eabihf 71 | # - env: TARGET=thumbv7m-none-eabi 72 | 73 | # Testing other channels 74 | - env: TARGET=x86_64-unknown-linux-gnu 75 | rust: nightly 76 | - env: TARGET=x86_64-apple-darwin 77 | os: osx 78 | rust: nightly 79 | 80 | before_install: 81 | - set -e 82 | - rustup self update 83 | 84 | install: 85 | - sh ci/install.sh 86 | - source ~/.cargo/env || true 87 | 88 | script: 89 | - bash ci/script.sh 90 | 91 | after_script: set +e 92 | 93 | before_deploy: 94 | - sh ci/before_deploy.sh 95 | 96 | deploy: 97 | api_key: 98 | secure: "LELZrRyfyoWcMCZR4XuZ1Cf/hSRWAbg6+H9+mGX2g2oy2k2gBNJOvRLcMByQ/14OvCx3nmZ2PKFbboqxdTMq3vHtJjuxW6ypz7XdJoGTW1MMsmJyRgfL8ux2InNGEectywJkjPyzbrGVqfnhetlFD2PdH9qV9oa0pF18UwYEsG2/oioiU5Sj/0bS8UFbLKoHc83osfVKLfO8tUXeCndnIE10X9e8TX0P1JbVKPYcNU9QSmlB4xMvLCCKyRr9k0sk2dZtcSBESc5iZinji9e2xmSQiYAoL5ttaWrZjpJJcPG9HwqAxa2NneFqIGVxUTYK7oTcajWUXSLYv+N+AIcF7A2LZnSJgLYwbkU9WcjA6Naf89w1dR3y0e6JFpBNvoivDXKOdW/MHy99dNZ/cCzh2eYMHa/eCFeiGUPZVjE0Y+AeAWBhNGQvzMDABmSU3SW8QmH56qjKChW0eWyhrKRJhesl/gRNf0facQj7qj7cXgTpENFRbVNW/4WjOxpuPR5gUNsSenqM7DXL3SqgyltouIMi6cfXjR/IJDEZdpxGB69GppKVGGXbj+xVNB/OjhfWqupg+0SQIl29IeV7eaO7ZGOblOMOXOCDViewDwTw9VPOBalRNmEGlzcZ/btNLb3M5tTFsbP7udgKHnVamMwR0O81O3IERU9Jix3E7+gdgik=" 99 | file_glob: true 100 | file: $CRATE_NAME-$TRAVIS_TAG-$TARGET.* 101 | on: 102 | condition: $TRAVIS_RUST_VERSION = stable 103 | tags: true 104 | provider: releases 105 | skip_cleanup: true 106 | 107 | cache: cargo 108 | before_cache: 109 | # Travis can't cache files that are not readable by "others" 110 | - chmod -R a+r $HOME/.cargo 111 | 112 | branches: 113 | only: 114 | # release tags 115 | - /^v\d+\.\d+\.\d+.*$/ 116 | - master 117 | 118 | notifications: 119 | email: 120 | on_success: never 121 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributions to `rq` are very welcome; please track contributions via 4 | the [issue tracker](https://github.com/dflemstr/rq/issues). 5 | 6 | All issues are marked as either `Bug`s or `Issue`s. They can also be 7 | tagged with an experience level `E-` which is one of `E-easy`, 8 | `E-medium`, `E-hard`, `E-mentor` and the most likely languages 9 | involved in the change `L-` which can be `L-rust`, `L-js` or `L-c`. 10 | 11 | `rq` is not directly affiliated with Spotify but the project still 12 | adheres to its 13 | [code of conduct](https://github.com/spotify/code-of-conduct/blob/master/code-of-conduct.md). 14 | 15 | # Development 16 | 17 | `rq` is mostly written in the [Rust programming language][rust]. 18 | Assuming that you have nothing installed, the easiest way to set 19 | things up is to use `rustup` (see [rustup.rs](https://www.rustup.rs/) 20 | for more info): 21 | 22 | curl -sSLf https://sh.rustup.rs | sh 23 | 24 | The Rust installer will give you further platform-specific 25 | instructions (e.g. if you're missing other development tools). 26 | 27 | You will need the clang development libraries. On a Debian based Linux system, 28 | you can get those like this: 29 | 30 | sudo apt install libclang-dev clang 31 | 32 | To build `rq`, navigate to the source directory. 33 | 34 | You will need a build of the V8 JavaScript engine as well. If 35 | your operating system package manager doesn't provide a package, 36 | you can download a build like this: 37 | 38 | wget "https://s3-eu-west-1.amazonaws.com/record-query/v8/$TARGET/5.7.441.1/v8-build.tar.gz" 39 | tar -xvf v8-build.tar.gz 40 | export V8_LIBS=$PWD/v8-build/lib/libv8uber.a 41 | export V8_SOURCE=$PWD/v8-build 42 | 43 | Now you can run the tests for the project (including JSDoc tests): 44 | 45 | cargo test 46 | 47 | A debug build of the executable can be created like so: 48 | 49 | cargo build 50 | 51 | It will be available in `target/debug/rq`. 52 | 53 | A release build can be created like so (might take a lot longer): 54 | 55 | cargo build --release 56 | 57 | It will be available in `target/release/rq`. 58 | 59 | # Cross-compiled builds 60 | 61 | The easiest way to create cross-compiled builds is to use the `./ci` script. 62 | 63 | Look in the Travis build config for available parameters. For example: 64 | 65 | TARGET=x86_64-unknown-linux-gnu USE_DOCKER=true ./ci test 66 | TARGET=x86_64-unknown-linux-gnu USE_DOCKER=true ./ci deploy 67 | 68 | [rust]: https://www.rust-lang.org/ 69 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "addr2line" 7 | version = "0.19.0" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" 10 | dependencies = [ 11 | "gimli", 12 | ] 13 | 14 | [[package]] 15 | name = "adler" 16 | version = "1.0.2" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" 19 | 20 | [[package]] 21 | name = "adler32" 22 | version = "1.2.0" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" 25 | 26 | [[package]] 27 | name = "aho-corasick" 28 | version = "1.0.1" 29 | source = "registry+https://github.com/rust-lang/crates.io-index" 30 | checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" 31 | dependencies = [ 32 | "memchr", 33 | ] 34 | 35 | [[package]] 36 | name = "ansi_term" 37 | version = "0.12.1" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" 40 | dependencies = [ 41 | "winapi", 42 | ] 43 | 44 | [[package]] 45 | name = "anyhow" 46 | version = "1.0.70" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" 49 | 50 | [[package]] 51 | name = "atty" 52 | version = "0.2.14" 53 | source = "registry+https://github.com/rust-lang/crates.io-index" 54 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 55 | dependencies = [ 56 | "hermit-abi", 57 | "libc", 58 | "winapi", 59 | ] 60 | 61 | [[package]] 62 | name = "autocfg" 63 | version = "1.1.0" 64 | source = "registry+https://github.com/rust-lang/crates.io-index" 65 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 66 | 67 | [[package]] 68 | name = "avro-rs" 69 | version = "0.6.6" 70 | source = "registry+https://github.com/rust-lang/crates.io-index" 71 | checksum = "755f3a531dfb40d903d811950433b475aa77ff919e0d67e615c414a121f0bfdc" 72 | dependencies = [ 73 | "byteorder", 74 | "crc", 75 | "digest", 76 | "failure", 77 | "libflate", 78 | "rand", 79 | "serde 1.0.160", 80 | "serde_json", 81 | "snap", 82 | ] 83 | 84 | [[package]] 85 | name = "backtrace" 86 | version = "0.3.67" 87 | source = "registry+https://github.com/rust-lang/crates.io-index" 88 | checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" 89 | dependencies = [ 90 | "addr2line", 91 | "cc", 92 | "cfg-if", 93 | "libc", 94 | "miniz_oxide", 95 | "object", 96 | "rustc-demangle", 97 | ] 98 | 99 | [[package]] 100 | name = "bitflags" 101 | version = "1.3.2" 102 | source = "registry+https://github.com/rust-lang/crates.io-index" 103 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 104 | 105 | [[package]] 106 | name = "build_const" 107 | version = "0.2.2" 108 | source = "registry+https://github.com/rust-lang/crates.io-index" 109 | checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7" 110 | 111 | [[package]] 112 | name = "built" 113 | version = "0.5.2" 114 | source = "registry+https://github.com/rust-lang/crates.io-index" 115 | checksum = "5b9c056b9ed43aee5e064b683aa1ec783e19c6acec7559e3ae931b7490472fbe" 116 | dependencies = [ 117 | "cargo-lock", 118 | ] 119 | 120 | [[package]] 121 | name = "byteorder" 122 | version = "1.4.3" 123 | source = "registry+https://github.com/rust-lang/crates.io-index" 124 | checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" 125 | 126 | [[package]] 127 | name = "cargo-lock" 128 | version = "8.0.3" 129 | source = "registry+https://github.com/rust-lang/crates.io-index" 130 | checksum = "031718ddb8f78aa5def78a09e90defe30151d1f6c672f937af4dd916429ed996" 131 | dependencies = [ 132 | "semver", 133 | "serde 1.0.160", 134 | "toml 0.5.11", 135 | "url", 136 | ] 137 | 138 | [[package]] 139 | name = "cc" 140 | version = "1.0.79" 141 | source = "registry+https://github.com/rust-lang/crates.io-index" 142 | checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" 143 | dependencies = [ 144 | "jobserver", 145 | ] 146 | 147 | [[package]] 148 | name = "cfg-if" 149 | version = "1.0.0" 150 | source = "registry+https://github.com/rust-lang/crates.io-index" 151 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 152 | 153 | [[package]] 154 | name = "clap" 155 | version = "2.34.0" 156 | source = "registry+https://github.com/rust-lang/crates.io-index" 157 | checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" 158 | dependencies = [ 159 | "ansi_term", 160 | "atty", 161 | "bitflags", 162 | "strsim", 163 | "textwrap", 164 | "unicode-width", 165 | "vec_map", 166 | ] 167 | 168 | [[package]] 169 | name = "core-foundation-sys" 170 | version = "0.8.4" 171 | source = "registry+https://github.com/rust-lang/crates.io-index" 172 | checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" 173 | 174 | [[package]] 175 | name = "crc" 176 | version = "1.8.1" 177 | source = "registry+https://github.com/rust-lang/crates.io-index" 178 | checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" 179 | dependencies = [ 180 | "build_const", 181 | ] 182 | 183 | [[package]] 184 | name = "crc32fast" 185 | version = "1.3.2" 186 | source = "registry+https://github.com/rust-lang/crates.io-index" 187 | checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" 188 | dependencies = [ 189 | "cfg-if", 190 | ] 191 | 192 | [[package]] 193 | name = "csv" 194 | version = "1.2.1" 195 | source = "registry+https://github.com/rust-lang/crates.io-index" 196 | checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" 197 | dependencies = [ 198 | "csv-core", 199 | "itoa 1.0.6", 200 | "ryu", 201 | "serde 1.0.160", 202 | ] 203 | 204 | [[package]] 205 | name = "csv-core" 206 | version = "0.1.10" 207 | source = "registry+https://github.com/rust-lang/crates.io-index" 208 | checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" 209 | dependencies = [ 210 | "memchr", 211 | ] 212 | 213 | [[package]] 214 | name = "digest" 215 | version = "0.8.1" 216 | source = "registry+https://github.com/rust-lang/crates.io-index" 217 | checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" 218 | dependencies = [ 219 | "generic-array", 220 | ] 221 | 222 | [[package]] 223 | name = "directories" 224 | version = "4.0.1" 225 | source = "registry+https://github.com/rust-lang/crates.io-index" 226 | checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210" 227 | dependencies = [ 228 | "dirs-sys", 229 | ] 230 | 231 | [[package]] 232 | name = "dirs-sys" 233 | version = "0.3.7" 234 | source = "registry+https://github.com/rust-lang/crates.io-index" 235 | checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" 236 | dependencies = [ 237 | "libc", 238 | "redox_users", 239 | "winapi", 240 | ] 241 | 242 | [[package]] 243 | name = "dtoa" 244 | version = "0.4.8" 245 | source = "registry+https://github.com/rust-lang/crates.io-index" 246 | checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" 247 | 248 | [[package]] 249 | name = "enum-iterator" 250 | version = "1.4.0" 251 | source = "registry+https://github.com/rust-lang/crates.io-index" 252 | checksum = "706d9e7cf1c7664859d79cd524e4e53ea2b67ea03c98cc2870c5e539695d597e" 253 | dependencies = [ 254 | "enum-iterator-derive", 255 | ] 256 | 257 | [[package]] 258 | name = "enum-iterator-derive" 259 | version = "1.2.0" 260 | source = "registry+https://github.com/rust-lang/crates.io-index" 261 | checksum = "355f93763ef7b0ae1c43c4d8eccc9d5848d84ad1a1d8ce61c421d1ac85a19d05" 262 | dependencies = [ 263 | "proc-macro2", 264 | "quote", 265 | "syn 1.0.109", 266 | ] 267 | 268 | [[package]] 269 | name = "env_logger" 270 | version = "0.7.1" 271 | source = "registry+https://github.com/rust-lang/crates.io-index" 272 | checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" 273 | dependencies = [ 274 | "atty", 275 | "humantime 1.3.0", 276 | "log", 277 | "regex", 278 | "termcolor", 279 | ] 280 | 281 | [[package]] 282 | name = "env_logger" 283 | version = "0.9.3" 284 | source = "registry+https://github.com/rust-lang/crates.io-index" 285 | checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" 286 | dependencies = [ 287 | "atty", 288 | "humantime 2.1.0", 289 | "log", 290 | "regex", 291 | "termcolor", 292 | ] 293 | 294 | [[package]] 295 | name = "failure" 296 | version = "0.1.8" 297 | source = "registry+https://github.com/rust-lang/crates.io-index" 298 | checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" 299 | dependencies = [ 300 | "backtrace", 301 | "failure_derive", 302 | ] 303 | 304 | [[package]] 305 | name = "failure_derive" 306 | version = "0.1.8" 307 | source = "registry+https://github.com/rust-lang/crates.io-index" 308 | checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" 309 | dependencies = [ 310 | "proc-macro2", 311 | "quote", 312 | "syn 1.0.109", 313 | "synstructure", 314 | ] 315 | 316 | [[package]] 317 | name = "form_urlencoded" 318 | version = "1.1.0" 319 | source = "registry+https://github.com/rust-lang/crates.io-index" 320 | checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" 321 | dependencies = [ 322 | "percent-encoding", 323 | ] 324 | 325 | [[package]] 326 | name = "fuchsia-cprng" 327 | version = "0.1.1" 328 | source = "registry+https://github.com/rust-lang/crates.io-index" 329 | checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" 330 | 331 | [[package]] 332 | name = "generic-array" 333 | version = "0.12.4" 334 | source = "registry+https://github.com/rust-lang/crates.io-index" 335 | checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" 336 | dependencies = [ 337 | "typenum", 338 | ] 339 | 340 | [[package]] 341 | name = "getrandom" 342 | version = "0.2.9" 343 | source = "registry+https://github.com/rust-lang/crates.io-index" 344 | checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" 345 | dependencies = [ 346 | "cfg-if", 347 | "libc", 348 | "wasi", 349 | ] 350 | 351 | [[package]] 352 | name = "getset" 353 | version = "0.1.2" 354 | source = "registry+https://github.com/rust-lang/crates.io-index" 355 | checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9" 356 | dependencies = [ 357 | "proc-macro-error", 358 | "proc-macro2", 359 | "quote", 360 | "syn 1.0.109", 361 | ] 362 | 363 | [[package]] 364 | name = "gimli" 365 | version = "0.27.2" 366 | source = "registry+https://github.com/rust-lang/crates.io-index" 367 | checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" 368 | 369 | [[package]] 370 | name = "git2" 371 | version = "0.16.1" 372 | source = "registry+https://github.com/rust-lang/crates.io-index" 373 | checksum = "ccf7f68c2995f392c49fffb4f95ae2c873297830eb25c6bc4c114ce8f4562acc" 374 | dependencies = [ 375 | "bitflags", 376 | "libc", 377 | "libgit2-sys", 378 | "log", 379 | "url", 380 | ] 381 | 382 | [[package]] 383 | name = "glob" 384 | version = "0.3.1" 385 | source = "registry+https://github.com/rust-lang/crates.io-index" 386 | checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" 387 | 388 | [[package]] 389 | name = "half" 390 | version = "1.8.2" 391 | source = "registry+https://github.com/rust-lang/crates.io-index" 392 | checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" 393 | 394 | [[package]] 395 | name = "hashbrown" 396 | version = "0.12.3" 397 | source = "registry+https://github.com/rust-lang/crates.io-index" 398 | checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" 399 | 400 | [[package]] 401 | name = "heck" 402 | version = "0.3.3" 403 | source = "registry+https://github.com/rust-lang/crates.io-index" 404 | checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" 405 | dependencies = [ 406 | "unicode-segmentation", 407 | ] 408 | 409 | [[package]] 410 | name = "hermit-abi" 411 | version = "0.1.19" 412 | source = "registry+https://github.com/rust-lang/crates.io-index" 413 | checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" 414 | dependencies = [ 415 | "libc", 416 | ] 417 | 418 | [[package]] 419 | name = "humantime" 420 | version = "1.3.0" 421 | source = "registry+https://github.com/rust-lang/crates.io-index" 422 | checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" 423 | dependencies = [ 424 | "quick-error", 425 | ] 426 | 427 | [[package]] 428 | name = "humantime" 429 | version = "2.1.0" 430 | source = "registry+https://github.com/rust-lang/crates.io-index" 431 | checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" 432 | 433 | [[package]] 434 | name = "idna" 435 | version = "0.3.0" 436 | source = "registry+https://github.com/rust-lang/crates.io-index" 437 | checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" 438 | dependencies = [ 439 | "unicode-bidi", 440 | "unicode-normalization", 441 | ] 442 | 443 | [[package]] 444 | name = "indexmap" 445 | version = "1.9.3" 446 | source = "registry+https://github.com/rust-lang/crates.io-index" 447 | checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" 448 | dependencies = [ 449 | "autocfg", 450 | "hashbrown", 451 | ] 452 | 453 | [[package]] 454 | name = "itoa" 455 | version = "0.4.8" 456 | source = "registry+https://github.com/rust-lang/crates.io-index" 457 | checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" 458 | 459 | [[package]] 460 | name = "itoa" 461 | version = "1.0.6" 462 | source = "registry+https://github.com/rust-lang/crates.io-index" 463 | checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" 464 | 465 | [[package]] 466 | name = "jobserver" 467 | version = "0.1.26" 468 | source = "registry+https://github.com/rust-lang/crates.io-index" 469 | checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" 470 | dependencies = [ 471 | "libc", 472 | ] 473 | 474 | [[package]] 475 | name = "lazy_static" 476 | version = "1.4.0" 477 | source = "registry+https://github.com/rust-lang/crates.io-index" 478 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 479 | 480 | [[package]] 481 | name = "libc" 482 | version = "0.2.142" 483 | source = "registry+https://github.com/rust-lang/crates.io-index" 484 | checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" 485 | 486 | [[package]] 487 | name = "libflate" 488 | version = "0.1.27" 489 | source = "registry+https://github.com/rust-lang/crates.io-index" 490 | checksum = "d9135df43b1f5d0e333385cb6e7897ecd1a43d7d11b91ac003f4d2c2d2401fdd" 491 | dependencies = [ 492 | "adler32", 493 | "crc32fast", 494 | "rle-decode-fast", 495 | "take_mut", 496 | ] 497 | 498 | [[package]] 499 | name = "libgit2-sys" 500 | version = "0.14.2+1.5.1" 501 | source = "registry+https://github.com/rust-lang/crates.io-index" 502 | checksum = "7f3d95f6b51075fe9810a7ae22c7095f12b98005ab364d8544797a825ce946a4" 503 | dependencies = [ 504 | "cc", 505 | "libc", 506 | "libz-sys", 507 | "pkg-config", 508 | ] 509 | 510 | [[package]] 511 | name = "libz-sys" 512 | version = "1.1.8" 513 | source = "registry+https://github.com/rust-lang/crates.io-index" 514 | checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" 515 | dependencies = [ 516 | "cc", 517 | "libc", 518 | "pkg-config", 519 | "vcpkg", 520 | ] 521 | 522 | [[package]] 523 | name = "linked-hash-map" 524 | version = "0.3.0" 525 | source = "registry+https://github.com/rust-lang/crates.io-index" 526 | checksum = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" 527 | dependencies = [ 528 | "serde 0.8.23", 529 | "serde_test", 530 | ] 531 | 532 | [[package]] 533 | name = "linked-hash-map" 534 | version = "0.5.6" 535 | source = "registry+https://github.com/rust-lang/crates.io-index" 536 | checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" 537 | 538 | [[package]] 539 | name = "log" 540 | version = "0.4.17" 541 | source = "registry+https://github.com/rust-lang/crates.io-index" 542 | checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" 543 | dependencies = [ 544 | "cfg-if", 545 | ] 546 | 547 | [[package]] 548 | name = "memchr" 549 | version = "2.5.0" 550 | source = "registry+https://github.com/rust-lang/crates.io-index" 551 | checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" 552 | 553 | [[package]] 554 | name = "memoffset" 555 | version = "0.6.5" 556 | source = "registry+https://github.com/rust-lang/crates.io-index" 557 | checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" 558 | dependencies = [ 559 | "autocfg", 560 | ] 561 | 562 | [[package]] 563 | name = "miniz_oxide" 564 | version = "0.6.2" 565 | source = "registry+https://github.com/rust-lang/crates.io-index" 566 | checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" 567 | dependencies = [ 568 | "adler", 569 | ] 570 | 571 | [[package]] 572 | name = "nix" 573 | version = "0.24.3" 574 | source = "registry+https://github.com/rust-lang/crates.io-index" 575 | checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" 576 | dependencies = [ 577 | "bitflags", 578 | "cfg-if", 579 | "libc", 580 | "memoffset", 581 | ] 582 | 583 | [[package]] 584 | name = "nom8" 585 | version = "0.2.0" 586 | source = "registry+https://github.com/rust-lang/crates.io-index" 587 | checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" 588 | dependencies = [ 589 | "memchr", 590 | ] 591 | 592 | [[package]] 593 | name = "ntapi" 594 | version = "0.4.1" 595 | source = "registry+https://github.com/rust-lang/crates.io-index" 596 | checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" 597 | dependencies = [ 598 | "winapi", 599 | ] 600 | 601 | [[package]] 602 | name = "num-traits" 603 | version = "0.1.43" 604 | source = "registry+https://github.com/rust-lang/crates.io-index" 605 | checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" 606 | dependencies = [ 607 | "num-traits 0.2.15", 608 | ] 609 | 610 | [[package]] 611 | name = "num-traits" 612 | version = "0.2.15" 613 | source = "registry+https://github.com/rust-lang/crates.io-index" 614 | checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" 615 | dependencies = [ 616 | "autocfg", 617 | ] 618 | 619 | [[package]] 620 | name = "object" 621 | version = "0.30.3" 622 | source = "registry+https://github.com/rust-lang/crates.io-index" 623 | checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" 624 | dependencies = [ 625 | "memchr", 626 | ] 627 | 628 | [[package]] 629 | name = "once_cell" 630 | version = "1.17.1" 631 | source = "registry+https://github.com/rust-lang/crates.io-index" 632 | checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" 633 | 634 | [[package]] 635 | name = "ordered-float" 636 | version = "3.6.0" 637 | source = "registry+https://github.com/rust-lang/crates.io-index" 638 | checksum = "13a384337e997e6860ffbaa83708b2ef329fd8c54cb67a5f64d421e0f943254f" 639 | dependencies = [ 640 | "num-traits 0.2.15", 641 | ] 642 | 643 | [[package]] 644 | name = "paste" 645 | version = "1.0.12" 646 | source = "registry+https://github.com/rust-lang/crates.io-index" 647 | checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" 648 | 649 | [[package]] 650 | name = "percent-encoding" 651 | version = "2.2.0" 652 | source = "registry+https://github.com/rust-lang/crates.io-index" 653 | checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" 654 | 655 | [[package]] 656 | name = "pest" 657 | version = "2.5.7" 658 | source = "registry+https://github.com/rust-lang/crates.io-index" 659 | checksum = "7b1403e8401ad5dedea73c626b99758535b342502f8d1e361f4a2dd952749122" 660 | dependencies = [ 661 | "thiserror", 662 | "ucd-trie", 663 | ] 664 | 665 | [[package]] 666 | name = "pkg-config" 667 | version = "0.3.26" 668 | source = "registry+https://github.com/rust-lang/crates.io-index" 669 | checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" 670 | 671 | [[package]] 672 | name = "proc-macro-error" 673 | version = "1.0.4" 674 | source = "registry+https://github.com/rust-lang/crates.io-index" 675 | checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" 676 | dependencies = [ 677 | "proc-macro-error-attr", 678 | "proc-macro2", 679 | "quote", 680 | "syn 1.0.109", 681 | "version_check", 682 | ] 683 | 684 | [[package]] 685 | name = "proc-macro-error-attr" 686 | version = "1.0.4" 687 | source = "registry+https://github.com/rust-lang/crates.io-index" 688 | checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" 689 | dependencies = [ 690 | "proc-macro2", 691 | "quote", 692 | "version_check", 693 | ] 694 | 695 | [[package]] 696 | name = "proc-macro2" 697 | version = "1.0.56" 698 | source = "registry+https://github.com/rust-lang/crates.io-index" 699 | checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" 700 | dependencies = [ 701 | "unicode-ident", 702 | ] 703 | 704 | [[package]] 705 | name = "protobuf" 706 | version = "2.28.0" 707 | source = "registry+https://github.com/rust-lang/crates.io-index" 708 | checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" 709 | 710 | [[package]] 711 | name = "quick-error" 712 | version = "1.2.3" 713 | source = "registry+https://github.com/rust-lang/crates.io-index" 714 | checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" 715 | 716 | [[package]] 717 | name = "quote" 718 | version = "1.0.26" 719 | source = "registry+https://github.com/rust-lang/crates.io-index" 720 | checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" 721 | dependencies = [ 722 | "proc-macro2", 723 | ] 724 | 725 | [[package]] 726 | name = "rand" 727 | version = "0.4.6" 728 | source = "registry+https://github.com/rust-lang/crates.io-index" 729 | checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" 730 | dependencies = [ 731 | "fuchsia-cprng", 732 | "libc", 733 | "rand_core 0.3.1", 734 | "rdrand", 735 | "winapi", 736 | ] 737 | 738 | [[package]] 739 | name = "rand_core" 740 | version = "0.3.1" 741 | source = "registry+https://github.com/rust-lang/crates.io-index" 742 | checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" 743 | dependencies = [ 744 | "rand_core 0.4.2", 745 | ] 746 | 747 | [[package]] 748 | name = "rand_core" 749 | version = "0.4.2" 750 | source = "registry+https://github.com/rust-lang/crates.io-index" 751 | checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" 752 | 753 | [[package]] 754 | name = "rdrand" 755 | version = "0.4.0" 756 | source = "registry+https://github.com/rust-lang/crates.io-index" 757 | checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" 758 | dependencies = [ 759 | "rand_core 0.3.1", 760 | ] 761 | 762 | [[package]] 763 | name = "record-query" 764 | version = "1.0.4" 765 | dependencies = [ 766 | "ansi_term", 767 | "atty", 768 | "avro-rs", 769 | "built", 770 | "csv", 771 | "directories", 772 | "dtoa", 773 | "env_logger 0.7.1", 774 | "env_logger 0.9.3", 775 | "failure", 776 | "glob", 777 | "itoa 0.4.8", 778 | "log", 779 | "nix", 780 | "ordered-float", 781 | "pest", 782 | "protobuf", 783 | "regex", 784 | "rmp", 785 | "rmpv", 786 | "serde 1.0.160", 787 | "serde-hjson", 788 | "serde-protobuf", 789 | "serde_cbor", 790 | "serde_json", 791 | "serde_yaml", 792 | "structopt", 793 | "toml 0.6.0", 794 | "vergen", 795 | "yaml-rust", 796 | ] 797 | 798 | [[package]] 799 | name = "redox_syscall" 800 | version = "0.2.16" 801 | source = "registry+https://github.com/rust-lang/crates.io-index" 802 | checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" 803 | dependencies = [ 804 | "bitflags", 805 | ] 806 | 807 | [[package]] 808 | name = "redox_users" 809 | version = "0.4.3" 810 | source = "registry+https://github.com/rust-lang/crates.io-index" 811 | checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" 812 | dependencies = [ 813 | "getrandom", 814 | "redox_syscall", 815 | "thiserror", 816 | ] 817 | 818 | [[package]] 819 | name = "regex" 820 | version = "1.8.1" 821 | source = "registry+https://github.com/rust-lang/crates.io-index" 822 | checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" 823 | dependencies = [ 824 | "aho-corasick", 825 | "memchr", 826 | "regex-syntax", 827 | ] 828 | 829 | [[package]] 830 | name = "regex-syntax" 831 | version = "0.7.1" 832 | source = "registry+https://github.com/rust-lang/crates.io-index" 833 | checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" 834 | 835 | [[package]] 836 | name = "rle-decode-fast" 837 | version = "1.0.3" 838 | source = "registry+https://github.com/rust-lang/crates.io-index" 839 | checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" 840 | 841 | [[package]] 842 | name = "rmp" 843 | version = "0.8.11" 844 | source = "registry+https://github.com/rust-lang/crates.io-index" 845 | checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f" 846 | dependencies = [ 847 | "byteorder", 848 | "num-traits 0.2.15", 849 | "paste", 850 | ] 851 | 852 | [[package]] 853 | name = "rmpv" 854 | version = "1.0.0" 855 | source = "registry+https://github.com/rust-lang/crates.io-index" 856 | checksum = "de8813b3a2f95c5138fe5925bfb8784175d88d6bff059ba8ce090aa891319754" 857 | dependencies = [ 858 | "num-traits 0.2.15", 859 | "rmp", 860 | ] 861 | 862 | [[package]] 863 | name = "rustc-demangle" 864 | version = "0.1.23" 865 | source = "registry+https://github.com/rust-lang/crates.io-index" 866 | checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" 867 | 868 | [[package]] 869 | name = "rustc_version" 870 | version = "0.4.0" 871 | source = "registry+https://github.com/rust-lang/crates.io-index" 872 | checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" 873 | dependencies = [ 874 | "semver", 875 | ] 876 | 877 | [[package]] 878 | name = "rustversion" 879 | version = "1.0.12" 880 | source = "registry+https://github.com/rust-lang/crates.io-index" 881 | checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" 882 | 883 | [[package]] 884 | name = "ryu" 885 | version = "1.0.13" 886 | source = "registry+https://github.com/rust-lang/crates.io-index" 887 | checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" 888 | 889 | [[package]] 890 | name = "semver" 891 | version = "1.0.17" 892 | source = "registry+https://github.com/rust-lang/crates.io-index" 893 | checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" 894 | dependencies = [ 895 | "serde 1.0.160", 896 | ] 897 | 898 | [[package]] 899 | name = "serde" 900 | version = "0.8.23" 901 | source = "registry+https://github.com/rust-lang/crates.io-index" 902 | checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" 903 | 904 | [[package]] 905 | name = "serde" 906 | version = "1.0.160" 907 | source = "registry+https://github.com/rust-lang/crates.io-index" 908 | checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" 909 | dependencies = [ 910 | "serde_derive", 911 | ] 912 | 913 | [[package]] 914 | name = "serde-hjson" 915 | version = "0.9.1" 916 | source = "registry+https://github.com/rust-lang/crates.io-index" 917 | checksum = "6a3a4e0ea8a88553209f6cc6cfe8724ecad22e1acf372793c27d995290fe74f8" 918 | dependencies = [ 919 | "lazy_static", 920 | "linked-hash-map 0.3.0", 921 | "num-traits 0.1.43", 922 | "regex", 923 | "serde 0.8.23", 924 | ] 925 | 926 | [[package]] 927 | name = "serde-protobuf" 928 | version = "0.8.2" 929 | source = "registry+https://github.com/rust-lang/crates.io-index" 930 | checksum = "386e6493f9a732832e6b595ac2757a3c8c9285f4ab0c27313b7696b1712c32e4" 931 | dependencies = [ 932 | "linked-hash-map 0.5.6", 933 | "log", 934 | "protobuf", 935 | "serde 1.0.160", 936 | "thiserror", 937 | ] 938 | 939 | [[package]] 940 | name = "serde_cbor" 941 | version = "0.11.2" 942 | source = "registry+https://github.com/rust-lang/crates.io-index" 943 | checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" 944 | dependencies = [ 945 | "half", 946 | "serde 1.0.160", 947 | ] 948 | 949 | [[package]] 950 | name = "serde_derive" 951 | version = "1.0.160" 952 | source = "registry+https://github.com/rust-lang/crates.io-index" 953 | checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" 954 | dependencies = [ 955 | "proc-macro2", 956 | "quote", 957 | "syn 2.0.15", 958 | ] 959 | 960 | [[package]] 961 | name = "serde_json" 962 | version = "1.0.96" 963 | source = "registry+https://github.com/rust-lang/crates.io-index" 964 | checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" 965 | dependencies = [ 966 | "itoa 1.0.6", 967 | "ryu", 968 | "serde 1.0.160", 969 | ] 970 | 971 | [[package]] 972 | name = "serde_spanned" 973 | version = "0.6.1" 974 | source = "registry+https://github.com/rust-lang/crates.io-index" 975 | checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" 976 | dependencies = [ 977 | "serde 1.0.160", 978 | ] 979 | 980 | [[package]] 981 | name = "serde_test" 982 | version = "0.8.23" 983 | source = "registry+https://github.com/rust-lang/crates.io-index" 984 | checksum = "110b3dbdf8607ec493c22d5d947753282f3bae73c0f56d322af1e8c78e4c23d5" 985 | dependencies = [ 986 | "serde 0.8.23", 987 | ] 988 | 989 | [[package]] 990 | name = "serde_yaml" 991 | version = "0.9.21" 992 | source = "registry+https://github.com/rust-lang/crates.io-index" 993 | checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" 994 | dependencies = [ 995 | "indexmap", 996 | "itoa 1.0.6", 997 | "ryu", 998 | "serde 1.0.160", 999 | "unsafe-libyaml", 1000 | ] 1001 | 1002 | [[package]] 1003 | name = "snap" 1004 | version = "0.2.5" 1005 | source = "registry+https://github.com/rust-lang/crates.io-index" 1006 | checksum = "95d697d63d44ad8b78b8d235bf85b34022a78af292c8918527c5f0cffdde7f43" 1007 | dependencies = [ 1008 | "byteorder", 1009 | "lazy_static", 1010 | ] 1011 | 1012 | [[package]] 1013 | name = "strsim" 1014 | version = "0.8.0" 1015 | source = "registry+https://github.com/rust-lang/crates.io-index" 1016 | checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" 1017 | 1018 | [[package]] 1019 | name = "structopt" 1020 | version = "0.3.26" 1021 | source = "registry+https://github.com/rust-lang/crates.io-index" 1022 | checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" 1023 | dependencies = [ 1024 | "clap", 1025 | "lazy_static", 1026 | "structopt-derive", 1027 | ] 1028 | 1029 | [[package]] 1030 | name = "structopt-derive" 1031 | version = "0.4.18" 1032 | source = "registry+https://github.com/rust-lang/crates.io-index" 1033 | checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" 1034 | dependencies = [ 1035 | "heck", 1036 | "proc-macro-error", 1037 | "proc-macro2", 1038 | "quote", 1039 | "syn 1.0.109", 1040 | ] 1041 | 1042 | [[package]] 1043 | name = "syn" 1044 | version = "1.0.109" 1045 | source = "registry+https://github.com/rust-lang/crates.io-index" 1046 | checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" 1047 | dependencies = [ 1048 | "proc-macro2", 1049 | "quote", 1050 | "unicode-ident", 1051 | ] 1052 | 1053 | [[package]] 1054 | name = "syn" 1055 | version = "2.0.15" 1056 | source = "registry+https://github.com/rust-lang/crates.io-index" 1057 | checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" 1058 | dependencies = [ 1059 | "proc-macro2", 1060 | "quote", 1061 | "unicode-ident", 1062 | ] 1063 | 1064 | [[package]] 1065 | name = "synstructure" 1066 | version = "0.12.6" 1067 | source = "registry+https://github.com/rust-lang/crates.io-index" 1068 | checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" 1069 | dependencies = [ 1070 | "proc-macro2", 1071 | "quote", 1072 | "syn 1.0.109", 1073 | "unicode-xid", 1074 | ] 1075 | 1076 | [[package]] 1077 | name = "sysinfo" 1078 | version = "0.27.8" 1079 | source = "registry+https://github.com/rust-lang/crates.io-index" 1080 | checksum = "a902e9050fca0a5d6877550b769abd2bd1ce8c04634b941dbe2809735e1a1e33" 1081 | dependencies = [ 1082 | "cfg-if", 1083 | "core-foundation-sys", 1084 | "libc", 1085 | "ntapi", 1086 | "once_cell", 1087 | "winapi", 1088 | ] 1089 | 1090 | [[package]] 1091 | name = "take_mut" 1092 | version = "0.2.2" 1093 | source = "registry+https://github.com/rust-lang/crates.io-index" 1094 | checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" 1095 | 1096 | [[package]] 1097 | name = "termcolor" 1098 | version = "1.2.0" 1099 | source = "registry+https://github.com/rust-lang/crates.io-index" 1100 | checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" 1101 | dependencies = [ 1102 | "winapi-util", 1103 | ] 1104 | 1105 | [[package]] 1106 | name = "textwrap" 1107 | version = "0.11.0" 1108 | source = "registry+https://github.com/rust-lang/crates.io-index" 1109 | checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" 1110 | dependencies = [ 1111 | "unicode-width", 1112 | ] 1113 | 1114 | [[package]] 1115 | name = "thiserror" 1116 | version = "1.0.40" 1117 | source = "registry+https://github.com/rust-lang/crates.io-index" 1118 | checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" 1119 | dependencies = [ 1120 | "thiserror-impl", 1121 | ] 1122 | 1123 | [[package]] 1124 | name = "thiserror-impl" 1125 | version = "1.0.40" 1126 | source = "registry+https://github.com/rust-lang/crates.io-index" 1127 | checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" 1128 | dependencies = [ 1129 | "proc-macro2", 1130 | "quote", 1131 | "syn 2.0.15", 1132 | ] 1133 | 1134 | [[package]] 1135 | name = "time" 1136 | version = "0.3.20" 1137 | source = "registry+https://github.com/rust-lang/crates.io-index" 1138 | checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" 1139 | dependencies = [ 1140 | "itoa 1.0.6", 1141 | "serde 1.0.160", 1142 | "time-core", 1143 | "time-macros", 1144 | ] 1145 | 1146 | [[package]] 1147 | name = "time-core" 1148 | version = "0.1.0" 1149 | source = "registry+https://github.com/rust-lang/crates.io-index" 1150 | checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" 1151 | 1152 | [[package]] 1153 | name = "time-macros" 1154 | version = "0.2.8" 1155 | source = "registry+https://github.com/rust-lang/crates.io-index" 1156 | checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" 1157 | dependencies = [ 1158 | "time-core", 1159 | ] 1160 | 1161 | [[package]] 1162 | name = "tinyvec" 1163 | version = "1.6.0" 1164 | source = "registry+https://github.com/rust-lang/crates.io-index" 1165 | checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" 1166 | dependencies = [ 1167 | "tinyvec_macros", 1168 | ] 1169 | 1170 | [[package]] 1171 | name = "tinyvec_macros" 1172 | version = "0.1.1" 1173 | source = "registry+https://github.com/rust-lang/crates.io-index" 1174 | checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" 1175 | 1176 | [[package]] 1177 | name = "toml" 1178 | version = "0.5.11" 1179 | source = "registry+https://github.com/rust-lang/crates.io-index" 1180 | checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" 1181 | dependencies = [ 1182 | "serde 1.0.160", 1183 | ] 1184 | 1185 | [[package]] 1186 | name = "toml" 1187 | version = "0.6.0" 1188 | source = "registry+https://github.com/rust-lang/crates.io-index" 1189 | checksum = "4fb9d890e4dc9298b70f740f615f2e05b9db37dce531f6b24fb77ac993f9f217" 1190 | dependencies = [ 1191 | "indexmap", 1192 | "serde 1.0.160", 1193 | "serde_spanned", 1194 | "toml_datetime", 1195 | "toml_edit", 1196 | ] 1197 | 1198 | [[package]] 1199 | name = "toml_datetime" 1200 | version = "0.5.1" 1201 | source = "registry+https://github.com/rust-lang/crates.io-index" 1202 | checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" 1203 | dependencies = [ 1204 | "serde 1.0.160", 1205 | ] 1206 | 1207 | [[package]] 1208 | name = "toml_edit" 1209 | version = "0.18.1" 1210 | source = "registry+https://github.com/rust-lang/crates.io-index" 1211 | checksum = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b" 1212 | dependencies = [ 1213 | "indexmap", 1214 | "nom8", 1215 | "serde 1.0.160", 1216 | "serde_spanned", 1217 | "toml_datetime", 1218 | ] 1219 | 1220 | [[package]] 1221 | name = "typenum" 1222 | version = "1.16.0" 1223 | source = "registry+https://github.com/rust-lang/crates.io-index" 1224 | checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" 1225 | 1226 | [[package]] 1227 | name = "ucd-trie" 1228 | version = "0.1.5" 1229 | source = "registry+https://github.com/rust-lang/crates.io-index" 1230 | checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" 1231 | 1232 | [[package]] 1233 | name = "unicode-bidi" 1234 | version = "0.3.13" 1235 | source = "registry+https://github.com/rust-lang/crates.io-index" 1236 | checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" 1237 | 1238 | [[package]] 1239 | name = "unicode-ident" 1240 | version = "1.0.8" 1241 | source = "registry+https://github.com/rust-lang/crates.io-index" 1242 | checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" 1243 | 1244 | [[package]] 1245 | name = "unicode-normalization" 1246 | version = "0.1.22" 1247 | source = "registry+https://github.com/rust-lang/crates.io-index" 1248 | checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" 1249 | dependencies = [ 1250 | "tinyvec", 1251 | ] 1252 | 1253 | [[package]] 1254 | name = "unicode-segmentation" 1255 | version = "1.10.1" 1256 | source = "registry+https://github.com/rust-lang/crates.io-index" 1257 | checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" 1258 | 1259 | [[package]] 1260 | name = "unicode-width" 1261 | version = "0.1.10" 1262 | source = "registry+https://github.com/rust-lang/crates.io-index" 1263 | checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" 1264 | 1265 | [[package]] 1266 | name = "unicode-xid" 1267 | version = "0.2.4" 1268 | source = "registry+https://github.com/rust-lang/crates.io-index" 1269 | checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" 1270 | 1271 | [[package]] 1272 | name = "unsafe-libyaml" 1273 | version = "0.2.8" 1274 | source = "registry+https://github.com/rust-lang/crates.io-index" 1275 | checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6" 1276 | 1277 | [[package]] 1278 | name = "url" 1279 | version = "2.3.1" 1280 | source = "registry+https://github.com/rust-lang/crates.io-index" 1281 | checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" 1282 | dependencies = [ 1283 | "form_urlencoded", 1284 | "idna", 1285 | "percent-encoding", 1286 | ] 1287 | 1288 | [[package]] 1289 | name = "vcpkg" 1290 | version = "0.2.15" 1291 | source = "registry+https://github.com/rust-lang/crates.io-index" 1292 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 1293 | 1294 | [[package]] 1295 | name = "vec_map" 1296 | version = "0.8.2" 1297 | source = "registry+https://github.com/rust-lang/crates.io-index" 1298 | checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" 1299 | 1300 | [[package]] 1301 | name = "vergen" 1302 | version = "7.5.1" 1303 | source = "registry+https://github.com/rust-lang/crates.io-index" 1304 | checksum = "f21b881cd6636ece9735721cf03c1fe1e774fe258683d084bb2812ab67435749" 1305 | dependencies = [ 1306 | "anyhow", 1307 | "cfg-if", 1308 | "enum-iterator", 1309 | "getset", 1310 | "git2", 1311 | "rustc_version", 1312 | "rustversion", 1313 | "sysinfo", 1314 | "thiserror", 1315 | "time", 1316 | ] 1317 | 1318 | [[package]] 1319 | name = "version_check" 1320 | version = "0.9.4" 1321 | source = "registry+https://github.com/rust-lang/crates.io-index" 1322 | checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" 1323 | 1324 | [[package]] 1325 | name = "wasi" 1326 | version = "0.11.0+wasi-snapshot-preview1" 1327 | source = "registry+https://github.com/rust-lang/crates.io-index" 1328 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 1329 | 1330 | [[package]] 1331 | name = "winapi" 1332 | version = "0.3.9" 1333 | source = "registry+https://github.com/rust-lang/crates.io-index" 1334 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 1335 | dependencies = [ 1336 | "winapi-i686-pc-windows-gnu", 1337 | "winapi-x86_64-pc-windows-gnu", 1338 | ] 1339 | 1340 | [[package]] 1341 | name = "winapi-i686-pc-windows-gnu" 1342 | version = "0.4.0" 1343 | source = "registry+https://github.com/rust-lang/crates.io-index" 1344 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 1345 | 1346 | [[package]] 1347 | name = "winapi-util" 1348 | version = "0.1.5" 1349 | source = "registry+https://github.com/rust-lang/crates.io-index" 1350 | checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" 1351 | dependencies = [ 1352 | "winapi", 1353 | ] 1354 | 1355 | [[package]] 1356 | name = "winapi-x86_64-pc-windows-gnu" 1357 | version = "0.4.0" 1358 | source = "registry+https://github.com/rust-lang/crates.io-index" 1359 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 1360 | 1361 | [[package]] 1362 | name = "yaml-rust" 1363 | version = "0.4.5" 1364 | source = "registry+https://github.com/rust-lang/crates.io-index" 1365 | checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" 1366 | dependencies = [ 1367 | "linked-hash-map 0.5.6", 1368 | ] 1369 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = ["David Flemström "] 3 | categories = ["command-line-interface", "command-line-utilities", "development-tools", "text-processing", "value-formatting"] 4 | description = "A tool for doing record analysis and transformation" 5 | documentation = "https://github.com/dflemstr/rq/blob/master/README.md" 6 | homepage = "https://github.com/dflemstr/rq" 7 | keywords = ["command-line-tool", "cli", "javascript", "record", "query"] 8 | license = "Apache-2.0" 9 | name = "record-query" 10 | readme = "README.md" 11 | repository = "https://github.com/dflemstr/rq" 12 | version = "1.0.4" 13 | edition = "2018" 14 | 15 | [badges] 16 | 17 | [badges.travis-ci] 18 | branch = "master" 19 | repository = "dflemstr/rq" 20 | 21 | [build-dependencies] 22 | built = "0.5.2" 23 | env_logger = "0.9.3" 24 | regex = "1.8.1" 25 | vergen = "7.5.1" 26 | 27 | [dependencies] 28 | ansi_term = "0.12.1" 29 | atty = "0.2.14" 30 | csv = "1.2.1" 31 | directories = "4.0.1" 32 | dtoa = "0.4.8" 33 | env_logger = "0.7.1" 34 | failure = "0.1.8" 35 | glob = "0.3.1" 36 | itoa = "0.4.8" 37 | log = "0.4.17" 38 | nix = "0.24.3" 39 | ordered-float = "3.6.0" 40 | pest = "2.5.7" 41 | protobuf = "2.28.0" 42 | rmp = "0.8.11" 43 | rmpv = "1.0.0" 44 | serde = "1.0.160" 45 | serde-hjson = "0.9.1" 46 | serde-protobuf = "0.8.2" 47 | serde_cbor = "0.11.2" 48 | serde_json = "1.0.96" 49 | serde_yaml = "0.9.21" 50 | structopt = "0.3.26" 51 | yaml-rust = "0.4.5" 52 | 53 | [dependencies.avro-rs] 54 | version = "0.6.6" 55 | features = ["snappy"] 56 | 57 | [dependencies.toml] 58 | version = "0.6.0" 59 | features = ["preserve_order"] 60 | 61 | [profile.release] 62 | lto = true 63 | codegen-units = 1 64 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `rq` [![Build Status](https://travis-ci.org/dflemstr/rq.svg?branch=master)](https://travis-ci.org/dflemstr/rq) [![Build status](https://ci.appveyor.com/api/projects/status/aq916pu1odthadeh?svg=true)](https://ci.appveyor.com/project/dflemstr/rq) [![Crates.io](https://img.shields.io/crates/v/record-query.svg)](https://crates.io/crates/record-query) [![Language (Rust)](https://img.shields.io/badge/powered_by-Rust-blue.svg)](http://www.rust-lang.org/) 2 | **NOTE**: `rq` is in very low maintenance mode as my day job is taking up 3 | a lot of my time. I will try my best to merge pull requests but will 4 | not drive active development of this crate. 5 | 6 | **NOTE**: `rq` no longer ships with query support and a Javascript 7 | engine is not included; instead, it focuses exclusively on format 8 | transformation. You can still pipe into a runtime like node.js if 9 | you need Javascript evaluation. Please see [this issue](https://github.com/dflemstr/rq/issues/208) 10 | to discuss introducing a new query language. 11 | 12 | This is the home of the tool called `rq` (record query). It's a tool 13 | that's used for performing queries on streams of records in various 14 | formats. 15 | 16 | The goal is to make ad-hoc exploration of data sets easy without 17 | having to use more heavy-weight tools like SQL/MapReduce/custom 18 | programs. `rq` fills a similar niche as tools like `awk` or `sed`, 19 | but works with structured (record) data instead of text. 20 | 21 | It was created with love out of the best parts of Rust, and is 22 | distributed as a dependency-free binary on many operating systems and 23 | architectures. 24 | 25 | ## Quick links 26 | 27 | - [Installation](doc/installation.md) — How to install `rq`. 28 | - [Tutorial](doc/tutorial.md) — Learn `rq` from scratch. 29 | - [Protobuf](doc/protobuf.md) — Configure Protobuf specifics. 30 | - [Development](CONTRIBUTING.md) — Contribute to `rq`. 31 | 32 | ## Format support status 33 | 34 | | Format | Read | Write | 35 | |-------------------------|------|-------| 36 | | Apache Avro | ✔️ | ✔️ | 37 | | CBOR | ✔️ | ✔️ | 38 | | JSON | ✔️ | ✔️ | 39 | | MessagePack | ✔️ | ✔️ | 40 | | Google Protocol Buffers | ✔️ | ✖️ | 41 | | YAML | ✔️ | ✔️ | 42 | | TOML | ✔️ | ✔️ | 43 | | Raw (plain text) | ✔️ | ✔️ | 44 | | CSV | ✔️ | ✔️ | 45 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | # Based on the "trust" template v0.1.2 2 | # https://github.com/japaric/trust/tree/v0.1.2 3 | 4 | environment: 5 | global: 6 | RUST_VERSION: stable 7 | 8 | CRATE_NAME: rq 9 | 10 | matrix: 11 | # MinGW 12 | - TARGET: i686-pc-windows-gnu 13 | - TARGET: x86_64-pc-windows-gnu 14 | 15 | # MSVC 16 | - TARGET: i686-pc-windows-msvc 17 | - TARGET: x86_64-pc-windows-msvc 18 | 19 | # Testing other channels 20 | - TARGET: x86_64-pc-windows-gnu 21 | RUST_VERSION: nightly 22 | - TARGET: x86_64-pc-windows-msvc 23 | RUST_VERSION: nightly 24 | 25 | install: 26 | - ps: >- 27 | If ($Env:TARGET -eq 'x86_64-pc-windows-gnu') { 28 | $Env:PATH += ';C:\msys64\mingw64\bin' 29 | } ElseIf ($Env:TARGET -eq 'i686-pc-windows-gnu') { 30 | $Env:PATH += ';C:\msys64\mingw32\bin' 31 | } 32 | - curl -sSf -o rustup-init.exe https://win.rustup.rs/ 33 | - rustup-init.exe -y --default-host %TARGET% --default-toolchain %RUST_VERSION% 34 | - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin 35 | - rustc -Vv 36 | - cargo -V 37 | 38 | test_script: 39 | # we don't run the "test phase" when doing deploys 40 | - if [%APPVEYOR_REPO_TAG%]==[false] ( 41 | cargo build --target %TARGET% && 42 | cargo build --target %TARGET% --release && 43 | cargo test --target %TARGET% && 44 | cargo test --target %TARGET% --release 45 | ) 46 | 47 | before_deploy: 48 | - cargo rustc --target %TARGET% --release --bin rq -- -C lto 49 | - ps: ci\before_deploy.ps1 50 | 51 | deploy: 52 | artifact: /.*\.zip/ 53 | auth_token: 54 | secure: VoQcuAqdS8I7e57x5AupOxbnk1cZbaCZI+sd/WnL9obtq24iAnGdFxG9TWudbKhb 55 | description: '' 56 | on: 57 | RUST_VERSION: stable 58 | appveyor_repo_tag: true 59 | provider: GitHub 60 | 61 | cache: 62 | - C:\Users\appveyor\.cargo\registry 63 | - target 64 | 65 | branches: 66 | only: 67 | # Release tags 68 | - /^v\d+\.\d+\.\d+.*$/ 69 | - master 70 | 71 | notifications: 72 | - provider: Email 73 | on_build_success: false 74 | 75 | # Building is done in the test phase, so we disable Appveyor's build phase. 76 | build: false 77 | -------------------------------------------------------------------------------- /build.rs: -------------------------------------------------------------------------------- 1 | use vergen::{vergen, Config}; 2 | 3 | fn main() { 4 | // Setup the flags, toggling off the 'SEMVER_FROM_CARGO_PKG' flag 5 | let mut flags = Config::default(); 6 | *flags.build_mut().semver_mut() = false; 7 | 8 | // Generate the 'cargo:' key output 9 | vergen(flags).expect("Unable to generate the cargo keys!") 10 | } 11 | -------------------------------------------------------------------------------- /ci/before_deploy.ps1: -------------------------------------------------------------------------------- 1 | # This script takes care of packaging the build artifacts that will go in the 2 | # release zipfile 3 | 4 | $SRC_DIR = $PWD.Path 5 | $STAGE = [System.Guid]::NewGuid().ToString() 6 | 7 | Set-Location $ENV:Temp 8 | New-Item -Type Directory -Name $STAGE 9 | Set-Location $STAGE 10 | 11 | $ZIP = "$SRC_DIR\$($Env:CRATE_NAME)-$($Env:APPVEYOR_REPO_TAG_NAME)-$($Env:TARGET).zip" 12 | 13 | Copy-Item "$SRC_DIR\target\$($Env:TARGET)\release\rq.exe" '.\' 14 | 15 | 7z a "$ZIP" * 16 | 17 | Push-AppveyorArtifact "$ZIP" 18 | 19 | Remove-Item *.* -Force 20 | Set-Location .. 21 | Remove-Item $STAGE 22 | Set-Location $SRC_DIR 23 | -------------------------------------------------------------------------------- /ci/before_deploy.sh: -------------------------------------------------------------------------------- 1 | # This script takes care of building your crate and packaging it for release 2 | 3 | set -ex 4 | 5 | main() { 6 | local src=$(pwd) \ 7 | stage= 8 | 9 | case $TRAVIS_OS_NAME in 10 | linux) 11 | stage=$(mktemp -d) 12 | ;; 13 | osx) 14 | stage=$(mktemp -d -t tmp) 15 | ;; 16 | esac 17 | 18 | test -f Cargo.lock || cargo generate-lockfile 19 | 20 | cross rustc --bin rq --target $TARGET --release -- -C lto 21 | 22 | cp target/$TARGET/release/rq $stage/ 23 | 24 | cd $stage 25 | tar czf $src/$CRATE_NAME-$TRAVIS_TAG-$TARGET.tar.gz * 26 | cd $src 27 | 28 | rm -rf $stage 29 | } 30 | 31 | main 32 | -------------------------------------------------------------------------------- /ci/install.sh: -------------------------------------------------------------------------------- 1 | set -ex 2 | 3 | main() { 4 | local target= 5 | if [ $TRAVIS_OS_NAME = linux ]; then 6 | target=x86_64-unknown-linux-musl 7 | sort=sort 8 | else 9 | target=x86_64-apple-darwin 10 | sort=gsort # for `sort --sort-version`, from brew's coreutils. 11 | fi 12 | 13 | # Builds for iOS are done on OSX, but require the specific target to be 14 | # installed. 15 | case $TARGET in 16 | aarch64-apple-ios) 17 | rustup target install aarch64-apple-ios 18 | ;; 19 | armv7-apple-ios) 20 | rustup target install armv7-apple-ios 21 | ;; 22 | armv7s-apple-ios) 23 | rustup target install armv7s-apple-ios 24 | ;; 25 | i386-apple-ios) 26 | rustup target install i386-apple-ios 27 | ;; 28 | x86_64-apple-ios) 29 | rustup target install x86_64-apple-ios 30 | ;; 31 | esac 32 | 33 | # This fetches latest stable release 34 | local tag=$(git ls-remote --tags --refs --exit-code https://github.com/japaric/cross \ 35 | | cut -d/ -f3 \ 36 | | grep -E '^v[0.1.0-9.]+$' \ 37 | | $sort --version-sort \ 38 | | tail -n1) 39 | curl -LSfs https://japaric.github.io/trust/install.sh | \ 40 | sh -s -- \ 41 | --force \ 42 | --git japaric/cross \ 43 | --tag $tag \ 44 | --target $target 45 | } 46 | 47 | main 48 | -------------------------------------------------------------------------------- /ci/script.sh: -------------------------------------------------------------------------------- 1 | # This script takes care of testing your crate 2 | 3 | set -ex 4 | 5 | main() { 6 | cross build --target $TARGET 7 | cross build --target $TARGET --release 8 | 9 | if [ ! -z $DISABLE_TESTS ]; then 10 | return 11 | fi 12 | 13 | cross test --target $TARGET 14 | cross test --target $TARGET --release 15 | } 16 | 17 | # we don't run the "test phase" when doing deploys 18 | if [ -z $TRAVIS_TAG ]; then 19 | main 20 | fi 21 | -------------------------------------------------------------------------------- /doc/image/highlighting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dflemstr/rq/eb2f033051f7068cb2a6a85dac9e47204cc88cf2/doc/image/highlighting.png -------------------------------------------------------------------------------- /doc/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | There are many different ways to install `rq`, listed from most preferred 4 | to least preferred. 5 | 6 | TODO: this section will soon be updated! 7 | 8 | * [Generic](#generic) (Up to date, fast) 9 | * [Cargo](#cargo) (Stable releases, slow) 10 | * [GitHub releases](#github-releases) (Stable releases, fast) 11 | * [Arch Linux](#arch-linux) (Up to date, fast) 12 | * [Mac OS X](#mac-os-x) (Out of date, slow) 13 | * [Nix](#nix) (Up to date, slow) 14 | 15 | ## Generic 16 | 17 | There is a generic best-effort installer available via the dreaded 18 | `curl | bash` method. This is the preferred method, because you don't 19 | need to compile `rq` from scratch, and you always get the latest 20 | version. 21 | 22 | curl -LSfs https://japaric.github.io/trust/install.sh | sh -s -- --git dflemstr/rq 23 | 24 | ## Cargo 25 | 26 | There is a crate available on [crates.io](https://crates.io/), so just run: 27 | 28 | cargo install record-query 29 | 30 | ## GitHub releases 31 | 32 | There are tagged releases of `rq` fairly infrequently. You can 33 | download pre-built binaries from the 34 | [GitHub releases](https://github.com/dflemstr/rq/releases) page. Note 35 | that these might be very out of date compared to `master`. 36 | 37 | ## Arch Linux 38 | 39 | There is an official package in the [community] repository: 40 | 41 | pacman -S rq 42 | 43 | ## Mac OS X 44 | 45 | There is a Homebrew tap available. Add it like this: 46 | 47 | brew tap dflemstr/tools 48 | 49 | This will let you install the latest version of `rq` (recommended): 50 | 51 | brew install --HEAD rq 52 | 53 | Note that the compilation might take some time, use `-v` for details. 54 | 55 | If you for some reason want the last tagged release of `rq` (might be 56 | severely out of date): 57 | 58 | brew install rq 59 | 60 | ## Nix 61 | 62 | `rq` is available in nixpkgs. You can install it via `nix-env`: 63 | 64 | nix-env -i rq 65 | 66 | Or add to packages list if you use [Home Manager](https://github.com/rycee/home-manager): 67 | 68 | home.packages = [ pkgs.rq ] 69 | -------------------------------------------------------------------------------- /doc/protobuf.md: -------------------------------------------------------------------------------- 1 | # Protobuf 2 | 3 | The Google Protocol Buffer support in `rq` is special because Protobuf 4 | requires an external schema to parse messages. 5 | 6 | `rq` maintains its own database of Protobuf schemata that is used to 7 | parse messages. You can add a schema to the database and all 8 | definitions in that schema will be made available. The schemata all 9 | share the same namespace so you can't provide conflicting definitions 10 | for messages. 11 | 12 | ## Adding new schemata 13 | 14 | Adding new schemata to the database is simple: 15 | 16 | rq protobuf add myschema.proto 17 | 18 | This stashes away the schema to be used the next time you run `rq` 19 | with the `-p` flag. 20 | 21 | Some schemata need to be in specific directories because of references 22 | by other schema files. `rq` will by default use the relative file 23 | name specified in the invocation to save the file internally. That 24 | means that if you call `rq` like so: 25 | 26 | rq protobuf add foo/bar/schema.proto 27 | 28 | ...then the schema will be stored internally with the given relative 29 | path of `foo/bar/schema.proto`. You can control this behavior with 30 | the `--base` flag, so this: 31 | 32 | rq protobuf add foo/bar/schema.proto --base foo 33 | 34 | ...will store the schema as `bar/schema.proto`. 35 | 36 | ## Deserializing messages 37 | 38 | You specify the fully qualified message name when deserializing 39 | Protobuf: 40 | 41 | rq -p .foo.bar.Person 42 | 43 | The leading `.` is needed to disambiguate namespace/package aliases, 44 | which are yet to be implemented. 45 | -------------------------------------------------------------------------------- /doc/tutorial.md: -------------------------------------------------------------------------------- 1 | # Tutorial 2 | 3 | This assumes that `rq` is installed. See 4 | [installation](installation.md) for more details on how to do that if 5 | you want to follow along. 6 | 7 | ## Input/Output 8 | 9 | `rq` reads record data from stdin, and writes transformed data to 10 | stdout. By default, it uses JSON for the input and output format, and 11 | returns each input record unmodified: 12 | 13 | $ rq <<< 'null true {"a": 2.5}' 14 | null 15 | true 16 | {"a":2.5} 17 | 18 | ## Highlighting 19 | 20 | This Markdown document doesn't do the `rq` output justice. The output 21 | of `rq` is actually very colorful! 22 | 23 | ![highlighting](image/highlighting.png) 24 | 25 | ## Record formats 26 | 27 | You can configure the input and output formats to use with flags (see 28 | `rq --help` for details). A lower-case single-letter flag sets the 29 | input format, and an upper-case single-letter flag sets the output 30 | format. For example, to read JSON and output CBOR, pass `-jC` and to 31 | read CBOR and output JSON, pass `-cJ`. This can be used to build a 32 | not-very-useful conversion pipeline that round-trips to CBOR (maybe 33 | you could pipe it through `gzip` and `ssh` in-between and it might be 34 | worth it): 35 | 36 | $ (rq -jC | rq -cJ) <<< 'null true {"a": 2.5}' 37 | null 38 | true 39 | {"a":2.5} 40 | 41 | Some format flags take an argument to configure them, for example 42 | Google Protocol Buffers: 43 | 44 | $ rq protobuf add example.proto 45 | $ rq -p .example.Person < person.pb 46 | {"name":"John","age":34} 47 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | # Copyright 2016 David Flemström. 3 | # 4 | # Based on https://sh.rustup.rs, which is: 5 | # Copyright 2016 The Rust Project Developers 6 | # Licensed under the Apache License, Version 2.0 7 | # (http://www.apache.org/licenses/LICENSE-2.0) 8 | 9 | base='https://s3-eu-west-1.amazonaws.com/record-query/record-query' 10 | 11 | msg() { 12 | printf "\33[1mrq:\33[0m %s\n" "$*" >&2 13 | } 14 | 15 | err() { 16 | msg "$@" 17 | exit 1 18 | } 19 | 20 | interactive=true 21 | path=$( (command -v rq | grep -Fv '/usr/bin/') || echo /usr/local/bin/rq) 22 | 23 | while [[ $# -gt 1 ]] 24 | do 25 | case "$1" in 26 | -y|--yes) 27 | interactive=false 28 | ;; 29 | -o|--output|-p|--path) 30 | path=$2 31 | shift 32 | ;; 33 | *) 34 | ;; 35 | esac 36 | shift 37 | done 38 | 39 | msg "Welcome to the rq installer!" 40 | msg 41 | 42 | cpu="$(uname -m)" 43 | os="$(uname -s)" 44 | 45 | # Darwin `uname -s` lies 46 | if [ "$os" = Darwin ] && [ "$cpu" = i386 ] 47 | then 48 | if sysctl hw.optional.x86_64 | grep -q ': 1' 49 | then 50 | cpu=x86_64 51 | fi 52 | fi 53 | 54 | case "$os" in 55 | Linux) 56 | os=unknown-linux-gnu ;; 57 | FreeBSD) 58 | os=unknown-freebsd ;; 59 | DragonFly) 60 | os=unknown-dragonfly ;; 61 | Darwin) 62 | os=apple-darwin ;; 63 | MINGW* | MSYS* | CYGWIN*) 64 | os=pc-windows-gnu ;; 65 | *) 66 | err "unrecognized OS type: $os" ;; 67 | esac 68 | 69 | case "$cpu" in 70 | i386 | i486 | i686 | i786 | x86) 71 | cpu=i686 ;; 72 | xscale | arm) 73 | cpu=arm ;; 74 | armv6l) 75 | cpu=arm 76 | os="${os}eabihf" ;; 77 | armv7l) 78 | cpu=armv7 79 | os="${os}eabihf" ;; 80 | aarch64) 81 | cpu=aarch64 ;; 82 | x86_64 | x86-64 | x64 | amd64) 83 | cpu=x86_64 ;; 84 | *) 85 | err "unrecognized CPU type: $cpu" ;; 86 | esac 87 | 88 | # Detect 64-bit linux with 32-bit user land 89 | if [ "$os" = unknown-linux-gnu ] && [ "$cpu" = x86_64 ] 90 | then 91 | bin_to_probe="/usr/bin/env" 92 | if [ -e "$bin_to_probe" ] 93 | then 94 | file -L "$bin_to_probe" | grep -q "x86[_-]64" 95 | if [ $? != 0 ] 96 | then 97 | cpu=i686 98 | fi 99 | fi 100 | fi 101 | 102 | arch="$cpu-$os" 103 | 104 | msg "Detected your architecture to be $arch" 105 | 106 | # musl mappings 107 | case "$arch" in 108 | x86_64-unknown-linux-gnu) 109 | musl_arch=x86_64-unknown-linux-musl ;; 110 | i686-unknown-linux-gnu) 111 | musl_arch=i686-unknown-linux-musl ;; 112 | arm-unknown-linux-gnueabi) 113 | musl_arch=arm-unknown-linux-musleabi ;; 114 | arm-unknown-linux-gnueabihf) 115 | musl_arch=arm-unknown-linux-musleabihf ;; 116 | armv7-unknown-linux-gnueabihf) 117 | musl_arch=armv7-unknown-linux-musleabihf ;; 118 | esac 119 | 120 | if [ -n "$musl_arch" ] 121 | then 122 | if [ "$interactive" = true ] 123 | then 124 | msg 'You can install the glibc or musl version of rq:' 125 | msg 126 | msg ' • The musl version is statically linked and with zero' 127 | msg ' dependencies (recommended).' 128 | msg ' • The glibc version is slightly smaller but depends on' 129 | msg ' recent versions of libstdc++ and glibc that you might' 130 | msg ' not have installed.' 131 | msg 132 | msg 'Which one do you prefer?' 133 | 134 | options=(musl glibc) 135 | PS3='Choice: ' 136 | select opt in "${options[@]}" 137 | do 138 | case "$opt" in 139 | musl) 140 | arch="$musl_arch"; break ;; 141 | glibc) 142 | break ;; 143 | *) 144 | msg "Invalid choice" ;; 145 | esac 146 | done < /dev/tty 147 | else 148 | msg 'Detected that your platform supports musl!' 149 | arch="$musl_arch" 150 | fi 151 | msg "Using architecture $arch" 152 | fi 153 | 154 | url="$base/$arch/rq" 155 | 156 | if [ "$interactive" = true ] 157 | then 158 | msg "Where should rq be installed? (default: $path)" 159 | read -rp 'Path: ' new_path < /dev/tty 160 | if [ -n "$new_path" ] 161 | then 162 | path=$(eval echo "$new_path") 163 | fi 164 | fi 165 | 166 | if [ -f "$path" ] 167 | then 168 | if command -v md5 > /dev/null 169 | then md5tool=md5 170 | elif command -v md5sum > /dev/null 171 | then md5tool=md5sum 172 | fi 173 | 174 | if command -v python2 > /dev/null 175 | then pythontool=python2 176 | elif command -v python > /dev/null 177 | then pythontool=python 178 | fi 179 | 180 | if [ -n "$md5tool" ] 181 | then 182 | checksum=$("$pythontool" - "$path" <, 30 | 31 | /// A query indicating how to transform each record. 32 | pub arg_query: Option, 33 | 34 | /// Force stylistic output formatting. Can be one of 'compact', 35 | /// 'readable' (with color) or 'indented' (without color) and the default is 36 | /// inferred from the terminal environment. 37 | #[structopt(long = "format")] 38 | pub flag_format: Option, 39 | #[structopt(long = "codec")] 40 | pub flag_codec: Option, 41 | 42 | /// Input is an Apache Avro container file. 43 | #[structopt(short = "a", long = "input-avro")] 44 | pub flag_input_avro: bool, 45 | /// Input is a series of CBOR values. 46 | #[structopt(short = "c", long = "input-cbor")] 47 | pub flag_input_cbor: bool, 48 | /// Input is white-space separated JSON values (default). 49 | #[structopt(short = "j", long = "input-json")] 50 | pub flag_input_json: bool, 51 | /// Input is CSV. 52 | #[structopt(short = "v", long = "input-csv")] 53 | pub flag_input_csv: bool, 54 | /// Input is formatted as MessagePack. 55 | #[structopt(short = "m", long = "input-message-pack")] 56 | pub flag_input_message_pack: bool, 57 | #[structopt(short = "p", long = "input-protobuf")] 58 | pub flag_input_protobuf: Option, 59 | /// Input is plain text. 60 | #[structopt(short = "r", long = "input-raw")] 61 | pub flag_input_raw: bool, 62 | /// Input is formatted as TOML document. 63 | #[structopt(short = "t", long = "input-toml")] 64 | pub flag_input_toml: bool, 65 | /// Input is a series of YAML documents. 66 | #[structopt(short = "y", long = "input-yaml")] 67 | pub flag_input_yaml: bool, 68 | 69 | #[structopt(short = "A", long = "output-avro")] 70 | pub flag_output_avro: Option, 71 | #[structopt(short = "C", long = "output-cbor")] 72 | pub flag_output_cbor: bool, 73 | #[structopt(short = "J", long = "output-json")] 74 | pub flag_output_json: bool, 75 | #[structopt(short = "R", long = "output-raw")] 76 | pub flag_output_raw: bool, 77 | #[structopt(short = "V", long = "output-csv")] 78 | pub flag_output_csv: bool, 79 | #[structopt(short = "M", long = "output-message-pack")] 80 | pub flag_output_message_pack: bool, 81 | #[structopt(short = "P", long = "output-protobuf")] 82 | pub flag_output_protobuf: Option, 83 | #[structopt(short = "T", long = "output-toml")] 84 | pub flag_output_toml: bool, 85 | #[structopt(short = "Y", long = "output-yaml")] 86 | pub flag_output_yaml: bool, 87 | 88 | #[structopt(short = "l", long = "log")] 89 | pub flag_log: Option, 90 | #[structopt(short = "q", long = "quiet")] 91 | pub flag_quiet: bool, 92 | #[structopt(long = "trace")] 93 | pub flag_trace: bool, 94 | } 95 | 96 | #[derive(Debug, StructOpt)] 97 | pub enum Subcmd { 98 | #[structopt(name = "protobuf")] 99 | Protobuf { 100 | #[structopt(subcommand)] 101 | subcmd: ProtobufSubcmd, 102 | }, 103 | } 104 | 105 | #[derive(Debug, StructOpt)] 106 | pub enum ProtobufSubcmd { 107 | #[structopt(name = "add")] 108 | Add { 109 | schema: path::PathBuf, 110 | #[structopt(short = "b", long = "base")] 111 | base: Option, 112 | }, 113 | } 114 | 115 | #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] 116 | pub enum Format { 117 | Compact, 118 | Readable, 119 | Indented, 120 | } 121 | 122 | fn main() { 123 | use structopt::StructOpt; 124 | 125 | let args: Options = match Options::clap().get_matches_safe() { 126 | Err(e) => { 127 | match e.kind { 128 | structopt::clap::ErrorKind::HelpDisplayed => set_ran_cmd("help").unwrap(), 129 | structopt::clap::ErrorKind::VersionDisplayed => set_ran_cmd("version").unwrap(), 130 | _ => (), 131 | } 132 | e.exit() 133 | } 134 | Ok(a) => Options::from_clap(&a), 135 | }; 136 | 137 | setup_log(args.flag_log.as_ref().map(String::as_ref), args.flag_quiet); 138 | 139 | main_with_args(&args).unwrap_or_else(|e| log_error(&args, &e)); 140 | } 141 | 142 | fn main_with_args(args: &Options) -> rq::error::Result<()> { 143 | match args.subcmd { 144 | Some(Subcmd::Protobuf { ref subcmd }) => match subcmd { 145 | ProtobufSubcmd::Add { schema, base } => { 146 | let base = base 147 | .as_ref() 148 | .map_or_else(|| path::Path::new("."), |p| p.as_path()); 149 | let paths = rq::config::Paths::new()?; 150 | rq::proto_index::add_file(&paths, base, schema) 151 | } 152 | }, 153 | None => run(args), 154 | } 155 | } 156 | 157 | fn run(args: &Options) -> rq::error::Result<()> { 158 | let stdin = io::stdin(); 159 | let mut input = stdin.lock(); 160 | 161 | if let Some(ref name) = args.flag_input_protobuf { 162 | let paths = rq::config::Paths::new()?; 163 | let proto_descriptors = load_descriptors(&paths)?; 164 | let stream = protobuf::CodedInputStream::new(&mut input); 165 | let source = rq::value::protobuf::source(&proto_descriptors, name, stream)?; 166 | run_source(args, source) 167 | } else if args.flag_input_avro { 168 | let source = rq::value::avro::source(&mut input)?; 169 | run_source(args, source) 170 | } else if args.flag_input_cbor { 171 | let source = rq::value::cbor::source(&mut input); 172 | run_source(args, source) 173 | } else if args.flag_input_message_pack { 174 | let source = rq::value::messagepack::source(&mut input); 175 | run_source(args, source) 176 | } else if args.flag_input_toml { 177 | let source = rq::value::toml::source(&mut input)?; 178 | run_source(args, source) 179 | } else if args.flag_input_yaml { 180 | let source = rq::value::yaml::source(&mut input); 181 | run_source(args, source) 182 | } else if args.flag_input_raw { 183 | let source = rq::value::raw::source(&mut input); 184 | run_source(args, source) 185 | } else if args.flag_input_csv { 186 | if env::args().skip(1).any(|v| v == "-v") && !has_ran_cmd("help")? { 187 | warn!("You started rq -v, which puts it in CSV input mode."); 188 | warn!("It's now waiting for CSV input, which might not be what you wanted."); 189 | warn!( 190 | "Specify --input-csv explicitly or run rq --help once to suppress this \ 191 | warning." 192 | ); 193 | } 194 | let source = rq::value::csv::source(&mut input); 195 | run_source(args, source) 196 | } else { 197 | if !args.flag_input_json && !has_ran_cmd("help")? { 198 | warn!("You started rq without any input flags, which puts it in JSON input mode."); 199 | warn!("It's now waiting for JSON input, which might not be what you wanted."); 200 | warn!( 201 | "Specify (-j|--input-json) explicitly or run rq --help once to suppress this \ 202 | warning." 203 | ); 204 | } 205 | let source = rq::value::json::source(&mut input); 206 | run_source(args, source) 207 | } 208 | } 209 | 210 | fn run_source(args: &Options, source: I) -> rq::error::Result<()> 211 | where 212 | I: rq::value::Source, 213 | { 214 | let mut output = io::stdout(); 215 | 216 | let format = args.flag_format.unwrap_or_else(infer_format); 217 | 218 | macro_rules! dispatch_format { 219 | ($compact:expr, $readable:expr, $indented:expr) => { 220 | match format { 221 | Format::Compact => { 222 | let sink = $compact(&mut output); 223 | run_source_sink(source, sink) 224 | } 225 | Format::Readable => { 226 | let sink = $readable(&mut output); 227 | run_source_sink(source, sink) 228 | } 229 | Format::Indented => { 230 | let sink = $indented(&mut output); 231 | run_source_sink(source, sink) 232 | } 233 | } 234 | }; 235 | } 236 | 237 | if args.flag_output_protobuf.is_some() { 238 | Err(rq::error::Error::unimplemented( 239 | "protobuf serialization".to_owned(), 240 | )) 241 | } else if let Some(ref schema_filename) = args.flag_output_avro { 242 | use std::str::FromStr; 243 | 244 | let schema = read_avro_schema_from_file(path::Path::new(schema_filename))?; 245 | let codec_string = if let Some(ref c) = args.flag_codec { 246 | c.as_str() 247 | } else { 248 | "null" 249 | }; 250 | let codec = if let Ok(v) = avro_rs::Codec::from_str(codec_string) { 251 | v 252 | } else { 253 | return Err(rq::error::Error::Message(format!( 254 | "illegal Avro codec: {}", 255 | codec_string 256 | ))); 257 | }; 258 | let sink = rq::value::avro::sink(&schema, &mut output, codec)?; 259 | run_source_sink(source, sink) 260 | } else if args.flag_output_cbor { 261 | let sink = rq::value::cbor::sink(&mut output); 262 | run_source_sink(source, sink) 263 | } else if args.flag_output_message_pack { 264 | let sink = rq::value::messagepack::sink(&mut output); 265 | run_source_sink(source, sink) 266 | } else if args.flag_output_toml { 267 | // TODO: add TOML ugly printing eventually; now it's always "readable" 268 | dispatch_format!( 269 | rq::value::toml::sink, 270 | rq::value::toml::sink, 271 | rq::value::toml::sink 272 | ) 273 | } else if args.flag_output_yaml { 274 | // TODO: add YAML ugly printing eventually; now it's always "readable" 275 | dispatch_format!( 276 | rq::value::yaml::sink, 277 | rq::value::yaml::sink, 278 | rq::value::yaml::sink 279 | ) 280 | } else if args.flag_output_raw { 281 | let sink = rq::value::raw::sink(&mut output); 282 | run_source_sink(source, sink) 283 | } else if args.flag_output_csv { 284 | let sink = rq::value::csv::sink(&mut output); 285 | run_source_sink(source, sink) 286 | } else { 287 | dispatch_format!( 288 | rq::value::json::sink_compact, 289 | rq::value::json::sink_readable, 290 | rq::value::json::sink_indented 291 | ) 292 | } 293 | } 294 | 295 | fn read_avro_schema_from_file(path: &path::Path) -> rq::error::Result { 296 | let mut file = fs::File::open(path)?; 297 | let mut buffer = String::new(); 298 | file.read_to_string(&mut buffer)?; 299 | avro_rs::Schema::parse_str(&buffer) 300 | .map_err(|e| rq::error::Error::Avro(rq::error::Avro::downcast(e))) 301 | } 302 | 303 | fn run_source_sink(mut source: I, mut sink: O) -> rq::error::Result<()> 304 | where 305 | I: rq::value::Source, 306 | O: rq::value::Sink, 307 | { 308 | while let Some(result) = rq::value::Source::read(&mut source)? { 309 | sink.write(result)?; 310 | } 311 | Ok(()) 312 | } 313 | 314 | fn load_descriptors( 315 | paths: &rq::config::Paths, 316 | ) -> rq::error::Result { 317 | let descriptors_proto = rq::proto_index::compile_descriptor_set(paths)?; 318 | Ok(serde_protobuf::descriptor::Descriptors::from_proto( 319 | &descriptors_proto, 320 | )) 321 | } 322 | 323 | fn infer_format() -> Format { 324 | if atty::is(atty::Stream::Stdout) { 325 | Format::Readable 326 | } else { 327 | Format::Compact 328 | } 329 | } 330 | 331 | fn has_ran_cmd(cmd: &str) -> rq::error::Result { 332 | let paths = match rq::config::Paths::new() { 333 | Ok(paths) => paths, 334 | Err(_) => return Ok(false), 335 | }; 336 | paths 337 | .find_config(&format!("{}{}", "has-ran-", cmd)) 338 | .map(|v| !v.is_empty()) 339 | .map_err(From::from) 340 | } 341 | 342 | fn set_ran_cmd(cmd: &str) -> rq::error::Result<()> { 343 | let paths = match rq::config::Paths::new() { 344 | Ok(paths) => paths, 345 | Err(_) => return Ok(()), 346 | }; 347 | 348 | let file = paths.preferred_config(format!("{}{}", "has-ran-", cmd)); 349 | 350 | if let Some(parent) = file.parent() { 351 | fs::create_dir_all(parent)?; 352 | } 353 | 354 | fs::File::create(&file)?; 355 | 356 | Ok(()) 357 | } 358 | 359 | fn log_error(args: &Options, error: &rq::error::Error) { 360 | use failure::Fail; 361 | 362 | let main_str = format!("{}", error); 363 | let mut main_lines = main_str.lines(); 364 | error!("Encountered: {}", main_lines.next().unwrap()); 365 | for line in main_lines { 366 | error!(" {}", line); 367 | } 368 | for e in ::iter_causes(error) { 369 | let sub_str = format!("{}", e); 370 | let mut sub_lines = sub_str.lines(); 371 | error!("Caused by: {}", sub_lines.next().unwrap()); 372 | for line in sub_lines { 373 | error!(" {}", line); 374 | } 375 | } 376 | 377 | if args.flag_trace || env::var("RUST_BACKTRACE").as_ref().map(String::as_str) == Ok("1") { 378 | error!(""); 379 | if let Some(backtrace) = error.backtrace() { 380 | error!("Backtrace:"); 381 | for line in format!("{:?}", backtrace).lines() { 382 | error!(" {}", line); 383 | } 384 | } else { 385 | error!("(No backtrace available)"); 386 | } 387 | } else { 388 | error!("(Re-run with --trace or RUST_BACKTRACE=1 for a backtrace)"); 389 | } 390 | } 391 | 392 | fn setup_log(spec: Option<&str>, quiet: bool) { 393 | let mut builder = env_logger::Builder::new(); 394 | 395 | if quiet { 396 | builder.filter(None, log::LevelFilter::Off); 397 | } else if let Some(s) = spec { 398 | builder.parse_filters(s); 399 | } else if let Ok(s) = env::var("RUST_LOG") { 400 | builder.parse_filters(&s); 401 | } else { 402 | builder.filter(None, log::LevelFilter::Info); 403 | }; 404 | 405 | builder.format(format_log_record); 406 | 407 | builder.init(); 408 | } 409 | 410 | impl str::FromStr for Format { 411 | type Err = failure::Error; 412 | 413 | fn from_str(s: &str) -> Result { 414 | match s { 415 | "compact" => Ok(Self::Compact), 416 | "readable" => Ok(Self::Readable), 417 | "indented" => Ok(Self::Indented), 418 | _ => Err(failure::err_msg(format!("unrecognized format: {}", s))), 419 | } 420 | } 421 | } 422 | 423 | fn format_log_record( 424 | formatter: &mut env_logger::fmt::Formatter, 425 | record: &log::Record, 426 | ) -> io::Result<()> { 427 | use ansi_term::ANSIStrings; 428 | use ansi_term::Colour; 429 | use ansi_term::Style; 430 | 431 | if atty::is(atty::Stream::Stderr) { 432 | let normal = Style::new(); 433 | let (front, back) = match record.level() { 434 | log::Level::Error => (Colour::Red.normal(), Colour::Red.dimmed()), 435 | log::Level::Warn => (Colour::Yellow.normal(), Colour::Yellow.dimmed()), 436 | log::Level::Info => (Colour::Blue.normal(), Colour::Blue.dimmed()), 437 | log::Level::Debug => (Colour::Purple.normal(), Colour::Purple.dimmed()), 438 | log::Level::Trace => (Colour::White.dimmed(), Colour::Black.normal()), 439 | }; 440 | 441 | let strings = &[ 442 | back.paint("["), 443 | front.paint(format!("{}", record.level())), 444 | back.paint("]"), 445 | normal.paint(" "), 446 | back.paint("["), 447 | front.paint(record.module_path().unwrap_or("")), 448 | back.paint("]"), 449 | normal.paint(" "), 450 | front.paint(format!("{}", record.args())), 451 | ]; 452 | 453 | writeln!(formatter, "{}", ANSIStrings(strings)) 454 | } else { 455 | writeln!( 456 | formatter, 457 | "[{}] [{}] {}", 458 | record.level(), 459 | record.module_path().unwrap_or(""), 460 | record.args() 461 | ) 462 | } 463 | } 464 | 465 | #[cfg(test)] 466 | mod test { 467 | use super::*; 468 | 469 | fn parse_args(args: &[&str]) -> Options { 470 | use structopt::StructOpt; 471 | let a = Options::from_iter_safe(args.iter()).unwrap(); 472 | println!("{:?}", a); 473 | a 474 | } 475 | 476 | #[test] 477 | fn test_docopt_kitchen_sink() { 478 | let a = parse_args(&["rq", "-l", "info", "-jP", ".foo.Bar", "select x"]); 479 | assert!(a.flag_input_json); 480 | assert_eq!(a.flag_output_protobuf, Some(".foo.Bar".to_owned())); 481 | assert_eq!(a.flag_log, Some("info".to_owned())); 482 | assert_eq!(a.arg_query, Some("select x".to_owned())); 483 | } 484 | 485 | #[test] 486 | fn test_docopt_no_args() { 487 | parse_args(&["rq"]); 488 | } 489 | 490 | #[test] 491 | #[cfg_attr( 492 | all(target_arch = "x86", target_pointer_width = "32", target_env = "musl"), 493 | ignore 494 | )] 495 | #[should_panic(expected = "Help")] 496 | fn test_docopt_help() { 497 | parse_args(&["rq", "--help"]); 498 | } 499 | 500 | #[test] 501 | fn test_docopt_input_json() { 502 | let a = parse_args(&["rq", "-j"]); 503 | assert!(a.flag_input_json); 504 | } 505 | 506 | #[test] 507 | fn test_docopt_input_json_long() { 508 | let a = parse_args(&["rq", "--input-json"]); 509 | assert!(a.flag_input_json); 510 | } 511 | 512 | #[test] 513 | fn test_docopt_output_json() { 514 | let a = parse_args(&["rq", "-J"]); 515 | assert!(a.flag_output_json); 516 | } 517 | 518 | #[test] 519 | fn test_docopt_output_json_long() { 520 | let a = parse_args(&["rq", "--output-json"]); 521 | assert!(a.flag_output_json); 522 | } 523 | 524 | #[test] 525 | fn test_docopt_input_raw() { 526 | let a = parse_args(&["rq", "-r"]); 527 | assert!(a.flag_input_raw); 528 | } 529 | 530 | #[test] 531 | fn test_docopt_input_raw_long() { 532 | let a = parse_args(&["rq", "--input-raw"]); 533 | assert!(a.flag_input_raw); 534 | } 535 | 536 | #[test] 537 | fn test_docopt_output_raw() { 538 | let a = parse_args(&["rq", "-R"]); 539 | assert!(a.flag_output_raw); 540 | } 541 | 542 | #[test] 543 | fn test_docopt_output_raw_long() { 544 | let a = parse_args(&["rq", "--output-raw"]); 545 | assert!(a.flag_output_raw); 546 | } 547 | 548 | #[test] 549 | fn test_docopt_input_csv() { 550 | let a = parse_args(&["rq", "-v"]); 551 | assert!(a.flag_input_csv); 552 | } 553 | 554 | #[test] 555 | fn test_docopt_input_csv_long() { 556 | let a = parse_args(&["rq", "--input-csv"]); 557 | assert!(a.flag_input_csv); 558 | } 559 | 560 | #[test] 561 | fn test_docopt_output_csv() { 562 | let a = parse_args(&["rq", "-V"]); 563 | assert!(a.flag_output_csv); 564 | } 565 | 566 | #[test] 567 | fn test_docopt_output_csv_long() { 568 | let a = parse_args(&["rq", "--output-csv"]); 569 | assert!(a.flag_output_csv); 570 | } 571 | 572 | #[test] 573 | fn test_docopt_input_cbor() { 574 | let a = parse_args(&["rq", "-c"]); 575 | assert!(a.flag_input_cbor); 576 | } 577 | 578 | #[test] 579 | fn test_docopt_input_cbor_long() { 580 | let a = parse_args(&["rq", "--input-cbor"]); 581 | assert!(a.flag_input_cbor); 582 | } 583 | 584 | #[test] 585 | fn test_docopt_output_cbor() { 586 | let a = parse_args(&["rq", "-C"]); 587 | assert!(a.flag_output_cbor); 588 | } 589 | 590 | #[test] 591 | fn test_docopt_output_cbor_long() { 592 | let a = parse_args(&["rq", "--output-cbor"]); 593 | assert!(a.flag_output_cbor); 594 | } 595 | 596 | #[test] 597 | fn test_docopt_input_protobuf() { 598 | let a = parse_args(&["rq", "-p", ".foo.Bar"]); 599 | assert_eq!(a.flag_input_protobuf, Some(".foo.Bar".to_owned())); 600 | } 601 | 602 | #[test] 603 | fn test_docopt_input_protobuf_long() { 604 | let a = parse_args(&["rq", "--input-protobuf", ".foo.Bar"]); 605 | assert_eq!(a.flag_input_protobuf, Some(".foo.Bar".to_owned())); 606 | } 607 | 608 | #[test] 609 | fn test_docopt_output_protobuf() { 610 | let a = parse_args(&["rq", "-P", ".foo.Bar"]); 611 | assert_eq!(a.flag_output_protobuf, Some(".foo.Bar".to_owned())); 612 | } 613 | 614 | #[test] 615 | fn test_docopt_output_protobuf_long() { 616 | let a = parse_args(&["rq", "--output-protobuf", ".foo.Bar"]); 617 | assert_eq!(a.flag_output_protobuf, Some(".foo.Bar".to_owned())); 618 | } 619 | 620 | #[test] 621 | fn test_docopt_protobuf_add_schema() { 622 | let a = parse_args(&["rq", "-l", "info", "protobuf", "add", "schema.proto"]); 623 | assert_eq!(a.flag_log, Some("info".to_owned())); 624 | assert_eq!( 625 | Some(path::PathBuf::from("schema.proto")), 626 | match a.subcmd { 627 | Some(Subcmd::Protobuf { subcmd }) => match subcmd { 628 | ProtobufSubcmd::Add { schema, .. } => Some(schema), 629 | }, 630 | _ => None, 631 | } 632 | ); 633 | } 634 | 635 | #[test] 636 | fn test_docopt_format_compact() { 637 | let a = parse_args(&["rq", "--format", "compact"]); 638 | assert_eq!(a.flag_format, Some(Format::Compact)); 639 | } 640 | 641 | #[test] 642 | fn test_docopt_format_readable() { 643 | let a = parse_args(&["rq", "--format", "readable"]); 644 | assert_eq!(a.flag_format, Some(Format::Readable)); 645 | } 646 | 647 | #[test] 648 | fn test_docopt_format_indented() { 649 | let a = parse_args(&["rq", "--format", "indented"]); 650 | assert_eq!(a.flag_format, Some(Format::Indented)); 651 | } 652 | } 653 | -------------------------------------------------------------------------------- /src/config.rs: -------------------------------------------------------------------------------- 1 | use crate::error; 2 | 3 | use glob; 4 | use std::env; 5 | use std::path; 6 | 7 | #[derive(Debug)] 8 | pub struct Paths { 9 | config: path::PathBuf, 10 | cache: path::PathBuf, 11 | data: path::PathBuf, 12 | } 13 | 14 | impl Paths { 15 | pub fn new() -> error::Result { 16 | match env::var_os("RQ_SYSTEM_DIR") { 17 | Some(basepath) => { 18 | let basepath = path::Path::new(&basepath); 19 | Ok(Self { 20 | config: basepath.join("config"), 21 | cache: basepath.join("cache"), 22 | data: basepath.join("data"), 23 | }) 24 | }, 25 | None => match directories::ProjectDirs::from("io", "dflemstr", "rq") { 26 | Some(dirs) => Ok(Self { 27 | config: dirs.config_dir().into(), 28 | cache: dirs.cache_dir().into(), 29 | data: dirs.data_dir().into(), 30 | }), 31 | None => Err(error::Error::Message( 32 | "The environment variable RQ_SYSTEM_DIR is unspecified and no home directory is known".to_string() 33 | )), 34 | }, 35 | } 36 | } 37 | 38 | pub fn preferred_config

(&self, path: P) -> path::PathBuf 39 | where 40 | P: AsRef, 41 | { 42 | let mut result = self.config.clone(); 43 | result.push(path); 44 | result 45 | } 46 | 47 | pub fn preferred_cache

(&self, path: P) -> path::PathBuf 48 | where 49 | P: AsRef, 50 | { 51 | let mut result = self.cache.clone(); 52 | result.push(path); 53 | result 54 | } 55 | 56 | pub fn preferred_data

(&self, path: P) -> path::PathBuf 57 | where 58 | P: AsRef, 59 | { 60 | let mut result = self.data.clone(); 61 | result.push(path); 62 | result 63 | } 64 | 65 | pub fn find_config(&self, pattern: &str) -> error::Result> { 66 | find(&self.config, pattern) 67 | } 68 | 69 | pub fn find_data(&self, pattern: &str) -> error::Result> { 70 | find(&self.data, pattern) 71 | } 72 | } 73 | 74 | fn find(home: &path::Path, pattern: &str) -> error::Result> { 75 | let mut result = Vec::new(); 76 | run_pattern(home, pattern, &mut result)?; 77 | Ok(result) 78 | } 79 | 80 | fn run_pattern( 81 | dir: &path::Path, 82 | pattern: &str, 83 | result: &mut Vec, 84 | ) -> error::Result<()> { 85 | let full_pattern = format!("{}/{}", dir.to_string_lossy(), pattern); 86 | 87 | for entry in glob::glob(&full_pattern)? { 88 | result.push(entry?); 89 | } 90 | 91 | Ok(()) 92 | } 93 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | use csv; 2 | use glob; 3 | use protobuf; 4 | use rmpv; 5 | use serde_cbor; 6 | use serde_hjson; 7 | use serde_json; 8 | use serde_protobuf; 9 | use serde_yaml; 10 | use std::io; 11 | use std::string; 12 | use toml; 13 | #[cfg(feature = "v8")] 14 | use v8; 15 | use yaml_rust; 16 | 17 | use std::result; 18 | 19 | pub type Result = result::Result; 20 | 21 | #[derive(Debug, Fail)] 22 | pub enum Error { 23 | #[fail(display = "protobuf error")] 24 | Protobuf(#[cause] serde_protobuf::error::Error), 25 | #[fail(display = "IO error")] 26 | Io(#[cause] io::Error), 27 | #[fail(display = "UTF-8 error")] 28 | Utf8(#[cause] string::FromUtf8Error), 29 | #[fail(display = "native protobuf error")] 30 | NativeProtobuf(#[cause] protobuf::ProtobufError), 31 | #[fail(display = "MessagePack encode error")] 32 | MessagePackEncode(#[cause] rmpv::encode::Error), 33 | #[fail(display = "Avro error")] 34 | Avro(#[cause] Avro), 35 | #[fail(display = "CBOR error")] 36 | Cbor(#[cause] serde_cbor::error::Error), 37 | #[fail(display = "HJSON error")] 38 | Hjson(#[cause] serde_hjson::Error), 39 | #[fail(display = "JSON error")] 40 | Json(#[cause] serde_json::Error), 41 | #[fail(display = "YAML error")] 42 | Yaml(#[cause] serde_yaml::Error), 43 | #[fail(display = "YAML scan error")] 44 | YamlScan(#[cause] yaml_rust::ScanError), 45 | #[fail(display = "TOML deserialize error")] 46 | TomlDeserialize(#[cause] toml::de::Error), 47 | #[fail(display = "TOML serialize error")] 48 | TomlSerialize(#[cause] toml::ser::Error), 49 | #[fail(display = "glob error")] 50 | Glob(#[cause] glob::GlobError), 51 | #[fail(display = "glob pattern error")] 52 | GlobPattern(#[cause] glob::PatternError), 53 | #[fail(display = "CSV error")] 54 | Csv(#[cause] csv::Error), 55 | #[fail(display = "MessagePack decode error")] 56 | MessagePackDecode(#[cause] rmpv::decode::Error), 57 | #[fail(display = "unimplemented: {}", msg)] 58 | Unimplemented { msg: String }, 59 | #[fail(display = "illegal state: {}", msg)] 60 | IllegalState { msg: String }, 61 | #[fail(display = "format error: {}", msg)] 62 | Format { msg: String }, 63 | #[fail(display = "internal error: {}", _0)] 64 | Internal(&'static str), 65 | #[fail(display = "{}", _0)] 66 | Message(String), 67 | } 68 | 69 | #[derive(Debug, Fail)] 70 | pub enum Avro { 71 | #[fail(display = "decode error")] 72 | Decode(#[cause] avro_rs::DecodeError), 73 | #[fail(display = "error when parsing schema")] 74 | ParseSchema(#[cause] avro_rs::ParseSchemaError), 75 | #[fail(display = "schema resolution error")] 76 | SchemaResolution(#[cause] avro_rs::SchemaResolutionError), 77 | #[fail(display = "validation error")] 78 | Validation(#[cause] avro_rs::ValidationError), 79 | #[fail(display = "{}", message)] 80 | Custom { message: String }, 81 | } 82 | 83 | impl Error { 84 | pub fn unimplemented(msg: String) -> Self { 85 | Self::Unimplemented { msg } 86 | } 87 | 88 | pub fn illegal_state(msg: String) -> Self { 89 | Self::IllegalState { msg } 90 | } 91 | } 92 | 93 | impl Avro { 94 | pub fn downcast(error: failure::Error) -> Self { 95 | let error = match error.downcast::() { 96 | Ok(error) => return Self::Decode(error), 97 | Err(error) => error, 98 | }; 99 | 100 | let error = match error.downcast::() { 101 | Ok(error) => return Self::ParseSchema(error), 102 | Err(error) => error, 103 | }; 104 | 105 | let error = match error.downcast::() { 106 | Ok(error) => return Self::SchemaResolution(error), 107 | Err(error) => error, 108 | }; 109 | 110 | let error = match error.downcast::() { 111 | Ok(error) => return Self::Validation(error), 112 | Err(error) => error, 113 | }; 114 | 115 | Self::Custom { 116 | message: error.to_string(), 117 | } 118 | } 119 | } 120 | 121 | macro_rules! gen_from { 122 | ($t:ty, $i:ident) => { 123 | impl From<$t> for Error { 124 | fn from(e: $t) -> Self { 125 | Self::$i(e) 126 | } 127 | } 128 | }; 129 | } 130 | 131 | gen_from!(serde_protobuf::error::Error, Protobuf); 132 | gen_from!(io::Error, Io); 133 | #[cfg(feature = "js")] 134 | gen_from!(v8::error::Error, Js); 135 | gen_from!(string::FromUtf8Error, Utf8); 136 | gen_from!(protobuf::ProtobufError, NativeProtobuf); 137 | gen_from!(rmpv::encode::Error, MessagePackEncode); 138 | gen_from!(serde_cbor::error::Error, Cbor); 139 | gen_from!(serde_hjson::Error, Hjson); 140 | gen_from!(serde_json::Error, Json); 141 | gen_from!(serde_yaml::Error, Yaml); 142 | gen_from!(yaml_rust::ScanError, YamlScan); 143 | gen_from!(toml::de::Error, TomlDeserialize); 144 | gen_from!(toml::ser::Error, TomlSerialize); 145 | gen_from!(glob::GlobError, Glob); 146 | gen_from!(glob::PatternError, GlobPattern); 147 | gen_from!(csv::Error, Csv); 148 | gen_from!(rmpv::decode::Error, MessagePackDecode); 149 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! `rq` (Record Query) is a library and command-line tool for manipulating structured (record) 2 | //! data. 3 | 4 | // For pest parser generation 5 | #![recursion_limit = "1024"] 6 | #![deny(warnings)] 7 | #![deny(clippy::all)] 8 | #![deny( 9 | missing_debug_implementations, 10 | trivial_casts, 11 | trivial_numeric_casts, 12 | unused_extern_crates, 13 | unused_import_braces, 14 | unused_qualifications 15 | )] 16 | 17 | #[macro_use] 18 | extern crate failure; 19 | #[macro_use] 20 | extern crate log; 21 | #[macro_use] 22 | extern crate pest; 23 | 24 | pub mod config; 25 | pub mod error; 26 | pub mod proto_index; 27 | pub mod value; 28 | 29 | pub const VERSION: &str = env!("VERGEN_GIT_SEMVER"); 30 | 31 | #[doc(hidden)] 32 | #[deprecated(since = "1.0.1", note = "use VERSION instead")] 33 | pub const GIT_VERSION: &str = VERSION; 34 | -------------------------------------------------------------------------------- /src/proto_index.rs: -------------------------------------------------------------------------------- 1 | use crate::config; 2 | use crate::error; 3 | 4 | use protobuf; 5 | use std::cmp; 6 | use std::fs; 7 | use std::path; 8 | use std::process; 9 | 10 | pub fn add_file( 11 | paths: &config::Paths, 12 | relative_to: &path::Path, 13 | file: &path::Path, 14 | ) -> error::Result<()> { 15 | let rel_file = file 16 | .strip_prefix(relative_to) 17 | .unwrap_or_else(|_| file.file_name().map_or(file, path::Path::new)); 18 | let target = paths.preferred_data("proto").join(rel_file); 19 | 20 | if let Some(parent) = target.parent() { 21 | trace!("Creating directory {:?}", parent); 22 | fs::create_dir_all(parent)?; 23 | } 24 | 25 | fs::copy(file, &target)?; 26 | info!("Added proto file as {:?}", target); 27 | Ok(()) 28 | } 29 | 30 | pub fn compile_descriptor_set( 31 | paths: &config::Paths, 32 | ) -> error::Result { 33 | let proto_includes = paths.find_data("proto")?; 34 | let proto_files = paths.find_data("proto/**/*.proto")?; 35 | let cache = paths.preferred_cache("descriptor-cache.pb"); 36 | 37 | debug!("Proto includes: {:?}", proto_includes); 38 | debug!("Proto files: {:?}", proto_files); 39 | debug!("Proto cache location: {:?}", cache); 40 | 41 | if is_cache_stale(&cache, &proto_files)? { 42 | info!("Proto descriptor cache is stale; recomputing"); 43 | 44 | if let Some(parent) = cache.parent() { 45 | trace!("Creating directory {:?}", parent); 46 | fs::create_dir_all(parent)?; 47 | } 48 | 49 | let include_args = proto_includes 50 | .into_iter() 51 | .map(|p| format!("-I{}", p.to_string_lossy())) 52 | .collect::>(); 53 | 54 | let status = process::Command::new("protoc") 55 | .arg("-o") 56 | .arg(&cache) 57 | .args(&include_args) 58 | .args(&proto_files) 59 | .status()?; 60 | if !status.success() { 61 | panic!("protoc descriptor compilation failed"); 62 | } 63 | 64 | trace!("Proto descriptor cache regenerated"); 65 | } 66 | 67 | let mut cache_file = fs::File::open(&cache)?; 68 | let descriptor_set = protobuf::Message::parse_from_reader(&mut cache_file)?; 69 | 70 | trace!("Successfully parsed descriptor set from cache"); 71 | 72 | Ok(descriptor_set) 73 | } 74 | 75 | fn is_cache_stale

(cache: &path::Path, proto_files: &[P]) -> error::Result 76 | where 77 | P: AsRef, 78 | { 79 | if cache.exists() { 80 | let cache_metadata = fs::metadata(&cache)?; 81 | let cache_mtime = cache_metadata.modified()?; 82 | let mut max_proto_mtime = std::time::SystemTime::UNIX_EPOCH; 83 | 84 | for proto_file in proto_files.iter() { 85 | let proto_metadata = fs::metadata(&proto_file)?; 86 | let proto_mtime = proto_metadata.modified()?; 87 | max_proto_mtime = cmp::max(max_proto_mtime, proto_mtime); 88 | } 89 | 90 | Ok(cache_mtime < max_proto_mtime) 91 | } else { 92 | Ok(true) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/value/avro.rs: -------------------------------------------------------------------------------- 1 | use crate::error; 2 | use crate::value; 3 | use avro_rs; 4 | use std; 5 | use std::fmt; 6 | use std::io; 7 | 8 | pub struct Source<'a, R>(avro_rs::Reader<'a, R>) 9 | where 10 | R: io::Read; 11 | 12 | pub struct Sink<'a, W>(avro_rs::Writer<'a, W>) 13 | where 14 | W: io::Write; 15 | 16 | #[inline] 17 | pub fn source<'a, R>(r: R) -> error::Result> 18 | where 19 | R: io::Read, 20 | { 21 | Ok(Source(avro_rs::Reader::new(r).map_err(|e| { 22 | error::Error::Avro(error::Avro::downcast(e)) 23 | })?)) 24 | } 25 | 26 | #[inline] 27 | pub fn sink(schema: &avro_rs::Schema, w: W, codec: avro_rs::Codec) -> error::Result> 28 | where 29 | W: io::Write, 30 | { 31 | Ok(Sink(avro_rs::Writer::with_codec(schema, w, codec))) 32 | } 33 | 34 | impl<'a, R> value::Source for Source<'a, R> 35 | where 36 | R: io::Read, 37 | { 38 | #[inline] 39 | fn read(&mut self) -> error::Result> { 40 | match self.0.next() { 41 | Some(Ok(v)) => Ok(Some(value_from_avro(v))), 42 | Some(Err(e)) => Err(error::Error::Avro(error::Avro::downcast(e))), 43 | None => Ok(None), 44 | } 45 | } 46 | } 47 | 48 | fn value_from_avro(value: avro_rs::types::Value) -> value::Value { 49 | use avro_rs::types::Value; 50 | match value { 51 | Value::Null => value::Value::Unit, 52 | Value::Boolean(v) => value::Value::Bool(v), 53 | Value::Int(v) => value::Value::I32(v), 54 | Value::Long(v) => value::Value::I64(v), 55 | Value::Float(v) => value::Value::from_f32(v), 56 | Value::Double(v) => value::Value::from_f64(v), 57 | Value::Bytes(v) | Value::Fixed(_, v) => value::Value::Bytes(v), 58 | Value::String(v) | Value::Enum(_, v) => value::Value::String(v), 59 | Value::Union(boxed) => value_from_avro(*boxed), 60 | Value::Array(v) => value::Value::Sequence(v.into_iter().map(value_from_avro).collect()), 61 | Value::Map(v) => value::Value::Map( 62 | v.into_iter() 63 | .map(|(k, v)| (value::Value::String(k), value_from_avro(v))) 64 | .collect(), 65 | ), 66 | Value::Record(v) => value::Value::Map( 67 | v.into_iter() 68 | .map(|(k, v)| (value::Value::String(k), value_from_avro(v))) 69 | .collect(), 70 | ), 71 | } 72 | } 73 | 74 | impl<'a, W> value::Sink for Sink<'a, W> 75 | where 76 | W: io::Write, 77 | { 78 | #[inline] 79 | fn write(&mut self, value: value::Value) -> error::Result<()> { 80 | self.0 81 | .append(value_to_avro(value)?) 82 | .map_err(|e| error::Error::Avro(error::Avro::downcast(e)))?; 83 | Ok(()) 84 | } 85 | } 86 | 87 | fn value_to_avro(value: value::Value) -> error::Result { 88 | use avro_rs::types::Value; 89 | use std::convert::TryFrom; 90 | match value { 91 | value::Value::Unit => Ok(Value::Null), 92 | value::Value::Bool(v) => Ok(Value::Boolean(v)), 93 | 94 | value::Value::I8(v) => Ok(Value::Int(i32::from(v))), 95 | value::Value::I16(v) => Ok(Value::Int(i32::from(v))), 96 | value::Value::I32(v) => Ok(Value::Int(v)), 97 | value::Value::I64(v) => Ok(Value::Long(v)), 98 | 99 | value::Value::U8(v) => Ok(Value::Int(i32::from(v))), 100 | value::Value::U16(v) => Ok(Value::Int(i32::from(v))), 101 | value::Value::U32(v) => Ok(Value::Long(i64::from(v))), 102 | value::Value::U64(v) => { 103 | if let Ok(v) = i64::try_from(v) { 104 | Ok(Value::Long(v)) 105 | } else { 106 | Err(error::Error::Format { 107 | msg: format!( 108 | "Avro output does not support unsigned 64 bit integer: {}", 109 | v 110 | ), 111 | }) 112 | } 113 | } 114 | 115 | value::Value::F32(ordered_float::OrderedFloat(v)) => Ok(Value::Float(v)), 116 | value::Value::F64(ordered_float::OrderedFloat(v)) => Ok(Value::Double(v)), 117 | 118 | value::Value::Char(v) => Ok(Value::String(format!("{}", v))), 119 | value::Value::String(v) => Ok(Value::String(v)), 120 | value::Value::Bytes(v) => Ok(Value::Bytes(v)), 121 | 122 | value::Value::Sequence(v) => Ok(Value::Array( 123 | v.into_iter() 124 | .map(value_to_avro) 125 | .collect::>>()?, 126 | )), 127 | value::Value::Map(v) => Ok(Value::Record( 128 | v.into_iter() 129 | .map(|(k, v)| match (value_to_string(k), value_to_avro(v)) { 130 | (Ok(k), Ok(v)) => Ok((k, v)), 131 | (Ok(_), Err(e)) | (Err(e), Ok(_)) | (Err(_), Err(e)) => Err(e), 132 | }) 133 | .collect::>>()?, 134 | )), 135 | } 136 | } 137 | 138 | fn value_to_string(value: value::Value) -> error::Result { 139 | match value { 140 | value::Value::Char(v) => Ok(format!("{}", v)), 141 | value::Value::String(v) => Ok(v), 142 | x => Err(error::Error::Format { 143 | msg: format!("Avro can only output string keys, got: {:?}", x), 144 | }), 145 | } 146 | } 147 | 148 | impl<'a, R> fmt::Debug for Source<'a, R> 149 | where 150 | R: io::Read, 151 | { 152 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 153 | f.debug_struct("AvroSource").finish() 154 | } 155 | } 156 | 157 | impl<'a, W> fmt::Debug for Sink<'a, W> 158 | where 159 | W: io::Write, 160 | { 161 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 162 | f.debug_struct("AvroSink").finish() 163 | } 164 | } 165 | 166 | impl<'a, W> Drop for Sink<'a, W> 167 | where 168 | W: io::Write, 169 | { 170 | fn drop(&mut self) { 171 | match self.0.flush() { 172 | Ok(_) => (), 173 | Err(error) => panic!("{}", error), 174 | } 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /src/value/cbor.rs: -------------------------------------------------------------------------------- 1 | use crate::error; 2 | 3 | use crate::value; 4 | use serde; 5 | use serde_cbor; 6 | use std::fmt; 7 | use std::io; 8 | 9 | pub struct Source(serde_cbor::de::Deserializer>) 10 | where 11 | R: io::Read; 12 | 13 | pub struct Sink(serde_cbor::ser::Serializer>) 14 | where 15 | W: io::Write; 16 | 17 | #[inline] 18 | pub fn source(r: R) -> Source 19 | where 20 | R: io::Read, 21 | { 22 | Source(serde_cbor::de::Deserializer::new( 23 | serde_cbor::de::IoRead::new(r), 24 | )) 25 | } 26 | 27 | #[inline] 28 | pub fn sink(w: W) -> Sink 29 | where 30 | W: io::Write, 31 | { 32 | Sink(serde_cbor::ser::Serializer::new( 33 | serde_cbor::ser::IoWrite::new(w), 34 | )) 35 | } 36 | 37 | impl value::Source for Source 38 | where 39 | R: io::Read, 40 | { 41 | #[inline] 42 | fn read(&mut self) -> error::Result> { 43 | match serde::Deserialize::deserialize(&mut self.0) { 44 | Ok(v) => Ok(Some(v)), 45 | Err(e) => match e.classify() { 46 | serde_cbor::error::Category::Eof => Ok(None), 47 | _ => Err(error::Error::from(e)), 48 | }, 49 | } 50 | } 51 | } 52 | 53 | impl value::Sink for Sink 54 | where 55 | W: io::Write, 56 | { 57 | #[inline] 58 | fn write(&mut self, v: value::Value) -> error::Result<()> { 59 | serde::Serialize::serialize(&v, &mut self.0).map_err(From::from) 60 | } 61 | } 62 | 63 | impl fmt::Debug for Source 64 | where 65 | R: io::Read, 66 | { 67 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 68 | f.debug_struct("CborSource").finish() 69 | } 70 | } 71 | 72 | impl fmt::Debug for Sink 73 | where 74 | W: io::Write, 75 | { 76 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 77 | f.debug_struct("CborSink").finish() 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/value/csv.rs: -------------------------------------------------------------------------------- 1 | use crate::error; 2 | use crate::value; 3 | use csv; 4 | use ordered_float; 5 | use std::fmt; 6 | use std::io; 7 | 8 | pub struct Source(csv::StringRecordsIntoIter) 9 | where 10 | R: io::Read; 11 | 12 | pub struct Sink(csv::Writer) 13 | where 14 | W: io::Write; 15 | 16 | #[inline] 17 | pub fn source(r: R) -> Source 18 | where 19 | R: io::Read, 20 | { 21 | Source( 22 | csv::ReaderBuilder::new() 23 | .has_headers(false) 24 | .from_reader(r) 25 | .into_records(), 26 | ) 27 | } 28 | 29 | #[inline] 30 | pub fn sink(w: W) -> Sink 31 | where 32 | W: io::Write, 33 | { 34 | Sink(csv::Writer::from_writer(w)) 35 | } 36 | 37 | impl value::Source for Source 38 | where 39 | R: io::Read, 40 | { 41 | #[inline] 42 | fn read(&mut self) -> error::Result> { 43 | match self.0.next() { 44 | Some(Ok(v)) => Ok(Some(value::Value::Sequence( 45 | v.iter() 46 | .map(|s| value::Value::String(s.to_string())) 47 | .collect(), 48 | ))), 49 | Some(Err(e)) => Err(error::Error::from(e)), 50 | None => Ok(None), 51 | } 52 | } 53 | } 54 | 55 | impl value::Sink for Sink 56 | where 57 | W: io::Write, 58 | { 59 | #[inline] 60 | fn write(&mut self, value: value::Value) -> error::Result<()> { 61 | match value { 62 | value::Value::Sequence(seq) => { 63 | let record: Vec = seq 64 | .into_iter() 65 | .map(value_to_csv) 66 | .collect::>>()?; 67 | self.0.write_record(record)?; 68 | Ok(()) 69 | } 70 | x => Err(error::Error::Format { 71 | msg: format!("csv can only output sequences, got: {:?}", x), 72 | }), 73 | } 74 | } 75 | } 76 | 77 | fn value_to_csv(value: value::Value) -> error::Result { 78 | match value { 79 | value::Value::Unit => Err(error::Error::Format { 80 | msg: "csv cannot output nested Unit".to_owned(), 81 | }), 82 | value::Value::Bool(v) => Ok(v.to_string()), 83 | 84 | value::Value::I8(v) => Ok(v.to_string()), 85 | value::Value::I16(v) => Ok(v.to_string()), 86 | value::Value::I32(v) => Ok(v.to_string()), 87 | value::Value::I64(v) => Ok(v.to_string()), 88 | 89 | value::Value::U8(v) => Ok(v.to_string()), 90 | value::Value::U16(v) => Ok(v.to_string()), 91 | value::Value::U32(v) => Ok(v.to_string()), 92 | value::Value::U64(v) => Ok(v.to_string()), 93 | 94 | value::Value::F32(ordered_float::OrderedFloat(v)) => Ok(v.to_string()), 95 | value::Value::F64(ordered_float::OrderedFloat(v)) => Ok(v.to_string()), 96 | 97 | value::Value::Char(v) => Ok(v.to_string()), 98 | value::Value::String(v) => Ok(v), 99 | value::Value::Bytes(_) => Err(error::Error::Format { 100 | msg: "csv cannot output nested bytes".to_owned(), 101 | }), 102 | 103 | value::Value::Sequence(_) => Err(error::Error::Format { 104 | msg: "csv cannot output nested sequences".to_owned(), 105 | }), 106 | value::Value::Map(_) => Err(error::Error::Format { 107 | msg: "csv cannot output nested maps".to_owned(), 108 | }), 109 | } 110 | } 111 | 112 | impl fmt::Debug for Source 113 | where 114 | R: io::Read, 115 | { 116 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 117 | f.debug_struct("CsvSource").finish() 118 | } 119 | } 120 | 121 | impl fmt::Debug for Sink 122 | where 123 | W: io::Write, 124 | { 125 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 126 | f.debug_struct("CsvSink").finish() 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/value/json.rs: -------------------------------------------------------------------------------- 1 | use ansi_term; 2 | use dtoa; 3 | 4 | use crate::error; 5 | use crate::value; 6 | use itoa; 7 | use serde; 8 | use serde_json; 9 | use std::fmt; 10 | use std::io; 11 | use std::str; 12 | 13 | pub struct Source<'de, R>( 14 | serde_json::StreamDeserializer<'de, serde_json::de::IoRead, value::Value>, 15 | ) 16 | where 17 | R: io::Read; 18 | 19 | pub struct Sink(W, F) 20 | where 21 | W: io::Write, 22 | F: Clone + serde_json::ser::Formatter; 23 | 24 | #[derive(Clone, Debug)] 25 | pub struct ReadableFormatter { 26 | current_indent: usize, 27 | is_in_object_key: bool, 28 | has_value: bool, 29 | 30 | null_style: ansi_term::Style, 31 | 32 | true_style: ansi_term::Style, 33 | false_style: ansi_term::Style, 34 | 35 | number_style: ansi_term::Style, 36 | 37 | string_quote_style: ansi_term::Style, 38 | string_char_style: ansi_term::Style, 39 | string_escape_style: ansi_term::Style, 40 | 41 | array_bracket_style: ansi_term::Style, 42 | array_comma_style: ansi_term::Style, 43 | 44 | object_brace_style: ansi_term::Style, 45 | object_colon_style: ansi_term::Style, 46 | object_comma_style: ansi_term::Style, 47 | object_key_quote_style: ansi_term::Style, 48 | object_key_char_style: ansi_term::Style, 49 | object_key_escape_style: ansi_term::Style, 50 | } 51 | 52 | #[inline] 53 | pub fn source<'de, R>(r: R) -> Source<'de, R> 54 | where 55 | R: io::Read, 56 | { 57 | Source(serde_json::Deserializer::new(serde_json::de::IoRead::new(r)).into_iter()) 58 | } 59 | 60 | #[inline] 61 | pub fn sink_compact(w: W) -> Sink 62 | where 63 | W: io::Write, 64 | { 65 | Sink(w, serde_json::ser::CompactFormatter) 66 | } 67 | 68 | #[inline] 69 | pub fn sink_readable(w: W) -> Sink 70 | where 71 | W: io::Write, 72 | { 73 | Sink(w, ReadableFormatter::new()) 74 | } 75 | 76 | #[inline] 77 | pub fn sink_indented<'a, W>(w: W) -> Sink> 78 | where 79 | W: io::Write, 80 | { 81 | Sink(w, serde_json::ser::PrettyFormatter::new()) 82 | } 83 | 84 | impl<'de, R> value::Source for Source<'de, R> 85 | where 86 | R: io::Read, 87 | { 88 | #[inline] 89 | fn read(&mut self) -> error::Result> { 90 | match self.0.next() { 91 | Some(Ok(v)) => Ok(Some(v)), 92 | Some(Err(e)) => Err(error::Error::from(e)), 93 | None => Ok(None), 94 | } 95 | } 96 | } 97 | 98 | impl value::Sink for Sink 99 | where 100 | W: io::Write, 101 | F: Clone + serde_json::ser::Formatter, 102 | { 103 | #[inline] 104 | fn write(&mut self, v: value::Value) -> error::Result<()> { 105 | { 106 | let mut serializer = 107 | serde_json::ser::Serializer::with_formatter(&mut self.0, self.1.clone()); 108 | serde::Serialize::serialize(&v, &mut serializer)?; 109 | } 110 | self.0.write_all(b"\n")?; 111 | Ok(()) 112 | } 113 | } 114 | 115 | impl ReadableFormatter { 116 | fn new() -> Self { 117 | use ansi_term::{Colour, Style}; 118 | 119 | Self { 120 | current_indent: 0, 121 | is_in_object_key: false, 122 | has_value: false, 123 | 124 | null_style: Colour::Black.dimmed().bold().italic(), 125 | 126 | true_style: Colour::Green.bold().italic(), 127 | false_style: Colour::Red.bold().italic(), 128 | 129 | number_style: Colour::Blue.normal(), 130 | 131 | string_quote_style: Colour::Green.dimmed(), 132 | string_char_style: Colour::Green.normal(), 133 | string_escape_style: Colour::Green.dimmed(), 134 | 135 | array_bracket_style: Style::default().bold(), 136 | array_comma_style: Style::default().bold(), 137 | 138 | object_brace_style: Style::default().bold(), 139 | object_colon_style: Style::default().bold(), 140 | object_comma_style: Style::default().bold(), 141 | object_key_quote_style: Colour::Blue.dimmed(), 142 | object_key_char_style: Colour::Blue.normal(), 143 | object_key_escape_style: Colour::Blue.dimmed(), 144 | } 145 | } 146 | 147 | /// Writes an integer value like `-123` to the specified writer. 148 | #[inline] 149 | fn write_integer(&mut self, mut writer: &mut W, value: I) -> io::Result<()> 150 | where 151 | W: io::Write + ?Sized, 152 | I: itoa::Integer, 153 | { 154 | write!(writer, "{}", self.number_style.prefix())?; 155 | itoa::write(&mut writer, value)?; 156 | write!(writer, "{}", self.number_style.suffix())?; 157 | Ok(()) 158 | } 159 | 160 | /// Writes a floating point value like `-31.26e+12` to the 161 | /// specified writer. 162 | #[inline] 163 | fn write_floating(&mut self, mut writer: &mut W, value: F) -> io::Result<()> 164 | where 165 | W: io::Write + ?Sized, 166 | F: dtoa::Floating, 167 | { 168 | write!(writer, "{}", self.number_style.prefix())?; 169 | dtoa::write(&mut writer, value)?; 170 | write!(writer, "{}", self.number_style.suffix())?; 171 | Ok(()) 172 | } 173 | } 174 | 175 | impl serde_json::ser::Formatter for ReadableFormatter { 176 | /// Writes a `null` value to the specified writer. 177 | #[inline] 178 | fn write_null(&mut self, writer: &mut W) -> io::Result<()> 179 | where 180 | W: io::Write + ?Sized, 181 | { 182 | write!(writer, "{}", self.null_style.paint("null")).map_err(From::from) 183 | } 184 | 185 | /// Writes a `true` or `false` value to the specified writer. 186 | #[inline] 187 | fn write_bool(&mut self, writer: &mut W, value: bool) -> io::Result<()> 188 | where 189 | W: io::Write + ?Sized, 190 | { 191 | let s = if value { 192 | self.true_style.paint("true") 193 | } else { 194 | self.false_style.paint("false") 195 | }; 196 | write!(writer, "{}", s).map_err(From::from) 197 | } 198 | 199 | #[inline] 200 | fn write_i8(&mut self, writer: &mut W, value: i8) -> io::Result<()> 201 | where 202 | W: io::Write + ?Sized, 203 | { 204 | self.write_integer(writer, value) 205 | } 206 | 207 | #[inline] 208 | fn write_i16(&mut self, writer: &mut W, value: i16) -> io::Result<()> 209 | where 210 | W: io::Write + ?Sized, 211 | { 212 | self.write_integer(writer, value) 213 | } 214 | 215 | #[inline] 216 | fn write_i32(&mut self, writer: &mut W, value: i32) -> io::Result<()> 217 | where 218 | W: io::Write + ?Sized, 219 | { 220 | self.write_integer(writer, value) 221 | } 222 | 223 | #[inline] 224 | fn write_i64(&mut self, writer: &mut W, value: i64) -> io::Result<()> 225 | where 226 | W: io::Write + ?Sized, 227 | { 228 | self.write_integer(writer, value) 229 | } 230 | 231 | #[inline] 232 | fn write_u8(&mut self, writer: &mut W, value: u8) -> io::Result<()> 233 | where 234 | W: io::Write + ?Sized, 235 | { 236 | self.write_integer(writer, value) 237 | } 238 | 239 | #[inline] 240 | fn write_u16(&mut self, writer: &mut W, value: u16) -> io::Result<()> 241 | where 242 | W: io::Write + ?Sized, 243 | { 244 | self.write_integer(writer, value) 245 | } 246 | 247 | #[inline] 248 | fn write_u32(&mut self, writer: &mut W, value: u32) -> io::Result<()> 249 | where 250 | W: io::Write + ?Sized, 251 | { 252 | self.write_integer(writer, value) 253 | } 254 | 255 | #[inline] 256 | fn write_u64(&mut self, writer: &mut W, value: u64) -> io::Result<()> 257 | where 258 | W: io::Write + ?Sized, 259 | { 260 | self.write_integer(writer, value) 261 | } 262 | 263 | #[inline] 264 | fn write_f32(&mut self, writer: &mut W, value: f32) -> io::Result<()> 265 | where 266 | W: io::Write + ?Sized, 267 | { 268 | self.write_floating(writer, value) 269 | } 270 | 271 | #[inline] 272 | fn write_f64(&mut self, writer: &mut W, value: f64) -> io::Result<()> 273 | where 274 | W: io::Write + ?Sized, 275 | { 276 | self.write_floating(writer, value) 277 | } 278 | 279 | /// Called before each series of `write_string_fragment` and 280 | /// `write_char_escape`. Writes a `"` to the specified writer. 281 | #[inline] 282 | fn begin_string(&mut self, writer: &mut W) -> io::Result<()> 283 | where 284 | W: io::Write + ?Sized, 285 | { 286 | let style = if self.is_in_object_key { 287 | self.object_key_quote_style 288 | } else { 289 | self.string_quote_style 290 | }; 291 | 292 | write!(writer, "{}", style.paint("\"")).map_err(From::from) 293 | } 294 | 295 | /// Called after each series of `write_string_fragment` and 296 | /// `write_char_escape`. Writes a `"` to the specified writer. 297 | #[inline] 298 | fn end_string(&mut self, writer: &mut W) -> io::Result<()> 299 | where 300 | W: io::Write + ?Sized, 301 | { 302 | let style = if self.is_in_object_key { 303 | self.object_key_quote_style 304 | } else { 305 | self.string_quote_style 306 | }; 307 | 308 | write!(writer, "{}", style.paint("\"")).map_err(From::from) 309 | } 310 | 311 | /// Writes a string fragment that doesn't need any escaping to the 312 | /// specified writer. 313 | #[inline] 314 | fn write_string_fragment(&mut self, writer: &mut W, fragment: &str) -> io::Result<()> 315 | where 316 | W: io::Write + ?Sized, 317 | { 318 | let style = if self.is_in_object_key { 319 | self.object_key_char_style 320 | } else { 321 | self.string_char_style 322 | }; 323 | 324 | write!(writer, "{}", style.paint(fragment)).map_err(From::from) 325 | } 326 | 327 | /// Writes a character escape code to the specified writer. 328 | #[inline] 329 | fn write_char_escape( 330 | &mut self, 331 | writer: &mut W, 332 | char_escape: serde_json::ser::CharEscape, 333 | ) -> io::Result<()> 334 | where 335 | W: io::Write + ?Sized, 336 | { 337 | use serde_json::ser::CharEscape::*; 338 | 339 | let style = if self.is_in_object_key { 340 | self.object_key_escape_style 341 | } else { 342 | self.string_escape_style 343 | }; 344 | 345 | let s = match char_escape { 346 | Quote => "\\\"", 347 | ReverseSolidus => "\\\\", 348 | Solidus => "\\/", 349 | Backspace => "\\b", 350 | FormFeed => "\\f", 351 | LineFeed => "\\n", 352 | CarriageReturn => "\\r", 353 | Tab => "\\t", 354 | AsciiControl(byte) => { 355 | static HEX_DIGITS: [u8; 16] = *b"0123456789abcdef"; 356 | let bytes = &[ 357 | b'\\', 358 | b'u', 359 | b'0', 360 | b'0', 361 | HEX_DIGITS[(byte >> 4) as usize], 362 | HEX_DIGITS[(byte & 0xF) as usize], 363 | ]; 364 | let s = unsafe { str::from_utf8_unchecked(bytes) }; 365 | 366 | // Need to return early because of allocated String 367 | return write!(writer, "{}", style.paint(s)).map_err(From::from); 368 | } 369 | }; 370 | 371 | write!(writer, "{}", style.paint(s)).map_err(From::from) 372 | } 373 | 374 | /// Called before every array. Writes a `[` to the specified 375 | /// writer. 376 | #[inline] 377 | fn begin_array(&mut self, writer: &mut W) -> io::Result<()> 378 | where 379 | W: io::Write + ?Sized, 380 | { 381 | self.current_indent += 1; 382 | self.has_value = false; 383 | 384 | write!(writer, "{}", self.array_bracket_style.paint("[")).map_err(From::from) 385 | } 386 | 387 | /// Called after every array. Writes a `]` to the specified 388 | /// writer. 389 | #[inline] 390 | fn end_array(&mut self, writer: &mut W) -> io::Result<()> 391 | where 392 | W: io::Write + ?Sized, 393 | { 394 | self.current_indent -= 1; 395 | 396 | if self.has_value { 397 | writeln!(writer)?; 398 | indent(writer, self.current_indent)?; 399 | } 400 | 401 | write!(writer, "{}", self.array_bracket_style.paint("]")).map_err(From::from) 402 | } 403 | 404 | /// Called before every array value. Writes a `,` if needed to 405 | /// the specified writer. 406 | #[inline] 407 | fn begin_array_value(&mut self, writer: &mut W, first: bool) -> io::Result<()> 408 | where 409 | W: io::Write + ?Sized, 410 | { 411 | if !first { 412 | write!(writer, "{}", self.array_comma_style.paint(","))?; 413 | } 414 | 415 | writeln!(writer)?; 416 | indent(writer, self.current_indent)?; 417 | Ok(()) 418 | } 419 | 420 | /// Called after every array value. 421 | #[inline] 422 | fn end_array_value(&mut self, _writer: &mut W) -> io::Result<()> 423 | where 424 | W: io::Write + ?Sized, 425 | { 426 | self.has_value = true; 427 | Ok(()) 428 | } 429 | 430 | /// Called before every object. Writes a `{` to the specified 431 | /// writer. 432 | #[inline] 433 | fn begin_object(&mut self, writer: &mut W) -> io::Result<()> 434 | where 435 | W: io::Write + ?Sized, 436 | { 437 | self.current_indent += 1; 438 | self.has_value = false; 439 | 440 | write!(writer, "{}", self.object_brace_style.paint("{")).map_err(From::from) 441 | } 442 | 443 | /// Called after every object. Writes a `}` to the specified 444 | /// writer. 445 | #[inline] 446 | fn end_object(&mut self, writer: &mut W) -> io::Result<()> 447 | where 448 | W: io::Write + ?Sized, 449 | { 450 | self.current_indent -= 1; 451 | 452 | if self.has_value { 453 | writeln!(writer)?; 454 | indent(writer, self.current_indent)?; 455 | } 456 | 457 | write!(writer, "{}", self.object_brace_style.paint("}")).map_err(From::from) 458 | } 459 | 460 | /// Called before every object key. 461 | #[inline] 462 | fn begin_object_key(&mut self, writer: &mut W, first: bool) -> io::Result<()> 463 | where 464 | W: io::Write + ?Sized, 465 | { 466 | self.is_in_object_key = true; 467 | 468 | if !first { 469 | write!(writer, "{}", self.object_comma_style.paint(","))?; 470 | } 471 | 472 | writeln!(writer)?; 473 | indent(writer, self.current_indent)?; 474 | Ok(()) 475 | } 476 | 477 | /// Called after every object key. A `:` should be written to the 478 | /// specified writer by either this method or 479 | /// `begin_object_value`. 480 | #[inline] 481 | fn end_object_key(&mut self, _writer: &mut W) -> io::Result<()> 482 | where 483 | W: io::Write + ?Sized, 484 | { 485 | self.is_in_object_key = false; 486 | Ok(()) 487 | } 488 | 489 | /// Called before every object value. A `:` should be written to 490 | /// the specified writer by either this method or 491 | /// `end_object_key`. 492 | #[inline] 493 | fn begin_object_value(&mut self, writer: &mut W) -> io::Result<()> 494 | where 495 | W: io::Write + ?Sized, 496 | { 497 | write!(writer, "{}", self.object_colon_style.paint(": ")).map_err(From::from) 498 | } 499 | 500 | /// Called after every object value. 501 | #[inline] 502 | fn end_object_value(&mut self, _writer: &mut W) -> io::Result<()> 503 | where 504 | W: io::Write + ?Sized, 505 | { 506 | self.has_value = true; 507 | Ok(()) 508 | } 509 | } 510 | 511 | fn indent(wr: &mut W, n: usize) -> io::Result<()> 512 | where 513 | W: io::Write + ?Sized, 514 | { 515 | for _ in 0..n { 516 | wr.write_all(b" ")?; 517 | } 518 | 519 | Ok(()) 520 | } 521 | 522 | impl<'de, R> fmt::Debug for Source<'de, R> 523 | where 524 | R: io::Read, 525 | { 526 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 527 | f.debug_struct("CsvSource").finish() 528 | } 529 | } 530 | 531 | impl fmt::Debug for Sink 532 | where 533 | W: io::Write, 534 | F: Clone + serde_json::ser::Formatter, 535 | { 536 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 537 | f.debug_struct("JsonSink").finish() 538 | } 539 | } 540 | -------------------------------------------------------------------------------- /src/value/messagepack.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | 3 | use ordered_float; 4 | use rmpv; 5 | 6 | use crate::error; 7 | use crate::value; 8 | 9 | #[derive(Debug)] 10 | pub struct MessagePackSource(R) 11 | where 12 | R: io::Read; 13 | 14 | #[derive(Debug)] 15 | pub struct MessagePackSink(W) 16 | where 17 | W: io::Write; 18 | 19 | #[inline] 20 | pub fn source(r: R) -> MessagePackSource 21 | where 22 | R: io::Read, 23 | { 24 | MessagePackSource(r) 25 | } 26 | 27 | #[inline] 28 | pub fn sink(w: W) -> MessagePackSink 29 | where 30 | W: io::Write, 31 | { 32 | MessagePackSink(w) 33 | } 34 | 35 | impl value::Source for MessagePackSource 36 | where 37 | R: io::Read, 38 | { 39 | #[inline] 40 | fn read(&mut self) -> error::Result> { 41 | use rmpv::decode::Error; 42 | 43 | match rmpv::decode::value::read_value(&mut self.0) { 44 | Ok(v) => Ok(Some(value_from_message_pack(v)?)), 45 | Err(Error::InvalidMarkerRead(ref e)) if e.kind() == io::ErrorKind::UnexpectedEof => { 46 | Ok(None) 47 | } 48 | Err(e) => Err(error::Error::MessagePackDecode(e)), 49 | } 50 | } 51 | } 52 | 53 | impl value::Sink for MessagePackSink 54 | where 55 | W: io::Write, 56 | { 57 | #[inline] 58 | fn write(&mut self, v: value::Value) -> error::Result<()> { 59 | rmpv::encode::write_value(&mut self.0, &value_to_message_pack(v)).map_err(From::from) 60 | } 61 | } 62 | 63 | fn value_from_message_pack(value: rmpv::Value) -> error::Result { 64 | use rmpv::Value; 65 | match value { 66 | Value::Nil => Ok(value::Value::Unit), 67 | Value::Boolean(v) => Ok(value::Value::Bool(v)), 68 | Value::Integer(i) if i.is_u64() => Ok(value::Value::U64(i.as_u64().unwrap())), 69 | Value::Integer(i) if i.is_i64() => Ok(value::Value::I64(i.as_i64().unwrap())), 70 | Value::Integer(_) => unreachable!(), 71 | Value::F32(v) => Ok(value::Value::from_f32(v)), 72 | Value::F64(v) => Ok(value::Value::from_f64(v)), 73 | Value::String(v) => { 74 | if v.is_err() { 75 | Err(error::Error::Format { 76 | msg: v.as_err().unwrap().to_string(), 77 | }) 78 | } else { 79 | Ok(value::Value::String(v.into_str().unwrap())) 80 | } 81 | } 82 | Value::Ext(_, v) | Value::Binary(v) => Ok(value::Value::Bytes(v)), 83 | Value::Array(v) => Ok(value::Value::Sequence( 84 | v.into_iter() 85 | .map(value_from_message_pack) 86 | .collect::>()?, 87 | )), 88 | Value::Map(v) => Ok(value::Value::Map( 89 | v.into_iter() 90 | .map(|(k, v)| Ok((value_from_message_pack(k)?, value_from_message_pack(v)?))) 91 | .collect::>()?, 92 | )), 93 | } 94 | } 95 | 96 | fn value_to_message_pack(value: value::Value) -> rmpv::Value { 97 | use rmpv::Value; 98 | match value { 99 | value::Value::Unit => Value::Nil, 100 | value::Value::Bool(v) => Value::Boolean(v), 101 | 102 | value::Value::I8(v) => Value::Integer(v.into()), 103 | value::Value::I16(v) => Value::Integer(v.into()), 104 | value::Value::I32(v) => Value::Integer(v.into()), 105 | value::Value::I64(v) => Value::Integer(v.into()), 106 | 107 | value::Value::U8(v) => Value::Integer(v.into()), 108 | value::Value::U16(v) => Value::Integer(v.into()), 109 | value::Value::U32(v) => Value::Integer(v.into()), 110 | value::Value::U64(v) => Value::Integer(v.into()), 111 | 112 | value::Value::F32(ordered_float::OrderedFloat(v)) => Value::F32(v), 113 | value::Value::F64(ordered_float::OrderedFloat(v)) => Value::F64(v), 114 | 115 | value::Value::Char(v) => Value::String(format!("{}", v).into()), 116 | value::Value::String(v) => Value::String(v.into()), 117 | value::Value::Bytes(v) => Value::Binary(v), 118 | 119 | value::Value::Sequence(v) => { 120 | Value::Array(v.into_iter().map(value_to_message_pack).collect()) 121 | } 122 | value::Value::Map(v) => Value::Map( 123 | v.into_iter() 124 | .map(|(k, v)| (value_to_message_pack(k), value_to_message_pack(v))) 125 | .collect(), 126 | ), 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/value/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::error; 2 | 3 | use ordered_float; 4 | use serde; 5 | use serde_json; 6 | use std::fmt; 7 | use std::io; 8 | 9 | pub mod avro; 10 | pub mod cbor; 11 | pub mod csv; 12 | pub mod json; 13 | pub mod messagepack; 14 | pub mod protobuf; 15 | pub mod raw; 16 | pub mod toml; 17 | pub mod yaml; 18 | 19 | #[derive(Debug)] 20 | pub enum Value { 21 | Unit, 22 | Bool(bool), 23 | 24 | I8(i8), 25 | I16(i16), 26 | I32(i32), 27 | I64(i64), 28 | 29 | U8(u8), 30 | U16(u16), 31 | U32(u32), 32 | U64(u64), 33 | 34 | F32(ordered_float::OrderedFloat), 35 | F64(ordered_float::OrderedFloat), 36 | 37 | Char(char), 38 | String(String), 39 | Bytes(Vec), 40 | 41 | Sequence(Vec), 42 | 43 | // A sequence of pairs is appropriate for representing an object/a map: 44 | // there is no need to deduplicate keys, and it is nice to preserve order. 45 | Map(Vec<(Value, Value)>), 46 | } 47 | 48 | pub trait Source { 49 | fn read(&mut self) -> error::Result>; 50 | } 51 | 52 | pub trait Sink { 53 | fn write(&mut self, v: Value) -> error::Result<()>; 54 | } 55 | 56 | struct ValueVisitor; 57 | 58 | impl Value { 59 | pub fn to_json(&self, mut w: &mut W) -> error::Result<()> 60 | where 61 | W: io::Write, 62 | { 63 | serde_json::to_writer(&mut w, self)?; 64 | w.write_all(&[10])?; // Newline 65 | Ok(()) 66 | } 67 | 68 | pub fn from_f32(v: f32) -> Self { 69 | Self::F32(ordered_float::OrderedFloat(v)) 70 | } 71 | 72 | pub fn from_f64(v: f64) -> Self { 73 | Self::F64(ordered_float::OrderedFloat(v)) 74 | } 75 | } 76 | 77 | impl fmt::Display for Value { 78 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 79 | match *self { 80 | Self::Unit => write!(f, "()"), 81 | Self::Bool(v) => write!(f, "{}", v), 82 | 83 | Self::I8(v) => write!(f, "{}", v), 84 | Self::I16(v) => write!(f, "{}", v), 85 | Self::I32(v) => write!(f, "{}", v), 86 | Self::I64(v) => write!(f, "{}", v), 87 | 88 | Self::U8(v) => write!(f, "{}", v), 89 | Self::U16(v) => write!(f, "{}", v), 90 | Self::U32(v) => write!(f, "{}", v), 91 | Self::U64(v) => write!(f, "{}", v), 92 | 93 | Self::F32(v) => write!(f, "{}", v), 94 | Self::F64(v) => write!(f, "{}", v), 95 | 96 | Self::Char(v) => write!(f, "{}", v), 97 | Self::String(ref v) => write!(f, "{}", v), 98 | Self::Bytes(ref v) => { 99 | for b in v { 100 | write!(f, "{:02x}", b)?; 101 | } 102 | Ok(()) 103 | } 104 | 105 | Self::Sequence(ref seq) => { 106 | let mut needs_sep = false; 107 | write!(f, "[")?; 108 | for v in seq { 109 | if needs_sep { 110 | write!(f, ", ")?; 111 | } 112 | write!(f, "{}", v)?; 113 | needs_sep = true; 114 | } 115 | write!(f, "]")?; 116 | Ok(()) 117 | } 118 | Self::Map(ref map) => { 119 | let mut needs_sep = false; 120 | write!(f, "{{")?; 121 | for (k, v) in map { 122 | if needs_sep { 123 | write!(f, ", ")?; 124 | } 125 | write!(f, "{}: {}", k, v)?; 126 | needs_sep = true; 127 | } 128 | write!(f, "}}")?; 129 | Ok(()) 130 | } 131 | } 132 | } 133 | } 134 | 135 | impl serde::ser::Serialize for Value { 136 | #[inline] 137 | fn serialize(&self, s: S) -> Result 138 | where 139 | S: serde::ser::Serializer, 140 | { 141 | match *self { 142 | Self::Unit => ().serialize(s), 143 | Self::Bool(v) => v.serialize(s), 144 | 145 | Self::I8(v) => v.serialize(s), 146 | Self::I16(v) => v.serialize(s), 147 | Self::I32(v) => v.serialize(s), 148 | Self::I64(v) => v.serialize(s), 149 | 150 | Self::U8(v) => v.serialize(s), 151 | Self::U16(v) => v.serialize(s), 152 | Self::U32(v) => v.serialize(s), 153 | Self::U64(v) => v.serialize(s), 154 | 155 | Self::F32(v) => v.serialize(s), 156 | Self::F64(v) => v.serialize(s), 157 | 158 | Self::Char(v) => v.serialize(s), 159 | Self::String(ref v) => v.serialize(s), 160 | Self::Bytes(ref v) => v.serialize(s), 161 | 162 | Self::Sequence(ref v) => v.serialize(s), 163 | Self::Map(ref v) => { 164 | use serde::ser::SerializeMap; 165 | let mut s = s.serialize_map(Some(v.len()))?; 166 | for (key, value) in v { 167 | SerializeMap::serialize_entry(&mut s, key, value)?; 168 | } 169 | SerializeMap::end(s) 170 | } 171 | } 172 | } 173 | } 174 | 175 | impl<'de> serde::de::Deserialize<'de> for Value { 176 | #[inline] 177 | fn deserialize(d: D) -> Result 178 | where 179 | D: serde::Deserializer<'de>, 180 | { 181 | d.deserialize_any(ValueVisitor) 182 | } 183 | } 184 | 185 | impl<'de> serde::de::Visitor<'de> for ValueVisitor { 186 | type Value = Value; 187 | 188 | #[inline] 189 | fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { 190 | write!(f, "any value") 191 | } 192 | 193 | #[inline] 194 | fn visit_bool(self, v: bool) -> Result 195 | where 196 | E: serde::de::Error, 197 | { 198 | Ok(Value::Bool(v)) 199 | } 200 | 201 | #[inline] 202 | fn visit_i8(self, v: i8) -> Result 203 | where 204 | E: serde::de::Error, 205 | { 206 | Ok(Value::I8(v)) 207 | } 208 | 209 | #[inline] 210 | fn visit_i16(self, v: i16) -> Result 211 | where 212 | E: serde::de::Error, 213 | { 214 | Ok(Value::I16(v)) 215 | } 216 | 217 | #[inline] 218 | fn visit_i32(self, v: i32) -> Result 219 | where 220 | E: serde::de::Error, 221 | { 222 | Ok(Value::I32(v)) 223 | } 224 | 225 | #[inline] 226 | fn visit_i64(self, v: i64) -> Result 227 | where 228 | E: serde::de::Error, 229 | { 230 | Ok(Value::I64(v)) 231 | } 232 | 233 | #[inline] 234 | fn visit_u8(self, v: u8) -> Result 235 | where 236 | E: serde::de::Error, 237 | { 238 | Ok(Value::U8(v)) 239 | } 240 | 241 | #[inline] 242 | fn visit_u16(self, v: u16) -> Result 243 | where 244 | E: serde::de::Error, 245 | { 246 | Ok(Value::U16(v)) 247 | } 248 | 249 | #[inline] 250 | fn visit_u32(self, v: u32) -> Result 251 | where 252 | E: serde::de::Error, 253 | { 254 | Ok(Value::U32(v)) 255 | } 256 | 257 | #[inline] 258 | fn visit_u64(self, v: u64) -> Result 259 | where 260 | E: serde::de::Error, 261 | { 262 | Ok(Value::U64(v)) 263 | } 264 | 265 | #[inline] 266 | fn visit_f32(self, v: f32) -> Result 267 | where 268 | E: serde::de::Error, 269 | { 270 | Ok(Value::from_f32(v)) 271 | } 272 | 273 | #[inline] 274 | fn visit_f64(self, v: f64) -> Result 275 | where 276 | E: serde::de::Error, 277 | { 278 | Ok(Value::from_f64(v)) 279 | } 280 | 281 | #[inline] 282 | fn visit_char(self, v: char) -> Result 283 | where 284 | E: serde::de::Error, 285 | { 286 | Ok(Value::Char(v)) 287 | } 288 | 289 | #[inline] 290 | fn visit_str(self, v: &str) -> Result 291 | where 292 | E: serde::de::Error, 293 | { 294 | Ok(Value::String(v.to_owned())) 295 | } 296 | 297 | #[inline] 298 | fn visit_string(self, v: String) -> Result 299 | where 300 | E: serde::de::Error, 301 | { 302 | Ok(Value::String(v)) 303 | } 304 | 305 | #[inline] 306 | fn visit_bytes(self, v: &[u8]) -> Result 307 | where 308 | E: serde::de::Error, 309 | { 310 | Ok(Value::Bytes(v.to_vec())) 311 | } 312 | 313 | #[inline] 314 | fn visit_byte_buf(self, v: Vec) -> Result 315 | where 316 | E: serde::de::Error, 317 | { 318 | Ok(Value::Bytes(v)) 319 | } 320 | 321 | #[inline] 322 | fn visit_none(self) -> Result 323 | where 324 | E: serde::de::Error, 325 | { 326 | Ok(Value::Unit) 327 | } 328 | 329 | #[inline] 330 | fn visit_some(self, d: D) -> Result 331 | where 332 | D: serde::de::Deserializer<'de>, 333 | { 334 | serde::de::Deserialize::deserialize(d) 335 | } 336 | 337 | #[inline] 338 | fn visit_unit(self) -> Result 339 | where 340 | E: serde::de::Error, 341 | { 342 | Ok(Value::Unit) 343 | } 344 | 345 | #[inline] 346 | fn visit_seq(self, mut v: V) -> Result 347 | where 348 | V: serde::de::SeqAccess<'de>, 349 | { 350 | let mut values = v.size_hint().map_or(Vec::new(), Vec::with_capacity); 351 | 352 | while let Some(element) = v.next_element()? { 353 | values.push(element); 354 | } 355 | 356 | Ok(Value::Sequence(values)) 357 | } 358 | 359 | #[inline] 360 | fn visit_map(self, mut v: V) -> Result 361 | where 362 | V: serde::de::MapAccess<'de>, 363 | { 364 | let mut values = Vec::new(); 365 | 366 | while let Some(entry) = v.next_entry()? { 367 | values.push(entry); 368 | } 369 | 370 | Ok(Value::Map(values)) 371 | } 372 | } 373 | -------------------------------------------------------------------------------- /src/value/protobuf.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use crate::error; 4 | use protobuf; 5 | use serde; 6 | 7 | use crate::value; 8 | use serde_protobuf; 9 | use serde_protobuf::descriptor; 10 | 11 | pub struct Source<'a>(serde_protobuf::de::Deserializer<'a>, bool); 12 | 13 | #[inline] 14 | pub fn source<'a>( 15 | descriptors: &'a descriptor::Descriptors, 16 | message_name: &str, 17 | input: protobuf::CodedInputStream<'a>, 18 | ) -> error::Result> { 19 | let de = serde_protobuf::de::Deserializer::for_named_message(descriptors, message_name, input)?; 20 | Ok(Source(de, true)) 21 | } 22 | 23 | impl<'a> value::Source for Source<'a> { 24 | #[inline] 25 | fn read(&mut self) -> error::Result> { 26 | if self.1 { 27 | self.1 = false; 28 | match serde::Deserialize::deserialize(&mut self.0) 29 | .map_err(serde_protobuf::error::CompatError::into_error) 30 | { 31 | Ok(v) => Ok(Some(v)), 32 | Err(serde_protobuf::error::Error::EndOfStream) => Ok(None), 33 | Err(e) => Err(error::Error::from(e)), 34 | } 35 | } else { 36 | Ok(None) 37 | } 38 | } 39 | } 40 | 41 | impl<'a> fmt::Debug for Source<'a> { 42 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 43 | f.debug_struct("ProtobufSource").finish() 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/value/raw.rs: -------------------------------------------------------------------------------- 1 | use crate::error; 2 | use crate::value; 3 | use std::io; 4 | 5 | #[derive(Debug)] 6 | pub struct Source(io::Lines>) 7 | where 8 | R: io::Read; 9 | 10 | #[derive(Debug)] 11 | pub struct Sink(io::LineWriter) 12 | where 13 | W: io::Write; 14 | 15 | #[inline] 16 | pub fn source(r: R) -> Source 17 | where 18 | R: io::Read, 19 | { 20 | use std::io::BufRead; 21 | Source(io::BufReader::new(r).lines()) 22 | } 23 | 24 | #[inline] 25 | pub fn sink(w: W) -> Sink 26 | where 27 | W: io::Write, 28 | { 29 | Sink(io::LineWriter::new(w)) 30 | } 31 | 32 | impl value::Source for Source 33 | where 34 | R: io::Read, 35 | { 36 | #[inline] 37 | fn read(&mut self) -> error::Result> { 38 | match self.0.next() { 39 | Some(Ok(v)) => Ok(Some(value::Value::String(v))), 40 | Some(Err(e)) => Err(error::Error::from(e)), 41 | None => Ok(None), 42 | } 43 | } 44 | } 45 | 46 | impl value::Sink for Sink 47 | where 48 | W: io::Write, 49 | { 50 | #[inline] 51 | fn write(&mut self, value: value::Value) -> error::Result<()> { 52 | use std::io::Write; 53 | match value { 54 | value::Value::String(s) => { 55 | self.0.write_all(s.as_bytes())?; 56 | self.0.write_all(b"\n")?; 57 | Ok(()) 58 | } 59 | value::Value::Bytes(b) => { 60 | self.0.write_all(&b)?; 61 | self.0.write_all(b"\n")?; 62 | Ok(()) 63 | } 64 | value::Value::Char(c) => { 65 | writeln!(self.0, "{}", c)?; 66 | Ok(()) 67 | } 68 | x => Err(error::Error::Format { 69 | msg: format!("raw can only output strings, bytes and chars, got: {:?}", x), 70 | }), 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/value/toml.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | 3 | use serde; 4 | use toml; 5 | 6 | use crate::error; 7 | use crate::value; 8 | 9 | #[derive(Debug)] 10 | pub struct Source(Option); 11 | 12 | #[derive(Debug)] 13 | pub struct Sink(W); 14 | 15 | #[inline] 16 | pub fn source(mut r: R) -> error::Result 17 | where 18 | R: io::Read, 19 | { 20 | let mut string = String::new(); 21 | r.read_to_string(&mut string)?; 22 | Ok(Source(Some(string))) 23 | } 24 | 25 | #[inline] 26 | pub fn sink(w: W) -> Sink 27 | where 28 | W: io::Write, 29 | { 30 | Sink(w) 31 | } 32 | 33 | impl value::Source for Source { 34 | #[inline] 35 | fn read(&mut self) -> error::Result> { 36 | match self.0.take() { 37 | Some(v) => { 38 | let de = toml::de::Deserializer::new(v.as_str()); 39 | match serde::Deserialize::deserialize(de) { 40 | Ok(v) => Ok(Some(v)), 41 | Err(e) => Err(error::Error::from(e)), 42 | } 43 | } 44 | None => Ok(None), 45 | } 46 | } 47 | } 48 | 49 | impl value::Sink for Sink 50 | where 51 | W: io::Write, 52 | { 53 | #[inline] 54 | fn write(&mut self, value: value::Value) -> error::Result<()> { 55 | let mut string = String::new(); 56 | { 57 | let ser = toml::ser::Serializer::new(&mut string); 58 | serde::Serialize::serialize(&value, ser)?; 59 | } 60 | 61 | self.0.write_all(string.as_bytes())?; 62 | self.0.write_all(b"\n")?; 63 | Ok(()) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/value/yaml.rs: -------------------------------------------------------------------------------- 1 | use crate::error; 2 | use crate::value; 3 | use serde_yaml; 4 | use std::io; 5 | 6 | #[derive(Debug)] 7 | pub struct Source(Option); 8 | 9 | #[derive(Debug)] 10 | pub struct Sink(W) 11 | where 12 | W: io::Write; 13 | 14 | #[inline] 15 | pub fn source(r: R) -> Source 16 | where 17 | R: io::Read, 18 | { 19 | Source(Some(r)) 20 | } 21 | 22 | #[inline] 23 | pub fn sink(w: W) -> Sink 24 | where 25 | W: io::Write, 26 | { 27 | Sink(w) 28 | } 29 | 30 | impl value::Source for Source 31 | where 32 | R: io::Read, 33 | { 34 | #[inline] 35 | fn read(&mut self) -> error::Result> { 36 | if let Some(r) = self.0.take() { 37 | match serde_yaml::from_reader(r) { 38 | Ok(v) => Ok(Some(v)), 39 | Err(e) => Err(error::Error::from(e)), 40 | } 41 | } else { 42 | Ok(None) 43 | } 44 | } 45 | } 46 | 47 | impl value::Sink for Sink 48 | where 49 | W: io::Write, 50 | { 51 | #[inline] 52 | fn write(&mut self, value: value::Value) -> error::Result<()> { 53 | serde_yaml::to_writer(&mut self.0, &value)?; 54 | self.0.write_all(b"\n")?; 55 | Ok(()) 56 | } 57 | } 58 | --------------------------------------------------------------------------------