├── .circleci └── config.yml ├── .dockerignore ├── .gitignore ├── .travis.yml ├── BUILDING.md ├── CONTRIBUTING.md ├── Cargo.toml ├── LICENSE.TXT ├── README.md ├── ROADMAP.md ├── docs ├── DataFusion Contributors.pdf └── sql_support.md ├── examples └── csv_sql.rs ├── img └── datafusion-logo.png ├── scripts ├── circle │ ├── build-examples.sh │ ├── build-release.sh │ ├── build.sh │ └── test.sh ├── docker │ ├── base │ │ ├── Dockerfile │ │ └── build.sh │ ├── console │ │ ├── Dockerfile │ │ ├── all_types_flat.csv │ │ ├── all_types_flat.parquet │ │ ├── build.sh │ │ ├── console.sh │ │ └── uk_cities.csv │ └── worker │ │ ├── Dockerfile │ │ └── build.sh ├── release.sh └── smoketest.sh ├── src ├── bin │ └── console │ │ ├── linereader.rs │ │ └── main.rs ├── dfparser.rs ├── execution │ ├── aggregate.rs │ ├── context.rs │ ├── datasource.rs │ ├── error.rs │ ├── expression.rs │ ├── filter.rs │ ├── mod.rs │ ├── physicalplan.rs │ ├── projection.rs │ ├── relation.rs │ └── value.rs ├── lib.rs ├── logicalplan.rs └── sqlplanner.rs ├── test └── data │ ├── aggregate_test_1.csv │ ├── aggregate_test_2.csv │ ├── all_types.csv │ ├── all_types_flat.csv │ ├── all_types_flat.parquet │ ├── alltypes_plain.parquet │ ├── example1.ndjson │ ├── expected │ ├── c_float32_cast.csv │ ├── c_float32_cast_uint32.csv │ ├── c_float32_high.csv │ ├── c_float32_high_uint32.csv │ ├── c_float32_low.csv │ ├── c_float32_low_uint32.csv │ ├── c_float64_cast.csv │ ├── c_float64_high.csv │ ├── c_float64_low.csv │ ├── c_int16_cast.csv │ ├── c_int16_negative.csv │ ├── c_int16_positive.csv │ ├── c_int32_cast.csv │ ├── c_int32_negative.csv │ ├── c_int32_positive.csv │ ├── c_int64_cast.csv │ ├── c_int64_negative.csv │ ├── c_int64_positive.csv │ ├── c_int8_cast.csv │ ├── c_int8_col_eq.csv │ ├── c_int8_col_gt.csv │ ├── c_int8_col_gteq.csv │ ├── c_int8_col_lt.csv │ ├── c_int8_col_lteq.csv │ ├── c_int8_col_noteq.csv │ ├── c_int8_eq.csv │ ├── c_int8_gt.csv │ ├── c_int8_gteq.csv │ ├── c_int8_lt.csv │ ├── c_int8_lteq.csv │ ├── c_int8_negative.csv │ ├── c_int8_noteq.csv │ ├── c_int8_positive.csv │ ├── c_int8_range_exclusive.csv │ ├── c_int8_range_inclusive.csv │ ├── c_int8_scalar_gt.csv │ ├── c_uint16_cast.csv │ ├── c_uint32_cast.csv │ ├── c_uint64_cast.csv │ ├── c_uint8_cast.csv │ ├── csv_aggregate_all_types.csv │ ├── csv_aggregate_by_c_bool.csv │ ├── csv_query_all_types.csv │ ├── is_not_null_csv.csv │ ├── is_null_csv.csv │ ├── numerics_divide.csv │ ├── numerics_divide_f64.csv │ ├── numerics_minus.csv │ ├── numerics_minus_f64.csv │ ├── numerics_modulo.csv │ ├── numerics_modulo_f64.csv │ ├── numerics_multiply.csv │ ├── numerics_multiply_f64.csv │ ├── numerics_plus.csv │ ├── numerics_plus_f64.csv │ ├── parquet_aggregate_all_types.csv │ ├── parquet_query_all_types.csv │ ├── test_cast.csv │ ├── test_chaining_functions.csv │ ├── test_df_udf_udt.csv │ ├── test_filter.csv │ ├── test_limit.csv │ ├── test_simple_predicate.csv │ ├── test_sql_min_max.csv │ ├── test_sql_udf_udt.csv │ └── test_sqrt.csv │ ├── null_test.csv │ ├── numerics.csv │ ├── people.csv │ ├── smoketest-expected.txt │ ├── smoketest.sql │ ├── uk_cities.csv │ └── uk_cities.parquet └── tests └── sql.rs /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | refresh_cache: 4 | docker: 5 | - image: rust:latest 6 | steps: 7 | - checkout 8 | - restore_cache: 9 | keys: 10 | - cargo-v1-{{ checksum "Cargo.toml" }} 11 | - run: cargo update 12 | - run: cargo fetch 13 | - save_cache: 14 | key: cargo-v1-{{ checksum "Cargo.toml" }} 15 | paths: 16 | - /usr/local/cargo/registry 17 | - /usr/local/cargo/git 18 | build_stable: 19 | docker: 20 | - image: rust:latest 21 | steps: 22 | - checkout 23 | - restore_cache: 24 | keys: 25 | - cargo-v1-{{ checksum "Cargo.toml" }} 26 | - run: 27 | name: Building Main Code 28 | command: sh scripts/circle/build.sh 29 | - run: 30 | name: Building Examples 31 | command: sh scripts/circle/build-examples.sh 32 | - run: 33 | name: Building Release 34 | command: sh scripts/circle/build-release.sh 35 | - save_cache: 36 | key: v1-release 37 | paths: 38 | - ~/project/target/release 39 | - run: 40 | name: Test Code 41 | command: sh scripts/circle/test.sh 42 | build_nightly: 43 | docker: 44 | - image: rustlang/rust:nightly 45 | steps: 46 | - checkout 47 | - restore_cache: 48 | keys: 49 | - cargo-v1-{{ checksum "Cargo.toml" }} 50 | - run: 51 | name: Nightly Version Info 52 | command: rustc --version; cargo --version 53 | - run: 54 | name: Building Main Code 55 | command: sh scripts/circle/build.sh 56 | - run: 57 | name: Building Examples 58 | command: sh scripts/circle/build-examples.sh 59 | - run: 60 | name: Test Code 61 | command: sh scripts/circle/test.sh 62 | do_deploy: 63 | docker: 64 | - image: rust:latest 65 | steps: 66 | - checkout 67 | - restore_cache: 68 | keys: 69 | - v1-release 70 | - setup_remote_docker: 71 | docker_layer_caching: true 72 | - run: 73 | name: Install Docker client 74 | command: | 75 | set -x 76 | VER="17.03.0-ce" 77 | curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz 78 | tar -xz -C /tmp -f /tmp/docker-$VER.tgz 79 | mv /tmp/docker/* /usr/bin 80 | # - run: 81 | # name: Deploy Docker Container 82 | # command: | 83 | # sh scripts/docker/base/build.sh 84 | # sh scripts/docker/worker/build.sh 85 | # sh scripts/docker/console/build.sh 86 | workflows: 87 | version: 2 88 | stable: 89 | jobs: 90 | - refresh_cache 91 | - build_stable: 92 | requires: 93 | - refresh_cache 94 | - do_deploy: 95 | requires: 96 | - build_stable 97 | filters: 98 | branches: 99 | only: master 100 | nightly: 101 | jobs: 102 | - refresh_cache 103 | - build_nightly: 104 | requires: 105 | - refresh_cache 106 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .circleci 2 | .idea 3 | bin 4 | target 5 | test -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # rust 2 | out 3 | target 4 | Cargo.lock 5 | 6 | # IntelliJ 7 | .idea/ 8 | *.iml 9 | *.ipr 10 | *.iws 11 | 12 | # vscode 13 | .vscode/ 14 | 15 | # test output 16 | _*.csv 17 | _*.txt 18 | _*.quiver 19 | temp 20 | 21 | # test data 22 | datasets 23 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | sudo: required 3 | cache: cargo 4 | language: rust 5 | addons: 6 | apt: 7 | packages: 8 | - libcurl4-openssl-dev 9 | - libelf-dev 10 | - libdw-dev 11 | - binutils-dev 12 | - cmake 13 | sources: 14 | - kalakris-cmake 15 | 16 | rust: 17 | - nightly 18 | 19 | before_script: 20 | - pip install 'travis-cargo<0.2' --user && export PATH=$HOME/.local/bin:$PATH 21 | - cargo install --force cargo-travis && export PATH=$HOME/.cargo/bin:$PATH 22 | 23 | script: 24 | - travis-cargo build 25 | - travis-cargo test 26 | 27 | after_success: 28 | - if [ "$TRAVIS_RUST_VERSION" == "nightly" ]; then 29 | cargo bench; 30 | fi 31 | - cargo coveralls --verbose --exclude-pattern '/datafusion/src/bin' 32 | 33 | 34 | env: 35 | global: 36 | - TRAVIS_CARGO_NIGHTLY_FEATURE="" 37 | -------------------------------------------------------------------------------- /BUILDING.md: -------------------------------------------------------------------------------- 1 | 2 | # Building DataFusion 3 | 4 | ## Prerequisites 5 | 6 | - Rust nightly 7 | - Thrift (required by `parquet-rs` crate) - instructions [here](https://github.com/sunchao/parquet-rs/) 8 | 9 | ## Building Locally 10 | 11 | Generally, you just need to run cargo. 12 | 13 | ```bash 14 | $ cargo build 15 | ``` 16 | 17 | ## Smoketest 18 | 19 | Run this before committing code to make sure everything is working. 20 | 21 | ```bash 22 | ./scripts/smoketest.sh 23 | ``` -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | 3 | - File an issue 4 | - Fork the repo 5 | - Make your changes on a branch in your repo 6 | - Make sure the project builds without any compiler warnings by running these commands 7 | - `cargo build --examples` 8 | - Make sure all tests pass: 9 | - `cargo test` 10 | - Submit a pull request 11 | 12 | By submitting code you agree to the terms of the Apache 2.0 licence for the submitted code. -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "datafusion" 3 | description = "DataFusion is a modern distributed compute platform that uses Apache Arrow as the memory model" 4 | version = "0.6.0" 5 | authors = ["Andy Grove "] 6 | homepage = "https://github.com/andygrove/datafusion" 7 | documentation = "https://datafusion.rs/docs/datafusion" 8 | keywords = [ "distributed", "query", "data", "processing", "sql" ] 9 | repository = "https://github.com/andygrove/datafusion" 10 | license = "Apache-2.0" 11 | include = [ 12 | "src/**/*.rs", 13 | "Cargo.toml", 14 | ] 15 | edition = "2018" 16 | 17 | [lib] 18 | name = "datafusion" 19 | path = "src/lib.rs" 20 | 21 | [[bin]] 22 | name = "console" 23 | path = "src/bin/console/main.rs" 24 | 25 | [dependencies] 26 | clap = "2.31.2" 27 | fnv = "1.0.3" 28 | arrow = "0.12.0" 29 | parquet = "0.12.0" 30 | datafusion-rustyline = "2.0.0-alpha-20180628" 31 | serde = { version = "1.0.80", features = ["alloc", "rc"] } 32 | serde_derive = "1.0.80" 33 | serde_json = "1.0.33" 34 | sqlparser = "0.2.1" 35 | 36 | [dev-dependencies] 37 | criterion = "0.2.0" 38 | 39 | -------------------------------------------------------------------------------- /LICENSE.TXT: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 3 | [![Version](https://img.shields.io/crates/v/datafusion.svg)](https://crates.io/crates/datafusion) 4 | [![Build Status](https://travis-ci.org/andygrove/datafusion.svg?branch=master)](https://travis-ci.org/andygrove/datafusion) 5 | [![Coverage Status](https://coveralls.io/repos/github/andygrove/datafusion/badge.svg?branch=master)](https://coveralls.io/github/andygrove/datafusion?branch=master) 6 | [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/datafusion-rs) 7 | 8 | # DataFusion: Modern Distributed Compute Platform implemented in Rust 9 | 10 | 11 | 12 | DataFusion is an attempt at building a modern distributed compute platform in Rust, leveraging [Apache Arrow](https://arrow.apache.org/) as the memory model. 13 | 14 | NOTE: DataFusion was donated to the Apache Arrow project in February 2019. Source is [here](https://github.com/apache/arrow/tree/master/rust/datafusion). 15 | 16 | See my article [How To Build a Modern Distributed Compute Platform](https://andygrove.io/how_to_build_a_modern_distributed_compute_platform/) to learn about the design and my motivation for building this. The TL;DR is that this project is a great way to learn about building a query engine but this is quite early and not usable for any real world work just yet. 17 | 18 | # Status 19 | 20 | The current code supports single-threaded execution of limited SQL queries (projection, selection, and aggregates) against CSV files. Parquet files will be supported shortly. 21 | 22 | To use DataFusion as a crate dependency, add the following to your Cargo.toml: 23 | 24 | ```toml 25 | [dependencies] 26 | datafusion = "0.6.0" 27 | ``` 28 | 29 | Here is a brief example for running a SQL query against a CSV file. See the [examples](examples) directory for full examples. 30 | 31 | ```rust 32 | fn main() { 33 | // create local execution context 34 | let mut ctx = ExecutionContext::new(); 35 | 36 | // define schema for data source (csv file) 37 | let schema = Arc::new(Schema::new(vec![ 38 | Field::new("city", DataType::Utf8, false), 39 | Field::new("lat", DataType::Float64, false), 40 | Field::new("lng", DataType::Float64, false), 41 | ])); 42 | 43 | // register csv file with the execution context 44 | let csv_datasource = CsvDataSource::new("test/data/uk_cities.csv", schema.clone(), 1024); 45 | ctx.register_datasource("cities", Rc::new(RefCell::new(csv_datasource))); 46 | 47 | // simple projection and selection 48 | let sql = "SELECT city, lat, lng FROM cities WHERE lat > 51.0 AND lat < 53"; 49 | 50 | // execute the query 51 | let relation = ctx.sql(&sql).unwrap(); 52 | 53 | // display the relation 54 | let mut results = relation.borrow_mut(); 55 | 56 | while let Some(batch) = results.next().unwrap() { 57 | 58 | println!( 59 | "RecordBatch has {} rows and {} columns", 60 | batch.num_rows(), 61 | batch.num_columns() 62 | ); 63 | 64 | let city = batch 65 | .column(0) 66 | .as_any() 67 | .downcast_ref::() 68 | .unwrap(); 69 | 70 | let lat = batch 71 | .column(1) 72 | .as_any() 73 | .downcast_ref::() 74 | .unwrap(); 75 | 76 | let lng = batch 77 | .column(2) 78 | .as_any() 79 | .downcast_ref::() 80 | .unwrap(); 81 | 82 | for i in 0..batch.num_rows() { 83 | let city_name: String = String::from_utf8(city.get_value(i).to_vec()).unwrap(); 84 | 85 | println!( 86 | "City: {}, Latitude: {}, Longitude: {}", 87 | city_name, 88 | lat.value(i), 89 | lng.value(i), 90 | ); 91 | } 92 | } 93 | } 94 | ``` 95 | 96 | # Roadmap 97 | 98 | See [ROADMAP.md](ROADMAP.md) for the full roadmap. 99 | 100 | # Prerequisites 101 | 102 | - Rust nightly (required by `parquet-rs` crate) 103 | 104 | # Building DataFusion 105 | 106 | See [BUILDING.md](/BUILDING.md). 107 | 108 | # Gitter 109 | 110 | There is a [Gitter channel](https://gitter.im/datafusion-rs/Lobby) where you can ask questions about the project or make feature suggestions too. 111 | 112 | # Contributing 113 | 114 | Contributors are welcome! Please see [CONTRIBUTING.md](/CONTRIBUTING.md) for details. 115 | 116 | 117 | 118 | -------------------------------------------------------------------------------- /ROADMAP.md: -------------------------------------------------------------------------------- 1 | # DataFusion Roadmap 2 | 3 | ## 0.6.x - Feature parity with original POC (in progress) 4 | 5 | I recently updated the code to use my fork of Arrow and this required considerable rework so I am currently working on restoring functionality that was previously working. 6 | 7 | - [x] Upgrade to Apache Arrow 0.12.0 8 | - [x] Allow query to be executed against Arrow CSV reader 9 | - [ ] Allow query to be executed against Arrow Parquet reader 10 | - [ ] Implement project push-down so that only necessary columns are loaded into memory 11 | - [x] Logical query plan definition 12 | - [x] SQL Parser 13 | - [x] Query planner 14 | - [x] Projection 15 | - [x] Selection 16 | - [x] Simple aggregate queries with optional GROUP BY 17 | - [x] Support for MIN/MAX 18 | - [x] Support for SUM 19 | - [ ] Support for COUNT 20 | - [ ] Support for COUNT(DISTINCT) 21 | - [ ] ORDER BY 22 | - [ ] Support `CREATE EXTERNAL TABLE` SQL to register data sources 23 | - [ ] SQL console and Docker image for standalone use / easy testing and benchmarking 24 | 25 | 26 | ## 0.7.0 - Mature query execution 27 | 28 | The goal of this release is to support a larger percentage of real world queries and to focus on improved unit testing to ensure correctness of query execution. 29 | 30 | - Scalar UDFs 31 | - Array UDFs 32 | - Support nested objects with dot notation 33 | - JOIN support (hash join and sort merge join) 34 | - Better unit tests / smoke test / performance tests 35 | 36 | ## 0.8.0 - Parallel processing / partitions 37 | 38 | - Parallel execution using threads (async/await) 39 | - Partitioning 40 | - Query optimizer improvements 41 | 42 | ## 0.9.0 - Groundwork for distributed queries 43 | 44 | - Serializable logical query plan (in protobuf format) 45 | - Worker node that can receive and execute plan against local files 46 | - Write query output to local files or return results in protobuf and/or IPC format 47 | - Consider supporting Hive protocol to allow JDBC/ODBC clients to submit queries to a single node 48 | 49 | ## 1.0.0 - Distributed queries 50 | 51 | This release will allow queries to be executed against a cluster, supporting interactive queries that return results in Arrow format or write results to disk. 52 | 53 | - Distributed query planner 54 | - Worker can delegate portions of query plan to other workers 55 | - Data source meta-data 56 | - Kubernetes support for spinning up worker nodes 57 | 58 | ## 1.1.0 - Usability 59 | 60 | - Web user interface / better tools / monitoring etc 61 | 62 | ## Backlog / TBD 63 | 64 | - Support for S3 65 | - Support for HDFS 66 | - Authentication/authorization 67 | - Encryption at rest and in transit 68 | -------------------------------------------------------------------------------- /docs/DataFusion Contributors.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygrove/datafusion-archive/219ca3f61f0d751779b4f30c02aa5fdcf8a40d12/docs/DataFusion Contributors.pdf -------------------------------------------------------------------------------- /docs/sql_support.md: -------------------------------------------------------------------------------- 1 | # DataFusion SQL Support 2 | -------------------------------------------------------------------------------- /examples/csv_sql.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::cell::RefCell; 16 | use std::rc::Rc; 17 | use std::sync::Arc; 18 | 19 | extern crate arrow; 20 | extern crate datafusion; 21 | 22 | use arrow::array::{BinaryArray, Float64Array}; 23 | use arrow::datatypes::{DataType, Field, Schema}; 24 | 25 | use datafusion::execution::context::ExecutionContext; 26 | use datafusion::execution::datasource::CsvDataSource; 27 | 28 | /// This example demonstrates executing a simple query against an Arrow data source and fetching results 29 | fn main() { 30 | // create local execution context 31 | let mut ctx = ExecutionContext::new(); 32 | 33 | // define schema for data source (csv file) 34 | let schema = Arc::new(Schema::new(vec![ 35 | Field::new("city", DataType::Utf8, false), 36 | Field::new("lat", DataType::Float64, false), 37 | Field::new("lng", DataType::Float64, false), 38 | ])); 39 | 40 | // register csv file with the execution context 41 | let csv_datasource = CsvDataSource::new("test/data/uk_cities.csv", schema.clone(), 1024); 42 | ctx.register_datasource("cities", Rc::new(RefCell::new(csv_datasource))); 43 | 44 | // simple projection and selection 45 | let sql = "SELECT city, lat, lng FROM cities WHERE lat > 51.0 AND lat < 53"; 46 | 47 | // execute the query 48 | let relation = ctx.sql(&sql).unwrap(); 49 | 50 | // display the relation 51 | let mut results = relation.borrow_mut(); 52 | 53 | while let Some(batch) = results.next().unwrap() { 54 | println!( 55 | "RecordBatch has {} rows and {} columns", 56 | batch.num_rows(), 57 | batch.num_columns() 58 | ); 59 | 60 | let city = batch 61 | .column(0) 62 | .as_any() 63 | .downcast_ref::() 64 | .unwrap(); 65 | 66 | let lat = batch 67 | .column(1) 68 | .as_any() 69 | .downcast_ref::() 70 | .unwrap(); 71 | 72 | let lng = batch 73 | .column(2) 74 | .as_any() 75 | .downcast_ref::() 76 | .unwrap(); 77 | 78 | for i in 0..batch.num_rows() { 79 | let city_name: String = String::from_utf8(city.value(i).to_vec()).unwrap(); 80 | 81 | println!( 82 | "City: {}, Latitude: {}, Longitude: {}", 83 | city_name, 84 | lat.value(i), 85 | lng.value(i), 86 | ); 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /img/datafusion-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygrove/datafusion-archive/219ca3f61f0d751779b4f30c02aa5fdcf8a40d12/img/datafusion-logo.png -------------------------------------------------------------------------------- /scripts/circle/build-examples.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Builds examples 4 | 5 | set -e 6 | 7 | cargo build --examples 8 | cargo run --example sql_query 9 | cargo run --example dataframe 10 | -------------------------------------------------------------------------------- /scripts/circle/build-release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Builds the release version 4 | 5 | cargo build --release 6 | -------------------------------------------------------------------------------- /scripts/circle/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Build the project 4 | 5 | set -e 6 | 7 | cargo build 8 | 9 | -------------------------------------------------------------------------------- /scripts/circle/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Tests the code branch 4 | 5 | set -e 6 | 7 | cargo test 8 | -------------------------------------------------------------------------------- /scripts/docker/base/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | RUN apt-get update 4 | RUN apt-get install -y curl vim pkg-config libssl-dev 5 | RUN curl https://sh.rustup.rs -sSf | sh -s -- -y 6 | RUN ~/.cargo/bin/rustup default nightly 7 | -------------------------------------------------------------------------------- /scripts/docker/base/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | docker build -f scripts/docker/base/Dockerfile -t datafusionrs/base . -------------------------------------------------------------------------------- /scripts/docker/console/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM datafusionrs/base:latest 2 | 3 | # Copy the source files into the image 4 | RUN mkdir -p /tmp/datafusion_source 5 | RUN mkdir -p /tmp/datafusion_source/src 6 | RUN mkdir -p /tmp/datafusion_source/examples 7 | 8 | ADD src /tmp/datafusion_source/src/ 9 | ADD examples /tmp/datafusion_source/examples/ 10 | ADD Cargo.toml /tmp/datafusion_source/ 11 | 12 | # Build the release 13 | RUN cd /tmp/datafusion_source ; . ~/.cargo/env ; cargo build --release 14 | 15 | ## Prepare directories for copy of the DataFusion binaries, and logging directory 16 | RUN mkdir -p /opt/datafusion/bin && \ 17 | mkdir -p /opt/datafusion/data && \ 18 | mkdir -p /var/log/datafusion 19 | 20 | RUN cp /tmp/datafusion_source/target/release/console /opt/datafusion/bin/ 21 | 22 | # Add some sample files 23 | ADD scripts/docker/console/all_types_flat.csv /opt/datafusion/data/ 24 | ADD scripts/docker/console/all_types_flat.parquet /opt/datafusion/data/ 25 | ADD scripts/docker/console/uk_cities.csv /opt/datafusion/data/ 26 | 27 | # Delete sources 28 | RUN rm -rf /tmp/datafusion_source 29 | 30 | ADD scripts/docker/console/console.sh /usr/bin/datafusion-console.sh 31 | 32 | ENTRYPOINT [ "/usr/bin/datafusion-console.sh" ] 33 | 34 | 35 | -------------------------------------------------------------------------------- /scripts/docker/console/all_types_flat.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygrove/datafusion-archive/219ca3f61f0d751779b4f30c02aa5fdcf8a40d12/scripts/docker/console/all_types_flat.parquet -------------------------------------------------------------------------------- /scripts/docker/console/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # get the product version from Cargo.toml 5 | DATAFUSION_VERSION=`grep version Cargo.toml | head -1 | awk -F' ' '{ print $3 }' | sed 's/\"//g'` 6 | 7 | # build the image 8 | echo "Building docker image for DataFusion Console ${DATAFUSION_VERSION} ..." 9 | sleep 0.2 10 | docker build -f scripts/docker/console/Dockerfile -t "datafusionrs/console:${DATAFUSION_VERSION}" . 11 | 12 | # tag as latest 13 | docker tag "datafusionrs/console:${DATAFUSION_VERSION}" "datafusionrs/console:latest" 14 | -------------------------------------------------------------------------------- /scripts/docker/console/console.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source /root/.cargo/env 3 | export LD_LIBRARY_PATH=$(rustc --print sysroot)/lib 4 | cd /opt/datafusion 5 | ./bin/console "$@" -------------------------------------------------------------------------------- /scripts/docker/console/uk_cities.csv: -------------------------------------------------------------------------------- 1 | "Elgin, Scotland, the UK",57.653484,-3.335724 2 | "Stoke-on-Trent, Staffordshire, the UK",53.002666,-2.179404 3 | "Solihull, Birmingham, UK",52.412811,-1.778197 4 | "Cardiff, Cardiff county, UK",51.481583,-3.179090 5 | "Eastbourne, East Sussex, UK",50.768036,0.290472 6 | "Oxford, Oxfordshire, UK",51.752022,-1.257677 7 | "London, UK",51.509865,-0.118092 8 | "Swindon, Swindon, UK",51.568535,-1.772232 9 | "Gravesend, Kent, UK",51.441883,0.370759 10 | "Northampton, Northamptonshire, UK",52.240479,-0.902656 11 | "Rugby, Warwickshire, UK",52.370876,-1.265032 12 | "Sutton Coldfield, West Midlands, UK",52.570385,-1.824042 13 | "Harlow, Essex, UK",51.772938,0.102310 14 | "Aberdeen, Aberdeen City, UK",57.149651,-2.099075 15 | "Swansea, Swansea, UK",51.621441,-3.943646 16 | "Chesterfield, Derbyshire, UK",53.235046,-1.421629 17 | "Londonderry, Derry, UK",55.006763,-7.318268 18 | "Salisbury, Wiltshire, UK",51.068787,-1.794472 19 | "Weymouth, Dorset, UK",50.614429,-2.457621 20 | "Wolverhampton, West Midlands, UK",52.591370,-2.110748 21 | "Preston, Lancashire, UK",53.765762,-2.692337 22 | "Bournemouth, UK",50.720806,-1.904755 23 | "Doncaster, South Yorkshire, UK",53.522820,-1.128462 24 | "Ayr, South Ayrshire, UK",55.458565,-4.629179 25 | "Hastings, East Sussex, UK",50.854259,0.573453 26 | "Bedford, UK",52.136436,-0.460739 27 | "Basildon, Essex, UK",51.572376,0.470009 28 | "Chippenham, Wiltshire, UK",51.458057,-2.116074 29 | "Belfast, UK",54.607868,-5.926437 30 | "Uckfield, East Sussex, UK",50.967941,0.085831 31 | "Worthing, West Sussex, UK",50.825024,-0.383835 32 | "Leeds, West Yorkshire, UK",53.801277,-1.548567 33 | "Kendal, Cumbria, UK",54.328506,-2.743870 34 | "Plymouth, UK",50.376289,-4.143841 35 | "Haverhill, Suffolk, UK",52.080875,0.444517 36 | "Frankton, Warwickshire, UK",52.328415,-1.377561 37 | "Inverness, the UK",57.477772,-4.224721 -------------------------------------------------------------------------------- /scripts/docker/worker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM datafusionrs/base:latest 2 | 3 | # Copy the source files into the image 4 | RUN mkdir -p /tmp/datafusion_source 5 | RUN mkdir -p /tmp/datafusion_source/src 6 | RUN mkdir -p /tmp/datafusion_source/examples 7 | RUN mkdir -p /tmp/datafusion_source/benches 8 | 9 | ADD src /tmp/datafusion_source/src/ 10 | ADD examples /tmp/datafusion_source/examples/ 11 | ADD benches /tmp/datafusion_source/benches/ 12 | ADD Cargo.toml /tmp/datafusion_source/ 13 | 14 | # Build the release 15 | RUN cd /tmp/datafusion_source ; . ~/.cargo/env ; cargo build --release 16 | 17 | ## Prepare directories for copy of the DataFusion binaries, and logging directory 18 | RUN mkdir -p /opt/datafusion/bin && \ 19 | mkdir -p /opt/datafusion/www && \ 20 | mkdir -p /opt/datafusion/www/css && \ 21 | mkdir -p /var/log/datafusion 22 | 23 | RUN cp /tmp/datafusion_source/target/release/worker /opt/datafusion/bin/ 24 | RUN cp /tmp/datafusion_source/src/bin/worker/*.html /opt/datafusion/www/ 25 | RUN cp /tmp/datafusion_source/src/bin/worker/css/* /opt/datafusion/www/css/ 26 | 27 | RUN rm -rf /tmp/datafusion_source 28 | 29 | # Expose port 8080 for the worker, if the worker is run 30 | EXPOSE 8080 31 | 32 | # Export /var/datafusion for data and logs 33 | VOLUME [ "/var/datafusion/data", "/var/datafusion/logs" ] 34 | 35 | ENTRYPOINT [ "/opt/datafusion/bin/worker" ] 36 | 37 | CMD [ "--help" ] 38 | 39 | 40 | -------------------------------------------------------------------------------- /scripts/docker/worker/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # get the product version from Cargo.toml 5 | DATAFUSION_VERSION=`grep version Cargo.toml | head -1 | awk -F' ' '{ print $3 }' | sed 's/\"//g'` 6 | 7 | # build the image 8 | echo "Building docker image for DataFusion Worker ${DATAFUSION_VERSION} ..." 9 | sleep 0.2 10 | docker build -f scripts/docker/worker/Dockerfile -t "datafusionrs/worker:${DATAFUSION_VERSION}" . 11 | 12 | # tag as latest 13 | docker tag "datafusionrs/worker:${DATAFUSION_VERSION}" "datafusionrs/worker:latest" 14 | -------------------------------------------------------------------------------- /scripts/release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | DATAFUSION_VERSION=`grep version Cargo.toml | head -1 | awk -F' ' '{ print $3 }' | sed 's/\"//g'` 6 | 7 | echo Version: $DATAFUSION_VERSION 8 | 9 | # Make sure there are no uncommitted changes 10 | git diff-index --quiet HEAD -- 11 | 12 | # Run tests 13 | cargo test 14 | 15 | # Run examples 16 | cargo run --example csv_sql 17 | cargo run --example csv_dataframe 18 | cargo run --example ndjson_sql 19 | cargo run --example parquet_sql 20 | cargo run --example parquet_dataframe 21 | 22 | # Build Docker image 23 | ./scripts/docker/console/build.sh 24 | 25 | # Publish crate 26 | cargo publish 27 | 28 | # Push Docker image 29 | docker push datafusionrs/console:$DATAFUSION_VERSION 30 | docker push datafusionrs/console:latest 31 | 32 | # Tag release 33 | git tag $DATAFUSION_VERSION 34 | git push origin :$DATAFUSION_VERSION 35 | -------------------------------------------------------------------------------- /scripts/smoketest.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | test_dir="target/smoketest" 4 | expected_file="test/data/smoketest-expected.txt" 5 | output_file="${test_dir}/smoketest_output.txt" 6 | 7 | mkdir -p $test_dir 8 | 9 | function cleanup { 10 | echo "CLEANUP: Removing ${test_dir}" 11 | rm -rf $test_dir 12 | } 13 | trap cleanup EXIT 14 | 15 | # run tests 16 | cargo fmt 17 | cargo test 18 | 19 | #TODO: check output from examples 20 | cargo run --example csv_sql 21 | #cargo run --example csv_dataframe 22 | #cargo run --example parquet_sql 23 | #cargo run --example parquet_dataframe 24 | 25 | # run benchmarks 26 | #cargo bench 27 | 28 | ## NOTE that distributed queries are broken since moving to Arrow ... will be be fixed later 29 | 30 | #./scripts/docker/worker/build.sh 31 | ./scripts/docker/console/build.sh 32 | # 33 | ## stop etcd 34 | #docker kill etcd 35 | #docker rm etcd 36 | # 37 | ## stop datafusion worker 38 | #docker kill datafusion 39 | #docker rm datafusion 40 | # 41 | ## run etcd 42 | #docker run -d -v /usr/share/ca-certificates/:/etc/ssl/certs -p 4001:4001 -p 2380:2380 -p 2379:2379 \ 43 | # --name etcd quay.io/coreos/etcd:v2.3.8 \ 44 | # -name etcd0 \ 45 | # -advertise-client-urls http://${HostIP}:2379,http://${HostIP}:4001 \ 46 | # -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 \ 47 | # -initial-advertise-peer-urls http://${HostIP}:2380 \ 48 | # -listen-peer-urls http://0.0.0.0:2380 \ 49 | # -initial-cluster-token etcd-cluster-1 \ 50 | # -initial-cluster etcd0=http://${HostIP}:2380 \ 51 | # -initial-cluster-state new 52 | # 53 | ## give etcd a chance to start up 54 | #sleep 2 55 | # 56 | ## run worker 57 | #docker run --network=host -d -p 8088:8088 \ 58 | # -v`pwd`/test/data:/var/datafusion/data \ 59 | # --name datafusion datafusionrs/worker:latest \ 60 | # --etcd http://127.0.0.1:2379 \ 61 | # --bind 127.0.0.1:8088 \ 62 | # --data_dir /var/datafusion/data \ 63 | # --webroot /opt/datafusion/www 64 | # 65 | ## give the worker a chance to start up 66 | #sleep 2 67 | # 68 | # run the console in interactive mode and run a test script 69 | docker run \ 70 | --network=host \ 71 | -v`pwd`/test/data:/test/data \ 72 | -it datafusionrs/console:latest \ 73 | --script /test/data/smoketest.sql \ 74 | > $output_file 75 | 76 | echo "###### Smoketest output" 77 | 78 | cat $output_file 79 | 80 | echo "###### Verifying smoke test results" 81 | 82 | file_diff="$(diff -bBZ -I seconds $output_file $expected_file)" 83 | if [ -n "$file_diff" ] 84 | then 85 | echo "${file_diff}" 86 | echo "ERROR: smoke test output differs from expected output" 87 | exit 1 88 | fi 89 | echo "SUCCESS: smoke test successfully executed" 90 | 91 | -------------------------------------------------------------------------------- /src/bin/console/linereader.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | extern crate datafusion_rustyline; 16 | use self::datafusion_rustyline::error::ReadlineError; 17 | use self::datafusion_rustyline::Editor; 18 | 19 | const DEFAULT_PROMPT: &'static str = "datafusion> "; 20 | const CONTINUE_PROMPT: &'static str = "> "; 21 | 22 | #[cfg(target_family = "unix")] 23 | pub enum LineResult { 24 | Break, 25 | Input(String), 26 | } 27 | 28 | #[cfg(target_family = "unix")] 29 | pub struct LineReader<'a> { 30 | reader: Editor<()>, 31 | prompt: &'a str, 32 | } 33 | 34 | #[cfg(target_family = "unix")] 35 | impl<'a> LineReader<'a> { 36 | pub fn new() -> Self { 37 | LineReader { 38 | reader: Editor::<()>::new(), 39 | prompt: DEFAULT_PROMPT, 40 | } 41 | } 42 | 43 | pub fn set_prompt(&mut self, prompt: &'a str) { 44 | self.prompt = prompt; 45 | } 46 | 47 | pub fn read_lines(&mut self) -> Option { 48 | let mut result = String::new(); 49 | 50 | // if rl.load_history("history.txt").is_err() { 51 | // println!("No previous history."); 52 | // } 53 | loop { 54 | let line = self.reader.readline(self.prompt); 55 | 56 | match line { 57 | Ok(i) => { 58 | let j = i.as_str().trim_end(); 59 | result.push_str(j); 60 | 61 | match j { 62 | "quit" | "exit" => { 63 | return Some(LineResult::Break); 64 | } 65 | _ => { 66 | // Handle the two types of statements, Default and Continue. 67 | // CONTINUE: are statements that don't end with a semicolon 68 | // DEFAULT: are statements that end with a semicolon 69 | // and can be returned to being executed. 70 | if j.ends_with(';') { 71 | self.set_prompt(DEFAULT_PROMPT); 72 | break; 73 | } else { 74 | self.set_prompt(CONTINUE_PROMPT); 75 | result.push_str(" "); 76 | continue; 77 | } 78 | } 79 | } 80 | } 81 | Err(ReadlineError::Interrupted) => { 82 | println!("CTRL-C"); 83 | break; 84 | } 85 | Err(ReadlineError::Eof) => { 86 | println!("CTRL-D"); 87 | return Some(LineResult::Break); 88 | } 89 | Err(err) => { 90 | println!("Error: {:?}", err); 91 | break; 92 | } 93 | }; 94 | } 95 | //self.reader.save_history("history.txt").unwrap(); 96 | 97 | if !result.trim().is_empty() { 98 | //self.reader.add_history_entry(&result); 99 | } 100 | 101 | // Return the command without semicolon 102 | Some(LineResult::Input(result[..result.len() - 1].to_string())) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/bin/console/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | extern crate clap; 16 | extern crate datafusion; 17 | 18 | use std::fs::File; 19 | use std::io::BufRead; 20 | use std::io::BufReader; 21 | use std::str; 22 | use std::time::Instant; 23 | 24 | use clap::{App, Arg}; 25 | //use datafusion::functions::geospatial::st_astext::*; 26 | //use datafusion::functions::geospatial::st_point::*; 27 | //use datafusion::functions::math::*; 28 | use datafusion::dfparser::DFASTNode::CreateExternalTable; 29 | use datafusion::dfparser::DFParser; 30 | use datafusion::execution::context::ExecutionContext; 31 | mod linereader; 32 | 33 | const VERSION: &'static str = env!("CARGO_PKG_VERSION"); 34 | 35 | #[cfg(target_family = "unix")] 36 | fn setup_console(cmdline: clap::ArgMatches) { 37 | //parse args 38 | //let etcd_endpoints = cmdline.value_of("ETCD").unwrap(); 39 | let mut console = Console::new(/*etcd_endpoints.to_string()*/); 40 | 41 | match cmdline.value_of("SCRIPT") { 42 | Some(filename) => match File::open(filename) { 43 | Ok(f) => { 44 | let mut cmd_buffer = String::new(); 45 | let reader = BufReader::new(&f); 46 | for line in reader.lines() { 47 | match line { 48 | Ok(cmd) => { 49 | cmd_buffer.push_str(&cmd); 50 | if cmd_buffer.as_str().ends_with(";") { 51 | console.execute(&cmd_buffer[0..cmd_buffer.len() - 2]); 52 | cmd_buffer = String::new(); 53 | } 54 | } 55 | Err(e) => println!("Error: {}", e), 56 | } 57 | } 58 | if cmd_buffer.as_str().ends_with(";") { 59 | console.execute(&cmd_buffer[0..cmd_buffer.len() - 2]); 60 | } 61 | } 62 | Err(e) => println!("Could not open file {}: {}", filename, e), 63 | }, 64 | _ => { 65 | let mut reader = linereader::LineReader::new(); 66 | loop { 67 | let result = reader.read_lines(); 68 | match result { 69 | Some(line) => match line { 70 | linereader::LineResult::Break => break, 71 | linereader::LineResult::Input(command) => console.execute(&command), 72 | }, 73 | None => (), 74 | } 75 | } 76 | } 77 | } 78 | } 79 | 80 | #[cfg(target_family = "windows")] 81 | fn setup_console(cmdline: clap::ArgMatches) { 82 | panic!("Console is not supported on windows!") 83 | } 84 | 85 | fn main() { 86 | println!("DataFusion Console"); 87 | // println!(""); 88 | // println!("Enter SQL statements terminated with semicolon, or 'quit' to leave."); 89 | // println!(""); 90 | 91 | let cmdline = App::new("DataFusion Console") 92 | .version(VERSION) 93 | // .arg( 94 | // Arg::with_name("ETCD") 95 | // .help("etcd endpoints") 96 | // .short("e") 97 | // .long("etcd") 98 | // .value_name("URL") 99 | // .required(true) 100 | // .takes_value(true), 101 | // ) 102 | .arg( 103 | Arg::with_name("SCRIPT") 104 | .help("SQL script to run") 105 | .short("s") 106 | .long("script") 107 | .required(false) 108 | .takes_value(true), 109 | ) 110 | .get_matches(); 111 | setup_console(cmdline); 112 | } 113 | 114 | /// Interactive SQL console 115 | struct Console { 116 | ctx: ExecutionContext, 117 | } 118 | 119 | impl Console { 120 | /// Create a new instance of the console 121 | fn new() -> Self { 122 | let ctx = ExecutionContext::new(); 123 | // ctx.register_scalar_function(Rc::new(STPointFunc {})); 124 | // ctx.register_scalar_function(Rc::new(STAsText {})); 125 | // ctx.register_scalar_function(Rc::new(SqrtFunction {})); 126 | Console { ctx } 127 | } 128 | 129 | /// Execute a SQL statement or console command 130 | fn execute(&mut self, sql: &str) { 131 | println!("Executing query ..."); 132 | 133 | let timer = Instant::now(); 134 | 135 | // parse the SQL 136 | match DFParser::parse_sql(String::from(sql)) { 137 | Ok(ast) => match ast { 138 | CreateExternalTable { .. } => { 139 | self.ctx.sql(&sql).unwrap(); 140 | //println!("Registered schema with execution context"); 141 | () 142 | } 143 | _ => match self.ctx.sql(sql) { 144 | Ok(_result) => { 145 | let elapsed = timer.elapsed(); 146 | let _elapsed_seconds = 147 | elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 / 1000000000.0; 148 | } 149 | Err(e) => println!("Error: {:?}", e), 150 | }, 151 | }, 152 | Err(e) => println!("Error: {:?}", e), 153 | } 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/dfparser.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! SQL Parser 16 | //! 17 | //! Note that most SQL parsing is now delegated to the sqlparser crate, which handles ANSI SQL but 18 | //! this module contains DataFusion-specific SQL extensions. 19 | 20 | use sqlparser::dialect::*; 21 | use sqlparser::sqlast::*; 22 | use sqlparser::sqlparser::*; 23 | use sqlparser::sqltokenizer::*; 24 | 25 | macro_rules! parser_err { 26 | ($MSG:expr) => { 27 | Err(ParserError::ParserError($MSG.to_string())) 28 | }; 29 | } 30 | 31 | #[derive(Debug, Clone)] 32 | pub enum FileType { 33 | NdJson, 34 | Parquet, 35 | CSV, 36 | } 37 | 38 | #[derive(Debug, Clone)] 39 | pub enum DFASTNode { 40 | /// ANSI SQL AST node 41 | ANSI(ASTNode), 42 | /// DDL for creating an external table in DataFusion 43 | CreateExternalTable { 44 | /// Table name 45 | name: String, 46 | /// Optional schema 47 | columns: Vec, 48 | /// File type (Parquet, NDJSON, CSV) 49 | file_type: FileType, 50 | /// Header row? 51 | header_row: bool, 52 | /// Path to file 53 | location: String, 54 | }, 55 | } 56 | 57 | /// SQL Parser 58 | pub struct DFParser { 59 | parser: Parser, 60 | } 61 | 62 | impl DFParser { 63 | /// Parse the specified tokens 64 | pub fn new(sql: String) -> Result { 65 | let dialect = GenericSqlDialect {}; 66 | let mut tokenizer = Tokenizer::new(&dialect, &sql); 67 | let tokens = tokenizer.tokenize()?; 68 | Ok(DFParser { 69 | parser: Parser::new(tokens), 70 | }) 71 | } 72 | 73 | /// Parse a SQL statement and produce an Abstract Syntax Tree (AST) 74 | pub fn parse_sql(sql: String) -> Result { 75 | let mut parser = DFParser::new(sql)?; 76 | parser.parse() 77 | } 78 | 79 | /// Parse a new expression 80 | pub fn parse(&mut self) -> Result { 81 | self.parse_expr(0) 82 | } 83 | 84 | /// Parse tokens until the precedence changes 85 | fn parse_expr(&mut self, precedence: u8) -> Result { 86 | let mut expr = self.parse_prefix()?; 87 | loop { 88 | let next_precedence = self.parser.get_next_precedence()?; 89 | if precedence >= next_precedence { 90 | break; 91 | } 92 | 93 | if let Some(infix_expr) = self.parse_infix(expr.clone(), next_precedence)? { 94 | expr = infix_expr; 95 | } 96 | } 97 | Ok(expr) 98 | } 99 | 100 | /// Parse an expression prefix 101 | fn parse_prefix(&mut self) -> Result { 102 | if self 103 | .parser 104 | .parse_keywords(vec!["CREATE", "EXTERNAL", "TABLE"]) 105 | { 106 | match self.parser.next_token() { 107 | Some(Token::Identifier(id)) => { 108 | // parse optional column list (schema) 109 | let mut columns = vec![]; 110 | if self.parser.consume_token(&Token::LParen) { 111 | loop { 112 | if let Some(Token::Identifier(column_name)) = self.parser.next_token() { 113 | if let Ok(data_type) = self.parser.parse_data_type() { 114 | let allow_null = 115 | if self.parser.parse_keywords(vec!["NOT", "NULL"]) { 116 | false 117 | } else if self.parser.parse_keyword("NULL") { 118 | true 119 | } else { 120 | true 121 | }; 122 | 123 | match self.parser.peek_token() { 124 | Some(Token::Comma) => { 125 | self.parser.next_token(); 126 | columns.push(SQLColumnDef { 127 | name: column_name, 128 | data_type: data_type, 129 | allow_null, 130 | default: None, 131 | is_primary: false, 132 | is_unique: false, 133 | }); 134 | } 135 | Some(Token::RParen) => { 136 | self.parser.next_token(); 137 | columns.push(SQLColumnDef { 138 | name: column_name, 139 | data_type: data_type, 140 | allow_null, 141 | default: None, 142 | is_primary: false, 143 | is_unique: false, 144 | }); 145 | break; 146 | } 147 | _ => { 148 | return parser_err!( 149 | "Expected ',' or ')' after column definition" 150 | ); 151 | } 152 | } 153 | } else { 154 | return parser_err!( 155 | "Error parsing data type in column definition" 156 | ); 157 | } 158 | } else { 159 | return parser_err!("Error parsing column name"); 160 | } 161 | } 162 | } 163 | 164 | //println!("Parsed {} column defs", columns.len()); 165 | 166 | let mut headers = true; 167 | let file_type: FileType = 168 | if self.parser.parse_keywords(vec!["STORED", "AS", "CSV"]) { 169 | if self.parser.parse_keywords(vec!["WITH", "HEADER", "ROW"]) { 170 | headers = true; 171 | } else if self.parser.parse_keywords(vec!["WITHOUT", "HEADER", "ROW"]) { 172 | headers = false; 173 | } 174 | FileType::CSV 175 | } else if self.parser.parse_keywords(vec!["STORED", "AS", "NDJSON"]) { 176 | FileType::NdJson 177 | } else if self.parser.parse_keywords(vec!["STORED", "AS", "PARQUET"]) { 178 | FileType::Parquet 179 | } else { 180 | return parser_err!(format!( 181 | "Expected 'STORED AS' clause, found {:?}", 182 | self.parser.peek_token() 183 | )); 184 | }; 185 | 186 | let location: String = if self.parser.parse_keywords(vec!["LOCATION"]) { 187 | self.parser.parse_literal_string()? 188 | } else { 189 | return parser_err!("Missing 'LOCATION' clause"); 190 | }; 191 | 192 | Ok(DFASTNode::CreateExternalTable { 193 | name: id, 194 | columns, 195 | file_type, 196 | header_row: headers, 197 | location, 198 | }) 199 | } 200 | _ => parser_err!(format!( 201 | "Unexpected token after CREATE EXTERNAL TABLE: {:?}", 202 | self.parser.peek_token() 203 | )), 204 | } 205 | } else { 206 | Ok(DFASTNode::ANSI(self.parser.parse_prefix()?)) 207 | } 208 | } 209 | 210 | pub fn parse_infix( 211 | &mut self, 212 | _expr: DFASTNode, 213 | _precedence: u8, 214 | ) -> Result, ParserError> { 215 | // match expr { 216 | // DFASTNode::ANSI(ansi) => { 217 | // //DFASTNode::ANSI(self.parser.parse_infix(ansi, precedence)?) 218 | // }) 219 | 220 | unimplemented!() 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /src/execution/context.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::cell::RefCell; 16 | use std::collections::HashMap; 17 | use std::rc::Rc; 18 | use std::sync::Arc; 19 | 20 | use arrow::datatypes::{Field, Schema}; 21 | 22 | use super::super::dfparser::{DFASTNode, DFParser}; 23 | use super::super::logicalplan::*; 24 | use super::super::sqlplanner::{SchemaProvider, SqlToRel}; 25 | use super::aggregate::AggregateRelation; 26 | use super::datasource::DataSource; 27 | use super::error::{ExecutionError, Result}; 28 | use super::expression::*; 29 | use super::filter::FilterRelation; 30 | use super::projection::ProjectRelation; 31 | use super::relation::{DataSourceRelation, Relation}; 32 | 33 | pub struct ExecutionContext { 34 | datasources: Rc>>>>, 35 | } 36 | 37 | impl ExecutionContext { 38 | pub fn new() -> Self { 39 | Self { 40 | datasources: Rc::new(RefCell::new(HashMap::new())), 41 | } 42 | } 43 | 44 | pub fn sql(&mut self, sql: &str) -> Result>> { 45 | let ast = DFParser::parse_sql(String::from(sql))?; 46 | 47 | match ast { 48 | // DFASTNode::CreateExternalTable { 49 | // name, 50 | // columns, 51 | // file_type, 52 | // header_row, 53 | // location, 54 | // } => { 55 | // let fields: Vec = columns 56 | // .iter() 57 | // .map(|c| Field::new(&c.name, convert_data_type(&c.data_type), c.allow_null)) 58 | // .collect(); 59 | // let schema = Schema::new(fields); 60 | // 61 | // let df = match file_type { 62 | // FileType::CSV => self.load_csv(&location, &schema, header_row, None)?, 63 | // FileType::NdJson => self.load_ndjson(&location, &schema, None)?, 64 | // FileType::Parquet => self.load_parquet(&location, None)?, 65 | // }; 66 | // 67 | // self.register(&name, df); 68 | // 69 | // //TODO: not sure what to return here 70 | // Ok(Rc::new(DF::new( 71 | // self.clone(), 72 | // Rc::new(LogicalPlan::EmptyRelation { 73 | // schema: Rc::new(Schema::empty()), 74 | // }), 75 | // ))) 76 | // } 77 | DFASTNode::ANSI(ansi) => { 78 | let schema_provider: Rc = Rc::new(ExecutionContextSchemaProvider { 79 | datasources: self.datasources.clone(), 80 | }); 81 | 82 | // create a query planner 83 | let query_planner = SqlToRel::new(schema_provider); 84 | 85 | // plan the query (create a logical relational plan) 86 | let plan = query_planner.sql_to_rel(&ansi)?; 87 | //println!("Logical plan: {:?}", plan); 88 | 89 | let optimized_plan = plan; //push_down_projection(&plan, &HashSet::new()); 90 | //println!("Optimized logical plan: {:?}", new_plan); 91 | 92 | let relation = self.execute(&optimized_plan)?; 93 | 94 | Ok(relation) 95 | } 96 | _ => unimplemented!(), 97 | } 98 | } 99 | 100 | pub fn register_datasource(&mut self, name: &str, ds: Rc>) { 101 | self.datasources.borrow_mut().insert(name.to_string(), ds); 102 | } 103 | 104 | pub fn execute(&mut self, plan: &LogicalPlan) -> Result>> { 105 | println!("Logical plan: {:?}", plan); 106 | 107 | match *plan { 108 | // LogicalPlan::EmptyRelation { .. } => Ok(Box::new(DataSourceRelation { 109 | // schema: Schema::new(vec![]), 110 | // ds: Rc::new(RefCell::new(EmptyRelation::new())), 111 | // })), 112 | // 113 | // LogicalPlan::Sort { .. } => unimplemented!(), 114 | LogicalPlan::TableScan { ref table_name, .. } => { 115 | match self.datasources.borrow().get(table_name) { 116 | Some(ds) => { 117 | //TODO: projection 118 | Ok(Rc::new(RefCell::new(DataSourceRelation::new(ds.clone())))) 119 | } 120 | _ => Err(ExecutionError::General(format!( 121 | "No table registered as '{}'", 122 | table_name 123 | ))), 124 | } 125 | } 126 | LogicalPlan::Selection { 127 | ref expr, 128 | ref input, 129 | } => { 130 | let input_rel = self.execute(input)?; 131 | let input_schema = input_rel.as_ref().borrow().schema().clone(); 132 | let runtime_expr = compile_scalar_expr(&self, expr, &input_schema)?; 133 | let rel = FilterRelation::new( 134 | input_rel, 135 | runtime_expr, /*.get_func().clone()*/ 136 | input_schema, 137 | ); 138 | Ok(Rc::new(RefCell::new(rel))) 139 | } 140 | LogicalPlan::Projection { 141 | ref expr, 142 | ref input, 143 | .. 144 | } => { 145 | let input_rel = self.execute(input)?; 146 | 147 | let input_schema = input_rel.as_ref().borrow().schema().clone(); 148 | 149 | let project_columns: Vec = exprlist_to_fields(&expr, &input_schema); 150 | 151 | let project_schema = Arc::new(Schema::new(project_columns)); 152 | 153 | let compiled_expr: Result> = expr 154 | .iter() 155 | .map(|e| compile_scalar_expr(&self, e, &input_schema)) 156 | .collect(); 157 | 158 | let rel = ProjectRelation::new(input_rel, compiled_expr?, project_schema); 159 | 160 | Ok(Rc::new(RefCell::new(rel))) 161 | } 162 | LogicalPlan::Aggregate { 163 | ref input, 164 | ref group_expr, 165 | ref aggr_expr, 166 | .. 167 | } => { 168 | let input_rel = self.execute(&input)?; 169 | 170 | let input_schema = input_rel.as_ref().borrow().schema().clone(); 171 | 172 | let compiled_group_expr_result: Result> = group_expr 173 | .iter() 174 | .map(|e| compile_scalar_expr(&self, e, &input_schema)) 175 | .collect(); 176 | let compiled_group_expr = compiled_group_expr_result?; 177 | 178 | let compiled_aggr_expr_result: Result> = aggr_expr 179 | .iter() 180 | .map(|e| compile_expr(&self, e, &input_schema)) 181 | .collect(); 182 | let compiled_aggr_expr = compiled_aggr_expr_result?; 183 | 184 | let rel = AggregateRelation::new( 185 | Arc::new(Schema::empty()), //(expr_to_field(&compiled_group_expr, &input_schema))), 186 | input_rel, 187 | compiled_group_expr, 188 | compiled_aggr_expr, 189 | ); 190 | 191 | Ok(Rc::new(RefCell::new(rel))) 192 | } 193 | 194 | _ => unimplemented!(), 195 | } 196 | } 197 | } 198 | 199 | #[derive(Debug, Clone)] 200 | pub enum ExecutionResult { 201 | Unit, 202 | Count(usize), 203 | Str(String), 204 | } 205 | 206 | pub fn expr_to_field(e: &Expr, input_schema: &Schema) -> Field { 207 | match e { 208 | Expr::Column(i) => input_schema.fields()[*i].clone(), 209 | Expr::Literal(ref lit) => Field::new("lit", lit.get_datatype(), true), 210 | Expr::ScalarFunction { 211 | ref name, 212 | ref return_type, 213 | .. 214 | } => Field::new(&name, return_type.clone(), true), 215 | Expr::AggregateFunction { 216 | ref name, 217 | ref return_type, 218 | .. 219 | } => Field::new(&name, return_type.clone(), true), 220 | Expr::Cast { ref data_type, .. } => Field::new("cast", data_type.clone(), true), 221 | Expr::BinaryExpr { 222 | ref left, 223 | ref right, 224 | .. 225 | } => { 226 | let left_type = left.get_type(input_schema); 227 | let right_type = right.get_type(input_schema); 228 | Field::new( 229 | "binary_expr", 230 | get_supertype(&left_type, &right_type).unwrap(), 231 | true, 232 | ) 233 | } 234 | _ => unimplemented!("Cannot determine schema type for expression {:?}", e), 235 | } 236 | } 237 | 238 | pub fn exprlist_to_fields(expr: &Vec, input_schema: &Schema) -> Vec { 239 | expr.iter() 240 | .map(|e| expr_to_field(e, input_schema)) 241 | .collect() 242 | } 243 | 244 | struct ExecutionContextSchemaProvider { 245 | datasources: Rc>>>>, 246 | } 247 | impl SchemaProvider for ExecutionContextSchemaProvider { 248 | fn get_table_meta(&self, name: &str) -> Option> { 249 | match self.datasources.borrow().get(name) { 250 | Some(ds) => Some(ds.borrow().schema().clone()), 251 | None => None, 252 | } 253 | } 254 | 255 | fn get_function_meta(&self, _name: &str) -> Option> { 256 | unimplemented!() 257 | } 258 | } 259 | -------------------------------------------------------------------------------- /src/execution/datasource.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! Data sources 16 | 17 | use std::fs::File; 18 | use std::rc::Rc; 19 | use std::sync::Arc; 20 | 21 | use arrow::csv; 22 | use arrow::datatypes::Schema; 23 | use arrow::record_batch::RecordBatch; 24 | 25 | use super::error::Result; 26 | 27 | pub trait DataSource { 28 | fn schema(&self) -> &Arc; 29 | fn next(&mut self) -> Result>; 30 | } 31 | 32 | /// CSV data source 33 | pub struct CsvDataSource { 34 | schema: Arc, 35 | reader: csv::Reader, 36 | } 37 | 38 | impl CsvDataSource { 39 | pub fn new(filename: &str, schema: Arc, batch_size: usize) -> Self { 40 | let file = File::open(filename).unwrap(); 41 | let reader = csv::Reader::new(file, schema.clone(), true, batch_size, None); 42 | Self { schema, reader } 43 | } 44 | 45 | pub fn from_reader(schema: Arc, reader: csv::Reader) -> Self { 46 | Self { schema, reader } 47 | } 48 | } 49 | 50 | impl DataSource for CsvDataSource { 51 | fn schema(&self) -> &Arc { 52 | &self.schema 53 | } 54 | 55 | fn next(&mut self) -> Result> { 56 | Ok(self.reader.next()?) 57 | } 58 | } 59 | 60 | //pub struct DataSourceIterator { 61 | // pub ds: Rc>, 62 | //} 63 | // 64 | //impl DataSourceIterator { 65 | // pub fn new(ds: Rc>) -> Self { 66 | // DataSourceIterator { ds } 67 | // } 68 | //} 69 | // 70 | //impl Iterator for DataSourceIterator { 71 | // type Item = Result>; 72 | // 73 | // fn next(&mut self) -> Option { 74 | // self.ds.borrow_mut().next() 75 | // } 76 | //} 77 | 78 | #[derive(Serialize, Deserialize, Clone)] 79 | pub enum DataSourceMeta { 80 | /// Represents a CSV file with a provided schema 81 | CsvFile { 82 | filename: String, 83 | schema: Rc, 84 | has_header: bool, 85 | projection: Option>, 86 | }, 87 | /// Represents a Parquet file that contains schema information 88 | ParquetFile { 89 | filename: String, 90 | schema: Rc, 91 | projection: Option>, 92 | }, 93 | } 94 | -------------------------------------------------------------------------------- /src/execution/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! Error types 16 | 17 | use std::io::Error; 18 | use std::result; 19 | 20 | use arrow::error::ArrowError; 21 | 22 | use sqlparser::sqlparser::ParserError; 23 | 24 | pub type Result = result::Result; 25 | 26 | #[derive(Debug)] 27 | pub enum ExecutionError { 28 | IoError(Error), 29 | ParserError(ParserError), 30 | General(String), 31 | InvalidColumn(String), 32 | NotImplemented(String), 33 | InternalError(String), 34 | ArrowError(ArrowError), 35 | ExecutionError(String), 36 | } 37 | 38 | impl From for ExecutionError { 39 | fn from(e: Error) -> Self { 40 | ExecutionError::IoError(e) 41 | } 42 | } 43 | 44 | impl From for ExecutionError { 45 | fn from(e: String) -> Self { 46 | ExecutionError::General(e) 47 | } 48 | } 49 | 50 | impl From<&'static str> for ExecutionError { 51 | fn from(e: &'static str) -> Self { 52 | ExecutionError::General(e.to_string()) 53 | } 54 | } 55 | 56 | impl From for ExecutionError { 57 | fn from(e: ArrowError) -> Self { 58 | ExecutionError::ArrowError(e) 59 | } 60 | } 61 | 62 | impl From for ExecutionError { 63 | fn from(e: ParserError) -> Self { 64 | ExecutionError::ParserError(e) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/execution/filter.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! Execution of a filter (predicate) 16 | 17 | use std::cell::RefCell; 18 | use std::rc::Rc; 19 | use std::sync::Arc; 20 | 21 | use arrow::array::{Array, ArrayRef, BinaryArray, BooleanArray, Float64Array}; 22 | use arrow::datatypes::{DataType, Schema}; 23 | use arrow::record_batch::RecordBatch; 24 | 25 | use super::error::{ExecutionError, Result}; 26 | use super::expression::RuntimeExpr; 27 | use super::relation::Relation; 28 | 29 | pub struct FilterRelation { 30 | schema: Arc, 31 | input: Rc>, 32 | expr: RuntimeExpr, 33 | } 34 | 35 | impl FilterRelation { 36 | pub fn new(input: Rc>, expr: RuntimeExpr, schema: Arc) -> Self { 37 | Self { 38 | schema, 39 | input, 40 | expr, 41 | } 42 | } 43 | } 44 | 45 | impl Relation for FilterRelation { 46 | fn next(&mut self) -> Result> { 47 | match self.input.borrow_mut().next()? { 48 | Some(batch) => { 49 | // evaluate the filter expression against the batch 50 | match self.expr.get_func()(&batch)? 51 | .as_any() 52 | .downcast_ref::() 53 | { 54 | Some(filter_bools) => { 55 | let filtered_columns: Result> = (0..batch.num_columns()) 56 | .map(|i| filter(batch.column(i), &filter_bools)) 57 | .collect(); 58 | 59 | let filtered_batch: RecordBatch = 60 | RecordBatch::new(Arc::new(Schema::empty()), filtered_columns?); 61 | 62 | Ok(Some(filtered_batch)) 63 | } 64 | _ => Err(ExecutionError::ExecutionError( 65 | "Filter expression did not evaluate to boolean".to_string(), 66 | )), 67 | } 68 | } 69 | None => Ok(None), 70 | } 71 | } 72 | 73 | fn schema(&self) -> &Arc { 74 | &self.schema 75 | } 76 | } 77 | 78 | //TODO: move into Arrow array_ops 79 | fn filter(array: &Arc, filter: &BooleanArray) -> Result { 80 | let a = array.as_ref(); 81 | 82 | match a.data_type() { 83 | DataType::Float64 => { 84 | let b = a.as_any().downcast_ref::().unwrap(); 85 | let mut builder = Float64Array::builder(b.len()); 86 | for i in 0..b.len() { 87 | if filter.value(i) { 88 | builder.append_value(b.value(i))?; 89 | } 90 | } 91 | Ok(Arc::new(builder.finish())) 92 | } 93 | DataType::Utf8 => { 94 | //TODO: this is inefficient and we should improve the Arrow impl to help make this more concise 95 | let b = a.as_any().downcast_ref::().unwrap(); 96 | let mut values: Vec = Vec::with_capacity(b.len()); 97 | for i in 0..b.len() { 98 | if filter.value(i) { 99 | values.push(b.get_string(i)); 100 | } 101 | } 102 | let tmp: Vec<&str> = values.iter().map(|s| s.as_str()).collect(); 103 | Ok(Arc::new(BinaryArray::from(tmp))) 104 | } 105 | other => Err(ExecutionError::ExecutionError(format!( 106 | "filter not supported for {:?}", 107 | other 108 | ))), 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/execution/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod aggregate; 16 | pub mod context; 17 | pub mod datasource; 18 | pub mod error; 19 | pub mod expression; 20 | pub mod filter; 21 | pub mod physicalplan; 22 | pub mod projection; 23 | pub mod relation; 24 | pub mod value; 25 | -------------------------------------------------------------------------------- /src/execution/physicalplan.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::super::logicalplan::LogicalPlan; 16 | use std::rc::Rc; 17 | 18 | #[derive(Serialize, Deserialize, Debug, Clone)] 19 | pub enum PhysicalPlan { 20 | /// Run a query and return the results to the client 21 | Interactive { 22 | plan: Rc, 23 | }, 24 | /// Execute a logical plan and write the output to a file 25 | Write { 26 | plan: Rc, 27 | filename: String, 28 | kind: String, 29 | }, 30 | Show { 31 | plan: Rc, 32 | count: usize, 33 | }, 34 | } 35 | -------------------------------------------------------------------------------- /src/execution/projection.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! Execution of a projection 16 | 17 | use std::cell::RefCell; 18 | use std::rc::Rc; 19 | use std::sync::Arc; 20 | 21 | use arrow::array::ArrayRef; 22 | use arrow::datatypes::{Field, Schema}; 23 | use arrow::record_batch::RecordBatch; 24 | 25 | use super::error::Result; 26 | use super::expression::RuntimeExpr; 27 | use super::relation::Relation; 28 | 29 | pub struct ProjectRelation { 30 | schema: Arc, 31 | input: Rc>, 32 | expr: Vec, 33 | } 34 | 35 | impl ProjectRelation { 36 | pub fn new(input: Rc>, expr: Vec, schema: Arc) -> Self { 37 | ProjectRelation { 38 | input, 39 | expr, 40 | schema, 41 | } 42 | } 43 | } 44 | 45 | impl Relation for ProjectRelation { 46 | fn next(&mut self) -> Result> { 47 | match self.input.borrow_mut().next()? { 48 | Some(batch) => { 49 | let projected_columns: Result> = 50 | self.expr.iter().map(|e| e.get_func()(&batch)).collect(); 51 | 52 | let schema = Schema::new( 53 | self.expr 54 | .iter() 55 | .map(|e| Field::new(&e.get_name(), e.get_type(), true)) 56 | .collect(), 57 | ); 58 | 59 | let projected_batch: RecordBatch = 60 | RecordBatch::new(Arc::new(schema), projected_columns?); 61 | 62 | Ok(Some(projected_batch)) 63 | } 64 | None => Ok(None), 65 | } 66 | } 67 | 68 | fn schema(&self) -> &Arc { 69 | &self.schema 70 | } 71 | } 72 | 73 | #[cfg(test)] 74 | mod tests { 75 | use super::super::super::logicalplan::Expr; 76 | use super::super::context::ExecutionContext; 77 | use super::super::datasource::CsvDataSource; 78 | use super::super::expression; 79 | use super::super::relation::DataSourceRelation; 80 | use super::*; 81 | use arrow::datatypes::{DataType, Field, Schema}; 82 | 83 | #[test] 84 | fn project_all_columns() { 85 | let schema = Arc::new(Schema::new(vec![ 86 | Field::new("id", DataType::Int32, false), 87 | Field::new("first_name", DataType::Utf8, false), 88 | ])); 89 | let ds = CsvDataSource::new("test/data/people.csv", schema.clone(), 1024); 90 | let relation = Rc::new(RefCell::new(DataSourceRelation::new(Rc::new( 91 | RefCell::new(ds), 92 | )))); 93 | let context = ExecutionContext::new(); 94 | 95 | let projection_expr = 96 | vec![expression::compile_expr(&context, &Expr::Column(0), schema.as_ref()).unwrap()]; 97 | 98 | let mut projection = ProjectRelation::new(relation, projection_expr, schema); 99 | let batch = projection.next().unwrap().unwrap(); 100 | assert_eq!(1, batch.num_columns()); 101 | 102 | assert_eq!("id", batch.schema().field(0).name()); 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /src/execution/relation.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::cell::RefCell; 16 | use std::rc::Rc; 17 | use std::sync::Arc; 18 | 19 | use arrow::datatypes::Schema; 20 | use arrow::record_batch::RecordBatch; 21 | 22 | use super::datasource::DataSource; 23 | use super::error::Result; 24 | 25 | /// trait for all relations (a relation is essentially just an iterator over rows with 26 | /// a known schema) 27 | pub trait Relation { 28 | fn next(&mut self) -> Result>; 29 | 30 | /// get the schema for this relation 31 | fn schema(&self) -> &Arc; 32 | } 33 | 34 | pub struct DataSourceRelation { 35 | schema: Arc, 36 | ds: Rc>, 37 | } 38 | 39 | impl DataSourceRelation { 40 | pub fn new(ds: Rc>) -> Self { 41 | let schema = ds.borrow().schema().clone(); 42 | Self { ds, schema } 43 | } 44 | } 45 | 46 | impl Relation for DataSourceRelation { 47 | fn next(&mut self) -> Result> { 48 | self.ds.borrow_mut().next() 49 | } 50 | 51 | fn schema(&self) -> &Arc { 52 | &self.schema 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/execution/value.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //use std::fmt; 16 | //use std::ops::Add; 17 | //use std::rc::Rc; 18 | // 19 | //use arrow::array::{Int32Array, Int64Array}; 20 | //use arrow::datatypes::DataType; 21 | // 22 | //use super::error::{ExecutionError, Result}; 23 | 24 | //impl ScalarValue { 25 | // pub fn get_datatype(&self) -> DataType { 26 | // match *self { 27 | // ScalarValue::Boolean(_) => DataType::Boolean, 28 | // ScalarValue::UInt8(_) => DataType::UInt8, 29 | // ScalarValue::UInt16(_) => DataType::UInt16, 30 | // ScalarValue::UInt32(_) => DataType::UInt32, 31 | // ScalarValue::UInt64(_) => DataType::UInt64, 32 | // ScalarValue::Int8(_) => DataType::Int8, 33 | // ScalarValue::Int16(_) => DataType::Int16, 34 | // ScalarValue::Int32(_) => DataType::Int32, 35 | // ScalarValue::Int64(_) => DataType::Int64, 36 | // ScalarValue::Float32(_) => DataType::Float32, 37 | // ScalarValue::Float64(_) => DataType::Float64, 38 | // ScalarValue::Utf8(_) => DataType::Utf8, 39 | // ScalarValue::Struct(_) => unimplemented!(), 40 | // ScalarValue::Null => unimplemented!(), 41 | // } 42 | // } 43 | //} 44 | // 45 | //pub fn can_coerce_from(left: &DataType, other: &DataType) -> bool { 46 | // use self::DataType::*; 47 | // match left { 48 | // Int8 => match other { 49 | // Int8 => true, 50 | // _ => false, 51 | // }, 52 | // Int16 => match other { 53 | // Int8 | Int16 => true, 54 | // _ => false, 55 | // }, 56 | // Int32 => match other { 57 | // Int8 | Int16 | Int32 => true, 58 | // _ => false, 59 | // }, 60 | // Int64 => match other { 61 | // Int8 | Int16 | Int32 | Int64 => true, 62 | // _ => false, 63 | // }, 64 | // UInt8 => match other { 65 | // UInt8 => true, 66 | // _ => false, 67 | // }, 68 | // UInt16 => match other { 69 | // UInt8 | UInt16 => true, 70 | // _ => false, 71 | // }, 72 | // UInt32 => match other { 73 | // UInt8 | UInt16 | UInt32 => true, 74 | // _ => false, 75 | // }, 76 | // UInt64 => match other { 77 | // UInt8 | UInt16 | UInt32 | UInt64 => true, 78 | // _ => false, 79 | // }, 80 | // Float32 => match other { 81 | // Int8 | Int16 | Int32 | Int64 => true, 82 | // UInt8 | UInt16 | UInt32 | UInt64 => true, 83 | // Float32 => true, 84 | // _ => false, 85 | // }, 86 | // Float64 => match other { 87 | // Int8 | Int16 | Int32 | Int64 => true, 88 | // UInt8 | UInt16 | UInt32 | UInt64 => true, 89 | // Float32 | Float64 => true, 90 | // _ => false, 91 | // }, 92 | // _ => false, 93 | // } 94 | //} 95 | // 96 | //macro_rules! primitive_accessor { 97 | // ($NAME:ident, $VARIANT:ident, $TY:ty) => { 98 | // pub fn $NAME(&self) -> Result<$TY> { 99 | // match self { 100 | // ScalarValue::$VARIANT(v) => Ok(*v), 101 | // other => Err(ExecutionError::General(format!("Cannot access scalar value {:?} as {}", other, stringify!($VARIANT)))) 102 | // } 103 | // } 104 | // } 105 | //} 106 | // 107 | //impl ScalarValue { 108 | // primitive_accessor!(get_bool, Boolean, bool); 109 | // primitive_accessor!(get_i8, Int8, i8); 110 | // primitive_accessor!(get_i16, Int16, i16); 111 | // primitive_accessor!(get_i32, Int32, i32); 112 | // primitive_accessor!(get_i64, Int64, i64); 113 | // primitive_accessor!(get_u8, UInt8, u8); 114 | // primitive_accessor!(get_u16, UInt16, u16); 115 | // primitive_accessor!(get_u32, UInt32, u32); 116 | // primitive_accessor!(get_u64, UInt64, u64); 117 | // primitive_accessor!(get_f32, Float32, f32); 118 | // primitive_accessor!(get_f64, Float64, f64); 119 | // 120 | // pub fn get_string(&self) -> Result<&String> { 121 | // match *self { 122 | // ScalarValue::Utf8(ref v) => Ok(v), 123 | // _ => Err(ExecutionError::from("TBD")), 124 | // } 125 | // } 126 | // 127 | // pub fn get_struct(&self) -> Result<&Vec> { 128 | // match *self { 129 | // ScalarValue::Struct(ref v) => Ok(v), 130 | // _ => Err(ExecutionError::from("TBD")), 131 | // } 132 | // } 133 | //} 134 | // 135 | //impl Add for ScalarValue { 136 | // type Output = ScalarValue; 137 | // 138 | // fn add(self, rhs: ScalarValue) -> ScalarValue { 139 | // assert_eq!(self.get_datatype(), rhs.get_datatype()); 140 | // match self { 141 | // ScalarValue::UInt8(x) => ScalarValue::UInt8(x + rhs.get_u8().unwrap()), 142 | // ScalarValue::UInt16(x) => ScalarValue::UInt16(x + rhs.get_u16().unwrap()), 143 | // ScalarValue::UInt32(x) => ScalarValue::UInt32(x + rhs.get_u32().unwrap()), 144 | // ScalarValue::UInt64(x) => ScalarValue::UInt64(x + rhs.get_u64().unwrap()), 145 | // ScalarValue::Float32(x) => ScalarValue::Float32(x + rhs.get_f32().unwrap()), 146 | // ScalarValue::Float64(x) => ScalarValue::Float64(x + rhs.get_f64().unwrap()), 147 | // ScalarValue::Int8(x) => ScalarValue::Int8(x.saturating_add(rhs.get_i8().unwrap())), 148 | // ScalarValue::Int16(x) => ScalarValue::Int16(x.saturating_add(rhs.get_i16().unwrap())), 149 | // ScalarValue::Int32(x) => ScalarValue::Int32(x.saturating_add(rhs.get_i32().unwrap())), 150 | // ScalarValue::Int64(x) => ScalarValue::Int64(x.saturating_add(rhs.get_i64().unwrap())), 151 | // _ => panic!("Unsupported type for addition"), 152 | // } 153 | // } 154 | //} 155 | // 156 | //impl fmt::Display for ScalarValue { 157 | // fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 158 | // match self { 159 | // ScalarValue::Null => write!(f, "NULL"), 160 | // ScalarValue::Boolean(v) => write!(f, "{}", v), 161 | // ScalarValue::Int8(v) => write!(f, "{}", v), 162 | // ScalarValue::Int16(v) => write!(f, "{}", v), 163 | // ScalarValue::Int32(v) => write!(f, "{}", v), 164 | // ScalarValue::Int64(v) => write!(f, "{}", v), 165 | // ScalarValue::UInt8(v) => write!(f, "{}", v), 166 | // ScalarValue::UInt16(v) => write!(f, "{}", v), 167 | // ScalarValue::UInt32(v) => write!(f, "{}", v), 168 | // ScalarValue::UInt64(v) => write!(f, "{}", v), 169 | // ScalarValue::Float32(v) => write!(f, "{}", v), 170 | // ScalarValue::Float64(v) => write!(f, "{}", v), 171 | // ScalarValue::Utf8(ref v) => write!(f, "{}", v), 172 | // ScalarValue::Struct(ref v) => { 173 | // for i in 0..v.len() { 174 | // if i > 0 { 175 | // write!(f, ", ")?; 176 | // } 177 | // write!(f, "{}", v[i])?; 178 | // } 179 | // Ok(()) 180 | // } 181 | // } 182 | // } 183 | //} 184 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! DataFusion is a modern distributed compute platform implemented in Rust that uses Apache Arrow 16 | //! as the memory model 17 | 18 | extern crate arrow; 19 | #[macro_use] 20 | extern crate serde_derive; 21 | extern crate serde_json; 22 | extern crate sqlparser; 23 | 24 | pub mod dfparser; 25 | pub mod execution; 26 | pub mod logicalplan; 27 | pub mod sqlplanner; 28 | -------------------------------------------------------------------------------- /src/logicalplan.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! Logical query plan 16 | 17 | use std::fmt; 18 | use std::fmt::{Error, Formatter}; 19 | use std::rc::Rc; 20 | use std::sync::Arc; 21 | 22 | use arrow::datatypes::*; 23 | 24 | #[derive(Serialize, Deserialize, Debug, Clone)] 25 | pub enum FunctionType { 26 | Scalar, 27 | Aggregate, 28 | } 29 | 30 | #[derive(Debug, Clone)] 31 | pub struct FunctionMeta { 32 | name: String, 33 | args: Vec, 34 | return_type: DataType, 35 | function_type: FunctionType, 36 | } 37 | 38 | impl FunctionMeta { 39 | pub fn new( 40 | name: String, 41 | args: Vec, 42 | return_type: DataType, 43 | function_type: FunctionType, 44 | ) -> Self { 45 | FunctionMeta { 46 | name, 47 | args, 48 | return_type, 49 | function_type, 50 | } 51 | } 52 | pub fn name(&self) -> &String { 53 | &self.name 54 | } 55 | pub fn args(&self) -> &Vec { 56 | &self.args 57 | } 58 | pub fn return_type(&self) -> &DataType { 59 | &self.return_type 60 | } 61 | pub fn function_type(&self) -> &FunctionType { 62 | &self.function_type 63 | } 64 | } 65 | 66 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] 67 | pub enum Operator { 68 | Eq, 69 | NotEq, 70 | Lt, 71 | LtEq, 72 | Gt, 73 | GtEq, 74 | Plus, 75 | Minus, 76 | Multiply, 77 | Divide, 78 | Modulus, 79 | And, 80 | Or, 81 | Not, 82 | Like, 83 | NotLike, 84 | } 85 | 86 | impl Operator { 87 | /// Get the result type of applying this operation to its left and right inputs 88 | pub fn get_datatype(&self, l: &Expr, _r: &Expr, schema: &Schema) -> DataType { 89 | //TODO: implement correctly, just go with left side for now 90 | l.get_type(schema).clone() 91 | } 92 | } 93 | 94 | /// ScalarValue enumeration 95 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] 96 | pub enum ScalarValue { 97 | Null, 98 | Boolean(bool), 99 | Float32(f32), 100 | Float64(f64), 101 | Int8(i8), 102 | Int16(i16), 103 | Int32(i32), 104 | Int64(i64), 105 | UInt8(u8), 106 | UInt16(u16), 107 | UInt32(u32), 108 | UInt64(u64), 109 | Utf8(Rc), 110 | Struct(Vec), 111 | } 112 | 113 | impl ScalarValue { 114 | pub fn get_datatype(&self) -> DataType { 115 | match *self { 116 | ScalarValue::Boolean(_) => DataType::Boolean, 117 | ScalarValue::UInt8(_) => DataType::UInt8, 118 | ScalarValue::UInt16(_) => DataType::UInt16, 119 | ScalarValue::UInt32(_) => DataType::UInt32, 120 | ScalarValue::UInt64(_) => DataType::UInt64, 121 | ScalarValue::Int8(_) => DataType::Int8, 122 | ScalarValue::Int16(_) => DataType::Int16, 123 | ScalarValue::Int32(_) => DataType::Int32, 124 | ScalarValue::Int64(_) => DataType::Int64, 125 | ScalarValue::Float32(_) => DataType::Float32, 126 | ScalarValue::Float64(_) => DataType::Float64, 127 | ScalarValue::Utf8(_) => DataType::Utf8, 128 | ScalarValue::Struct(_) => unimplemented!(), 129 | ScalarValue::Null => unimplemented!(), 130 | } 131 | } 132 | } 133 | 134 | /// Relation Expression 135 | #[derive(Serialize, Deserialize, Clone, PartialEq)] 136 | pub enum Expr { 137 | /// index into a value within the row or complex value 138 | Column(usize), 139 | /// literal value 140 | Literal(ScalarValue), 141 | /// binary expression e.g. "age > 21" 142 | BinaryExpr { 143 | left: Rc, 144 | op: Operator, 145 | right: Rc, 146 | }, 147 | /// unary IS NOT NULL 148 | IsNotNull(Rc), 149 | /// unary IS NULL 150 | IsNull(Rc), 151 | /// cast a value to a different type 152 | Cast { expr: Rc, data_type: DataType }, 153 | /// sort expression 154 | Sort { expr: Rc, asc: bool }, 155 | /// scalar function 156 | ScalarFunction { 157 | name: String, 158 | args: Vec, 159 | return_type: DataType, 160 | }, 161 | /// aggregate function 162 | AggregateFunction { 163 | name: String, 164 | args: Vec, 165 | return_type: DataType, 166 | }, 167 | } 168 | 169 | impl Expr { 170 | pub fn get_type(&self, schema: &Schema) -> DataType { 171 | match self { 172 | Expr::Column(n) => schema.field(*n).data_type().clone(), 173 | Expr::Literal(l) => l.get_datatype(), 174 | Expr::Cast { data_type, .. } => data_type.clone(), 175 | Expr::ScalarFunction { return_type, .. } => return_type.clone(), 176 | Expr::AggregateFunction { return_type, .. } => return_type.clone(), 177 | Expr::IsNull(_) => DataType::Boolean, 178 | Expr::IsNotNull(_) => DataType::Boolean, 179 | Expr::BinaryExpr { 180 | ref left, 181 | ref right, 182 | ref op, 183 | } => { 184 | match op { 185 | Operator::Eq | Operator::NotEq => DataType::Boolean, 186 | Operator::Lt | Operator::LtEq => DataType::Boolean, 187 | Operator::Gt | Operator::GtEq => DataType::Boolean, 188 | Operator::And | Operator::Or => DataType::Boolean, 189 | _ => { 190 | let left_type = left.get_type(schema); 191 | let right_type = right.get_type(schema); 192 | get_supertype(&left_type, &right_type).unwrap_or(DataType::Utf8) //TODO ??? 193 | } 194 | } 195 | } 196 | Expr::Sort { ref expr, .. } => expr.get_type(schema), 197 | } 198 | } 199 | 200 | pub fn cast_to(&self, cast_to_type: &DataType, schema: &Schema) -> Result { 201 | let this_type = self.get_type(schema); 202 | if this_type == *cast_to_type { 203 | Ok(self.clone()) 204 | } else if can_coerce_from(cast_to_type, &this_type) { 205 | Ok(Expr::Cast { 206 | expr: Rc::new(self.clone()), 207 | data_type: cast_to_type.clone(), 208 | }) 209 | } else { 210 | Err(format!( 211 | "Cannot automatically convert {:?} to {:?}", 212 | this_type, cast_to_type 213 | )) 214 | } 215 | } 216 | 217 | pub fn eq(&self, other: &Expr) -> Expr { 218 | Expr::BinaryExpr { 219 | left: Rc::new(self.clone()), 220 | op: Operator::Eq, 221 | right: Rc::new(other.clone()), 222 | } 223 | } 224 | 225 | pub fn not_eq(&self, other: &Expr) -> Expr { 226 | Expr::BinaryExpr { 227 | left: Rc::new(self.clone()), 228 | op: Operator::NotEq, 229 | right: Rc::new(other.clone()), 230 | } 231 | } 232 | 233 | pub fn gt(&self, other: &Expr) -> Expr { 234 | Expr::BinaryExpr { 235 | left: Rc::new(self.clone()), 236 | op: Operator::Gt, 237 | right: Rc::new(other.clone()), 238 | } 239 | } 240 | 241 | pub fn gt_eq(&self, other: &Expr) -> Expr { 242 | Expr::BinaryExpr { 243 | left: Rc::new(self.clone()), 244 | op: Operator::GtEq, 245 | right: Rc::new(other.clone()), 246 | } 247 | } 248 | 249 | pub fn lt(&self, other: &Expr) -> Expr { 250 | Expr::BinaryExpr { 251 | left: Rc::new(self.clone()), 252 | op: Operator::Lt, 253 | right: Rc::new(other.clone()), 254 | } 255 | } 256 | 257 | pub fn lt_eq(&self, other: &Expr) -> Expr { 258 | Expr::BinaryExpr { 259 | left: Rc::new(self.clone()), 260 | op: Operator::LtEq, 261 | right: Rc::new(other.clone()), 262 | } 263 | } 264 | } 265 | 266 | impl fmt::Debug for Expr { 267 | fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { 268 | match self { 269 | Expr::Column(i) => write!(f, "#{}", i), 270 | Expr::Literal(v) => write!(f, "{:?}", v), 271 | Expr::Cast { expr, data_type } => write!(f, "CAST({:?} AS {:?})", expr, data_type), 272 | Expr::IsNull(expr) => write!(f, "{:?} IS NULL", expr), 273 | Expr::IsNotNull(expr) => write!(f, "{:?} IS NOT NULL", expr), 274 | Expr::BinaryExpr { left, op, right } => write!(f, "{:?} {:?} {:?}", left, op, right), 275 | Expr::Sort { expr, asc } => { 276 | if *asc { 277 | write!(f, "{:?} ASC", expr) 278 | } else { 279 | write!(f, "{:?} DESC", expr) 280 | } 281 | } 282 | Expr::ScalarFunction { name, ref args, .. } => { 283 | write!(f, "{}(", name)?; 284 | for i in 0..args.len() { 285 | if i > 0 { 286 | write!(f, ", ")?; 287 | } 288 | write!(f, "{:?}", args[i])?; 289 | } 290 | 291 | write!(f, ")") 292 | } 293 | Expr::AggregateFunction { name, ref args, .. } => { 294 | write!(f, "{}(", name)?; 295 | for i in 0..args.len() { 296 | if i > 0 { 297 | write!(f, ", ")?; 298 | } 299 | write!(f, "{:?}", args[i])?; 300 | } 301 | 302 | write!(f, ")") 303 | } 304 | } 305 | } 306 | } 307 | 308 | /// The LogicalPlan represents different types of relations (such as Projection, Selection, etc) and 309 | /// can be created by the SQL query planner and the DataFrame API. 310 | #[derive(Serialize, Deserialize, Clone)] 311 | pub enum LogicalPlan { 312 | /// A relation that applies a row limit to its child relation 313 | Limit { 314 | limit: usize, 315 | input: Rc, 316 | schema: Arc, 317 | }, 318 | /// A Projection (essentially a SELECT with an expression list) 319 | Projection { 320 | expr: Vec, 321 | input: Rc, 322 | schema: Arc, 323 | }, 324 | /// A Selection (essentially a WHERE clause with a predicate expression) 325 | Selection { expr: Expr, input: Rc }, 326 | /// Represents a list of aggregate expressions with optional grouping expressions 327 | Aggregate { 328 | input: Rc, 329 | group_expr: Vec, 330 | aggr_expr: Vec, 331 | schema: Arc, 332 | }, 333 | /// Represents a list of sort expressions to be applied to a relation 334 | Sort { 335 | expr: Vec, 336 | input: Rc, 337 | schema: Arc, 338 | }, 339 | /// A table scan against a table that has been registered on a context 340 | TableScan { 341 | schema_name: String, 342 | table_name: String, 343 | schema: Arc, 344 | projection: Option>, 345 | }, 346 | /// An empty relation with an empty schema 347 | EmptyRelation { schema: Arc }, 348 | } 349 | 350 | impl LogicalPlan { 351 | /// Get a reference to the logical plan's schema 352 | pub fn schema(&self) -> &Arc { 353 | match self { 354 | LogicalPlan::EmptyRelation { schema } => &schema, 355 | LogicalPlan::TableScan { schema, .. } => &schema, 356 | LogicalPlan::Projection { schema, .. } => &schema, 357 | LogicalPlan::Selection { input, .. } => input.schema(), 358 | LogicalPlan::Aggregate { schema, .. } => &schema, 359 | LogicalPlan::Sort { schema, .. } => &schema, 360 | LogicalPlan::Limit { schema, .. } => &schema, 361 | } 362 | } 363 | } 364 | 365 | impl LogicalPlan { 366 | fn fmt_with_indent(&self, f: &mut Formatter, indent: usize) -> Result<(), Error> { 367 | if indent > 0 { 368 | writeln!(f)?; 369 | for _ in 0..indent { 370 | write!(f, " ")?; 371 | } 372 | } 373 | match *self { 374 | LogicalPlan::EmptyRelation { .. } => write!(f, "EmptyRelation"), 375 | LogicalPlan::TableScan { 376 | ref table_name, 377 | ref projection, 378 | .. 379 | } => write!(f, "TableScan: {} projection={:?}", table_name, projection), 380 | LogicalPlan::Projection { 381 | ref expr, 382 | ref input, 383 | .. 384 | } => { 385 | write!(f, "Projection: ")?; 386 | for i in 0..expr.len() { 387 | if i > 0 { 388 | write!(f, ", ")?; 389 | } 390 | write!(f, "{:?}", expr[i])?; 391 | } 392 | input.fmt_with_indent(f, indent + 1) 393 | } 394 | LogicalPlan::Selection { 395 | ref expr, 396 | ref input, 397 | .. 398 | } => { 399 | write!(f, "Selection: {:?}", expr)?; 400 | input.fmt_with_indent(f, indent + 1) 401 | } 402 | LogicalPlan::Aggregate { 403 | ref input, 404 | ref group_expr, 405 | ref aggr_expr, 406 | .. 407 | } => { 408 | write!( 409 | f, 410 | "Aggregate: groupBy=[{:?}], aggr=[{:?}]", 411 | group_expr, aggr_expr 412 | )?; 413 | input.fmt_with_indent(f, indent + 1) 414 | } 415 | LogicalPlan::Sort { 416 | ref input, 417 | ref expr, 418 | .. 419 | } => { 420 | write!(f, "Sort: ")?; 421 | for i in 0..expr.len() { 422 | if i > 0 { 423 | write!(f, ", ")?; 424 | } 425 | write!(f, "{:?}", expr[i])?; 426 | } 427 | input.fmt_with_indent(f, indent + 1) 428 | } 429 | LogicalPlan::Limit { 430 | ref input, limit, .. 431 | } => { 432 | write!(f, "Limit: {}", limit)?; 433 | input.fmt_with_indent(f, indent + 1) 434 | } 435 | } 436 | } 437 | } 438 | 439 | impl fmt::Debug for LogicalPlan { 440 | fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { 441 | self.fmt_with_indent(f, 0) 442 | } 443 | } 444 | 445 | //TODO move to Arrow DataType impl? 446 | pub fn get_supertype(l: &DataType, r: &DataType) -> Option { 447 | match _get_supertype(l, r) { 448 | Some(dt) => Some(dt), 449 | None => match _get_supertype(r, l) { 450 | Some(dt) => Some(dt), 451 | None => None, 452 | }, 453 | } 454 | } 455 | 456 | fn _get_supertype(l: &DataType, r: &DataType) -> Option { 457 | use self::DataType::*; 458 | match (l, r) { 459 | (UInt8, Int8) => Some(Int8), 460 | (UInt8, Int16) => Some(Int16), 461 | (UInt8, Int32) => Some(Int32), 462 | (UInt8, Int64) => Some(Int64), 463 | 464 | (UInt16, Int16) => Some(Int16), 465 | (UInt16, Int32) => Some(Int32), 466 | (UInt16, Int64) => Some(Int64), 467 | 468 | (UInt32, Int32) => Some(Int32), 469 | (UInt32, Int64) => Some(Int64), 470 | 471 | (UInt64, Int64) => Some(Int64), 472 | 473 | (Int8, UInt8) => Some(Int8), 474 | 475 | (Int16, UInt8) => Some(Int16), 476 | (Int16, UInt16) => Some(Int16), 477 | 478 | (Int32, UInt8) => Some(Int32), 479 | (Int32, UInt16) => Some(Int32), 480 | (Int32, UInt32) => Some(Int32), 481 | 482 | (Int64, UInt8) => Some(Int64), 483 | (Int64, UInt16) => Some(Int64), 484 | (Int64, UInt32) => Some(Int64), 485 | (Int64, UInt64) => Some(Int64), 486 | 487 | (UInt8, UInt8) => Some(UInt8), 488 | (UInt8, UInt16) => Some(UInt16), 489 | (UInt8, UInt32) => Some(UInt32), 490 | (UInt8, UInt64) => Some(UInt64), 491 | (UInt8, Float32) => Some(Float32), 492 | (UInt8, Float64) => Some(Float64), 493 | 494 | (UInt16, UInt8) => Some(UInt16), 495 | (UInt16, UInt16) => Some(UInt16), 496 | (UInt16, UInt32) => Some(UInt32), 497 | (UInt16, UInt64) => Some(UInt64), 498 | (UInt16, Float32) => Some(Float32), 499 | (UInt16, Float64) => Some(Float64), 500 | 501 | (UInt32, UInt8) => Some(UInt32), 502 | (UInt32, UInt16) => Some(UInt32), 503 | (UInt32, UInt32) => Some(UInt32), 504 | (UInt32, UInt64) => Some(UInt64), 505 | (UInt32, Float32) => Some(Float32), 506 | (UInt32, Float64) => Some(Float64), 507 | 508 | (UInt64, UInt8) => Some(UInt64), 509 | (UInt64, UInt16) => Some(UInt64), 510 | (UInt64, UInt32) => Some(UInt64), 511 | (UInt64, UInt64) => Some(UInt64), 512 | (UInt64, Float32) => Some(Float32), 513 | (UInt64, Float64) => Some(Float64), 514 | 515 | (Int8, Int8) => Some(Int8), 516 | (Int8, Int16) => Some(Int16), 517 | (Int8, Int32) => Some(Int32), 518 | (Int8, Int64) => Some(Int64), 519 | (Int8, Float32) => Some(Float32), 520 | (Int8, Float64) => Some(Float64), 521 | 522 | (Int16, Int8) => Some(Int16), 523 | (Int16, Int16) => Some(Int16), 524 | (Int16, Int32) => Some(Int32), 525 | (Int16, Int64) => Some(Int64), 526 | (Int16, Float32) => Some(Float32), 527 | (Int16, Float64) => Some(Float64), 528 | 529 | (Int32, Int8) => Some(Int32), 530 | (Int32, Int16) => Some(Int32), 531 | (Int32, Int32) => Some(Int32), 532 | (Int32, Int64) => Some(Int64), 533 | (Int32, Float32) => Some(Float32), 534 | (Int32, Float64) => Some(Float64), 535 | 536 | (Int64, Int8) => Some(Int64), 537 | (Int64, Int16) => Some(Int64), 538 | (Int64, Int32) => Some(Int64), 539 | (Int64, Int64) => Some(Int64), 540 | (Int64, Float32) => Some(Float32), 541 | (Int64, Float64) => Some(Float64), 542 | 543 | (Float32, Float32) => Some(Float32), 544 | (Float32, Float64) => Some(Float64), 545 | (Float64, Float32) => Some(Float64), 546 | (Float64, Float64) => Some(Float64), 547 | 548 | (Utf8, Utf8) => Some(Utf8), 549 | 550 | (Boolean, Boolean) => Some(Boolean), 551 | 552 | _ => None, 553 | } 554 | } 555 | 556 | pub fn can_coerce_from(left: &DataType, other: &DataType) -> bool { 557 | use self::DataType::*; 558 | match left { 559 | Int8 => match other { 560 | Int8 => true, 561 | _ => false, 562 | }, 563 | Int16 => match other { 564 | Int8 | Int16 => true, 565 | _ => false, 566 | }, 567 | Int32 => match other { 568 | Int8 | Int16 | Int32 => true, 569 | _ => false, 570 | }, 571 | Int64 => match other { 572 | Int8 | Int16 | Int32 | Int64 => true, 573 | _ => false, 574 | }, 575 | UInt8 => match other { 576 | UInt8 => true, 577 | _ => false, 578 | }, 579 | UInt16 => match other { 580 | UInt8 | UInt16 => true, 581 | _ => false, 582 | }, 583 | UInt32 => match other { 584 | UInt8 | UInt16 | UInt32 => true, 585 | _ => false, 586 | }, 587 | UInt64 => match other { 588 | UInt8 | UInt16 | UInt32 | UInt64 => true, 589 | _ => false, 590 | }, 591 | Float32 => match other { 592 | Int8 | Int16 | Int32 | Int64 => true, 593 | UInt8 | UInt16 | UInt32 | UInt64 => true, 594 | Float32 => true, 595 | _ => false, 596 | }, 597 | Float64 => match other { 598 | Int8 | Int16 | Int32 | Int64 => true, 599 | UInt8 | UInt16 | UInt32 | UInt64 => true, 600 | Float32 | Float64 => true, 601 | _ => false, 602 | }, 603 | _ => false, 604 | } 605 | } 606 | 607 | #[cfg(test)] 608 | mod tests { 609 | use super::*; 610 | use serde_json; 611 | 612 | #[test] 613 | fn serialize_plan() { 614 | let schema = Schema::new(vec![ 615 | Field::new("first_name", DataType::Utf8, false), 616 | Field::new("last_name", DataType::Utf8, false), 617 | Field::new( 618 | "address", 619 | DataType::Struct(vec![ 620 | Field::new("street", DataType::Utf8, false), 621 | Field::new("zip", DataType::UInt16, false), 622 | ]), 623 | false, 624 | ), 625 | ]); 626 | 627 | let plan = LogicalPlan::TableScan { 628 | schema_name: "".to_string(), 629 | table_name: "people".to_string(), 630 | schema: Arc::new(schema), 631 | projection: Some(vec![0, 1, 4]), 632 | }; 633 | 634 | let serialized = serde_json::to_string(&plan).unwrap(); 635 | 636 | assert_eq!( 637 | "{\"TableScan\":{\ 638 | \"schema_name\":\"\",\ 639 | \"table_name\":\"people\",\ 640 | \"schema\":{\"fields\":[\ 641 | {\"name\":\"first_name\",\"data_type\":\"Utf8\",\"nullable\":false},\ 642 | {\"name\":\"last_name\",\"data_type\":\"Utf8\",\"nullable\":false},\ 643 | {\"name\":\"address\",\"data_type\":{\"Struct\":\ 644 | [\ 645 | {\"name\":\"street\",\"data_type\":\"Utf8\",\"nullable\":false},\ 646 | {\"name\":\"zip\",\"data_type\":\"UInt16\",\"nullable\":false}]},\"nullable\":false}\ 647 | ]},\ 648 | \"projection\":[0,1,4]}}", 649 | serialized 650 | ); 651 | } 652 | } 653 | -------------------------------------------------------------------------------- /test/data/aggregate_test_1.csv: -------------------------------------------------------------------------------- 1 | a,b 2 | 1,2.2 3 | 1,1.1 4 | 2,4.4 5 | 2,5.5 6 | 2,3.3 7 | 3,1.0 8 | 3,2.0 9 | -------------------------------------------------------------------------------- /test/data/aggregate_test_2.csv: -------------------------------------------------------------------------------- 1 | a,b 2 | "one",2.2 3 | "one",1.1 4 | "two",4.4 5 | "two",5.5 6 | "two",3.3 7 | "three",1.0 8 | "three",2.0 9 | -------------------------------------------------------------------------------- /test/data/all_types.csv: -------------------------------------------------------------------------------- 1 | c_int,c_float,c_string 2 | 1,1.1,"1.11" 3 | 2,2.2,"2.22" -------------------------------------------------------------------------------- /test/data/all_types_flat.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygrove/datafusion-archive/219ca3f61f0d751779b4f30c02aa5fdcf8a40d12/test/data/all_types_flat.parquet -------------------------------------------------------------------------------- /test/data/alltypes_plain.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygrove/datafusion-archive/219ca3f61f0d751779b4f30c02aa5fdcf8a40d12/test/data/alltypes_plain.parquet -------------------------------------------------------------------------------- /test/data/example1.ndjson: -------------------------------------------------------------------------------- 1 | { "a": 1, "b": "this is a string", "c": 12.34 } 2 | { "a": 2, "b": "this is also a string", "c": 43.21 } 3 | { "a": 3, "b": "is this a string too?", "c": 0.0 } -------------------------------------------------------------------------------- /test/data/expected/c_float32_cast.csv: -------------------------------------------------------------------------------- 1 | 0.35193855 2 | 0.358496 3 | 0.16038722 4 | 0.13161778 5 | 0.029641032 6 | 0.29698944 7 | 0.4986608 8 | 0.4512084 9 | 0.44425493 10 | 0.48962414 11 | 0.45312655 12 | 0.25761652 13 | 0.016121447 14 | 0.04562819 15 | 0.18841183 16 | 0.30245435 17 | 0.33012044 18 | 0.15909088 19 | 0.49108446 20 | 0.11973965 21 | 0.23732436 22 | 0.47078454 23 | 0.12356734 24 | 0.06767273 25 | 0.49354666 26 | 0.0035754442 27 | 0.19058514 28 | 0.16995096 29 | 0.4722255 30 | 0.29327798 31 | 0.21719217 32 | 0.18611157 33 | 0.17169523 34 | 0.085885346 35 | 0.4516369 36 | 0.008584857 37 | 0.4037001 38 | 0.39962548 39 | 0.2180531 40 | 0.08403635 41 | 0.30247223 42 | 0.31087244 43 | 0.06814504 44 | 0.40330583 45 | 0.2127701 46 | 0.48187488 47 | 0.2555343 48 | 0.13049966 49 | 0.29213196 50 | 0.4889694 51 | 0.49604642 52 | 0.05585593 53 | 0.12730539 54 | 0.37161523 55 | 0.35318744 56 | 0.37926137 57 | 0.040717006 58 | 0.019938052 59 | 0.4021532 60 | 0.35121644 61 | 0.13883227 62 | 0.18730986 63 | 0.023733556 64 | 0.028709888 65 | 0.49014562 66 | 0.4566242 67 | 0.30840534 68 | 0.3797847 69 | 0.35682666 70 | 0.20281577 71 | 0.22660851 72 | 0.2918396 73 | 0.09570438 74 | 0.20388937 75 | 0.45021224 76 | 0.08662772 77 | 0.22055316 78 | 0.30986643 79 | 0.07456052 80 | 0.3827533 81 | 0.22262114 82 | 0.22438735 83 | 0.235416 84 | 0.3267156 85 | 0.07452023 86 | 0.09279096 87 | 0.34970838 88 | 0.18776399 89 | 0.10556418 90 | 0.10422468 91 | 0.17477554 92 | 0.4736452 93 | 0.47393662 94 | 0.22867829 95 | 0.01843369 96 | 0.053943872 97 | 0.18460625 98 | 0.41487402 99 | 0.46603346 100 | 0.3720122 101 | 0.17614293 102 | 0.18146902 103 | 0.036323726 104 | 0.3762604 105 | 0.3629434 106 | 0.0670923 107 | 0.24547786 108 | 0.36911923 109 | 0.039002776 110 | 0.4712336 111 | 0.1382814 112 | 0.15231031 113 | 0.3328836 114 | 0.2345826 115 | 0.2202034 116 | 0.10491228 117 | 0.23397148 118 | 0.0690732 119 | 0.37644255 120 | 0.46385628 121 | 0.4389583 122 | 0.43040425 123 | 0.031057596 124 | 0.41694415 125 | 0.26653832 126 | 0.21435279 127 | 0.081951976 128 | 0.15626663 129 | -------------------------------------------------------------------------------- /test/data/expected/c_float32_cast_uint32.csv: -------------------------------------------------------------------------------- 1 | 0.35193855 2 | 0.7267874 3 | 0.91558653 4 | 0.51842016 5 | 0.358496 6 | 0.6369339 7 | 0.16038722 8 | 0.5143737 9 | 0.81791216 10 | 0.62861645 11 | 0.9637603 12 | 0.6729386 13 | 0.5022637 14 | 0.7171558 15 | 0.56148523 16 | 0.7738669 17 | 0.13161778 18 | 0.029641032 19 | 0.29698944 20 | 0.4986608 21 | 0.4512084 22 | 0.6613548 23 | 0.44425493 24 | 0.48962414 25 | 0.7012702 26 | 0.58028233 27 | 0.67805874 28 | 0.45312655 29 | 0.95071757 30 | 0.59714806 31 | 0.25761652 32 | 0.76446146 33 | 0.016121447 34 | 0.76205754 35 | 0.83667356 36 | 0.04562819 37 | 0.6499184 38 | 0.18841183 39 | 0.30245435 40 | 0.94893974 41 | 0.33012044 42 | 0.9912891 43 | 0.15909088 44 | 0.49108446 45 | 0.11973965 46 | 0.7775541 47 | 0.64277047 48 | 0.23732436 49 | 0.5675772 50 | 0.47078454 51 | 0.12356734 52 | 0.06767273 53 | 0.7009651 54 | 0.49354666 55 | 0.0035754442 56 | 0.19058514 57 | 0.6016194 58 | 0.7760923 59 | 0.55648285 60 | 0.16995096 61 | 0.4722255 62 | 0.29327798 63 | 0.21719217 64 | 0.55365586 65 | 0.18611157 66 | 0.9451594 67 | 0.17169523 68 | 0.9329869 69 | 0.085885346 70 | 0.4516369 71 | 0.62046945 72 | 0.008584857 73 | 0.85783935 74 | 0.4037001 75 | 0.39962548 76 | 0.2180531 77 | 0.08403635 78 | 0.8448413 79 | 0.30247223 80 | 0.31087244 81 | 0.06814504 82 | 0.40330583 83 | 0.2127701 84 | 0.69673395 85 | 0.5773193 86 | 0.48187488 87 | 0.2555343 88 | 0.59320647 89 | 0.65183306 90 | 0.7753064 91 | 0.51078033 92 | 0.74783224 93 | 0.8460534 94 | 0.13049966 95 | 0.29213196 96 | 0.4889694 97 | 0.49604642 98 | 0.7106179 99 | 0.6409457 100 | 0.890248 101 | 0.5061584 102 | 0.55716026 103 | 0.05585593 104 | 0.12730539 105 | 0.93294895 106 | 0.9642894 107 | 0.7988619 108 | 0.5150462 109 | 0.37161523 110 | 0.8542008 111 | 0.57682896 112 | 0.9821998 113 | 0.9474292 114 | 0.35318744 115 | 0.37926137 116 | 0.75121415 117 | 0.040717006 118 | 0.5631632 119 | 0.6002374 120 | 0.019938052 121 | 0.7646779 122 | 0.6662327 123 | 0.82842743 124 | 0.4021532 125 | 0.53511935 126 | 0.8195417 127 | 0.35121644 128 | 0.5824734 129 | 0.76236916 130 | 0.13883227 131 | 0.18730986 132 | 0.023733556 133 | 0.028709888 134 | 0.54221433 135 | 0.49014562 136 | 0.96864915 137 | 0.9383131 138 | 0.4566242 139 | 0.30840534 140 | 0.3797847 141 | 0.8175702 142 | 0.7698803 143 | 0.7594864 144 | 0.73088354 145 | 0.35682666 146 | 0.8483804 147 | 0.6796592 148 | 0.7071798 149 | 0.20281577 150 | 0.60717714 151 | 0.22660851 152 | 0.2918396 153 | 0.09570438 154 | 0.20388937 155 | 0.8950938 156 | 0.45021224 157 | 0.88485366 158 | 0.88475823 159 | 0.08662772 160 | 0.6037678 161 | 0.22055316 162 | 0.30986643 163 | 0.07456052 164 | 0.3827533 165 | 0.8006082 166 | 0.56046015 167 | 0.7050925 168 | 0.22262114 169 | 0.22438735 170 | 0.235416 171 | 0.9539606 172 | 0.3267156 173 | 0.07452023 174 | 0.09279096 175 | 0.9524316 176 | 0.34970838 177 | 0.18776399 178 | 0.6182978 179 | 0.9424731 180 | 0.9966397 181 | 0.89963603 182 | 0.5033241 183 | 0.91039294 184 | 0.10556418 185 | 0.6383702 186 | 0.10422468 187 | 0.17477554 188 | 0.8527578 189 | 0.71685743 190 | 0.6111811 191 | 0.71527416 192 | 0.5438788 193 | 0.71885854 194 | 0.4736452 195 | 0.47393662 196 | 0.22867829 197 | 0.01843369 198 | 0.97064084 199 | 0.64356726 200 | 0.687745 201 | 0.80785763 202 | 0.053943872 203 | 0.18460625 204 | 0.41487402 205 | 0.46603346 206 | 0.3720122 207 | 0.5427082 208 | 0.17614293 209 | 0.18146902 210 | 0.8597146 211 | 0.036323726 212 | 0.82544696 213 | 0.78424275 214 | 0.3762604 215 | 0.3629434 216 | 0.9454982 217 | 0.6511946 218 | 0.5297432 219 | 0.5532096 220 | 0.8531791 221 | 0.0670923 222 | 0.24547786 223 | 0.8558959 224 | 0.7393128 225 | 0.36911923 226 | 0.5365573 227 | 0.039002776 228 | 0.52667034 229 | 0.7182265 230 | 0.8041417 231 | 0.4712336 232 | 0.1382814 233 | 0.15231031 234 | 0.3328836 235 | 0.83534896 236 | 0.62340224 237 | 0.2345826 238 | 0.2202034 239 | 0.10491228 240 | 0.23397148 241 | 0.745774 242 | 0.0690732 243 | 0.37644255 244 | 0.46385628 245 | 0.4389583 246 | 0.43040425 247 | 0.6307863 248 | 0.031057596 249 | 0.41694415 250 | 0.26653832 251 | 0.54456913 252 | 0.21435279 253 | 0.5686012 254 | 0.95081633 255 | 0.081951976 256 | 0.15626663 257 | -------------------------------------------------------------------------------- /test/data/expected/c_float32_high.csv: -------------------------------------------------------------------------------- 1 | 0.7267874 2 | 0.91558653 3 | 0.51842016 4 | 0.6369339 5 | 0.5143737 6 | 0.81791216 7 | 0.62861645 8 | 0.9637603 9 | 0.6729386 10 | 0.5022637 11 | 0.7171558 12 | 0.56148523 13 | 0.7738669 14 | 0.6613548 15 | 0.7012702 16 | 0.58028233 17 | 0.67805874 18 | 0.95071757 19 | 0.59714806 20 | 0.76446146 21 | 0.76205754 22 | 0.83667356 23 | 0.6499184 24 | 0.94893974 25 | 0.9912891 26 | 0.7775541 27 | 0.64277047 28 | 0.5675772 29 | 0.7009651 30 | 0.6016194 31 | 0.7760923 32 | 0.55648285 33 | 0.55365586 34 | 0.9451594 35 | 0.9329869 36 | 0.62046945 37 | 0.85783935 38 | 0.8448413 39 | 0.69673395 40 | 0.5773193 41 | 0.59320647 42 | 0.65183306 43 | 0.7753064 44 | 0.51078033 45 | 0.74783224 46 | 0.8460534 47 | 0.7106179 48 | 0.6409457 49 | 0.890248 50 | 0.5061584 51 | 0.55716026 52 | 0.93294895 53 | 0.9642894 54 | 0.7988619 55 | 0.5150462 56 | 0.8542008 57 | 0.57682896 58 | 0.9821998 59 | 0.9474292 60 | 0.75121415 61 | 0.5631632 62 | 0.6002374 63 | 0.7646779 64 | 0.6662327 65 | 0.82842743 66 | 0.53511935 67 | 0.8195417 68 | 0.5824734 69 | 0.76236916 70 | 0.54221433 71 | 0.96864915 72 | 0.9383131 73 | 0.8175702 74 | 0.7698803 75 | 0.7594864 76 | 0.73088354 77 | 0.8483804 78 | 0.6796592 79 | 0.7071798 80 | 0.60717714 81 | 0.8950938 82 | 0.88485366 83 | 0.88475823 84 | 0.6037678 85 | 0.8006082 86 | 0.56046015 87 | 0.7050925 88 | 0.9539606 89 | 0.9524316 90 | 0.6182978 91 | 0.9424731 92 | 0.9966397 93 | 0.89963603 94 | 0.5033241 95 | 0.91039294 96 | 0.6383702 97 | 0.8527578 98 | 0.71685743 99 | 0.6111811 100 | 0.71527416 101 | 0.5438788 102 | 0.71885854 103 | 0.97064084 104 | 0.64356726 105 | 0.687745 106 | 0.80785763 107 | 0.5427082 108 | 0.8597146 109 | 0.82544696 110 | 0.78424275 111 | 0.9454982 112 | 0.6511946 113 | 0.5297432 114 | 0.5532096 115 | 0.8531791 116 | 0.8558959 117 | 0.7393128 118 | 0.5365573 119 | 0.52667034 120 | 0.7182265 121 | 0.8041417 122 | 0.83534896 123 | 0.62340224 124 | 0.745774 125 | 0.6307863 126 | 0.54456913 127 | 0.5686012 128 | 0.95081633 129 | -------------------------------------------------------------------------------- /test/data/expected/c_float32_high_uint32.csv: -------------------------------------------------------------------------------- 1 | 0.35193855 2 | 0.7267874 3 | 0.91558653 4 | 0.51842016 5 | 0.358496 6 | 0.6369339 7 | 0.16038722 8 | 0.5143737 9 | 0.81791216 10 | 0.62861645 11 | 0.9637603 12 | 0.6729386 13 | 0.5022637 14 | 0.7171558 15 | 0.56148523 16 | 0.7738669 17 | 0.13161778 18 | 0.029641032 19 | 0.29698944 20 | 0.4986608 21 | 0.4512084 22 | 0.6613548 23 | 0.44425493 24 | 0.48962414 25 | 0.7012702 26 | 0.58028233 27 | 0.67805874 28 | 0.45312655 29 | 0.95071757 30 | 0.59714806 31 | 0.25761652 32 | 0.76446146 33 | 0.016121447 34 | 0.76205754 35 | 0.83667356 36 | 0.04562819 37 | 0.6499184 38 | 0.18841183 39 | 0.30245435 40 | 0.94893974 41 | 0.33012044 42 | 0.9912891 43 | 0.15909088 44 | 0.49108446 45 | 0.11973965 46 | 0.7775541 47 | 0.64277047 48 | 0.23732436 49 | 0.5675772 50 | 0.47078454 51 | 0.12356734 52 | 0.06767273 53 | 0.7009651 54 | 0.49354666 55 | 0.0035754442 56 | 0.19058514 57 | 0.6016194 58 | 0.7760923 59 | 0.55648285 60 | 0.16995096 61 | 0.4722255 62 | 0.29327798 63 | 0.21719217 64 | 0.55365586 65 | 0.18611157 66 | 0.9451594 67 | 0.17169523 68 | 0.9329869 69 | 0.085885346 70 | 0.4516369 71 | 0.62046945 72 | 0.008584857 73 | 0.85783935 74 | 0.4037001 75 | 0.39962548 76 | 0.2180531 77 | 0.08403635 78 | 0.8448413 79 | 0.30247223 80 | 0.31087244 81 | 0.06814504 82 | 0.40330583 83 | 0.2127701 84 | 0.69673395 85 | 0.5773193 86 | 0.48187488 87 | 0.2555343 88 | 0.59320647 89 | 0.65183306 90 | 0.7753064 91 | 0.51078033 92 | 0.74783224 93 | 0.8460534 94 | 0.13049966 95 | 0.29213196 96 | 0.4889694 97 | 0.49604642 98 | 0.7106179 99 | 0.6409457 100 | 0.890248 101 | 0.5061584 102 | 0.55716026 103 | 0.05585593 104 | 0.12730539 105 | 0.93294895 106 | 0.9642894 107 | 0.7988619 108 | 0.5150462 109 | 0.37161523 110 | 0.8542008 111 | 0.57682896 112 | 0.9821998 113 | 0.9474292 114 | 0.35318744 115 | 0.37926137 116 | 0.75121415 117 | 0.040717006 118 | 0.5631632 119 | 0.6002374 120 | 0.019938052 121 | 0.7646779 122 | 0.6662327 123 | 0.82842743 124 | 0.4021532 125 | 0.53511935 126 | 0.8195417 127 | 0.35121644 128 | 0.5824734 129 | 0.76236916 130 | 0.13883227 131 | 0.18730986 132 | 0.023733556 133 | 0.028709888 134 | 0.54221433 135 | 0.49014562 136 | 0.96864915 137 | 0.9383131 138 | 0.4566242 139 | 0.30840534 140 | 0.3797847 141 | 0.8175702 142 | 0.7698803 143 | 0.7594864 144 | 0.73088354 145 | 0.35682666 146 | 0.8483804 147 | 0.6796592 148 | 0.7071798 149 | 0.20281577 150 | 0.60717714 151 | 0.22660851 152 | 0.2918396 153 | 0.09570438 154 | 0.20388937 155 | 0.8950938 156 | 0.45021224 157 | 0.88485366 158 | 0.88475823 159 | 0.08662772 160 | 0.6037678 161 | 0.22055316 162 | 0.30986643 163 | 0.07456052 164 | 0.3827533 165 | 0.8006082 166 | 0.56046015 167 | 0.7050925 168 | 0.22262114 169 | 0.22438735 170 | 0.235416 171 | 0.9539606 172 | 0.3267156 173 | 0.07452023 174 | 0.09279096 175 | 0.9524316 176 | 0.34970838 177 | 0.18776399 178 | 0.6182978 179 | 0.9424731 180 | 0.9966397 181 | 0.89963603 182 | 0.5033241 183 | 0.91039294 184 | 0.10556418 185 | 0.6383702 186 | 0.10422468 187 | 0.17477554 188 | 0.8527578 189 | 0.71685743 190 | 0.6111811 191 | 0.71527416 192 | 0.5438788 193 | 0.71885854 194 | 0.4736452 195 | 0.47393662 196 | 0.22867829 197 | 0.01843369 198 | 0.97064084 199 | 0.64356726 200 | 0.687745 201 | 0.80785763 202 | 0.053943872 203 | 0.18460625 204 | 0.41487402 205 | 0.46603346 206 | 0.3720122 207 | 0.5427082 208 | 0.17614293 209 | 0.18146902 210 | 0.8597146 211 | 0.036323726 212 | 0.82544696 213 | 0.78424275 214 | 0.3762604 215 | 0.3629434 216 | 0.9454982 217 | 0.6511946 218 | 0.5297432 219 | 0.5532096 220 | 0.8531791 221 | 0.0670923 222 | 0.24547786 223 | 0.8558959 224 | 0.7393128 225 | 0.36911923 226 | 0.5365573 227 | 0.039002776 228 | 0.52667034 229 | 0.7182265 230 | 0.8041417 231 | 0.4712336 232 | 0.1382814 233 | 0.15231031 234 | 0.3328836 235 | 0.83534896 236 | 0.62340224 237 | 0.2345826 238 | 0.2202034 239 | 0.10491228 240 | 0.23397148 241 | 0.745774 242 | 0.0690732 243 | 0.37644255 244 | 0.46385628 245 | 0.4389583 246 | 0.43040425 247 | 0.6307863 248 | 0.031057596 249 | 0.41694415 250 | 0.26653832 251 | 0.54456913 252 | 0.21435279 253 | 0.5686012 254 | 0.95081633 255 | 0.081951976 256 | 0.15626663 257 | -------------------------------------------------------------------------------- /test/data/expected/c_float32_low.csv: -------------------------------------------------------------------------------- 1 | 0.35193855 2 | 0.358496 3 | 0.16038722 4 | 0.13161778 5 | 0.029641032 6 | 0.29698944 7 | 0.4986608 8 | 0.4512084 9 | 0.44425493 10 | 0.48962414 11 | 0.45312655 12 | 0.25761652 13 | 0.016121447 14 | 0.04562819 15 | 0.18841183 16 | 0.30245435 17 | 0.33012044 18 | 0.15909088 19 | 0.49108446 20 | 0.11973965 21 | 0.23732436 22 | 0.47078454 23 | 0.12356734 24 | 0.06767273 25 | 0.49354666 26 | 0.0035754442 27 | 0.19058514 28 | 0.16995096 29 | 0.4722255 30 | 0.29327798 31 | 0.21719217 32 | 0.18611157 33 | 0.17169523 34 | 0.085885346 35 | 0.4516369 36 | 0.008584857 37 | 0.4037001 38 | 0.39962548 39 | 0.2180531 40 | 0.08403635 41 | 0.30247223 42 | 0.31087244 43 | 0.06814504 44 | 0.40330583 45 | 0.2127701 46 | 0.48187488 47 | 0.2555343 48 | 0.13049966 49 | 0.29213196 50 | 0.4889694 51 | 0.49604642 52 | 0.05585593 53 | 0.12730539 54 | 0.37161523 55 | 0.35318744 56 | 0.37926137 57 | 0.040717006 58 | 0.019938052 59 | 0.4021532 60 | 0.35121644 61 | 0.13883227 62 | 0.18730986 63 | 0.023733556 64 | 0.028709888 65 | 0.49014562 66 | 0.4566242 67 | 0.30840534 68 | 0.3797847 69 | 0.35682666 70 | 0.20281577 71 | 0.22660851 72 | 0.2918396 73 | 0.09570438 74 | 0.20388937 75 | 0.45021224 76 | 0.08662772 77 | 0.22055316 78 | 0.30986643 79 | 0.07456052 80 | 0.3827533 81 | 0.22262114 82 | 0.22438735 83 | 0.235416 84 | 0.3267156 85 | 0.07452023 86 | 0.09279096 87 | 0.34970838 88 | 0.18776399 89 | 0.10556418 90 | 0.10422468 91 | 0.17477554 92 | 0.4736452 93 | 0.47393662 94 | 0.22867829 95 | 0.01843369 96 | 0.053943872 97 | 0.18460625 98 | 0.41487402 99 | 0.46603346 100 | 0.3720122 101 | 0.17614293 102 | 0.18146902 103 | 0.036323726 104 | 0.3762604 105 | 0.3629434 106 | 0.0670923 107 | 0.24547786 108 | 0.36911923 109 | 0.039002776 110 | 0.4712336 111 | 0.1382814 112 | 0.15231031 113 | 0.3328836 114 | 0.2345826 115 | 0.2202034 116 | 0.10491228 117 | 0.23397148 118 | 0.0690732 119 | 0.37644255 120 | 0.46385628 121 | 0.4389583 122 | 0.43040425 123 | 0.031057596 124 | 0.41694415 125 | 0.26653832 126 | 0.21435279 127 | 0.081951976 128 | 0.15626663 129 | -------------------------------------------------------------------------------- /test/data/expected/c_float32_low_uint32.csv: -------------------------------------------------------------------------------- 1 | 0.35193855 2 | 0.7267874 3 | 0.91558653 4 | 0.51842016 5 | 0.358496 6 | 0.6369339 7 | 0.16038722 8 | 0.5143737 9 | 0.81791216 10 | 0.62861645 11 | 0.9637603 12 | 0.6729386 13 | 0.5022637 14 | 0.7171558 15 | 0.56148523 16 | 0.7738669 17 | 0.13161778 18 | 0.029641032 19 | 0.29698944 20 | 0.4986608 21 | 0.4512084 22 | 0.6613548 23 | 0.44425493 24 | 0.48962414 25 | 0.7012702 26 | 0.58028233 27 | 0.67805874 28 | 0.45312655 29 | 0.95071757 30 | 0.59714806 31 | 0.25761652 32 | 0.76446146 33 | 0.016121447 34 | 0.76205754 35 | 0.83667356 36 | 0.04562819 37 | 0.6499184 38 | 0.18841183 39 | 0.30245435 40 | 0.94893974 41 | 0.33012044 42 | 0.9912891 43 | 0.15909088 44 | 0.49108446 45 | 0.11973965 46 | 0.7775541 47 | 0.64277047 48 | 0.23732436 49 | 0.5675772 50 | 0.47078454 51 | 0.12356734 52 | 0.06767273 53 | 0.7009651 54 | 0.49354666 55 | 0.0035754442 56 | 0.19058514 57 | 0.6016194 58 | 0.7760923 59 | 0.55648285 60 | 0.16995096 61 | 0.4722255 62 | 0.29327798 63 | 0.21719217 64 | 0.55365586 65 | 0.18611157 66 | 0.9451594 67 | 0.17169523 68 | 0.9329869 69 | 0.085885346 70 | 0.4516369 71 | 0.62046945 72 | 0.008584857 73 | 0.85783935 74 | 0.4037001 75 | 0.39962548 76 | 0.2180531 77 | 0.08403635 78 | 0.8448413 79 | 0.30247223 80 | 0.31087244 81 | 0.06814504 82 | 0.40330583 83 | 0.2127701 84 | 0.69673395 85 | 0.5773193 86 | 0.48187488 87 | 0.2555343 88 | 0.59320647 89 | 0.65183306 90 | 0.7753064 91 | 0.51078033 92 | 0.74783224 93 | 0.8460534 94 | 0.13049966 95 | 0.29213196 96 | 0.4889694 97 | 0.49604642 98 | 0.7106179 99 | 0.6409457 100 | 0.890248 101 | 0.5061584 102 | 0.55716026 103 | 0.05585593 104 | 0.12730539 105 | 0.93294895 106 | 0.9642894 107 | 0.7988619 108 | 0.5150462 109 | 0.37161523 110 | 0.8542008 111 | 0.57682896 112 | 0.9821998 113 | 0.9474292 114 | 0.35318744 115 | 0.37926137 116 | 0.75121415 117 | 0.040717006 118 | 0.5631632 119 | 0.6002374 120 | 0.019938052 121 | 0.7646779 122 | 0.6662327 123 | 0.82842743 124 | 0.4021532 125 | 0.53511935 126 | 0.8195417 127 | 0.35121644 128 | 0.5824734 129 | 0.76236916 130 | 0.13883227 131 | 0.18730986 132 | 0.023733556 133 | 0.028709888 134 | 0.54221433 135 | 0.49014562 136 | 0.96864915 137 | 0.9383131 138 | 0.4566242 139 | 0.30840534 140 | 0.3797847 141 | 0.8175702 142 | 0.7698803 143 | 0.7594864 144 | 0.73088354 145 | 0.35682666 146 | 0.8483804 147 | 0.6796592 148 | 0.7071798 149 | 0.20281577 150 | 0.60717714 151 | 0.22660851 152 | 0.2918396 153 | 0.09570438 154 | 0.20388937 155 | 0.8950938 156 | 0.45021224 157 | 0.88485366 158 | 0.88475823 159 | 0.08662772 160 | 0.6037678 161 | 0.22055316 162 | 0.30986643 163 | 0.07456052 164 | 0.3827533 165 | 0.8006082 166 | 0.56046015 167 | 0.7050925 168 | 0.22262114 169 | 0.22438735 170 | 0.235416 171 | 0.9539606 172 | 0.3267156 173 | 0.07452023 174 | 0.09279096 175 | 0.9524316 176 | 0.34970838 177 | 0.18776399 178 | 0.6182978 179 | 0.9424731 180 | 0.9966397 181 | 0.89963603 182 | 0.5033241 183 | 0.91039294 184 | 0.10556418 185 | 0.6383702 186 | 0.10422468 187 | 0.17477554 188 | 0.8527578 189 | 0.71685743 190 | 0.6111811 191 | 0.71527416 192 | 0.5438788 193 | 0.71885854 194 | 0.4736452 195 | 0.47393662 196 | 0.22867829 197 | 0.01843369 198 | 0.97064084 199 | 0.64356726 200 | 0.687745 201 | 0.80785763 202 | 0.053943872 203 | 0.18460625 204 | 0.41487402 205 | 0.46603346 206 | 0.3720122 207 | 0.5427082 208 | 0.17614293 209 | 0.18146902 210 | 0.8597146 211 | 0.036323726 212 | 0.82544696 213 | 0.78424275 214 | 0.3762604 215 | 0.3629434 216 | 0.9454982 217 | 0.6511946 218 | 0.5297432 219 | 0.5532096 220 | 0.8531791 221 | 0.0670923 222 | 0.24547786 223 | 0.8558959 224 | 0.7393128 225 | 0.36911923 226 | 0.5365573 227 | 0.039002776 228 | 0.52667034 229 | 0.7182265 230 | 0.8041417 231 | 0.4712336 232 | 0.1382814 233 | 0.15231031 234 | 0.3328836 235 | 0.83534896 236 | 0.62340224 237 | 0.2345826 238 | 0.2202034 239 | 0.10491228 240 | 0.23397148 241 | 0.745774 242 | 0.0690732 243 | 0.37644255 244 | 0.46385628 245 | 0.4389583 246 | 0.43040425 247 | 0.6307863 248 | 0.031057596 249 | 0.41694415 250 | 0.26653832 251 | 0.54456913 252 | 0.21435279 253 | 0.5686012 254 | 0.95081633 255 | 0.081951976 256 | 0.15626663 257 | -------------------------------------------------------------------------------- /test/data/expected/c_float64_cast.csv: -------------------------------------------------------------------------------- 1 | 0.24829962755879142 2 | 0.2375179592454354 3 | 0.4981203861148582 4 | 0.29422932552059233 5 | 0.17348140475392826 6 | 0.263043606194834 7 | 0.14779665189566116 8 | 0.1880123385211735 9 | 0.2592869241443291 10 | 0.33625578665565203 11 | 0.3765691217587185 12 | 0.08748462506515176 13 | 0.10782045437731824 14 | 0.3485618824317698 15 | 0.1779888254873827 16 | 0.057659034649668706 17 | 0.02795727978930862 18 | 0.4732095210027568 19 | 0.2921431977782333 20 | 0.22428236737824025 21 | 0.3282605064249652 22 | 0.3805155918372052 23 | 0.4171152142888477 24 | 0.2714871256536533 25 | 0.337610267576799 26 | 0.20829674198517034 27 | 0.09415645257989425 28 | 0.20419981170860768 29 | 0.3632467230669233 30 | 0.420481376975951 31 | 0.4504094938089088 32 | 0.2866919957795735 33 | 0.25232259590626993 34 | 0.020039612901084802 35 | 0.4517175352013263 36 | 0.3811163177553121 37 | 0.24211579179324005 38 | 0.28616553688834667 39 | 0.009589347996759257 40 | 0.2914030832205625 41 | 0.1815576433955779 42 | 0.09279902688553465 43 | 0.3156515718067934 44 | 0.35478892533333795 45 | 0.3052065612133401 46 | 0.4215452606660476 47 | 0.15135181769502282 48 | 0.434825030523886 49 | 0.21517202088141052 50 | 0.048453399897169014 51 | 0.3561153179185831 52 | 0.08914172928832309 53 | 0.3213157883357525 54 | 0.143335247625615 55 | 0.2508628317430849 56 | 0.26075374684018604 57 | 0.13857950330272173 58 | 0.31472051144986646 59 | 0.3147516785904575 60 | 0.03804835023179365 61 | 0.03864944648071922 62 | 0.44352114747118687 63 | 0.2899923531077021 64 | 0.01702500540212859 65 | 0.34755680027822267 66 | 0.4198956617315416 67 | 0.40837258850021263 68 | 0.3908420873239533 69 | 0.0945252022944818 70 | 0.42977725574305814 71 | 0.27593425461119236 72 | 0.3080164797755379 73 | 0.30168615595975146 74 | 0.1990353927336258 75 | 0.12304694505257907 76 | 0.2442881913524514 77 | 0.15391640344898394 78 | 0.4060677982338461 79 | 0.19772741284905904 80 | 0.05378145868940831 81 | 0.30441670021080536 82 | 0.16892493843508882 83 | 0.1855868367858331 84 | 0.34061056731209205 85 | 0.03993674675026826 86 | 0.3830327364641034 87 | 0.02844783454047195 88 | 0.3475227749887002 89 | 0.20398103766939302 90 | 0.479485457280151 91 | 0.26279223578429056 92 | 0.0598221750681448 93 | 0.18274288889404178 94 | 0.2122121824162333 95 | 0.2648091438304201 96 | 0.4490898393630405 97 | 0.15693491492054112 98 | 0.296694844829575 99 | 0.23315610007297094 100 | 0.2719236684679943 101 | 0.3703581922997703 102 | 0.32998426533103375 103 | 0.2651492629459913 104 | 0.04998213882456293 105 | 0.45721629419942944 106 | 0.09773734165086989 107 | 0.3966274533417897 108 | 0.05442721715046017 109 | 0.4512584059320437 110 | 0.28256220081287675 111 | 0.0946642619465724 112 | 0.4167184761641931 113 | 0.22098288969051738 114 | 0.09498698285175156 115 | 0.0014460175058255142 116 | 0.29754757702705936 117 | 0.12958961975675531 118 | 0.09465088299568925 119 | 0.4770979497091494 120 | 0.09262331822987424 121 | 0.23940023791448506 122 | 0.40824405676263453 123 | 0.01023964046913095 124 | 0.07898064162389484 125 | 0.2697580960717878 126 | 0.29764615198420163 127 | 0.2567099989300493 128 | 0.4616319718363274 129 | 0.11918711984988928 130 | 0.06844656232765345 131 | 0.2749355068497815 132 | 0.3274990362903428 133 | 0.04790633102108133 134 | 0.26469010287736106 135 | 0.40973968548511475 136 | 0.3054443670484893 137 | 0.11666006317601962 138 | -------------------------------------------------------------------------------- /test/data/expected/c_float64_high.csv: -------------------------------------------------------------------------------- 1 | 0.501613048032352 2 | 0.9848153564240777 3 | 0.5429001544619461 4 | 0.5096163904352368 5 | 0.5121134304724516 6 | 0.8102583970068632 7 | 0.9060969102791906 8 | 0.8663958325385142 9 | 0.5910969431428907 10 | 0.9987703202471936 11 | 0.6548679910576786 12 | 0.9719428637234058 13 | 0.7457059463748047 14 | 0.5279233199598208 15 | 0.6984254061219464 16 | 0.6396288470398015 17 | 0.9569563272783185 18 | 0.7013442366533913 19 | 0.857483012880279 20 | 0.6990022692847451 21 | 0.9628895636174963 22 | 0.5855272636174155 23 | 0.6277049685488401 24 | 0.5642848201640165 25 | 0.696608188191791 26 | 0.76214839914083 27 | 0.923795394851777 28 | 0.7626051219937391 29 | 0.5717575205156706 30 | 0.5644386692056418 31 | 0.7549911117119996 32 | 0.8562151131699631 33 | 0.6434247139812375 34 | 0.8761476467052858 35 | 0.675652242741229 36 | 0.5807528551152212 37 | 0.5637189738424164 38 | 0.9147462460144969 39 | 0.905490779234353 40 | 0.9678466125587091 41 | 0.7377421998597028 42 | 0.5129705241041475 43 | 0.5466370818473373 44 | 0.9814992515656239 45 | 0.6749794359634981 46 | 0.5016446783724774 47 | 0.8520859737571457 48 | 0.8305740538524686 49 | 0.8355453825918163 50 | 0.5023261268239142 51 | 0.7723502098850213 52 | 0.8277609838234431 53 | 0.8017213523855212 54 | 0.8627083185855362 55 | 0.6049396837870864 56 | 0.677618337887574 57 | 0.6568968892971336 58 | 0.9333612416875543 59 | 0.5904764978405083 60 | 0.6910549838842718 61 | 0.7452829547728306 62 | 0.7451840097388837 63 | 0.9806368435475767 64 | 0.639934628631153 65 | 0.7876082540140072 66 | 0.6797434264577749 67 | 0.8465942672091502 68 | 0.5890937225273637 69 | 0.7811886581582305 70 | 0.6881121482310578 71 | 0.7958555216678562 72 | 0.6158853671300282 73 | 0.9835923929247793 74 | 0.5613780027608319 75 | 0.5008493503109895 76 | 0.5573373437370076 77 | 0.9766644947415974 78 | 0.7100596889249792 79 | 0.9615190890855236 80 | 0.5355630566407764 81 | 0.7679634224261329 82 | 0.556372608878059 83 | 0.5607727803885736 84 | 0.6580880563970105 85 | 0.8709637544292466 86 | 0.566950855656357 87 | 0.8435500043206446 88 | 0.845653885856783 89 | 0.9753528491547591 90 | 0.7321159644886541 91 | 0.7085026394528402 92 | 0.789663420394937 93 | 0.5537035869173044 94 | 0.8172198386294783 95 | 0.6054332047911015 96 | 0.8034771236941767 97 | 0.7838036604719485 98 | 0.9210818477939374 99 | 0.5022242480091891 100 | 0.70908397935545 101 | 0.8246641436719843 102 | 0.9968826450233449 103 | 0.5076847755859427 104 | 0.879519270992787 105 | 0.5030474475649561 106 | 0.8338279864158362 107 | 0.6496595368142363 108 | 0.789289871346883 109 | 0.8874324151846106 110 | 0.7508437586551192 111 | 0.9058936980566197 112 | 0.5529858916888012 113 | 0.6453989782043966 114 | 0.7662250808178842 115 | 0.9264561736443667 116 | 0.5940351613053869 117 | 0.9590753194830887 118 | 0.7628660139787938 119 | 0.9699169373953178 120 | -------------------------------------------------------------------------------- /test/data/expected/c_float64_low.csv: -------------------------------------------------------------------------------- 1 | 0.24829962755879142 2 | 0.2375179592454354 3 | 0.4981203861148582 4 | 0.29422932552059233 5 | 0.17348140475392826 6 | 0.263043606194834 7 | 0.14779665189566116 8 | 0.1880123385211735 9 | 0.2592869241443291 10 | 0.33625578665565203 11 | 0.3765691217587185 12 | 0.08748462506515176 13 | 0.10782045437731824 14 | 0.3485618824317698 15 | 0.1779888254873827 16 | 0.057659034649668706 17 | 0.02795727978930862 18 | 0.4732095210027568 19 | 0.2921431977782333 20 | 0.22428236737824025 21 | 0.3282605064249652 22 | 0.3805155918372052 23 | 0.4171152142888477 24 | 0.2714871256536533 25 | 0.337610267576799 26 | 0.20829674198517034 27 | 0.09415645257989425 28 | 0.20419981170860768 29 | 0.3632467230669233 30 | 0.420481376975951 31 | 0.4504094938089088 32 | 0.2866919957795735 33 | 0.25232259590626993 34 | 0.020039612901084802 35 | 0.4517175352013263 36 | 0.3811163177553121 37 | 0.24211579179324005 38 | 0.28616553688834667 39 | 0.009589347996759257 40 | 0.2914030832205625 41 | 0.1815576433955779 42 | 0.09279902688553465 43 | 0.3156515718067934 44 | 0.35478892533333795 45 | 0.3052065612133401 46 | 0.4215452606660476 47 | 0.15135181769502282 48 | 0.434825030523886 49 | 0.21517202088141052 50 | 0.048453399897169014 51 | 0.3561153179185831 52 | 0.08914172928832309 53 | 0.3213157883357525 54 | 0.143335247625615 55 | 0.2508628317430849 56 | 0.26075374684018604 57 | 0.13857950330272173 58 | 0.31472051144986646 59 | 0.3147516785904575 60 | 0.03804835023179365 61 | 0.03864944648071922 62 | 0.44352114747118687 63 | 0.2899923531077021 64 | 0.01702500540212859 65 | 0.34755680027822267 66 | 0.4198956617315416 67 | 0.40837258850021263 68 | 0.3908420873239533 69 | 0.0945252022944818 70 | 0.42977725574305814 71 | 0.27593425461119236 72 | 0.3080164797755379 73 | 0.30168615595975146 74 | 0.1990353927336258 75 | 0.12304694505257907 76 | 0.2442881913524514 77 | 0.15391640344898394 78 | 0.4060677982338461 79 | 0.19772741284905904 80 | 0.05378145868940831 81 | 0.30441670021080536 82 | 0.16892493843508882 83 | 0.1855868367858331 84 | 0.34061056731209205 85 | 0.03993674675026826 86 | 0.3830327364641034 87 | 0.02844783454047195 88 | 0.3475227749887002 89 | 0.20398103766939302 90 | 0.479485457280151 91 | 0.26279223578429056 92 | 0.0598221750681448 93 | 0.18274288889404178 94 | 0.2122121824162333 95 | 0.2648091438304201 96 | 0.4490898393630405 97 | 0.15693491492054112 98 | 0.296694844829575 99 | 0.23315610007297094 100 | 0.2719236684679943 101 | 0.3703581922997703 102 | 0.32998426533103375 103 | 0.2651492629459913 104 | 0.04998213882456293 105 | 0.45721629419942944 106 | 0.09773734165086989 107 | 0.3966274533417897 108 | 0.05442721715046017 109 | 0.4512584059320437 110 | 0.28256220081287675 111 | 0.0946642619465724 112 | 0.4167184761641931 113 | 0.22098288969051738 114 | 0.09498698285175156 115 | 0.0014460175058255142 116 | 0.29754757702705936 117 | 0.12958961975675531 118 | 0.09465088299568925 119 | 0.4770979497091494 120 | 0.09262331822987424 121 | 0.23940023791448506 122 | 0.40824405676263453 123 | 0.01023964046913095 124 | 0.07898064162389484 125 | 0.2697580960717878 126 | 0.29764615198420163 127 | 0.2567099989300493 128 | 0.4616319718363274 129 | 0.11918711984988928 130 | 0.06844656232765345 131 | 0.2749355068497815 132 | 0.3274990362903428 133 | 0.04790633102108133 134 | 0.26469010287736106 135 | 0.40973968548511475 136 | 0.3054443670484893 137 | 0.11666006317601962 138 | -------------------------------------------------------------------------------- /test/data/expected/c_int16_cast.csv: -------------------------------------------------------------------------------- 1 | -1596 2 | -18088 3 | -23777 4 | -26421 5 | -17933 6 | -207 7 | -32096 8 | -23786 9 | -7250 10 | -28298 11 | -17142 12 | -8919 13 | -15788 14 | -1960 15 | -7315 16 | -1070 17 | -20058 18 | -32474 19 | -31483 20 | -31097 21 | -16719 22 | -31044 23 | -2274 24 | -21760 25 | -13131 26 | -6873 27 | -6413 28 | -24562 29 | -21595 30 | -17349 31 | -10115 32 | -25335 33 | -5378 34 | -30889 35 | -1577 36 | -23866 37 | -10662 38 | -12666 39 | -27263 40 | -9032 41 | -15298 42 | -3004 43 | -20089 44 | -12850 45 | -24097 46 | -17968 47 | -26923 48 | -20876 49 | -12911 50 | -22000 51 | -31540 52 | -7657 53 | -11401 54 | -27652 55 | -30230 56 | -10448 57 | -29466 58 | -919 59 | -7496 60 | -26361 61 | -863 62 | -30762 63 | -29714 64 | -3969 65 | -11885 66 | -25642 67 | -8021 68 | -27268 69 | -10778 70 | -8103 71 | -28137 72 | -29302 73 | -14913 74 | -13832 75 | -20467 76 | -807 77 | -20616 78 | -16278 79 | -11417 80 | -19203 81 | -21176 82 | -22147 83 | -21609 84 | -28391 85 | -30976 86 | -18867 87 | -12105 88 | -31264 89 | -1614 90 | -11837 91 | -24307 92 | -23003 93 | -24520 94 | -12272 95 | -15374 96 | -25665 97 | -237 98 | -24464 99 | -23749 100 | -13737 101 | -26548 102 | -32625 103 | -27102 104 | -3112 105 | -10964 106 | -10560 107 | -16889 108 | -31009 109 | -3536 110 | -21157 111 | -14983 112 | -6809 113 | -7007 114 | -24177 115 | -8253 116 | -17377 117 | -1996 118 | -25198 119 | -4313 120 | -3167 121 | -8255 122 | -10080 123 | -25602 124 | -26202 125 | -6041 126 | -3980 127 | -17832 128 | -4767 129 | -23091 130 | -10251 131 | -16678 132 | -8682 133 | -8039 134 | -------------------------------------------------------------------------------- /test/data/expected/c_int16_negative.csv: -------------------------------------------------------------------------------- 1 | -1596 2 | -18088 3 | -23777 4 | -26421 5 | -17933 6 | -207 7 | -32096 8 | -23786 9 | -7250 10 | -28298 11 | -17142 12 | -8919 13 | -15788 14 | -1960 15 | -7315 16 | -1070 17 | -20058 18 | -32474 19 | -31483 20 | -31097 21 | -16719 22 | -31044 23 | -2274 24 | -21760 25 | -13131 26 | -6873 27 | -6413 28 | -24562 29 | -21595 30 | -17349 31 | -10115 32 | -25335 33 | -5378 34 | -30889 35 | -1577 36 | -23866 37 | -10662 38 | -12666 39 | -27263 40 | -9032 41 | -15298 42 | -3004 43 | -20089 44 | -12850 45 | -24097 46 | -17968 47 | -26923 48 | -20876 49 | -12911 50 | -22000 51 | -31540 52 | -7657 53 | -11401 54 | -27652 55 | -30230 56 | -10448 57 | -29466 58 | -919 59 | -7496 60 | -26361 61 | -863 62 | -30762 63 | -29714 64 | -3969 65 | -11885 66 | -25642 67 | -8021 68 | -27268 69 | -10778 70 | -8103 71 | -28137 72 | -29302 73 | -14913 74 | -13832 75 | -20467 76 | -807 77 | -20616 78 | -16278 79 | -11417 80 | -19203 81 | -21176 82 | -22147 83 | -21609 84 | -28391 85 | -30976 86 | -18867 87 | -12105 88 | -31264 89 | -1614 90 | -11837 91 | -24307 92 | -23003 93 | -24520 94 | -12272 95 | -15374 96 | -25665 97 | -237 98 | -24464 99 | -23749 100 | -13737 101 | -26548 102 | -32625 103 | -27102 104 | -3112 105 | -10964 106 | -10560 107 | -16889 108 | -31009 109 | -3536 110 | -21157 111 | -14983 112 | -6809 113 | -7007 114 | -24177 115 | -8253 116 | -17377 117 | -1996 118 | -25198 119 | -4313 120 | -3167 121 | -8255 122 | -10080 123 | -25602 124 | -26202 125 | -6041 126 | -3980 127 | -17832 128 | -4767 129 | -23091 130 | -10251 131 | -16678 132 | -8682 133 | -8039 134 | -------------------------------------------------------------------------------- /test/data/expected/c_int16_positive.csv: -------------------------------------------------------------------------------- 1 | 13870 2 | 19157 3 | 27606 4 | 1081 5 | 25205 6 | 18860 7 | 19355 8 | 2090 9 | 4207 10 | 7947 11 | 6900 12 | 17805 13 | 2684 14 | 28213 15 | 4274 16 | 23978 17 | 15628 18 | 9342 19 | 14287 20 | 9562 21 | 570 22 | 13893 23 | 604 24 | 6754 25 | 6509 26 | 213 27 | 13815 28 | 6925 29 | 16511 30 | 29055 31 | 10888 32 | 7852 33 | 7745 34 | 23735 35 | 23105 36 | 10654 37 | 9059 38 | 7221 39 | 19242 40 | 30071 41 | 19214 42 | 26943 43 | 5773 44 | 3246 45 | 19407 46 | 23844 47 | 29438 48 | 1395 49 | 13276 50 | 8739 51 | 21345 52 | 22344 53 | 14071 54 | 25145 55 | 3488 56 | 17880 57 | 26885 58 | 11563 59 | 14356 60 | 10816 61 | 19155 62 | 32339 63 | 29279 64 | 6615 65 | 12739 66 | 139 67 | 30338 68 | 16524 69 | 3314 70 | 28827 71 | 114 72 | 12437 73 | 22998 74 | 5911 75 | 30310 76 | 12903 77 | 26041 78 | 1516 79 | 25644 80 | 21451 81 | 26637 82 | 8010 83 | 6272 84 | 11534 85 | 4475 86 | 28946 87 | 22159 88 | 32203 89 | 3720 90 | 19507 91 | 8524 92 | 31036 93 | 13685 94 | 28170 95 | 30224 96 | 11969 97 | 7273 98 | 1089 99 | 30633 100 | 23524 101 | 29569 102 | 20213 103 | 5235 104 | 13227 105 | 32257 106 | 31162 107 | 17320 108 | 7199 109 | 20535 110 | 8179 111 | 23984 112 | 7172 113 | 24518 114 | 32405 115 | 6703 116 | 19151 117 | 13460 118 | 21014 119 | 732 120 | 17637 121 | 8922 122 | 22197 123 | 13652 124 | -------------------------------------------------------------------------------- /test/data/expected/c_int32_cast.csv: -------------------------------------------------------------------------------- 1 | -1121737404 2 | -879181593 3 | -1373926101 4 | -1629763360 5 | -869481407 6 | -9255144 7 | -212839128 8 | -1804138479 9 | -1539182058 10 | -926560209 11 | -517239196 12 | -1822497989 13 | -1147990479 14 | -1962535867 15 | -176052757 16 | -1711315035 17 | -227928338 18 | -944863825 19 | -1623843875 20 | -1902144824 21 | -730280258 22 | -624438455 23 | -215433347 24 | -1930125995 25 | -2069387796 26 | -783427820 27 | -1190838950 28 | -1294672738 29 | -623428436 30 | -1570868257 31 | -938432496 32 | -305400046 33 | -2061016334 34 | -833058389 35 | -927977663 36 | -925151094 37 | -2035828826 38 | -591072145 39 | -958497859 40 | -1290460299 41 | -973684999 42 | -1636206929 43 | -1829072979 44 | -1250685124 45 | -234080791 46 | -2094903057 47 | -1506140672 48 | -2120652033 49 | -1195170308 50 | -400901845 51 | -1654744681 52 | -1652626809 53 | -866155263 54 | -1356299307 55 | -911659303 56 | -181588028 57 | -1032134770 58 | -1713674985 59 | -1571923721 60 | -419811620 61 | -170702331 62 | -116627502 63 | -1410928897 64 | -532088819 65 | -639925920 66 | -82113298 67 | -1215945719 68 | -326933763 69 | -2063214930 70 | -541826583 71 | -1858809022 72 | -1845742387 73 | -423519622 74 | -185268669 75 | -444620608 76 | -815697931 77 | -1064925075 78 | -1533566767 79 | -971971603 80 | -1068563589 81 | -124790736 82 | -2041871921 83 | -2139804706 84 | -724635501 85 | -338667711 86 | -1185471716 87 | -440187006 88 | -1726120170 89 | -30888687 90 | -1212956684 91 | -1432922043 92 | -657312568 93 | -1974203649 94 | -785739649 95 | -1258973346 96 | -1789326631 97 | -105765628 98 | -649198433 99 | -1185399346 100 | -1423713872 101 | -294801502 102 | -407962060 103 | -521968346 104 | -760249949 105 | -1252674704 106 | -51817396 107 | -371128363 108 | -64759520 109 | -538943732 110 | -1101801378 111 | -605725162 112 | -1126021273 113 | -1399308161 114 | -976064148 115 | -867274444 116 | -1884637579 117 | -150795199 118 | -926882849 119 | -176173288 120 | -150042007 121 | -228136768 122 | -894187338 123 | -516556536 124 | -------------------------------------------------------------------------------- /test/data/expected/c_int32_negative.csv: -------------------------------------------------------------------------------- 1 | -1121737404 2 | -879181593 3 | -1373926101 4 | -1629763360 5 | -869481407 6 | -9255144 7 | -212839128 8 | -1804138479 9 | -1539182058 10 | -926560209 11 | -517239196 12 | -1822497989 13 | -1147990479 14 | -1962535867 15 | -176052757 16 | -1711315035 17 | -227928338 18 | -944863825 19 | -1623843875 20 | -1902144824 21 | -730280258 22 | -624438455 23 | -215433347 24 | -1930125995 25 | -2069387796 26 | -783427820 27 | -1190838950 28 | -1294672738 29 | -623428436 30 | -1570868257 31 | -938432496 32 | -305400046 33 | -2061016334 34 | -833058389 35 | -927977663 36 | -925151094 37 | -2035828826 38 | -591072145 39 | -958497859 40 | -1290460299 41 | -973684999 42 | -1636206929 43 | -1829072979 44 | -1250685124 45 | -234080791 46 | -2094903057 47 | -1506140672 48 | -2120652033 49 | -1195170308 50 | -400901845 51 | -1654744681 52 | -1652626809 53 | -866155263 54 | -1356299307 55 | -911659303 56 | -181588028 57 | -1032134770 58 | -1713674985 59 | -1571923721 60 | -419811620 61 | -170702331 62 | -116627502 63 | -1410928897 64 | -532088819 65 | -639925920 66 | -82113298 67 | -1215945719 68 | -326933763 69 | -2063214930 70 | -541826583 71 | -1858809022 72 | -1845742387 73 | -423519622 74 | -185268669 75 | -444620608 76 | -815697931 77 | -1064925075 78 | -1533566767 79 | -971971603 80 | -1068563589 81 | -124790736 82 | -2041871921 83 | -2139804706 84 | -724635501 85 | -338667711 86 | -1185471716 87 | -440187006 88 | -1726120170 89 | -30888687 90 | -1212956684 91 | -1432922043 92 | -657312568 93 | -1974203649 94 | -785739649 95 | -1258973346 96 | -1789326631 97 | -105765628 98 | -649198433 99 | -1185399346 100 | -1423713872 101 | -294801502 102 | -407962060 103 | -521968346 104 | -760249949 105 | -1252674704 106 | -51817396 107 | -371128363 108 | -64759520 109 | -538943732 110 | -1101801378 111 | -605725162 112 | -1126021273 113 | -1399308161 114 | -976064148 115 | -867274444 116 | -1884637579 117 | -150795199 118 | -926882849 119 | -176173288 120 | -150042007 121 | -228136768 122 | -894187338 123 | -516556536 124 | -------------------------------------------------------------------------------- /test/data/expected/c_int32_positive.csv: -------------------------------------------------------------------------------- 1 | 1234231326 2 | 1508731446 3 | 164409925 4 | 1570913558 5 | 984961643 6 | 702694219 7 | 497049627 8 | 85696742 9 | 211059659 10 | 1555964510 11 | 1141579081 12 | 346465510 13 | 131563263 14 | 1606534704 15 | 1428177081 16 | 466035216 17 | 1178519874 18 | 490185344 19 | 1762894796 20 | 1931644210 21 | 1267012355 22 | 1173467820 23 | 1648116798 24 | 35863937 25 | 448423894 26 | 1605102767 27 | 1351161599 28 | 867309376 29 | 591727007 30 | 1914514519 31 | 167629554 32 | 966073370 33 | 1601416104 34 | 1336737969 35 | 767156003 36 | 1464391210 37 | 143247473 38 | 949431846 39 | 1008197199 40 | 1222653058 41 | 1180497866 42 | 1942365141 43 | 72591987 44 | 1317639664 45 | 1561153724 46 | 21286937 47 | 493535785 48 | 1017959972 49 | 569197401 50 | 1669476380 51 | 151610328 52 | 379197915 53 | 172848785 54 | 9422129 55 | 1161707988 56 | 1084496302 57 | 1473455669 58 | 644273448 59 | 1476287370 60 | 466420125 61 | 1683005790 62 | 93724289 63 | 37781033 64 | 1969366652 65 | 1513672192 66 | 63207346 67 | 1569780950 68 | 2089947233 69 | 1979987443 70 | 1103054626 71 | 2064685622 72 | 924073819 73 | 837139408 74 | 223202271 75 | 1030922026 76 | 249962800 77 | 1639737736 78 | 237902789 79 | 1182167074 80 | 1190087826 81 | 211523089 82 | 922271646 83 | 1994852586 84 | 450351994 85 | 507136253 86 | 519074099 87 | 2000826202 88 | 146696599 89 | 1427233750 90 | 1503663711 91 | 1261628174 92 | 1502118792 93 | 1533600891 94 | 1152857104 95 | 1011970251 96 | 2145139030 97 | 453803257 98 | 411945319 99 | 1774745418 100 | 468294998 101 | 394590508 102 | 1250564920 103 | 182024486 104 | 324975908 105 | 1679434520 106 | 871466697 107 | 1960736964 108 | 1808092021 109 | 792538835 110 | 353725066 111 | 150155779 112 | 1252730885 113 | 83141248 114 | 1350530810 115 | 1193737407 116 | 938166686 117 | 1889985445 118 | 623239332 119 | 1757736884 120 | 1607613295 121 | 407243213 122 | 498397434 123 | 1879854093 124 | 1736953712 125 | 1295793286 126 | 1672890683 127 | 1505737882 128 | 1640723130 129 | 1634350374 130 | 1033891533 131 | 833890130 132 | 1624375853 133 | 1397589747 134 | -------------------------------------------------------------------------------- /test/data/expected/c_int64_cast.csv: -------------------------------------------------------------------------------- 1 | -3707023917728695883 2 | -7958412917705956932 3 | -9153360329814749238 4 | -1539534406690282878 5 | -8012109879546454244 6 | -391842732204935620 7 | -796714470026579345 8 | -4164310042088722601 9 | -6252303781092136744 10 | -6585487401183055348 11 | -4146065602579576811 12 | -8094366414051042988 13 | -4906431753521378993 14 | -1152680452773224056 15 | -5154175125068656566 16 | -2209107919116321238 17 | -7679277413407079967 18 | -3906417007632767922 19 | -7954139520243645276 20 | -2142068600586430508 21 | -9136614041513084610 22 | -5846744023232872251 23 | -4346154021836583909 24 | -5954897226712164389 25 | -4010522296881539778 26 | -500409598773634350 27 | -5792585860452956181 28 | -4215474026183872190 29 | -8475892967406247745 30 | -7123230281893220309 31 | -3032310109863430091 32 | -1922381870112074274 33 | -5898663357950632544 34 | -6152654796407870961 35 | -4629701276086450132 36 | -6574922049684700616 37 | -6942403670248424465 38 | -416676517875626885 39 | -1979751288669640757 40 | -5473568741056175944 41 | -2018840238644402942 42 | -1505944159283906126 43 | -6750618225918461002 44 | -2078510067519764775 45 | -4523475253013105708 46 | -4166283961398142264 47 | -9216439232345632980 48 | -2090515699168836997 49 | -7718886336742930723 50 | -7823988326463117720 51 | -5539144888875980695 52 | -5139170029485337114 53 | -6712302346253743895 54 | -8101309980276055142 55 | -9122277453893808611 56 | -7962518126606180752 57 | -974771215430995249 58 | -3254320722874379362 59 | -1070128557870580220 60 | -4538677862736040277 61 | -5803603774572649287 62 | -5386022942906873035 63 | -997433072488275245 64 | -5250188028060280745 65 | -8505461947064856204 66 | -956832623771146513 67 | -7839649174118521193 68 | -544610854867257318 69 | -5605352300474289547 70 | -2974078663424361303 71 | -5239040597459791977 72 | -7437790280021405118 73 | -9042118254009968025 74 | -7954246658343292133 75 | -2717017794490574789 76 | -7593157191549944626 77 | -849066385018286446 78 | -5195432491042805854 79 | -1611897952838717131 80 | -6980373723786070110 81 | -4830074867727432466 82 | -3811638934217246215 83 | -3789283977526124832 84 | -2937718484805791768 85 | -2288471342949481916 86 | -4857670553866706619 87 | -7710681500716716325 88 | -7198140148660377265 89 | -3193250702534549782 90 | -1117803367748784547 91 | -3384152560995186100 92 | -5345375259489784071 93 | -6949848426361579234 94 | -8219198316803458907 95 | -3465333809121248905 96 | -4985229514747786958 97 | -5023265279643967337 98 | -752871478250714969 99 | -7774726185513849433 100 | -1236033946341067124 101 | -7853989582454992713 102 | -4610716468070162606 103 | -2103749563615537867 104 | -2674416610700795288 105 | -3375634575315459031 106 | -3974808341229739387 107 | -3283950910421578121 108 | -8873966780457438942 109 | -2222896549059064624 110 | -5512594733186640333 111 | -4999321731499002492 112 | -3201068453518124906 113 | -5866625933452838677 114 | -7243505533089654587 115 | -2932679718644882803 116 | -4562545035019281250 117 | -1896563881960380322 118 | -8955014781461314016 119 | -3245065061820959388 120 | -5609981245581940964 121 | -8116302398722627489 122 | -1182163242051036192 123 | -------------------------------------------------------------------------------- /test/data/expected/c_int64_negative.csv: -------------------------------------------------------------------------------- 1 | -3707023917728695883 2 | -7958412917705956932 3 | -9153360329814749238 4 | -1539534406690282878 5 | -8012109879546454244 6 | -391842732204935620 7 | -796714470026579345 8 | -4164310042088722601 9 | -6252303781092136744 10 | -6585487401183055348 11 | -4146065602579576811 12 | -8094366414051042988 13 | -4906431753521378993 14 | -1152680452773224056 15 | -5154175125068656566 16 | -2209107919116321238 17 | -7679277413407079967 18 | -3906417007632767922 19 | -7954139520243645276 20 | -2142068600586430508 21 | -9136614041513084610 22 | -5846744023232872251 23 | -4346154021836583909 24 | -5954897226712164389 25 | -4010522296881539778 26 | -500409598773634350 27 | -5792585860452956181 28 | -4215474026183872190 29 | -8475892967406247745 30 | -7123230281893220309 31 | -3032310109863430091 32 | -1922381870112074274 33 | -5898663357950632544 34 | -6152654796407870961 35 | -4629701276086450132 36 | -6574922049684700616 37 | -6942403670248424465 38 | -416676517875626885 39 | -1979751288669640757 40 | -5473568741056175944 41 | -2018840238644402942 42 | -1505944159283906126 43 | -6750618225918461002 44 | -2078510067519764775 45 | -4523475253013105708 46 | -4166283961398142264 47 | -9216439232345632980 48 | -2090515699168836997 49 | -7718886336742930723 50 | -7823988326463117720 51 | -5539144888875980695 52 | -5139170029485337114 53 | -6712302346253743895 54 | -8101309980276055142 55 | -9122277453893808611 56 | -7962518126606180752 57 | -974771215430995249 58 | -3254320722874379362 59 | -1070128557870580220 60 | -4538677862736040277 61 | -5803603774572649287 62 | -5386022942906873035 63 | -997433072488275245 64 | -5250188028060280745 65 | -8505461947064856204 66 | -956832623771146513 67 | -7839649174118521193 68 | -544610854867257318 69 | -5605352300474289547 70 | -2974078663424361303 71 | -5239040597459791977 72 | -7437790280021405118 73 | -9042118254009968025 74 | -7954246658343292133 75 | -2717017794490574789 76 | -7593157191549944626 77 | -849066385018286446 78 | -5195432491042805854 79 | -1611897952838717131 80 | -6980373723786070110 81 | -4830074867727432466 82 | -3811638934217246215 83 | -3789283977526124832 84 | -2937718484805791768 85 | -2288471342949481916 86 | -4857670553866706619 87 | -7710681500716716325 88 | -7198140148660377265 89 | -3193250702534549782 90 | -1117803367748784547 91 | -3384152560995186100 92 | -5345375259489784071 93 | -6949848426361579234 94 | -8219198316803458907 95 | -3465333809121248905 96 | -4985229514747786958 97 | -5023265279643967337 98 | -752871478250714969 99 | -7774726185513849433 100 | -1236033946341067124 101 | -7853989582454992713 102 | -4610716468070162606 103 | -2103749563615537867 104 | -2674416610700795288 105 | -3375634575315459031 106 | -3974808341229739387 107 | -3283950910421578121 108 | -8873966780457438942 109 | -2222896549059064624 110 | -5512594733186640333 111 | -4999321731499002492 112 | -3201068453518124906 113 | -5866625933452838677 114 | -7243505533089654587 115 | -2932679718644882803 116 | -4562545035019281250 117 | -1896563881960380322 118 | -8955014781461314016 119 | -3245065061820959388 120 | -5609981245581940964 121 | -8116302398722627489 122 | -1182163242051036192 123 | -------------------------------------------------------------------------------- /test/data/expected/c_int64_positive.csv: -------------------------------------------------------------------------------- 1 | 3923416169952057646 2 | 8925632445420578596 3 | 8842876120447285340 4 | 6569147229399117619 5 | 143228945056456631 6 | 20701021643807815 7 | 1056716721061520581 8 | 2355616434385030368 9 | 6535067329646013840 10 | 1419282613021825007 11 | 4443132472147709052 12 | 624487320592771035 13 | 6621485749046145630 14 | 4018628870944688215 15 | 2994687010251554643 16 | 164642702238865172 17 | 5355321136858340128 18 | 5394994146196081516 19 | 363962205986566052 20 | 6729598730405181612 21 | 5910019470167397237 22 | 5780921237045002271 23 | 3323691255778749741 24 | 8423434239078433009 25 | 6913488027442637238 26 | 1541550608598153956 27 | 4861802455287862336 28 | 8144498362863709452 29 | 4488234234349287441 30 | 7147952938476908636 31 | 3459639484266869426 32 | 3719016504427893406 33 | 245833550551406861 34 | 1666693256291559690 35 | 8902320515066654855 36 | 5957393991977287973 37 | 5064208619045430671 38 | 4617478254209884634 39 | 4165891724677523574 40 | 7050648459405897992 41 | 4771852763069643949 42 | 7120620435592122318 43 | 2144998523095333024 44 | 2796359392582420369 45 | 4565177683312531032 46 | 1320671179524366868 47 | 4411863822111419006 48 | 2224586585060278198 49 | 3328498683828456843 50 | 6640586874325022912 51 | 4491125346907381815 52 | 3224160792289524144 53 | 3066437650902048487 54 | 1576350744315291421 55 | 908482088768601596 56 | 1819445355373663874 57 | 4601093771773799249 58 | 321794347904500166 59 | 6598453538542729952 60 | 4112848153032561892 61 | 3821731361044833835 62 | 2120880350861883579 63 | 1292379374708880307 64 | 1164858801793800808 65 | 1356873541588406340 66 | 2776691022936185348 67 | 5700970230235297892 68 | 790867450853168714 69 | 5305041486930110117 70 | 9155374553061731286 71 | 9201665022718575466 72 | 7337279928358679796 73 | 60226721086805611 74 | 6291341386209588360 75 | 4665618113923515115 76 | 8484517148267817351 77 | 4607500669321780649 78 | 6298546146407188963 79 | 5285234520967596715 80 | 8265337405046498147 81 | 7701109651724719902 82 | 2765650454778188034 83 | 108153940198218877 84 | 5580648357785566471 85 | 7601043534721898233 86 | 8645682135229516630 87 | 4829839445128605002 88 | 7341782609509084299 89 | 405538164706495167 90 | 3826199197102548759 91 | 5333041377946667991 92 | 5469539346156519424 93 | 3944657029466120525 94 | 8263063637702236690 95 | 8124458865858010666 96 | 3473644655523422137 97 | 3039390349100660513 98 | 3117522684378328623 99 | 3642627760280722819 100 | 8675061408112971386 101 | 7252811393769297694 102 | 2281603937128163561 103 | 2724238383658753127 104 | 670456397367139848 105 | 7891961009888818461 106 | 6806138664214278948 107 | 1477808950836860726 108 | 6186298840390736967 109 | 7820193225032620891 110 | 7586340282937154032 111 | 3543203616318744714 112 | 7283651347220016313 113 | 8382086564535564431 114 | 20553134549368439 115 | 3194629316961088104 116 | 5412155990366864411 117 | 727226771807372723 118 | 4776742583075073262 119 | 3002013368078838828 120 | 5792681898786644575 121 | 6802013956830195359 122 | 7975414790775317613 123 | 2627066092072658678 124 | 9070365160125430370 125 | 5010692401093873922 126 | 2081307451813152858 127 | 5121349134294840037 128 | 664269413064484815 129 | 6079111845833599979 130 | 6258605060818382676 131 | 4788989286431932240 132 | 1297637434204179010 133 | 6946322014254064963 134 | 8354763653702371440 135 | -------------------------------------------------------------------------------- /test/data/expected/c_int8_cast.csv: -------------------------------------------------------------------------------- 1 | -60 2 | -61 3 | -27 4 | -78 5 | -37 6 | -89 7 | -51 8 | -93 9 | -34 10 | -109 11 | -54 12 | -5 13 | -83 14 | -111 15 | -70 16 | -52 17 | -49 18 | -100 19 | -104 20 | -61 21 | -29 22 | -14 23 | -60 24 | -84 25 | -107 26 | -3 27 | -16 28 | -96 29 | -72 30 | -7 31 | -106 32 | -49 33 | -54 34 | -34 35 | -21 36 | -116 37 | -91 38 | -50 39 | -84 40 | -121 41 | -94 42 | -29 43 | -86 44 | -115 45 | -14 46 | -83 47 | -20 48 | -13 49 | -102 50 | -123 51 | -50 52 | -15 53 | -51 54 | -68 55 | -48 56 | -81 57 | -53 58 | -13 59 | -89 60 | -78 61 | -78 62 | -45 63 | -47 64 | -88 65 | -46 66 | -124 67 | -3 68 | -21 69 | -103 70 | -1 71 | -102 72 | -68 73 | -41 74 | -36 75 | -5 76 | -27 77 | -112 78 | -47 79 | -58 80 | -61 81 | -96 82 | -73 83 | -10 84 | -26 85 | -15 86 | -14 87 | -128 88 | -50 89 | -61 90 | -42 91 | -127 92 | -111 93 | -27 94 | -37 95 | -9 96 | -76 97 | -97 98 | -56 99 | -127 100 | -45 101 | -119 102 | -122 103 | -19 104 | -39 105 | -34 106 | -54 107 | -41 108 | -106 109 | -77 110 | -90 111 | -99 112 | -111 113 | -11 114 | -58 115 | -108 116 | -32 117 | -25 118 | -5 119 | -13 120 | -108 121 | -39 122 | -84 123 | -23 124 | -69 125 | -45 126 | -35 127 | -8 128 | -105 129 | -116 130 | -68 131 | -5 132 | -------------------------------------------------------------------------------- /test/data/expected/c_int8_col_eq.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygrove/datafusion-archive/219ca3f61f0d751779b4f30c02aa5fdcf8a40d12/test/data/expected/c_int8_col_eq.csv -------------------------------------------------------------------------------- /test/data/expected/c_int8_col_gt.csv: -------------------------------------------------------------------------------- 1 | -60 2 | 74 3 | 57 4 | -27 5 | -37 6 | -93 7 | -34 8 | 70 9 | -5 10 | -111 11 | 3 12 | 60 13 | -100 14 | 16 15 | -104 16 | -61 17 | -14 18 | 52 19 | 104 20 | -107 21 | -3 22 | 46 23 | -96 24 | 15 25 | 75 26 | 50 27 | 39 28 | -49 29 | -54 30 | -21 31 | -116 32 | -91 33 | -84 34 | -121 35 | -94 36 | -29 37 | 124 38 | -115 39 | 30 40 | 110 41 | 39 42 | 42 43 | 55 44 | 14 45 | 13 46 | -102 47 | -68 48 | -81 49 | 90 50 | -89 51 | 4 52 | -78 53 | -45 54 | -46 55 | 77 56 | 92 57 | -124 58 | -3 59 | -21 60 | 66 61 | 79 62 | -102 63 | -68 64 | -41 65 | -36 66 | 3 67 | 87 68 | 18 69 | 60 70 | 76 71 | 61 72 | -112 73 | 96 74 | -58 75 | -61 76 | 56 77 | -73 78 | -10 79 | 47 80 | 117 81 | 20 82 | 25 83 | 30 84 | 2 85 | 52 86 | 99 87 | -128 88 | 48 89 | -61 90 | 108 91 | 83 92 | -76 93 | 39 94 | -56 95 | -127 96 | 30 97 | 122 98 | -119 99 | -122 100 | -39 101 | -34 102 | -54 103 | 65 104 | 92 105 | -41 106 | 122 107 | -106 108 | 51 109 | 31 110 | 80 111 | 123 112 | -111 113 | -58 114 | 98 115 | 76 116 | 6 117 | -108 118 | -32 119 | -25 120 | -13 121 | -108 122 | -39 123 | -84 124 | 23 125 | -69 126 | 103 127 | 0 128 | -105 129 | 42 130 | 123 131 | -116 132 | -68 133 | -5 134 | -------------------------------------------------------------------------------- /test/data/expected/c_int8_col_gteq.csv: -------------------------------------------------------------------------------- 1 | -60 2 | 74 3 | 57 4 | -27 5 | -37 6 | -93 7 | -34 8 | 70 9 | -5 10 | -111 11 | 3 12 | 60 13 | -100 14 | 16 15 | -104 16 | -61 17 | -14 18 | 52 19 | 104 20 | -107 21 | -3 22 | 46 23 | -96 24 | 15 25 | 75 26 | 50 27 | 39 28 | -49 29 | -54 30 | -21 31 | -116 32 | -91 33 | -84 34 | -121 35 | -94 36 | -29 37 | 124 38 | -115 39 | 30 40 | 110 41 | 39 42 | 42 43 | 55 44 | 14 45 | 13 46 | -102 47 | -68 48 | -81 49 | 90 50 | -89 51 | 4 52 | -78 53 | -45 54 | -46 55 | 77 56 | 92 57 | -124 58 | -3 59 | -21 60 | 66 61 | 79 62 | -102 63 | -68 64 | -41 65 | -36 66 | 3 67 | 87 68 | 18 69 | 60 70 | 76 71 | 61 72 | -112 73 | 96 74 | -58 75 | -61 76 | 56 77 | -73 78 | -10 79 | 47 80 | 117 81 | 20 82 | 25 83 | 30 84 | 2 85 | 52 86 | 99 87 | -128 88 | 48 89 | -61 90 | 108 91 | 83 92 | -76 93 | 39 94 | -56 95 | -127 96 | 30 97 | 122 98 | -119 99 | -122 100 | -39 101 | -34 102 | -54 103 | 65 104 | 92 105 | -41 106 | 122 107 | -106 108 | 51 109 | 31 110 | 80 111 | 123 112 | -111 113 | -58 114 | 98 115 | 76 116 | 6 117 | -108 118 | -32 119 | -25 120 | -13 121 | -108 122 | -39 123 | -84 124 | 23 125 | -69 126 | 103 127 | 0 128 | -105 129 | 42 130 | 123 131 | -116 132 | -68 133 | -5 134 | -------------------------------------------------------------------------------- /test/data/expected/c_int8_col_lt.csv: -------------------------------------------------------------------------------- 1 | -61 2 | -78 3 | 42 4 | -89 5 | -51 6 | 51 7 | -109 8 | 7 9 | -54 10 | 71 11 | 61 12 | -83 13 | -70 14 | -52 15 | 107 16 | -49 17 | 97 18 | 46 19 | -29 20 | -60 21 | 109 22 | -84 23 | 118 24 | -16 25 | 54 26 | 121 27 | -72 28 | 91 29 | -7 30 | -106 31 | -34 32 | 47 33 | -50 34 | -86 35 | -14 36 | -83 37 | -20 38 | -13 39 | 21 40 | 7 41 | 111 42 | -123 43 | -50 44 | -15 45 | 56 46 | -51 47 | -48 48 | 112 49 | 79 50 | 110 51 | 108 52 | 122 53 | -53 54 | 113 55 | -13 56 | 69 57 | 4 58 | -78 59 | 82 60 | -47 61 | -88 62 | 23 63 | -103 64 | -1 65 | 92 66 | 111 67 | 36 68 | -5 69 | 70 70 | 8 71 | 27 72 | -27 73 | -47 74 | -96 75 | 40 76 | 102 77 | -26 78 | 82 79 | -15 80 | 79 81 | 70 82 | 21 83 | -14 84 | 127 85 | 12 86 | 118 87 | 59 88 | -50 89 | -42 90 | -127 91 | -111 92 | -27 93 | -37 94 | -9 95 | 27 96 | 97 97 | -97 98 | -45 99 | 111 100 | -19 101 | 1 102 | -77 103 | -90 104 | -99 105 | -11 106 | 51 107 | 91 108 | 64 109 | 93 110 | 41 111 | 56 112 | 41 113 | 58 114 | 61 115 | -5 116 | -23 117 | 34 118 | 21 119 | 2 120 | -45 121 | -35 122 | -8 123 | 54 124 | -------------------------------------------------------------------------------- /test/data/expected/c_int8_col_lteq.csv: -------------------------------------------------------------------------------- 1 | -61 2 | -78 3 | 42 4 | -89 5 | -51 6 | 51 7 | -109 8 | 7 9 | -54 10 | 71 11 | 61 12 | -83 13 | -70 14 | -52 15 | 107 16 | -49 17 | 97 18 | 46 19 | -29 20 | -60 21 | 109 22 | -84 23 | 118 24 | -16 25 | 54 26 | 121 27 | -72 28 | 91 29 | -7 30 | -106 31 | -34 32 | 47 33 | -50 34 | -86 35 | -14 36 | -83 37 | -20 38 | -13 39 | 21 40 | 7 41 | 111 42 | -123 43 | -50 44 | -15 45 | 56 46 | -51 47 | -48 48 | 112 49 | 79 50 | 110 51 | 108 52 | 122 53 | -53 54 | 113 55 | -13 56 | 69 57 | 4 58 | -78 59 | 82 60 | -47 61 | -88 62 | 23 63 | -103 64 | -1 65 | 92 66 | 111 67 | 36 68 | -5 69 | 70 70 | 8 71 | 27 72 | -27 73 | -47 74 | -96 75 | 40 76 | 102 77 | -26 78 | 82 79 | -15 80 | 79 81 | 70 82 | 21 83 | -14 84 | 127 85 | 12 86 | 118 87 | 59 88 | -50 89 | -42 90 | -127 91 | -111 92 | -27 93 | -37 94 | -9 95 | 27 96 | 97 97 | -97 98 | -45 99 | 111 100 | -19 101 | 1 102 | -77 103 | -90 104 | -99 105 | -11 106 | 51 107 | 91 108 | 64 109 | 93 110 | 41 111 | 56 112 | 41 113 | 58 114 | 61 115 | -5 116 | -23 117 | 34 118 | 21 119 | 2 120 | -45 121 | -35 122 | -8 123 | 54 124 | -------------------------------------------------------------------------------- /test/data/expected/c_int8_col_noteq.csv: -------------------------------------------------------------------------------- 1 | -60 2 | 74 3 | 57 4 | -61 5 | -27 6 | -78 7 | -37 8 | 42 9 | -89 10 | -51 11 | -93 12 | -34 13 | 51 14 | -109 15 | 70 16 | 7 17 | -54 18 | 71 19 | -5 20 | 61 21 | -83 22 | -111 23 | 3 24 | -70 25 | 60 26 | -52 27 | 107 28 | -49 29 | 97 30 | -100 31 | 16 32 | -104 33 | 46 34 | -61 35 | -29 36 | -14 37 | -60 38 | 52 39 | 109 40 | -84 41 | 118 42 | 104 43 | -107 44 | -3 45 | 46 46 | -16 47 | -96 48 | 54 49 | 15 50 | 75 51 | 121 52 | -72 53 | 91 54 | -7 55 | 50 56 | 39 57 | -106 58 | -49 59 | -54 60 | -34 61 | -21 62 | -116 63 | 47 64 | -91 65 | -50 66 | -84 67 | -121 68 | -94 69 | -29 70 | 124 71 | -86 72 | -115 73 | 30 74 | 110 75 | -14 76 | 39 77 | 42 78 | -83 79 | 55 80 | -20 81 | 14 82 | -13 83 | 21 84 | 7 85 | 13 86 | -102 87 | 111 88 | -123 89 | -50 90 | -15 91 | 56 92 | -51 93 | -68 94 | -48 95 | 112 96 | 79 97 | -81 98 | 110 99 | 108 100 | 122 101 | -53 102 | 113 103 | 90 104 | -13 105 | -89 106 | 69 107 | 4 108 | 4 109 | -78 110 | -78 111 | -45 112 | 82 113 | -47 114 | -88 115 | 23 116 | -46 117 | 77 118 | 92 119 | -124 120 | -3 121 | -21 122 | -103 123 | -1 124 | 66 125 | 92 126 | 79 127 | -102 128 | -68 129 | -41 130 | 111 131 | -36 132 | 36 133 | 3 134 | -5 135 | 70 136 | 8 137 | 27 138 | 87 139 | 18 140 | 60 141 | 76 142 | -27 143 | 61 144 | -112 145 | 96 146 | -47 147 | -58 148 | -61 149 | -96 150 | 40 151 | 56 152 | -73 153 | -10 154 | 47 155 | 102 156 | -26 157 | 82 158 | -15 159 | 117 160 | 20 161 | 79 162 | 25 163 | 30 164 | 70 165 | 21 166 | 2 167 | 52 168 | -14 169 | 127 170 | 12 171 | 99 172 | 118 173 | -128 174 | 48 175 | 59 176 | -50 177 | -61 178 | -42 179 | 108 180 | -127 181 | -111 182 | -27 183 | 83 184 | -37 185 | -9 186 | 27 187 | -76 188 | 97 189 | 39 190 | -97 191 | -56 192 | -127 193 | 30 194 | -45 195 | 122 196 | -119 197 | -122 198 | 111 199 | -19 200 | -39 201 | -34 202 | -54 203 | 65 204 | 92 205 | -41 206 | 122 207 | -106 208 | 51 209 | 1 210 | 31 211 | 80 212 | -77 213 | -90 214 | 123 215 | -99 216 | -111 217 | -11 218 | -58 219 | 51 220 | 91 221 | 98 222 | 64 223 | 93 224 | 41 225 | 76 226 | 6 227 | 56 228 | -108 229 | -32 230 | 41 231 | 58 232 | -25 233 | 61 234 | -5 235 | -13 236 | -108 237 | -39 238 | -84 239 | 23 240 | -23 241 | 34 242 | 21 243 | -69 244 | 2 245 | 103 246 | -45 247 | -35 248 | 0 249 | -8 250 | -105 251 | 42 252 | 123 253 | -116 254 | -68 255 | -5 256 | 54 257 | -------------------------------------------------------------------------------- /test/data/expected/c_int8_eq.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygrove/datafusion-archive/219ca3f61f0d751779b4f30c02aa5fdcf8a40d12/test/data/expected/c_int8_eq.csv -------------------------------------------------------------------------------- /test/data/expected/c_int8_gt.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygrove/datafusion-archive/219ca3f61f0d751779b4f30c02aa5fdcf8a40d12/test/data/expected/c_int8_gt.csv -------------------------------------------------------------------------------- /test/data/expected/c_int8_gteq.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygrove/datafusion-archive/219ca3f61f0d751779b4f30c02aa5fdcf8a40d12/test/data/expected/c_int8_gteq.csv -------------------------------------------------------------------------------- /test/data/expected/c_int8_lt.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygrove/datafusion-archive/219ca3f61f0d751779b4f30c02aa5fdcf8a40d12/test/data/expected/c_int8_lt.csv -------------------------------------------------------------------------------- /test/data/expected/c_int8_lteq.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygrove/datafusion-archive/219ca3f61f0d751779b4f30c02aa5fdcf8a40d12/test/data/expected/c_int8_lteq.csv -------------------------------------------------------------------------------- /test/data/expected/c_int8_negative.csv: -------------------------------------------------------------------------------- 1 | -60 2 | -61 3 | -27 4 | -78 5 | -37 6 | -89 7 | -51 8 | -93 9 | -34 10 | -109 11 | -54 12 | -5 13 | -83 14 | -111 15 | -70 16 | -52 17 | -49 18 | -100 19 | -104 20 | -61 21 | -29 22 | -14 23 | -60 24 | -84 25 | -107 26 | -3 27 | -16 28 | -96 29 | -72 30 | -7 31 | -106 32 | -49 33 | -54 34 | -34 35 | -21 36 | -116 37 | -91 38 | -50 39 | -84 40 | -121 41 | -94 42 | -29 43 | -86 44 | -115 45 | -14 46 | -83 47 | -20 48 | -13 49 | -102 50 | -123 51 | -50 52 | -15 53 | -51 54 | -68 55 | -48 56 | -81 57 | -53 58 | -13 59 | -89 60 | -78 61 | -78 62 | -45 63 | -47 64 | -88 65 | -46 66 | -124 67 | -3 68 | -21 69 | -103 70 | -1 71 | -102 72 | -68 73 | -41 74 | -36 75 | -5 76 | -27 77 | -112 78 | -47 79 | -58 80 | -61 81 | -96 82 | -73 83 | -10 84 | -26 85 | -15 86 | -14 87 | -128 88 | -50 89 | -61 90 | -42 91 | -127 92 | -111 93 | -27 94 | -37 95 | -9 96 | -76 97 | -97 98 | -56 99 | -127 100 | -45 101 | -119 102 | -122 103 | -19 104 | -39 105 | -34 106 | -54 107 | -41 108 | -106 109 | -77 110 | -90 111 | -99 112 | -111 113 | -11 114 | -58 115 | -108 116 | -32 117 | -25 118 | -5 119 | -13 120 | -108 121 | -39 122 | -84 123 | -23 124 | -69 125 | -45 126 | -35 127 | -8 128 | -105 129 | -116 130 | -68 131 | -5 132 | -------------------------------------------------------------------------------- /test/data/expected/c_int8_noteq.csv: -------------------------------------------------------------------------------- 1 | -60 2 | 74 3 | 57 4 | -61 5 | -27 6 | -78 7 | -37 8 | 42 9 | -89 10 | -51 11 | -93 12 | -34 13 | 51 14 | -109 15 | 70 16 | 7 17 | -54 18 | 71 19 | -5 20 | 61 21 | -83 22 | -111 23 | 3 24 | -70 25 | 60 26 | -52 27 | 107 28 | -49 29 | 97 30 | -100 31 | 16 32 | -104 33 | 46 34 | -61 35 | -29 36 | -14 37 | -60 38 | 52 39 | 109 40 | -84 41 | 118 42 | 104 43 | -107 44 | -3 45 | 46 46 | -16 47 | -96 48 | 54 49 | 15 50 | 75 51 | 121 52 | -72 53 | 91 54 | -7 55 | 50 56 | 39 57 | -106 58 | -49 59 | -54 60 | -34 61 | -21 62 | -116 63 | 47 64 | -91 65 | -50 66 | -84 67 | -121 68 | -94 69 | -29 70 | 124 71 | -86 72 | -115 73 | 30 74 | 110 75 | -14 76 | 39 77 | 42 78 | -83 79 | 55 80 | -20 81 | 14 82 | -13 83 | 21 84 | 7 85 | 13 86 | -102 87 | 111 88 | -123 89 | -50 90 | -15 91 | 56 92 | -51 93 | -68 94 | -48 95 | 112 96 | 79 97 | -81 98 | 110 99 | 108 100 | 122 101 | -53 102 | 113 103 | 90 104 | -13 105 | -89 106 | 69 107 | 4 108 | 4 109 | -78 110 | -78 111 | -45 112 | 82 113 | -47 114 | -88 115 | 23 116 | -46 117 | 77 118 | 92 119 | -124 120 | -3 121 | -21 122 | -103 123 | -1 124 | 66 125 | 92 126 | 79 127 | -102 128 | -68 129 | -41 130 | 111 131 | -36 132 | 36 133 | 3 134 | -5 135 | 70 136 | 8 137 | 27 138 | 87 139 | 18 140 | 60 141 | 76 142 | -27 143 | 61 144 | -112 145 | 96 146 | -47 147 | -58 148 | -61 149 | -96 150 | 40 151 | 56 152 | -73 153 | -10 154 | 47 155 | 102 156 | -26 157 | 82 158 | -15 159 | 117 160 | 20 161 | 79 162 | 25 163 | 30 164 | 70 165 | 21 166 | 2 167 | 52 168 | -14 169 | 127 170 | 12 171 | 99 172 | 118 173 | -128 174 | 48 175 | 59 176 | -50 177 | -61 178 | -42 179 | 108 180 | -127 181 | -111 182 | -27 183 | 83 184 | -37 185 | -9 186 | 27 187 | -76 188 | 97 189 | 39 190 | -97 191 | -56 192 | -127 193 | 30 194 | -45 195 | 122 196 | -119 197 | -122 198 | 111 199 | -19 200 | -39 201 | -34 202 | -54 203 | 65 204 | 92 205 | -41 206 | 122 207 | -106 208 | 51 209 | 1 210 | 31 211 | 80 212 | -77 213 | -90 214 | 123 215 | -99 216 | -111 217 | -11 218 | -58 219 | 51 220 | 91 221 | 98 222 | 64 223 | 93 224 | 41 225 | 76 226 | 6 227 | 56 228 | -108 229 | -32 230 | 41 231 | 58 232 | -25 233 | 61 234 | -5 235 | -13 236 | -108 237 | -39 238 | -84 239 | 23 240 | -23 241 | 34 242 | 21 243 | -69 244 | 2 245 | 103 246 | -45 247 | -35 248 | -8 249 | -105 250 | 42 251 | 123 252 | -116 253 | -68 254 | -5 255 | 54 256 | -------------------------------------------------------------------------------- /test/data/expected/c_int8_positive.csv: -------------------------------------------------------------------------------- 1 | 74 2 | 57 3 | 42 4 | 51 5 | 70 6 | 7 7 | 71 8 | 61 9 | 3 10 | 60 11 | 107 12 | 97 13 | 16 14 | 46 15 | 52 16 | 109 17 | 118 18 | 104 19 | 46 20 | 54 21 | 15 22 | 75 23 | 121 24 | 91 25 | 50 26 | 39 27 | 47 28 | 124 29 | 30 30 | 110 31 | 39 32 | 42 33 | 55 34 | 14 35 | 21 36 | 7 37 | 13 38 | 111 39 | 56 40 | 112 41 | 79 42 | 110 43 | 108 44 | 122 45 | 113 46 | 90 47 | 69 48 | 4 49 | 4 50 | 82 51 | 23 52 | 77 53 | 92 54 | 66 55 | 92 56 | 79 57 | 111 58 | 36 59 | 3 60 | 70 61 | 8 62 | 27 63 | 87 64 | 18 65 | 60 66 | 76 67 | 61 68 | 96 69 | 40 70 | 56 71 | 47 72 | 102 73 | 82 74 | 117 75 | 20 76 | 79 77 | 25 78 | 30 79 | 70 80 | 21 81 | 2 82 | 52 83 | 127 84 | 12 85 | 99 86 | 118 87 | 48 88 | 59 89 | 108 90 | 83 91 | 27 92 | 97 93 | 39 94 | 30 95 | 122 96 | 111 97 | 65 98 | 92 99 | 122 100 | 51 101 | 1 102 | 31 103 | 80 104 | 123 105 | 51 106 | 91 107 | 98 108 | 64 109 | 93 110 | 41 111 | 76 112 | 6 113 | 56 114 | 41 115 | 58 116 | 61 117 | 23 118 | 34 119 | 21 120 | 2 121 | 103 122 | 0 123 | 42 124 | 123 125 | 54 126 | -------------------------------------------------------------------------------- /test/data/expected/c_int8_range_exclusive.csv: -------------------------------------------------------------------------------- 1 | 107 2 | 109 3 | 118 4 | 104 5 | 121 6 | 124 7 | 110 8 | 111 9 | 112 10 | 110 11 | 108 12 | 122 13 | 113 14 | 111 15 | 102 16 | 117 17 | 127 18 | 118 19 | 108 20 | 122 21 | 111 22 | 122 23 | 123 24 | 103 25 | 123 26 | -------------------------------------------------------------------------------- /test/data/expected/c_int8_range_inclusive.csv: -------------------------------------------------------------------------------- 1 | 74 2 | 57 3 | 42 4 | 51 5 | 70 6 | 7 7 | 71 8 | 61 9 | 3 10 | 60 11 | 97 12 | 16 13 | 46 14 | 52 15 | 46 16 | 54 17 | 15 18 | 75 19 | 91 20 | 50 21 | 39 22 | 47 23 | 30 24 | 39 25 | 42 26 | 55 27 | 14 28 | 21 29 | 7 30 | 13 31 | 56 32 | 79 33 | 90 34 | 69 35 | 4 36 | 4 37 | 82 38 | 23 39 | 77 40 | 92 41 | 66 42 | 92 43 | 79 44 | 36 45 | 3 46 | 70 47 | 8 48 | 27 49 | 87 50 | 18 51 | 60 52 | 76 53 | 61 54 | 96 55 | 40 56 | 56 57 | 47 58 | 82 59 | 20 60 | 79 61 | 25 62 | 30 63 | 70 64 | 21 65 | 2 66 | 52 67 | 12 68 | 99 69 | 48 70 | 59 71 | 83 72 | 27 73 | 97 74 | 39 75 | 30 76 | 65 77 | 92 78 | 51 79 | 31 80 | 80 81 | 51 82 | 91 83 | 98 84 | 64 85 | 93 86 | 41 87 | 76 88 | 6 89 | 56 90 | 41 91 | 58 92 | 61 93 | 23 94 | 34 95 | 21 96 | 2 97 | 42 98 | 54 99 | -------------------------------------------------------------------------------- /test/data/expected/c_int8_scalar_gt.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygrove/datafusion-archive/219ca3f61f0d751779b4f30c02aa5fdcf8a40d12/test/data/expected/c_int8_scalar_gt.csv -------------------------------------------------------------------------------- /test/data/expected/c_uint16_cast.csv: -------------------------------------------------------------------------------- 1 | 3 2 | 9 3 | 7 4 | 6 5 | 12 6 | 1 7 | 5 8 | 3 9 | 11 10 | 1 11 | 7 12 | 1 13 | 3 14 | 6 15 | 8 16 | 3 17 | 8 18 | 6 19 | 8 20 | 9 21 | 8 22 | 2 23 | 11 24 | 11 25 | 6 26 | 1 27 | 2 28 | 12 29 | 1 30 | 6 31 | 10 32 | 4 33 | 8 34 | 12 35 | 0 36 | 0 37 | 7 38 | 5 39 | 6 40 | 5 41 | 6 42 | 9 43 | 5 44 | 1 45 | 11 46 | 7 47 | 8 48 | 3 49 | 5 50 | 10 51 | 4 52 | 2 53 | 12 54 | 3 55 | 12 56 | 10 57 | 0 58 | 11 59 | 0 60 | 8 61 | 8 62 | 3 63 | 5 64 | 1 65 | 9 66 | 7 67 | 5 68 | 9 69 | 5 70 | 3 71 | 12 72 | 6 73 | 5 74 | 10 75 | 9 76 | 1 77 | 10 78 | 11 79 | 0 80 | 2 81 | 1 82 | 0 83 | 9 84 | 8 85 | 11 86 | 12 87 | 5 88 | 0 89 | 10 90 | 1 91 | 0 92 | 10 93 | 6 94 | 2 95 | 1 96 | 8 97 | 2 98 | 0 99 | 10 100 | 2 101 | 10 102 | 2 103 | 9 104 | 3 105 | 6 106 | 3 107 | 12 108 | 1 109 | 7 110 | 1 111 | 1 112 | 1 113 | 5 114 | 10 115 | 7 116 | 2 117 | 8 118 | 1 119 | 7 120 | 4 121 | 8 122 | 7 123 | 12 124 | 7 125 | 6 126 | 9 127 | 6 128 | 12 129 | 4 130 | 7 131 | 4 132 | 1 133 | 2 134 | 12 135 | 4 136 | 11 137 | 7 138 | 9 139 | 2 140 | 4 141 | 5 142 | 11 143 | 1 144 | 2 145 | 8 146 | 1 147 | 1 148 | 6 149 | 5 150 | 6 151 | 12 152 | 5 153 | 1 154 | 7 155 | 7 156 | 7 157 | 12 158 | 9 159 | 6 160 | 9 161 | 8 162 | 9 163 | 4 164 | 7 165 | 10 166 | 12 167 | 2 168 | 9 169 | 6 170 | 4 171 | 0 172 | 0 173 | 7 174 | 0 175 | 4 176 | 9 177 | 1 178 | 6 179 | 6 180 | 3 181 | 8 182 | 3 183 | 6 184 | 6 185 | 0 186 | 5 187 | 5 188 | 10 189 | 4 190 | 11 191 | 0 192 | 9 193 | 12 194 | 5 195 | 1 196 | 9 197 | 9 198 | 11 199 | 8 200 | 6 201 | 4 202 | 0 203 | 10 204 | 1 205 | 5 206 | 8 207 | 6 208 | 6 209 | 2 210 | 0 211 | 2 212 | 9 213 | 5 214 | 4 215 | 8 216 | 10 217 | 8 218 | 9 219 | 0 220 | 10 221 | 5 222 | 12 223 | 12 224 | 1 225 | 0 226 | 4 227 | 6 228 | 3 229 | 11 230 | 5 231 | 6 232 | 4 233 | 12 234 | 6 235 | 10 236 | 0 237 | 5 238 | 5 239 | 12 240 | 0 241 | 11 242 | 7 243 | 3 244 | 0 245 | 12 246 | 5 247 | 4 248 | 4 249 | 1 250 | 4 251 | 6 252 | 4 253 | 6 254 | 6 255 | 12 256 | 1 257 | -------------------------------------------------------------------------------- /test/data/expected/c_uint32_cast.csv: -------------------------------------------------------------------------------- 1 | 13 2 | 19 3 | 20 4 | 15 5 | 10 6 | 28 7 | 12 8 | 12 9 | 20 10 | 18 11 | 26 12 | 28 13 | 9 14 | 18 15 | 26 16 | 28 17 | 2 18 | 7 19 | 0 20 | 5 21 | 19 22 | 21 23 | 7 24 | 3 25 | 2 26 | 15 27 | 18 28 | 5 29 | 18 30 | 23 31 | 14 32 | 21 33 | 23 34 | 24 35 | 26 36 | 26 37 | 25 38 | 20 39 | 16 40 | 4 41 | 18 42 | 6 43 | 19 44 | 22 45 | 10 46 | 15 47 | 10 48 | 25 49 | 23 50 | 10 51 | 1 52 | 13 53 | 12 54 | 10 55 | 14 56 | 15 57 | 5 58 | 12 59 | 4 60 | 22 61 | 19 62 | 18 63 | 11 64 | 3 65 | 8 66 | 3 67 | 4 68 | 18 69 | 20 70 | 3 71 | 2 72 | 10 73 | 0 74 | 13 75 | 27 76 | 2 77 | 3 78 | 4 79 | 25 80 | 4 81 | 11 82 | 19 83 | 16 84 | 20 85 | 20 86 | 16 87 | 14 88 | 2 89 | 17 90 | 27 91 | 16 92 | 9 93 | 7 94 | 22 95 | 9 96 | 1 97 | 23 98 | 2 99 | 24 100 | 2 101 | 5 102 | 22 103 | 23 104 | 3 105 | 20 106 | 19 107 | 0 108 | 6 109 | 27 110 | 6 111 | 12 112 | 9 113 | 24 114 | 8 115 | 20 116 | 26 117 | 19 118 | 27 119 | 7 120 | 12 121 | 25 122 | 28 123 | 20 124 | 1 125 | 25 126 | 2 127 | 16 128 | 25 129 | 14 130 | 0 131 | 9 132 | 21 133 | 7 134 | 24 135 | 18 136 | 18 137 | 14 138 | 22 139 | 28 140 | 1 141 | 8 142 | 6 143 | 24 144 | 26 145 | 15 146 | 19 147 | 17 148 | 23 149 | 1 150 | 22 151 | 5 152 | 16 153 | 26 154 | 7 155 | 2 156 | 19 157 | 8 158 | 23 159 | 25 160 | 23 161 | 22 162 | 8 163 | 9 164 | 6 165 | 24 166 | 5 167 | 12 168 | 1 169 | 28 170 | 4 171 | 23 172 | 12 173 | 2 174 | 12 175 | 6 176 | 27 177 | 10 178 | 14 179 | 7 180 | 14 181 | 9 182 | 28 183 | 10 184 | 3 185 | 24 186 | 0 187 | 9 188 | 24 189 | 3 190 | 18 191 | 26 192 | 17 193 | 14 194 | 19 195 | 23 196 | 10 197 | 7 198 | 11 199 | 6 200 | 26 201 | 2 202 | 15 203 | 24 204 | 8 205 | 8 206 | 23 207 | 0 208 | 9 209 | 24 210 | 22 211 | 22 212 | 9 213 | 27 214 | 2 215 | 13 216 | 14 217 | 21 218 | 20 219 | 16 220 | 18 221 | 16 222 | 27 223 | 7 224 | 20 225 | 8 226 | 21 227 | 18 228 | 12 229 | 5 230 | 14 231 | 1 232 | 23 233 | 26 234 | 24 235 | 16 236 | 16 237 | 7 238 | 28 239 | 0 240 | 23 241 | 8 242 | 26 243 | 18 244 | 12 245 | 23 246 | 22 247 | 12 248 | 5 249 | 2 250 | 26 251 | 14 252 | 16 253 | 5 254 | 4 255 | 9 256 | 0 257 | -------------------------------------------------------------------------------- /test/data/expected/c_uint64_cast.csv: -------------------------------------------------------------------------------- 1 | 1531241020 2 | 206695022 3 | 818075781 4 | 584334584 5 | 1088152525 6 | 1675416048 7 | 868812003 8 | 1089699136 9 | 1361773026 10 | 1517414054 11 | 986672784 12 | 2130843314 13 | 270745351 14 | 1515848610 15 | 1932631261 16 | 1089538590 17 | 519817179 18 | 1284297466 19 | 2118139912 20 | 1734869757 21 | 1786835579 22 | 1853660290 23 | 1617498451 24 | 1344241364 25 | 311526785 26 | 1045609427 27 | 1255733166 28 | 483321637 29 | 174520530 30 | 28029682 31 | 1049003644 32 | 265370207 33 | 998526696 34 | 1476204031 35 | 883035612 36 | 1329614038 37 | 1261465891 38 | 657415491 39 | 694557375 40 | 1503653029 41 | 152873783 42 | 1321790172 43 | 1287225845 44 | 1088009139 45 | 1754701450 46 | 1092923336 47 | 498455696 48 | 1213693592 49 | 229745057 50 | 2139595848 51 | 998721306 52 | 588710295 53 | 1283109638 54 | 1780573279 55 | 1941178506 56 | 1145886737 57 | 768907620 58 | 1078985869 59 | 2136785513 60 | 1518342226 61 | 734416642 62 | 449851569 63 | 401313053 64 | 700348801 65 | 1789974023 66 | 831373904 67 | 1575527324 68 | 1770440563 69 | 1708128607 70 | 325594285 71 | 135734197 72 | 130466921 73 | 1406653387 74 | 351441140 75 | 1896057511 76 | 1037868643 77 | 1660899380 78 | 147870142 79 | 1303037366 80 | 1214458706 81 | 1471946189 82 | 1765602393 83 | 387372753 84 | 1581928087 85 | 2048754964 86 | 778075956 87 | 1904569058 88 | 554917662 89 | 1605770338 90 | 916115615 91 | 698633209 92 | 331717930 93 | 450726051 94 | 186639041 95 | 683974670 96 | 1926050631 97 | 159409222 98 | 668198746 99 | 1629969491 100 | 964921733 101 | 753159665 102 | 102069306 103 | 653707285 104 | 2059881038 105 | 409734065 106 | 1259996683 107 | 1839720673 108 | 1301693720 109 | 240962410 110 | 491938874 111 | 292523091 112 | 573870887 113 | 2101996079 114 | 1985551670 115 | 701041872 116 | 701083461 117 | 308099843 118 | 770426034 119 | 1527844932 120 | 1360802956 121 | 424937890 122 | 2048821577 123 | 714237037 124 | 1475093755 125 | 1975095528 126 | 1990728669 127 | 996835606 128 | 910769856 129 | 1229079897 130 | 702437653 131 | 774692787 132 | 166810683 133 | 2125241159 134 | 127665671 135 | 219535881 136 | 1727138559 137 | 1288123073 138 | 521409816 139 | 1729380619 140 | 1580335224 141 | 1825815563 142 | 219229032 143 | 500434828 144 | 1191561227 145 | 1186735829 146 | 1409639194 147 | 505461611 148 | 755389207 149 | 1516086503 150 | 1720613484 151 | 167732375 152 | 1793225363 153 | 6036300 154 | 1726331706 155 | 1100084362 156 | 1896145160 157 | 549691465 158 | 1239733478 159 | 2095680238 160 | 247029838 161 | 764261877 162 | 1062782840 163 | 1890302684 164 | 846560533 165 | 1762253986 166 | 110574873 167 | 864761995 168 | 1677399018 169 | 1942653741 170 | 1753166636 171 | 693877312 172 | 1932467860 173 | 457130070 174 | 1621197357 175 | 393046421 176 | 1694238063 177 | 86040064 178 | 1412164238 179 | 624873238 180 | 734893259 181 | 905397311 182 | 2091415377 183 | 361195459 184 | 929181548 185 | 2140323720 186 | 1449145810 187 | 1787062807 188 | 190356155 189 | 219242968 190 | 1994859279 191 | 884721435 192 | 96588005 193 | 2022855930 194 | 1477853573 195 | 163907572 196 | 1385062481 197 | 1968915629 198 | 276724257 199 | 1478402725 200 | 1947499328 201 | 751351361 202 | 2117512386 203 | 1545692391 204 | 1690286716 205 | 783628 206 | 614922851 207 | 1522475364 208 | 2125344243 209 | 1552813180 210 | 1857139486 211 | 1071339242 212 | 742819789 213 | 468916404 214 | 2026564449 215 | 326586959 216 | 1208384845 217 | 964499581 218 | 95742180 219 | 1071509982 220 | 1188807674 221 | 698906415 222 | 17570265 223 | 735489609 224 | 915091167 225 | 552057524 226 | 510613993 227 | 692406315 228 | 924528952 229 | 274430582 230 | 1751536026 231 | 1507465631 232 | 958006470 233 | 1045919070 234 | 1972845497 235 | 926210712 236 | 98375280 237 | 579584653 238 | 192413889 239 | 2034800013 240 | 1824726680 241 | 201318545 242 | 665464125 243 | 1477391018 244 | 27536489 245 | 1457100270 246 | 1783942973 247 | 1532666296 248 | 73962897 249 | 1207239595 250 | 828547331 251 | 744993075 252 | 792636561 253 | 1486287868 254 | 1733761461 255 | 1779584435 256 | 1725357266 257 | -------------------------------------------------------------------------------- /test/data/expected/c_uint8_cast.csv: -------------------------------------------------------------------------------- 1 | 2 2 | 4 3 | 0 4 | 1 5 | 4 6 | 0 7 | 2 8 | 3 9 | 4 10 | 1 11 | 2 12 | 1 13 | 1 14 | 4 15 | 3 16 | 1 17 | 1 18 | 1 19 | 1 20 | 1 21 | 3 22 | 0 23 | 2 24 | 2 25 | 2 26 | 2 27 | 2 28 | 4 29 | 4 30 | 4 31 | 2 32 | 2 33 | 0 34 | 0 35 | 0 36 | 4 37 | 3 38 | 3 39 | 0 40 | 0 41 | 1 42 | 1 43 | 4 44 | 1 45 | 3 46 | 2 47 | 4 48 | 3 49 | 4 50 | 2 51 | 1 52 | 1 53 | 1 54 | 0 55 | 3 56 | 2 57 | 2 58 | 3 59 | 3 60 | 2 61 | 1 62 | 1 63 | 2 64 | 4 65 | 0 66 | 0 67 | 1 68 | 4 69 | 3 70 | 3 71 | 3 72 | 3 73 | 1 74 | 4 75 | 1 76 | 3 77 | 3 78 | 0 79 | 0 80 | 3 81 | 2 82 | 1 83 | 4 84 | 4 85 | 4 86 | 0 87 | 0 88 | 4 89 | 4 90 | 2 91 | 0 92 | 1 93 | 2 94 | 4 95 | 1 96 | 3 97 | 0 98 | 3 99 | 1 100 | 1 101 | 4 102 | 3 103 | 2 104 | 2 105 | 4 106 | 2 107 | 0 108 | 2 109 | 0 110 | 4 111 | 2 112 | 0 113 | 4 114 | 2 115 | 1 116 | 0 117 | 4 118 | 4 119 | 3 120 | 0 121 | 2 122 | 1 123 | 0 124 | 3 125 | 1 126 | 3 127 | 3 128 | 2 129 | 2 130 | 1 131 | 4 132 | 0 133 | 1 134 | 1 135 | 4 136 | 1 137 | 2 138 | 2 139 | 0 140 | 2 141 | 1 142 | 4 143 | 2 144 | 2 145 | 2 146 | 4 147 | 4 148 | 1 149 | 0 150 | 2 151 | 3 152 | 0 153 | 3 154 | 4 155 | 3 156 | 0 157 | 0 158 | 2 159 | 1 160 | 2 161 | 3 162 | 4 163 | 3 164 | 1 165 | 0 166 | 3 167 | 4 168 | 0 169 | 3 170 | 1 171 | 1 172 | 3 173 | 0 174 | 4 175 | 2 176 | 4 177 | 3 178 | 4 179 | 4 180 | 1 181 | 0 182 | 2 183 | 2 184 | 2 185 | 1 186 | 1 187 | 3 188 | 3 189 | 0 190 | 2 191 | 2 192 | 4 193 | 0 194 | 1 195 | 3 196 | 2 197 | 1 198 | 1 199 | 4 200 | 4 201 | 4 202 | 0 203 | 3 204 | 3 205 | 4 206 | 2 207 | 3 208 | 4 209 | 1 210 | 2 211 | 4 212 | 1 213 | 0 214 | 3 215 | 0 216 | 3 217 | 2 218 | 0 219 | 1 220 | 3 221 | 3 222 | 3 223 | 0 224 | 4 225 | 4 226 | 1 227 | 1 228 | 3 229 | 1 230 | 1 231 | 1 232 | 3 233 | 3 234 | 2 235 | 1 236 | 0 237 | 2 238 | 3 239 | 4 240 | 3 241 | 0 242 | 2 243 | 4 244 | 0 245 | 1 246 | 0 247 | 2 248 | 2 249 | 3 250 | 3 251 | 1 252 | 2 253 | 4 254 | 0 255 | 4 256 | 4 257 | -------------------------------------------------------------------------------- /test/data/expected/csv_aggregate_all_types.csv: -------------------------------------------------------------------------------- 1 | 256,256,false,true,0,4,0,12,0,28,783628,2140323720,-128,127,-32625,32405,-2139804706,2145139030,-9216439232345632980,9201665022718575466,0.0035754442,0.9966397,0.0014460175058255142,0.9987703202471936,B⇞扯䩐糍䒷刪蝣錢쾴ਵ䒵⠮䀈磚줔⟱莈薾촆,B⇞扯䩐糍䒷刪蝣錢쾴ਵ䒵⠮䀈磚줔⟱莈薾촆 2 | -------------------------------------------------------------------------------- /test/data/expected/csv_aggregate_by_c_bool.csv: -------------------------------------------------------------------------------- 1 | false,0,4,0,12,0,28,6036300,2140323720,-127,123,-32474,32339,-2069387796,2000826202,-9216439232345632980,9201665022718575466,0.008584857,0.9966397,0.0014460175058255142,0.9848153564240777,Ĵ㭮慰ꌧ靵碉鴕₻镊䯊鄸넼覆⎈榓揦郠竤損ჸ,ힼઠ靶ゾ瀹蠓藋쫰䌩鸤휛䣧翹굗㪁迾뉪ꖈƋ屍 2 | true,0,4,0,12,0,28,783628,2139595848,-128,127,-32625,32405,-2139804706,2145139030,-9042118254009968025,9155374553061731286,0.0035754442,0.9912891,0.009589347996759257,0.9987703202471936,B⇞扯䩐糍䒷刪蝣錢쾴ਵ䒵⠮䀈磚줔⟱莈薾촆,흢곧᜾꿤㽼䀀ㇾ孑쥓襯笨蒌㬂隇੮訓ϸ돻컭栞 3 | -------------------------------------------------------------------------------- /test/data/expected/csv_query_all_types.csv: -------------------------------------------------------------------------------- 1 | true,1,6,7,1284297466,71,7947,-926560209,2355616434385030368,0.029641032,0.08748462506515176,൤筹뫓䍃ⶬ譓滄⇼窄흀쇲扴캄蕾ㄒ宦㵬클瓱郋 2 | false,0,8,23,998526696,46,9342,-944863825,-2209107919116321238,0.016121447,0.057659034649668706,瘕許速괋䚙ģ˿쏙肨꺏᡽䴂৵宒ꭂ熋ⶳ洪䆹䭀 3 | true,0,12,24,1476204031,-61,-1070,466035216,-7679277413407079967,0.76205754,0.02795727978930862,騫榵垮䢏ণሬ团㢱灸뾻䱣傇藗䘅Ⲣ럀ታ勖⇋魵 4 | true,1,4,1,998721306,121,213,-623428436,-5954897226712164389,0.12356734,0.09415645257989425,갗풜숓툨⁵셳ꕼ蠟ǂ긆໭滓螔ꠕ琪拸㇡ꪐ๞쀙 5 | true,1,8,19,734416642,-21,-17349,591727007,245833550551406861,0.4722255,0.020039612901084802,ᦟစ혟添龾Ⱋ⠲셗貁ꂷ䜢덽ⴣ饅싺틴❚筗踠໭ 6 | true,4,9,18,1770440563,-94,-1577,-927977663,5064208619045430671,0.9329869,0.009589347996759257,뾨㏑ꊍ㚸㚥䶩≑밒ꆫ期誅㾧䜹얍ᴭ캬희ằ뉬㠥 7 | false,4,10,13,351441140,110,-9032,143247473,-4629701276086450132,0.4037001,0.09279902688553465,࿝㠘酇ជ碆霠ነ窧靴㑶߄翨뒢ຑ㡭礉䜀쿙⏇佄 8 | true,2,1,27,916115615,-15,3246,1561153724,6640586874325022912,0.7753064,0.048453399897169014,粢릘⥅䎖䤱늱섞䌡顨෥⮶⦝莊㴑ല涵쓿驈嘤粽 9 | false,1,1,9,683974670,112,1395,1017959972,1576350744315291421,0.29213196,0.08914172928832309,㈕舶ƚ骩絽慈澅栄미䬎銠꫅宆腂稨牌꿐Ʋ힡ഗ 10 | true,0,7,27,240962410,-78,11563,379197915,-2090515699168836997,0.37161523,0.03804835023179365,㽿익櫀촅ꤏ琬睦ꁏ뽾承ང埛拇䤳៉噽齎࿈帆㤽 11 | true,4,1,6,491938874,-78,-7657,-1713674985,2120880350861883579,0.8542008,0.03864944648071922,轝功⃟釮辯쳄柺Жᵭ꺂彷現҉튼䠓왥熵칦ែᨷ 12 | false,1,7,20,701041872,23,32339,1161707988,2776691022936185348,0.37926137,0.01702500540212859,㏸㍍剝홖덺ꢴ謐縬鱮╹뚗窂䠚祣ᢏࡀⲝ馴∷ۣ 13 | false,1,7,28,2048821577,-103,29279,-1410928897,9201665022718575466,0.6662327,0.0945252022944818,濋잲鏷ᱨ僁臭텱魸딿で㮡레嚹㥭答켍葮蹎몫貜 14 | false,0,5,1,1516086503,-96,5911,249962800,-7839649174118521193,0.20281577,0.05378145868940831,໑何䠐ꦘꞍ牉؏悲Ⓐ쏝쳦掠빵듉䝨靳쭯콉趈㱦 15 | true,0,10,24,1762253986,21,8010,507136253,-1611897952838717131,0.8006082,0.03993674675026826,⻡⬕멊촛쬫帲澳寠狊鉹뵃瘹ꚍꭆ딛ꯞ೟␚쮊꣮ 16 | true,0,9,1,1677399018,-14,6272,-2139804706,7341782609509084299,0.22262114,0.02844783454047195,Ꭿ俲ऩ샇麕氒쌉挟᢬욦棌繾鹈瞀絲췖쮨쑴ꤌ瓦 17 | false,4,9,27,1694238063,-50,32203,-1726120170,5469539346156519424,0.34970838,0.0598221750681448,鲏쬺⋜ⳅ➤菡᎔쒛毴쭔핦ᖔ园⾨鏛㔶돨殓뤛븇 18 | false,3,1,23,163907572,122,-237,468294998,2724238383658753127,0.47393662,0.04998213882456293,爾㿥僅玎⓻ǟꧡ⪤珯弎作ⸯ㷽젖㲌䯉⶟샕ʿ쬁 19 | false,4,6,26,1947499328,-39,-13737,-105765628,7891961009888818461,0.687745,0.09773734165086989,ᬫθ覇Ꜳ⋯륟짧랕合ȣߛꤶ诃齼䃒䱛妙믣ㄣ늡 20 | true,4,5,8,783628,-41,-10964,1960736964,-752871478250714969,0.46603346,0.05442721715046017,孼㥨䍧葇菁售稉底ᐈ甌ᯘ铻≙驱䂒狴鳏鋄杦㾆 21 | false,1,9,9,742819789,-77,20213,-1423713872,3543203616318744714,0.82544696,0.0946642619465724,꓅攤ꄙ࿩藺篡躁㕵륦캝槎㇧쑜簆Њヵ숯됤䵬鵰 22 | true,0,8,13,326586959,-99,13227,-407962060,-4610716468070162606,0.3629434,0.09498698285175156,忠꒢蓅䏈뭀鍶⑍┙瑵쬩࡫兴峼팺ᐓ縪驌餋ⵤ㜜 23 | false,3,10,14,1208384845,-111,-6809,-521968346,20553134549368439,0.9454982,0.0014460175058255142,참띇Ḻএ᠆ࠚň쪴쁂䆎엲㥶ᓃᏑ᭏意஗敿㕐᧍ 24 | true,3,10,18,1188807674,91,17320,1193737407,4776742583075073262,0.8531791,0.09465088299568925,⩜㖌쭑世塳ጄ짯㆑Á儤閌닑ᆪ೪ಊ廞䏍一듉努 25 | true,4,1,20,915091167,41,8179,1889985445,5792681898786644575,0.7393128,0.09262331822987424,㹼섬널䮈஬㜥ꅛ紦哈쐤頶쏷榌䶙磕蕅䤇枯哼닏 26 | true,3,3,12,924528952,-108,-1996,1757736884,7975414790775317613,0.52667034,0.01023964046913095,샯꟨熟쒐쏥ᘊ擬낢훆⼨ꮥ臚绂꧴⺟佞為팊ᐉ朁 27 | false,1,6,1,1507465631,58,24518,1607613295,-8873966780457438942,0.4712336,0.07898064162389484,墻䥵ঢ៽릕枼鮴䛨ᎀ႐讫閹ऐ霫䍕蓿岫䡴鋥䪉 28 | true,1,12,23,1457100270,103,-3980,-926882849,-4562545035019281250,0.4389583,0.06844656232765345,ᐱ랿팢ꀔ⥘ѹ먧ᦌ싃鳭뮊バ匟븲譬憸⡳뫂ṥ竃 29 | false,3,1,2,1207239595,-8,22197,1033891533,4788989286431932240,0.41694415,0.04790633102108133,잎犣캘⮝㖄ấꌮᴟ友Ǟ੼옯(䀒稐胧ᠭⰧ圭ꈁ 30 | -------------------------------------------------------------------------------- /test/data/expected/is_not_null_csv.csv: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | 4 4 | 5 5 | -------------------------------------------------------------------------------- /test/data/expected/is_null_csv.csv: -------------------------------------------------------------------------------- 1 | 3 2 | -------------------------------------------------------------------------------- /test/data/expected/numerics_divide.csv: -------------------------------------------------------------------------------- 1 | 0,1,0.8,-1.4741783,1.57,1.25600004196167 2 | 1,2,2,1,1.5,1.2 3 | 0,0,0,0.525,1.05,0.8399999618530274 4 | -------------------------------------------------------------------------------- /test/data/expected/numerics_divide_f64.csv: -------------------------------------------------------------------------------- 1 | 0,1,0.8,-1.4741784037558687,1.57,1.256 2 | 1,2,2,1,1.5,1.2 3 | 0,0,0,0.525,1.05,0.8400000000000001 4 | -------------------------------------------------------------------------------- /test/data/expected/numerics_minus.csv: -------------------------------------------------------------------------------- 1 | -1,0,-0.5,5.2700005,1.1400001,0.6400001049041748 2 | 0,3,2.5,0,1,0.5 3 | -2,-2,-2.5,-1.9000001,0.099999905,-0.40000009536743164 4 | -------------------------------------------------------------------------------- /test/data/expected/numerics_minus_f64.csv: -------------------------------------------------------------------------------- 1 | -1,0,-0.5,5.27,1.1400000000000001,0.6400000000000001 2 | 0,3,2.5,0,1,0.5 3 | -2,-2,-2.5,-1.9,0.10000000000000009,-0.3999999999999999 4 | -------------------------------------------------------------------------------- /test/data/expected/numerics_modulo.csv: -------------------------------------------------------------------------------- 1 | 2,0,2,1.01,1.1400001,0.6400001049041748 2 | 0,1,0,0,1,0.5 3 | 0,0,0,2.1,0.099999905,2.0999999046325684 4 | -------------------------------------------------------------------------------- /test/data/expected/numerics_modulo_f64.csv: -------------------------------------------------------------------------------- 1 | 2,0,2,1.0100000000000002,1.1400000000000001,0.6400000000000001 2 | 0,1,0,0,1,0.5 3 | 0,0,0,2.1,0.10000000000000009,2.1 4 | -------------------------------------------------------------------------------- /test/data/expected/numerics_multiply.csv: -------------------------------------------------------------------------------- 1 | 6,4,5,-6.6882005,6.28,7.850000262260437 2 | 25,10,12.5,9,6,7.5 3 | 0,0,0,8.4,4.2,5.249999761581421 4 | -------------------------------------------------------------------------------- /test/data/expected/numerics_multiply_f64.csv: -------------------------------------------------------------------------------- 1 | 6,4,5,-6.6882,6.28,7.8500000000000005 2 | 25,10,12.5,9,6,7.5 3 | 0,0,0,8.4,4.2,5.25 4 | -------------------------------------------------------------------------------- /test/data/expected/numerics_plus.csv: -------------------------------------------------------------------------------- 1 | 5,4,4.5,1.01,5.1400003,5.640000104904175 2 | 10,7,7.5,6,5,5.5 3 | 2,2,2.5,6.1,4.1,4.599999904632568 4 | -------------------------------------------------------------------------------- /test/data/expected/numerics_plus_f64.csv: -------------------------------------------------------------------------------- 1 | 5,4,4.5,1.0100000000000002,5.140000000000001,5.640000000000001 2 | 10,7,7.5,6,5,5.5 3 | 2,2,2.5,6.1,4.1,4.6 4 | -------------------------------------------------------------------------------- /test/data/expected/parquet_aggregate_all_types.csv: -------------------------------------------------------------------------------- 1 | 256,256,false,true,0,4,0,12,0,28,783628,2140323720,-128,127,-32625,32405,-2139804706,2145139030,-9216439232345632980,9201665022718575466,0.0035754442,0.9966397,0.0014460175058255142,0.9987703202471936,씶꺙頫璂赫⦑㯇姹㨝恋鷉ᜀ鱕芿萺⽗㥋۹뿠,씶꺙頫璂赫⦑㯇姹㨝恋鷉ᜀ鱕芿萺⽗㥋۹뿠,-169,-289753,2147483647,269433464379590488,522,1482,3658,277238556498,125.8706,121.70651186599423 2 | -------------------------------------------------------------------------------- /test/data/expected/parquet_query_all_types.csv: -------------------------------------------------------------------------------- 1 | true,1,6,7,1284297466,71,7947,-926560209,2355616434385030368,0.029641032,0.08748462506515176,൤筹뫓䍃ⶬ譓滄⇼窄흀쇲扴캄蕾ㄒ宦㵬클瓱郋 2 | false,0,8,23,998526696,46,9342,-944863825,-2209107919116321238,0.016121447,0.057659034649668706,瘕許速괋䚙ģ˿쏙肨꺏᡽䴂৵宒ꭂ熋ⶳ洪䆹䭀 3 | true,0,12,24,1476204031,-61,-1070,466035216,-7679277413407079967,0.76205754,0.02795727978930862,騫榵垮䢏ণሬ团㢱灸뾻䱣傇藗䘅Ⲣ럀ታ勖⇋魵 4 | true,1,4,1,998721306,121,213,-623428436,-5954897226712164389,0.12356734,0.09415645257989425,갗풜숓툨⁵셳ꕼ蠟ǂ긆໭滓螔ꠕ琪拸㇡ꪐ๞쀙 5 | true,1,8,19,734416642,-21,-17349,591727007,245833550551406861,0.4722255,0.020039612901084802,ᦟစ혟添龾Ⱋ⠲셗貁ꂷ䜢덽ⴣ饅싺틴❚筗踠໭ 6 | true,4,9,18,1770440563,-94,-1577,-927977663,5064208619045430671,0.9329869,0.009589347996759257,뾨㏑ꊍ㚸㚥䶩≑밒ꆫ期誅㾧䜹얍ᴭ캬희ằ뉬㠥 7 | false,4,10,13,351441140,110,-9032,143247473,-4629701276086450132,0.4037001,0.09279902688553465,࿝㠘酇ជ碆霠ነ窧靴㑶߄翨뒢ຑ㡭礉䜀쿙⏇佄 8 | true,2,1,27,916115615,-15,3246,1561153724,6640586874325022912,0.7753064,0.048453399897169014,粢릘⥅䎖䤱늱섞䌡顨෥⮶⦝莊㴑ല涵쓿驈嘤粽 9 | false,1,1,9,683974670,112,1395,1017959972,1576350744315291421,0.29213196,0.08914172928832309,㈕舶ƚ骩絽慈澅栄미䬎銠꫅宆腂稨牌꿐Ʋ힡ഗ 10 | true,0,7,27,240962410,-78,11563,379197915,-2090515699168836997,0.37161523,0.03804835023179365,㽿익櫀촅ꤏ琬睦ꁏ뽾承ང埛拇䤳៉噽齎࿈帆㤽 11 | true,4,1,6,491938874,-78,-7657,-1713674985,2120880350861883579,0.8542008,0.03864944648071922,轝功⃟釮辯쳄柺Жᵭ꺂彷現҉튼䠓왥熵칦ែᨷ 12 | false,1,7,20,701041872,23,32339,1161707988,2776691022936185348,0.37926137,0.01702500540212859,㏸㍍剝홖덺ꢴ謐縬鱮╹뚗窂䠚祣ᢏࡀⲝ馴∷ۣ 13 | false,1,7,28,2048821577,-103,29279,-1410928897,9201665022718575466,0.6662327,0.0945252022944818,濋잲鏷ᱨ僁臭텱魸딿で㮡레嚹㥭答켍葮蹎몫貜 14 | false,0,5,1,1516086503,-96,5911,249962800,-7839649174118521193,0.20281577,0.05378145868940831,໑何䠐ꦘꞍ牉؏悲Ⓐ쏝쳦掠빵듉䝨靳쭯콉趈㱦 15 | true,0,10,24,1762253986,21,8010,507136253,-1611897952838717131,0.8006082,0.03993674675026826,⻡⬕멊촛쬫帲澳寠狊鉹뵃瘹ꚍꭆ딛ꯞ೟␚쮊꣮ 16 | true,0,9,1,1677399018,-14,6272,-2139804706,7341782609509084299,0.22262114,0.02844783454047195,Ꭿ俲ऩ샇麕氒쌉挟᢬욦棌繾鹈瞀絲췖쮨쑴ꤌ瓦 17 | false,4,9,27,1694238063,-50,32203,-1726120170,5469539346156519424,0.34970838,0.0598221750681448,鲏쬺⋜ⳅ➤菡᎔쒛毴쭔핦ᖔ园⾨鏛㔶돨殓뤛븇 18 | false,3,1,23,163907572,122,-237,468294998,2724238383658753127,0.47393662,0.04998213882456293,爾㿥僅玎⓻ǟꧡ⪤珯弎作ⸯ㷽젖㲌䯉⶟샕ʿ쬁 19 | false,4,6,26,1947499328,-39,-13737,-105765628,7891961009888818461,0.687745,0.09773734165086989,ᬫθ覇Ꜳ⋯륟짧랕合ȣߛꤶ诃齼䃒䱛妙믣ㄣ늡 20 | true,4,5,8,783628,-41,-10964,1960736964,-752871478250714969,0.46603346,0.05442721715046017,孼㥨䍧葇菁售稉底ᐈ甌ᯘ铻≙驱䂒狴鳏鋄杦㾆 21 | false,1,9,9,742819789,-77,20213,-1423713872,3543203616318744714,0.82544696,0.0946642619465724,꓅攤ꄙ࿩藺篡躁㕵륦캝槎㇧쑜簆Њヵ숯됤䵬鵰 22 | true,0,8,13,326586959,-99,13227,-407962060,-4610716468070162606,0.3629434,0.09498698285175156,忠꒢蓅䏈뭀鍶⑍┙瑵쬩࡫兴峼팺ᐓ縪驌餋ⵤ㜜 23 | false,3,10,14,1208384845,-111,-6809,-521968346,20553134549368439,0.9454982,0.0014460175058255142,참띇Ḻএ᠆ࠚň쪴쁂䆎엲㥶ᓃᏑ᭏意஗敿㕐᧍ 24 | true,3,10,18,1188807674,91,17320,1193737407,4776742583075073262,0.8531791,0.09465088299568925,⩜㖌쭑世塳ጄ짯㆑Á儤閌닑ᆪ೪ಊ廞䏍一듉努 25 | true,4,1,20,915091167,41,8179,1889985445,5792681898786644575,0.7393128,0.09262331822987424,㹼섬널䮈஬㜥ꅛ紦哈쐤頶쏷榌䶙磕蕅䤇枯哼닏 26 | true,3,3,12,924528952,-108,-1996,1757736884,7975414790775317613,0.52667034,0.01023964046913095,샯꟨熟쒐쏥ᘊ擬낢훆⼨ꮥ臚绂꧴⺟佞為팊ᐉ朁 27 | false,1,6,1,1507465631,58,24518,1607613295,-8873966780457438942,0.4712336,0.07898064162389484,墻䥵ঢ៽릕枼鮴䛨ᎀ႐讫閹ऐ霫䍕蓿岫䡴鋥䪉 28 | true,1,12,23,1457100270,103,-3980,-926882849,-4562545035019281250,0.4389583,0.06844656232765345,ᐱ랿팢ꀔ⥘ѹ먧ᦌ싃鳭뮊バ匟븲譬憸⡳뫂ṥ竃 29 | false,3,1,2,1207239595,-8,22197,1033891533,4788989286431932240,0.41694415,0.04790633102108133,잎犣캘⮝㖄ấꌮᴟ友Ǟ੼옯(䀒稐胧ᠭⰧ圭ꈁ 30 | -------------------------------------------------------------------------------- /test/data/expected/test_cast.csv: -------------------------------------------------------------------------------- 1 | 1,1,1,1,1.1,1.1,1.11,1.11 2 | 2,2,2,2,2.2,2.2,2.22,2.22 3 | -------------------------------------------------------------------------------- /test/data/expected/test_chaining_functions.csv: -------------------------------------------------------------------------------- 1 | POINT (57.653484 -3.335724) 2 | POINT (53.002666 -2.179404) 3 | POINT (52.412811 -1.778197) 4 | POINT (51.481583 -3.17909) 5 | POINT (50.768036 0.290472) 6 | POINT (51.752022 -1.257677) 7 | POINT (51.509865 -0.118092) 8 | POINT (51.568535 -1.772232) 9 | POINT (51.441883 0.370759) 10 | POINT (52.240479 -0.902656) 11 | POINT (52.370876 -1.265032) 12 | POINT (52.570385 -1.824042) 13 | POINT (51.772938 0.10231) 14 | POINT (57.149651 -2.099075) 15 | POINT (51.621441 -3.943646) 16 | POINT (53.235046 -1.421629) 17 | POINT (55.006763 -7.318268) 18 | POINT (51.068787 -1.794472) 19 | POINT (50.614429 -2.457621) 20 | POINT (52.59137 -2.110748) 21 | POINT (53.765762 -2.692337) 22 | POINT (50.720806 -1.904755) 23 | POINT (53.52282 -1.128462) 24 | POINT (55.458565 -4.629179) 25 | POINT (50.854259 0.573453) 26 | POINT (52.136436 -0.460739) 27 | POINT (51.572376 0.470009) 28 | POINT (51.458057 -2.116074) 29 | POINT (54.607868 -5.926437) 30 | POINT (50.967941 0.085831) 31 | POINT (50.825024 -0.383835) 32 | POINT (53.801277 -1.548567) 33 | POINT (54.328506 -2.74387) 34 | POINT (50.376289 -4.143841) 35 | POINT (52.080875 0.444517) 36 | POINT (52.328415 -1.377561) 37 | POINT (57.477772 -4.224721) 38 | -------------------------------------------------------------------------------- /test/data/expected/test_df_udf_udt.csv: -------------------------------------------------------------------------------- 1 | 57.653484, -3.335724 2 | 53.002666, -2.179404 3 | 52.412811, -1.778197 4 | 51.481583, -3.17909 5 | 50.768036, 0.290472 6 | 51.752022, -1.257677 7 | 51.509865, -0.118092 8 | 51.568535, -1.772232 9 | 51.441883, 0.370759 10 | 52.240479, -0.902656 11 | 52.370876, -1.265032 12 | 52.570385, -1.824042 13 | 51.772938, 0.10231 14 | 57.149651, -2.099075 15 | 51.621441, -3.943646 16 | 53.235046, -1.421629 17 | 55.006763, -7.318268 18 | 51.068787, -1.794472 19 | 50.614429, -2.457621 20 | 52.59137, -2.110748 21 | 53.765762, -2.692337 22 | 50.720806, -1.904755 23 | 53.52282, -1.128462 24 | 55.458565, -4.629179 25 | 50.854259, 0.573453 26 | 52.136436, -0.460739 27 | 51.572376, 0.470009 28 | 51.458057, -2.116074 29 | 54.607868, -5.926437 30 | 50.967941, 0.085831 31 | 50.825024, -0.383835 32 | 53.801277, -1.548567 33 | 54.328506, -2.74387 34 | 50.376289, -4.143841 35 | 52.080875, 0.444517 36 | 52.328415, -1.377561 37 | 57.477772, -4.224721 38 | -------------------------------------------------------------------------------- /test/data/expected/test_filter.csv: -------------------------------------------------------------------------------- 1 | Elgin, Scotland, the UK,57.653484,-3.335724 2 | Stoke-on-Trent, Staffordshire, the UK,53.002666,-2.179404 3 | Solihull, Birmingham, UK,52.412811,-1.778197 4 | Northampton, Northamptonshire, UK,52.240479,-0.902656 5 | Rugby, Warwickshire, UK,52.370876,-1.265032 6 | Sutton Coldfield, West Midlands, UK,52.570385,-1.824042 7 | Aberdeen, Aberdeen City, UK,57.149651,-2.099075 8 | Chesterfield, Derbyshire, UK,53.235046,-1.421629 9 | Londonderry, Derry, UK,55.006763,-7.318268 10 | Wolverhampton, West Midlands, UK,52.59137,-2.110748 11 | Preston, Lancashire, UK,53.765762,-2.692337 12 | Doncaster, South Yorkshire, UK,53.52282,-1.128462 13 | Ayr, South Ayrshire, UK,55.458565,-4.629179 14 | Bedford, UK,52.136436,-0.460739 15 | Belfast, UK,54.607868,-5.926437 16 | Leeds, West Yorkshire, UK,53.801277,-1.548567 17 | Kendal, Cumbria, UK,54.328506,-2.74387 18 | Haverhill, Suffolk, UK,52.080875,0.444517 19 | Frankton, Warwickshire, UK,52.328415,-1.377561 20 | Inverness, the UK,57.477772,-4.224721 21 | -------------------------------------------------------------------------------- /test/data/expected/test_limit.csv: -------------------------------------------------------------------------------- 1 | 1,1 2 | 2,1.4142135623730951 3 | 3,1.7320508075688772 4 | 4,2 5 | 5,2.23606797749979 6 | -------------------------------------------------------------------------------- /test/data/expected/test_simple_predicate.csv: -------------------------------------------------------------------------------- 1 | POINT (52.412811 -1.778197) 2 | POINT (51.481583 -3.17909) 3 | POINT (50.768036 0.290472) 4 | POINT (51.752022 -1.257677) 5 | POINT (51.509865 -0.118092) 6 | POINT (51.568535 -1.772232) 7 | POINT (51.441883 0.370759) 8 | POINT (52.240479 -0.902656) 9 | POINT (52.370876 -1.265032) 10 | POINT (52.570385 -1.824042) 11 | POINT (51.772938 0.10231) 12 | POINT (51.621441 -3.943646) 13 | POINT (51.068787 -1.794472) 14 | POINT (50.614429 -2.457621) 15 | POINT (52.59137 -2.110748) 16 | POINT (50.720806 -1.904755) 17 | POINT (50.854259 0.573453) 18 | POINT (52.136436 -0.460739) 19 | POINT (51.572376 0.470009) 20 | POINT (51.458057 -2.116074) 21 | POINT (50.967941 0.085831) 22 | POINT (50.825024 -0.383835) 23 | POINT (50.376289 -4.143841) 24 | POINT (52.080875 0.444517) 25 | POINT (52.328415 -1.377561) 26 | -------------------------------------------------------------------------------- /test/data/expected/test_sql_min_max.csv: -------------------------------------------------------------------------------- 1 | 50.376289,57.653484,-7.318268,0.573453 2 | -------------------------------------------------------------------------------- /test/data/expected/test_sql_udf_udt.csv: -------------------------------------------------------------------------------- 1 | 57.653484, -3.335724 2 | 53.002666, -2.179404 3 | 52.412811, -1.778197 4 | 51.481583, -3.17909 5 | 50.768036, 0.290472 6 | 51.752022, -1.257677 7 | 51.509865, -0.118092 8 | 51.568535, -1.772232 9 | 51.441883, 0.370759 10 | 52.240479, -0.902656 11 | 52.370876, -1.265032 12 | 52.570385, -1.824042 13 | 51.772938, 0.10231 14 | 57.149651, -2.099075 15 | 51.621441, -3.943646 16 | 53.235046, -1.421629 17 | 55.006763, -7.318268 18 | 51.068787, -1.794472 19 | 50.614429, -2.457621 20 | 52.59137, -2.110748 21 | 53.765762, -2.692337 22 | 50.720806, -1.904755 23 | 53.52282, -1.128462 24 | 55.458565, -4.629179 25 | 50.854259, 0.573453 26 | 52.136436, -0.460739 27 | 51.572376, 0.470009 28 | 51.458057, -2.116074 29 | 54.607868, -5.926437 30 | 50.967941, 0.085831 31 | 50.825024, -0.383835 32 | 53.801277, -1.548567 33 | 54.328506, -2.74387 34 | 50.376289, -4.143841 35 | 52.080875, 0.444517 36 | 52.328415, -1.377561 37 | 57.477772, -4.224721 38 | -------------------------------------------------------------------------------- /test/data/expected/test_sqrt.csv: -------------------------------------------------------------------------------- 1 | 1,1 2 | 2,1.4142135623730951 3 | 3,1.7320508075688772 4 | 4,2 5 | 5,2.23606797749979 6 | 6,2.449489742783178 7 | 7,2.6457513110645907 8 | 8,2.8284271247461903 9 | 9,3 10 | 10,3.1622776601683795 11 | -------------------------------------------------------------------------------- /test/data/null_test.csv: -------------------------------------------------------------------------------- 1 | c_int,c_float,c_string,c_bool 2 | 1,1.1,"1.11",true 3 | 2,2.2,"2.22",true 4 | 3,,"3.33",true 5 | 4,4.4,,false 6 | 5,6.6,"",false -------------------------------------------------------------------------------- /test/data/numerics.csv: -------------------------------------------------------------------------------- 1 | a,b,a_f,b_f 2 | 2,3,3.14,-2.13 3 | 5,5,3.0,3 4 | 0,2,2.1,4.00 -------------------------------------------------------------------------------- /test/data/people.csv: -------------------------------------------------------------------------------- 1 | id,first_name 2 | 1,Andy 3 | 2,Brian 4 | 3,Chris 5 | 4,Donna 6 | 5,Edward 7 | 6,Fiona 8 | 7,Gary 9 | 8,Helen 10 | 9,Irene 11 | 10,Juliet -------------------------------------------------------------------------------- /test/data/smoketest-expected.txt: -------------------------------------------------------------------------------- 1 | DataFusion Console 2 | Executing query ... 3 | Executing query ... 4 | POINT (52.412811 -1.778197) 5 | POINT (51.481583 -3.17909) 6 | POINT (50.768036 0.290472) 7 | POINT (51.752022 -1.257677) 8 | POINT (51.509865 -0.118092) 9 | POINT (51.568535 -1.772232) 10 | POINT (51.441883 0.370759) 11 | POINT (52.240479 -0.902656) 12 | POINT (52.370876 -1.265032) 13 | POINT (52.570385 -1.824042) 14 | POINT (51.772938 0.10231) 15 | POINT (51.621441 -3.943646) 16 | POINT (51.068787 -1.794472) 17 | POINT (50.614429 -2.457621) 18 | POINT (52.59137 -2.110748) 19 | POINT (50.720806 -1.904755) 20 | POINT (50.854259 0.573453) 21 | POINT (52.136436 -0.460739) 22 | POINT (51.572376 0.470009) 23 | POINT (51.458057 -2.116074) 24 | POINT (50.967941 0.085831) 25 | POINT (50.825024 -0.383835) 26 | POINT (50.376289 -4.143841) 27 | POINT (52.080875 0.444517) 28 | POINT (52.328415 -1.377561) 29 | Executing query ... 30 | POINT (57.653484 -3.335724) 31 | POINT (53.002666 -2.179404) 32 | POINT (57.149651 -2.099075) 33 | POINT (53.235046 -1.421629) 34 | POINT (55.006763 -7.318268) 35 | POINT (53.765762 -2.692337) 36 | POINT (53.52282 -1.128462) 37 | POINT (55.458565 -4.629179) 38 | POINT (54.607868 -5.926437) 39 | POINT (53.801277 -1.548567) 40 | POINT (54.328506 -2.74387) 41 | POINT (57.477772 -4.224721) 42 | -------------------------------------------------------------------------------- /test/data/smoketest.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE uk_cities (city VARCHAR(100), lat DOUBLE, lng DOUBLE) STORED AS CSV WITHOUT HEADER ROW LOCATION '/test/data/uk_cities.csv'; 2 | SELECT ST_AsText(ST_Point(lat, lng)) FROM uk_cities WHERE lat < 53.0; 3 | SELECT ST_AsText(ST_Point(lat, lng)) FROM uk_cities WHERE lat >= 53.0; -------------------------------------------------------------------------------- /test/data/uk_cities.csv: -------------------------------------------------------------------------------- 1 | "Elgin, Scotland, the UK",57.653484,-3.335724 2 | "Stoke-on-Trent, Staffordshire, the UK",53.002666,-2.179404 3 | "Solihull, Birmingham, UK",52.412811,-1.778197 4 | "Cardiff, Cardiff county, UK",51.481583,-3.179090 5 | "Eastbourne, East Sussex, UK",50.768036,0.290472 6 | "Oxford, Oxfordshire, UK",51.752022,-1.257677 7 | "London, UK",51.509865,-0.118092 8 | "Swindon, Swindon, UK",51.568535,-1.772232 9 | "Gravesend, Kent, UK",51.441883,0.370759 10 | "Northampton, Northamptonshire, UK",52.240479,-0.902656 11 | "Rugby, Warwickshire, UK",52.370876,-1.265032 12 | "Sutton Coldfield, West Midlands, UK",52.570385,-1.824042 13 | "Harlow, Essex, UK",51.772938,0.102310 14 | "Aberdeen, Aberdeen City, UK",57.149651,-2.099075 15 | "Swansea, Swansea, UK",51.621441,-3.943646 16 | "Chesterfield, Derbyshire, UK",53.235046,-1.421629 17 | "Londonderry, Derry, UK",55.006763,-7.318268 18 | "Salisbury, Wiltshire, UK",51.068787,-1.794472 19 | "Weymouth, Dorset, UK",50.614429,-2.457621 20 | "Wolverhampton, West Midlands, UK",52.591370,-2.110748 21 | "Preston, Lancashire, UK",53.765762,-2.692337 22 | "Bournemouth, UK",50.720806,-1.904755 23 | "Doncaster, South Yorkshire, UK",53.522820,-1.128462 24 | "Ayr, South Ayrshire, UK",55.458565,-4.629179 25 | "Hastings, East Sussex, UK",50.854259,0.573453 26 | "Bedford, UK",52.136436,-0.460739 27 | "Basildon, Essex, UK",51.572376,0.470009 28 | "Chippenham, Wiltshire, UK",51.458057,-2.116074 29 | "Belfast, UK",54.607868,-5.926437 30 | "Uckfield, East Sussex, UK",50.967941,0.085831 31 | "Worthing, West Sussex, UK",50.825024,-0.383835 32 | "Leeds, West Yorkshire, UK",53.801277,-1.548567 33 | "Kendal, Cumbria, UK",54.328506,-2.743870 34 | "Plymouth, UK",50.376289,-4.143841 35 | "Haverhill, Suffolk, UK",52.080875,0.444517 36 | "Frankton, Warwickshire, UK",52.328415,-1.377561 37 | "Inverness, the UK",57.477772,-4.224721 -------------------------------------------------------------------------------- /test/data/uk_cities.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andygrove/datafusion-archive/219ca3f61f0d751779b4f30c02aa5fdcf8a40d12/test/data/uk_cities.parquet -------------------------------------------------------------------------------- /tests/sql.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Grove Enterprises LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::cell::RefCell; 16 | use std::rc::Rc; 17 | use std::sync::Arc; 18 | 19 | extern crate arrow; 20 | extern crate datafusion; 21 | 22 | use arrow::array::*; 23 | use arrow::datatypes::{DataType, Field, Schema}; 24 | 25 | use datafusion::execution::context::ExecutionContext; 26 | use datafusion::execution::datasource::CsvDataSource; 27 | use datafusion::execution::relation::Relation; 28 | 29 | #[test] 30 | fn csv_query_with_predicate() { 31 | let mut ctx = ExecutionContext::new(); 32 | register_cities_csv(&mut ctx); 33 | let sql = "SELECT city, lat, lng, lat + lng FROM cities WHERE lat > 51.0 AND lat < 53"; 34 | let actual = execute(&mut ctx, sql); 35 | let expected= "\"Solihull, Birmingham, UK\"\t52.412811\t-1.778197\t50.634614\n\"Cardiff, Cardiff county, UK\"\t51.481583\t-3.17909\t48.302493\n\"Oxford, Oxfordshire, UK\"\t51.752022\t-1.257677\t50.494344999999996\n\"London, UK\"\t51.509865\t-0.118092\t51.391773\n\"Swindon, Swindon, UK\"\t51.568535\t-1.772232\t49.796302999999995\n\"Gravesend, Kent, UK\"\t51.441883\t0.370759\t51.812642\n\"Northampton, Northamptonshire, UK\"\t52.240479\t-0.902656\t51.337823\n\"Rugby, Warwickshire, UK\"\t52.370876\t-1.265032\t51.105844000000005\n\"Sutton Coldfield, West Midlands, UK\"\t52.570385\t-1.824042\t50.746343\n\"Harlow, Essex, UK\"\t51.772938\t0.10231\t51.875248000000006\n\"Swansea, Swansea, UK\"\t51.621441\t-3.943646\t47.677794999999996\n\"Salisbury, Wiltshire, UK\"\t51.068787\t-1.794472\t49.274315\n\"Wolverhampton, West Midlands, UK\"\t52.59137\t-2.110748\t50.480622\n\"Bedford, UK\"\t52.136436\t-0.460739\t51.67569700000001\n\"Basildon, Essex, UK\"\t51.572376\t0.470009\t52.042384999999996\n\"Chippenham, Wiltshire, UK\"\t51.458057\t-2.116074\t49.341983\n\"Haverhill, Suffolk, UK\"\t52.080875\t0.444517\t52.525392\n\"Frankton, Warwickshire, UK\"\t52.328415\t-1.377561\t50.950854\n".to_string(); 36 | assert_eq!(expected, actual); 37 | } 38 | 39 | #[test] 40 | fn csv_query_group_by_int_min_max() { 41 | let mut ctx = ExecutionContext::new(); 42 | let schema = Arc::new(Schema::new(vec![ 43 | Field::new("a", DataType::Int32, false), 44 | Field::new("b", DataType::Float64, false), 45 | ])); 46 | register_csv(&mut ctx, "t1", "test/data/aggregate_test_1.csv", &schema); 47 | //TODO add ORDER BY once supported, to make this test determistic 48 | let sql = "SELECT a, MIN(b), MAX(b) FROM t1 GROUP BY a"; 49 | let actual = execute(&mut ctx, sql); 50 | let expected = "2\t3.3\t5.5\n3\t1.0\t2.0\n1\t1.1\t2.2\n".to_string(); 51 | assert_eq!(expected, actual); 52 | } 53 | 54 | #[test] 55 | fn csv_query_group_by_string_min_max() { 56 | let mut ctx = ExecutionContext::new(); 57 | let schema = Arc::new(Schema::new(vec![ 58 | Field::new("a", DataType::Utf8, false), 59 | Field::new("b", DataType::Float64, false), 60 | ])); 61 | register_csv(&mut ctx, "t1", "test/data/aggregate_test_2.csv", &schema); 62 | //TODO add ORDER BY once supported, to make this test determistic 63 | let sql = "SELECT a, MIN(b), MAX(b) FROM t1 GROUP BY a"; 64 | let actual = execute(&mut ctx, sql); 65 | let expected = "\"three\"\t1.0\t2.0\n\"two\"\t3.3\t5.5\n\"one\"\t1.1\t2.2\n".to_string(); 66 | assert_eq!(expected, actual); 67 | } 68 | 69 | #[test] 70 | fn csv_query_cast() { 71 | let mut ctx = ExecutionContext::new(); 72 | register_cities_csv(&mut ctx); 73 | let sql = "SELECT CAST(lat AS int) FROM cities"; 74 | let actual = execute(&mut ctx, sql); 75 | let expected= "53\n52\n51\n50\n51\n51\n51\n51\n52\n52\n52\n51\n57\n51\n53\n55\n51\n50\n52\n53\n50\n53\n55\n50\n52\n51\n51\n54\n50\n50\n53\n54\n50\n52\n52\n57\n".to_string(); 76 | assert_eq!(expected, actual); 77 | } 78 | 79 | fn register_cities_csv(ctx: &mut ExecutionContext) { 80 | let schema = Arc::new(Schema::new(vec![ 81 | Field::new("city", DataType::Utf8, false), 82 | Field::new("lat", DataType::Float64, false), 83 | Field::new("lng", DataType::Float64, false), 84 | ])); 85 | 86 | register_csv(ctx, "cities", "test/data/uk_cities.csv", &schema); 87 | } 88 | 89 | fn register_csv(ctx: &mut ExecutionContext, name: &str, filename: &str, schema: &Arc) { 90 | let csv_datasource = CsvDataSource::new(filename, schema.clone(), 1024); 91 | ctx.register_datasource(name, Rc::new(RefCell::new(csv_datasource))); 92 | } 93 | 94 | /// Execute query and return result set as tab delimited string 95 | fn execute(ctx: &mut ExecutionContext, sql: &str) -> String { 96 | let results = ctx.sql(&sql).unwrap(); 97 | result_str(&results) 98 | } 99 | 100 | fn result_str(results: &Rc>) -> String { 101 | let mut relation = results.borrow_mut(); 102 | let mut str = String::new(); 103 | while let Some(batch) = relation.next().unwrap() { 104 | for row_index in 0..batch.num_rows() { 105 | for column_index in 0..batch.num_columns() { 106 | if column_index > 0 { 107 | str.push_str("\t"); 108 | } 109 | let column = batch.column(column_index); 110 | 111 | match column.data_type() { 112 | DataType::Int32 => { 113 | let array = column.as_any().downcast_ref::().unwrap(); 114 | str.push_str(&format!("{:?}", array.value(row_index))); 115 | } 116 | DataType::Float32 => { 117 | let array = column.as_any().downcast_ref::().unwrap(); 118 | str.push_str(&format!("{:?}", array.value(row_index))); 119 | } 120 | DataType::Float64 => { 121 | let array = column.as_any().downcast_ref::().unwrap(); 122 | str.push_str(&format!("{:?}", array.value(row_index))); 123 | } 124 | DataType::Utf8 => { 125 | let array = column.as_any().downcast_ref::().unwrap(); 126 | let s = String::from_utf8(array.value(row_index).to_vec()).unwrap(); 127 | 128 | str.push_str(&format!("{:?}", s)); 129 | } 130 | _ => str.push_str("???"), 131 | } 132 | } 133 | str.push_str("\n"); 134 | } 135 | } 136 | str 137 | } 138 | --------------------------------------------------------------------------------