├── .asf.yaml ├── .cargo └── config.toml ├── .dockerignore ├── .github └── workflows │ ├── build.yml │ ├── k8s.yml │ └── main.yml ├── .gitignore ├── .pre-commit-config.yaml ├── Cargo.lock ├── Cargo.toml ├── LICENSE.txt ├── NOTICE ├── README.md ├── datafusion_ray ├── __init__.py ├── core.py ├── friendly.py └── util.py ├── dev ├── create_license.py └── release │ ├── README.md │ ├── check-rat-report.py │ ├── create-tarball.sh │ ├── generate-changelog.py │ ├── rat_exclude_files.txt │ ├── release-tarball.sh │ ├── run-rat.sh │ └── verify-release-candidate.sh ├── docs ├── README.md ├── benchmarks │ └── df-ray-0.1.0rc1 │ │ ├── i4g.4xlarge │ │ ├── i4g.4xlarge_s3 │ │ ├── m7g.12x │ │ ├── m7g.12xlarge │ │ ├── m7g.12xlarge_s3 │ │ ├── m7gd.12x │ │ ├── m7gd.12xlarge │ │ └── r8g.8xlarge └── contributing.md ├── examples ├── http_csv.py └── tips.py ├── k8s ├── README.md ├── bench_toolbox.py ├── cmds.py ├── machine_prep.sh ├── pricing.py ├── pvcs.yaml.template ├── ray_cluster.yaml.template ├── ray_job.sh.template ├── requirements.txt.template ├── spark_job.yaml.template └── spark_tpcbench.py ├── pyproject.toml ├── src ├── codec.rs ├── context.rs ├── dataframe.rs ├── flight.rs ├── isolator.rs ├── lib.rs ├── max_rows.rs ├── physical.rs ├── pre_fetch.rs ├── processor_service.rs ├── proto │ ├── datafusion.proto │ ├── datafusion_common.proto │ ├── datafusion_ray.proto │ ├── generated │ │ ├── mod.rs │ │ └── protobuf.rs │ └── mod.rs ├── stage.rs ├── stage_reader.rs └── util.rs ├── testdata ├── tips │ └── tips.parquet └── tpch │ └── .gitignore └── tpch ├── make_data.py ├── queries ├── q1.sql ├── q10.sql ├── q11.sql ├── q12.sql ├── q13.sql ├── q14.sql ├── q15.sql ├── q16.sql ├── q17.sql ├── q18.sql ├── q19.sql ├── q2.sql ├── q20.sql ├── q21.sql ├── q22.sql ├── q3.sql ├── q4.sql ├── q5.sql ├── q6.sql ├── q7.sql ├── q8.sql └── q9.sql ├── requirements.txt └── tpcbench.py /.asf.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/.asf.yaml -------------------------------------------------------------------------------- /.cargo/config.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/.cargo/config.toml -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/.dockerignore -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/.github/workflows/build.yml -------------------------------------------------------------------------------- /.github/workflows/k8s.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/.github/workflows/k8s.yml -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/.github/workflows/main.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/Cargo.lock -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/Cargo.toml -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/LICENSE.txt -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/NOTICE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/README.md -------------------------------------------------------------------------------- /datafusion_ray/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/datafusion_ray/__init__.py -------------------------------------------------------------------------------- /datafusion_ray/core.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/datafusion_ray/core.py -------------------------------------------------------------------------------- /datafusion_ray/friendly.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/datafusion_ray/friendly.py -------------------------------------------------------------------------------- /datafusion_ray/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/datafusion_ray/util.py -------------------------------------------------------------------------------- /dev/create_license.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/dev/create_license.py -------------------------------------------------------------------------------- /dev/release/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/dev/release/README.md -------------------------------------------------------------------------------- /dev/release/check-rat-report.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/dev/release/check-rat-report.py -------------------------------------------------------------------------------- /dev/release/create-tarball.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/dev/release/create-tarball.sh -------------------------------------------------------------------------------- /dev/release/generate-changelog.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/dev/release/generate-changelog.py -------------------------------------------------------------------------------- /dev/release/rat_exclude_files.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/dev/release/rat_exclude_files.txt -------------------------------------------------------------------------------- /dev/release/release-tarball.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/dev/release/release-tarball.sh -------------------------------------------------------------------------------- /dev/release/run-rat.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/dev/release/run-rat.sh -------------------------------------------------------------------------------- /dev/release/verify-release-candidate.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/dev/release/verify-release-candidate.sh -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/docs/README.md -------------------------------------------------------------------------------- /docs/benchmarks/df-ray-0.1.0rc1/i4g.4xlarge: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/docs/benchmarks/df-ray-0.1.0rc1/i4g.4xlarge -------------------------------------------------------------------------------- /docs/benchmarks/df-ray-0.1.0rc1/i4g.4xlarge_s3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/docs/benchmarks/df-ray-0.1.0rc1/i4g.4xlarge_s3 -------------------------------------------------------------------------------- /docs/benchmarks/df-ray-0.1.0rc1/m7g.12x: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/docs/benchmarks/df-ray-0.1.0rc1/m7g.12x -------------------------------------------------------------------------------- /docs/benchmarks/df-ray-0.1.0rc1/m7g.12xlarge: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/docs/benchmarks/df-ray-0.1.0rc1/m7g.12xlarge -------------------------------------------------------------------------------- /docs/benchmarks/df-ray-0.1.0rc1/m7g.12xlarge_s3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/docs/benchmarks/df-ray-0.1.0rc1/m7g.12xlarge_s3 -------------------------------------------------------------------------------- /docs/benchmarks/df-ray-0.1.0rc1/m7gd.12x: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/docs/benchmarks/df-ray-0.1.0rc1/m7gd.12x -------------------------------------------------------------------------------- /docs/benchmarks/df-ray-0.1.0rc1/m7gd.12xlarge: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/docs/benchmarks/df-ray-0.1.0rc1/m7gd.12xlarge -------------------------------------------------------------------------------- /docs/benchmarks/df-ray-0.1.0rc1/r8g.8xlarge: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/docs/benchmarks/df-ray-0.1.0rc1/r8g.8xlarge -------------------------------------------------------------------------------- /docs/contributing.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/docs/contributing.md -------------------------------------------------------------------------------- /examples/http_csv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/examples/http_csv.py -------------------------------------------------------------------------------- /examples/tips.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/examples/tips.py -------------------------------------------------------------------------------- /k8s/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/k8s/README.md -------------------------------------------------------------------------------- /k8s/bench_toolbox.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/k8s/bench_toolbox.py -------------------------------------------------------------------------------- /k8s/cmds.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/k8s/cmds.py -------------------------------------------------------------------------------- /k8s/machine_prep.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/k8s/machine_prep.sh -------------------------------------------------------------------------------- /k8s/pricing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/k8s/pricing.py -------------------------------------------------------------------------------- /k8s/pvcs.yaml.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/k8s/pvcs.yaml.template -------------------------------------------------------------------------------- /k8s/ray_cluster.yaml.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/k8s/ray_cluster.yaml.template -------------------------------------------------------------------------------- /k8s/ray_job.sh.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/k8s/ray_job.sh.template -------------------------------------------------------------------------------- /k8s/requirements.txt.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/k8s/requirements.txt.template -------------------------------------------------------------------------------- /k8s/spark_job.yaml.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/k8s/spark_job.yaml.template -------------------------------------------------------------------------------- /k8s/spark_tpcbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/k8s/spark_tpcbench.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/pyproject.toml -------------------------------------------------------------------------------- /src/codec.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/codec.rs -------------------------------------------------------------------------------- /src/context.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/context.rs -------------------------------------------------------------------------------- /src/dataframe.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/dataframe.rs -------------------------------------------------------------------------------- /src/flight.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/flight.rs -------------------------------------------------------------------------------- /src/isolator.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/isolator.rs -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/lib.rs -------------------------------------------------------------------------------- /src/max_rows.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/max_rows.rs -------------------------------------------------------------------------------- /src/physical.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/physical.rs -------------------------------------------------------------------------------- /src/pre_fetch.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/pre_fetch.rs -------------------------------------------------------------------------------- /src/processor_service.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/processor_service.rs -------------------------------------------------------------------------------- /src/proto/datafusion.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/proto/datafusion.proto -------------------------------------------------------------------------------- /src/proto/datafusion_common.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/proto/datafusion_common.proto -------------------------------------------------------------------------------- /src/proto/datafusion_ray.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/proto/datafusion_ray.proto -------------------------------------------------------------------------------- /src/proto/generated/mod.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/proto/generated/mod.rs -------------------------------------------------------------------------------- /src/proto/generated/protobuf.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/proto/generated/protobuf.rs -------------------------------------------------------------------------------- /src/proto/mod.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/proto/mod.rs -------------------------------------------------------------------------------- /src/stage.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/stage.rs -------------------------------------------------------------------------------- /src/stage_reader.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/stage_reader.rs -------------------------------------------------------------------------------- /src/util.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/src/util.rs -------------------------------------------------------------------------------- /testdata/tips/tips.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/testdata/tips/tips.parquet -------------------------------------------------------------------------------- /testdata/tpch/.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tpch/make_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/make_data.py -------------------------------------------------------------------------------- /tpch/queries/q1.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q1.sql -------------------------------------------------------------------------------- /tpch/queries/q10.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q10.sql -------------------------------------------------------------------------------- /tpch/queries/q11.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q11.sql -------------------------------------------------------------------------------- /tpch/queries/q12.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q12.sql -------------------------------------------------------------------------------- /tpch/queries/q13.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q13.sql -------------------------------------------------------------------------------- /tpch/queries/q14.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q14.sql -------------------------------------------------------------------------------- /tpch/queries/q15.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q15.sql -------------------------------------------------------------------------------- /tpch/queries/q16.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q16.sql -------------------------------------------------------------------------------- /tpch/queries/q17.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q17.sql -------------------------------------------------------------------------------- /tpch/queries/q18.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q18.sql -------------------------------------------------------------------------------- /tpch/queries/q19.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q19.sql -------------------------------------------------------------------------------- /tpch/queries/q2.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q2.sql -------------------------------------------------------------------------------- /tpch/queries/q20.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q20.sql -------------------------------------------------------------------------------- /tpch/queries/q21.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q21.sql -------------------------------------------------------------------------------- /tpch/queries/q22.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q22.sql -------------------------------------------------------------------------------- /tpch/queries/q3.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q3.sql -------------------------------------------------------------------------------- /tpch/queries/q4.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q4.sql -------------------------------------------------------------------------------- /tpch/queries/q5.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q5.sql -------------------------------------------------------------------------------- /tpch/queries/q6.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q6.sql -------------------------------------------------------------------------------- /tpch/queries/q7.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q7.sql -------------------------------------------------------------------------------- /tpch/queries/q8.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q8.sql -------------------------------------------------------------------------------- /tpch/queries/q9.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/queries/q9.sql -------------------------------------------------------------------------------- /tpch/requirements.txt: -------------------------------------------------------------------------------- 1 | duckdb 2 | ray[default] 3 | maturin 4 | -------------------------------------------------------------------------------- /tpch/tpcbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/datafusion-ray/HEAD/tpch/tpcbench.py --------------------------------------------------------------------------------