├── benchmarks ├── .gitignore ├── queries │ ├── q6.sql │ ├── q17.sql │ ├── q14.sql │ ├── q4.sql │ ├── q13.sql │ ├── q3.sql │ ├── q5.sql │ ├── q1.sql │ ├── q15.sql │ ├── q11.sql │ ├── q16.sql │ ├── q18.sql │ ├── q10.sql │ ├── q12.sql │ ├── q9.sql │ ├── q2.sql │ ├── q21.sql │ ├── q20.sql │ ├── q22.sql │ ├── q8.sql │ ├── q7.sql │ └── q19.sql ├── entrypoint.sh ├── run.sh ├── tpchgen.dockerfile ├── .dockerignore └── tpch-gen.sh ├── .dockerignore ├── ballista ├── rust │ ├── .gitignore │ ├── scheduler │ │ ├── testdata │ │ │ ├── region │ │ │ │ └── region.tbl │ │ │ ├── nation │ │ │ │ └── nation.tbl │ │ │ ├── orders │ │ │ │ └── orders.tbl │ │ │ ├── part │ │ │ │ └── part.tbl │ │ │ ├── lineitem │ │ │ │ ├── partition0.tbl │ │ │ │ └── partition1.tbl │ │ │ └── supplier │ │ │ │ └── supplier.tbl │ │ ├── README.md │ │ └── build.rs │ ├── client │ │ └── src │ │ │ ├── lib.rs │ │ │ └── prelude.rs │ ├── executor │ │ ├── README.md │ │ ├── build.rs │ │ └── examples │ │ │ └── example_executor_config.toml │ ├── core │ │ ├── README.md │ │ ├── src │ │ │ ├── lib.rs │ │ │ └── execution_plans │ │ │ │ └── mod.rs │ │ └── build.rs │ └── .dockerignore ├── ui │ └── scheduler │ │ ├── .dockerignore │ │ ├── public │ │ ├── favicon.ico │ │ ├── logo192.png │ │ ├── logo512.png │ │ ├── manifest.json │ │ └── robots.txt │ │ ├── .gitignore │ │ ├── tsconfig.json │ │ ├── src │ │ ├── App.css │ │ ├── react-app-env.d.ts │ │ ├── setupTests.ts │ │ ├── App.test.tsx │ │ ├── index.css │ │ ├── components │ │ │ ├── Empty.tsx │ │ │ └── Footer.tsx │ │ └── reportWebVitals.ts │ │ └── index.d.ts └── docs │ ├── images │ └── query-execution.png │ └── integration-testing.md ├── datafusion ├── tests │ ├── example.csv │ ├── customer.csv │ ├── jsons │ │ ├── 1.json │ │ └── 2.json │ ├── aggregate_simple.csv │ ├── sql_integration.rs │ └── tpch-csv │ │ ├── nation.csv │ │ ├── orders.csv │ │ └── lineitem.csv ├── src │ ├── error.rs │ ├── datasource │ │ └── listing │ │ │ └── mod.rs │ ├── physical_plan │ │ ├── coercion_rule │ │ │ └── mod.rs │ │ └── expressions │ │ │ └── stats.rs │ ├── sql │ │ └── mod.rs │ ├── execution │ │ └── mod.rs │ ├── physical_optimizer │ │ └── mod.rs │ └── optimizer │ │ └── mod.rs ├── fuzz-utils │ └── Cargo.toml └── benches │ └── scalar.rs ├── docs ├── source │ ├── _templates │ │ ├── layout.html │ │ └── docs-sidebar.html │ ├── _static │ │ └── images │ │ │ ├── DataFusion-Logo-Dark.png │ │ │ ├── DataFusion-Logo-Light.png │ │ │ └── DataFusion-Logo-Background-White.png │ ├── user-guide │ │ ├── distributed │ │ │ ├── clients │ │ │ │ ├── python.md │ │ │ │ └── index.rst │ │ │ ├── index.rst │ │ │ └── deployment │ │ │ │ └── index.rst │ │ └── sql │ │ │ └── index.rst │ ├── python │ │ ├── api │ │ │ ├── dataframe.rst │ │ │ ├── functions.rst │ │ │ ├── expression.rst │ │ │ └── execution_context.rst │ │ └── api.rst │ └── specification │ │ └── rfcs │ │ └── template.md ├── .gitignore └── requirements.txt ├── .gitmodules ├── .gitattributes ├── .github ├── dependabot.yml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── pull_request_template.md └── workflows │ └── dev_pr │ └── labeler.yml ├── header ├── integration-tests ├── __init__.py ├── requirements.in ├── sqls │ ├── simple_select.sql │ ├── simple_union_all.sql │ ├── simple_sort.sql │ ├── simple_sort_nulls_order.sql │ ├── self_join_with_alias.sql │ ├── values_list.sql │ ├── simple_math_expressions.sql │ ├── simple_except.sql │ ├── simple_intersect.sql │ ├── simple_aggregation.sql │ ├── simple_except_all.sql │ ├── simple_intersect_all.sql │ ├── simple_group_by.sql │ ├── simple_window_full_aggregation.sql │ ├── simple_window_lead_built_in_functions.sql │ ├── simple_window_ranked_built_in_functions.sql │ ├── simple_window_ordered_aggregation.sql │ ├── simple_window_partition_aggregation.sql │ └── simple_window_partition_order_aggregation.sql └── create_test_table.sql ├── ci ├── conda_env_gandiva.yml ├── conda_env_gandiva_win.yml ├── conda_env_crossbow.txt ├── conda_env_unix.yml ├── scripts │ ├── csharp_build.sh │ ├── csharp_pack.sh │ ├── js_test.sh │ ├── release_test.sh │ ├── python_sdist_build.sh │ ├── csharp_test.sh │ ├── go_build.sh │ ├── ccache_setup.sh │ ├── r_build.sh │ ├── install_spark.sh │ ├── integration_hiveserver2.sh │ ├── r_deps.sh │ ├── js_build.sh │ ├── go_test.sh │ ├── ruby_test.sh │ ├── install_dask.sh │ ├── msys2_system_clean.sh │ ├── python_test.sh │ ├── r_sanitize.sh │ ├── util_checkout.sh │ ├── integration_kartothek.sh │ └── install_turbodbc.sh ├── conda_env_sphinx.yml ├── vcpkg │ ├── x64-windows-static-md-debug.cmake │ ├── x64-windows-static-md-release.cmake │ ├── x64-linux-static-debug.cmake │ ├── x64-linux-static-release.cmake │ ├── x64-osx-static-debug.cmake │ ├── x64-osx-static-release.cmake │ ├── arm64-linux-static-debug.cmake │ └── arm64-linux-static-release.cmake ├── docker │ ├── ubuntu-18.04-csharp.dockerfile │ ├── conda-python-dask.dockerfile │ ├── debian-10-go.dockerfile │ ├── linux-apt-ruby.dockerfile │ ├── conda-python-jpype.dockerfile │ ├── debian-10-js.dockerfile │ ├── conda-python-pandas.dockerfile │ ├── debian-9-java.dockerfile │ └── python-wheel-manylinux-test.dockerfile ├── conda_env_r.yml ├── conda_env_archery.yml ├── conda_env_cpp.yml └── conda_env_python.yml ├── .readthedocs.yml ├── datafusion-common ├── src │ └── lib.rs └── README.md ├── tokomak ├── src │ └── rules │ │ └── mod.rs └── Cargo.toml ├── dev ├── build-set-env.sh ├── build-ui.sh ├── docker │ └── ballista-arm64.Dockerfile ├── integration-tests.sh ├── build-ballista-docker.sh └── release │ ├── update_change_log-all.sh │ ├── update_change_log-python.sh │ └── update_change_log-datafusion.sh ├── rustfmt.toml ├── python └── README.md ├── CODE_OF_CONDUCT.md ├── .hadolint.yaml ├── .dir-locals.el ├── datafusion-cli ├── src │ └── lib.rs └── Dockerfile ├── Cargo.toml ├── CHANGELOG.md ├── datafusion-examples └── examples │ └── README.md └── .github_changelog_generator /benchmarks/.gitignore: -------------------------------------------------------------------------------- 1 | data -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | **target 3 | -------------------------------------------------------------------------------- /ballista/rust/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | temp -------------------------------------------------------------------------------- /datafusion/tests/example.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,2,3 -------------------------------------------------------------------------------- /ballista/ui/scheduler/.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | -------------------------------------------------------------------------------- /datafusion/tests/customer.csv: -------------------------------------------------------------------------------- 1 | andrew,100 2 | jorge,200 3 | andy,150 4 | paul,300 5 | -------------------------------------------------------------------------------- /ballista/docs/images/query-execution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datafusion-contrib/datafusion-tokomak/HEAD/ballista/docs/images/query-execution.png -------------------------------------------------------------------------------- /ballista/ui/scheduler/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datafusion-contrib/datafusion-tokomak/HEAD/ballista/ui/scheduler/public/favicon.ico -------------------------------------------------------------------------------- /ballista/ui/scheduler/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datafusion-contrib/datafusion-tokomak/HEAD/ballista/ui/scheduler/public/logo192.png -------------------------------------------------------------------------------- /ballista/ui/scheduler/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datafusion-contrib/datafusion-tokomak/HEAD/ballista/ui/scheduler/public/logo512.png -------------------------------------------------------------------------------- /docs/source/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "pydata_sphinx_theme/layout.html" %} 2 | 3 | {# Silence the navbar #} 4 | {% block docs_navbar %} 5 | {% endblock %} 6 | -------------------------------------------------------------------------------- /docs/source/_static/images/DataFusion-Logo-Dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datafusion-contrib/datafusion-tokomak/HEAD/docs/source/_static/images/DataFusion-Logo-Dark.png -------------------------------------------------------------------------------- /docs/source/_static/images/DataFusion-Logo-Light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datafusion-contrib/datafusion-tokomak/HEAD/docs/source/_static/images/DataFusion-Logo-Light.png -------------------------------------------------------------------------------- /docs/source/_static/images/DataFusion-Logo-Background-White.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datafusion-contrib/datafusion-tokomak/HEAD/docs/source/_static/images/DataFusion-Logo-Background-White.png -------------------------------------------------------------------------------- /datafusion/tests/jsons/1.json: -------------------------------------------------------------------------------- 1 | {"a":1, "b":[2.0, 1.3, -6.1], "c":[false, true], "d":"4"} 2 | {"a":-10, "b":[2.0, 1.3, -6.1], "c":[true, true], "d":"4"} 3 | {"a":2, "b":[2.0, null, -6.1], "c":[false, null], "d":"text"} 4 | {} -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "parquet-testing"] 2 | path = parquet-testing 3 | url = https://github.com/apache/parquet-testing.git 4 | [submodule "testing"] 5 | path = testing 6 | url = https://github.com/apache/arrow-testing 7 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | r/R/RcppExports.R linguist-generated=true 2 | r/R/arrowExports.R linguist-generated=true 3 | r/src/RcppExports.cpp linguist-generated=true 4 | r/src/arrowExports.cpp linguist-generated=true 5 | r/man/*.Rd linguist-generated=true 6 | 7 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: cargo 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | open-pull-requests-limit: 10 8 | target-branch: master 9 | labels: [auto-dependencies] 10 | -------------------------------------------------------------------------------- /benchmarks/queries/q6.sql: -------------------------------------------------------------------------------- 1 | select 2 | sum(l_extendedprice * l_discount) as revenue 3 | from 4 | lineitem 5 | where 6 | l_shipdate >= date '1994-01-01' 7 | and l_shipdate < date '1995-01-01' 8 | and l_discount between 0.06 - 0.01 and 0.06 + 0.01 9 | and l_quantity < 24; -------------------------------------------------------------------------------- /benchmarks/queries/q17.sql: -------------------------------------------------------------------------------- 1 | select 2 | sum(l_extendedprice) / 7.0 as avg_yearly 3 | from 4 | lineitem, 5 | part 6 | where 7 | p_partkey = l_partkey 8 | and p_brand = 'Brand#23' 9 | and p_container = 'MED BOX' 10 | and l_quantity < ( 11 | select 12 | 0.2 * avg(l_quantity) 13 | from 14 | lineitem 15 | where 16 | l_partkey = p_partkey 17 | ); -------------------------------------------------------------------------------- /ballista/ui/scheduler/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /ballista/rust/scheduler/testdata/region/region.tbl: -------------------------------------------------------------------------------- 1 | 0|AFRICA|lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to | 2 | 1|AMERICA|hs use ironic, even requests. s| 3 | 2|ASIA|ges. thinly even pinto beans ca| 4 | 3|EUROPE|ly final courts cajole furiously final excuse| 5 | 4|MIDDLE EAST|uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl| 6 | -------------------------------------------------------------------------------- /benchmarks/queries/q14.sql: -------------------------------------------------------------------------------- 1 | select 2 | 100.00 * sum(case 3 | when p_type like 'PROMO%' 4 | then l_extendedprice * (1 - l_discount) 5 | else 0 6 | end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue 7 | from 8 | lineitem, 9 | part 10 | where 11 | l_partkey = p_partkey 12 | and l_shipdate >= date '1995-09-01' 13 | and l_shipdate < date '1995-10-01'; -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 16 | **Expected behavior** 17 | A clear and concise description of what you expected to happen. 18 | 19 | **Additional context** 20 | Add any other context about the problem here. 21 | -------------------------------------------------------------------------------- /benchmarks/queries/q4.sql: -------------------------------------------------------------------------------- 1 | select 2 | o_orderpriority, 3 | count(*) as order_count 4 | from 5 | orders 6 | where 7 | o_orderdate >= '1993-07-01' 8 | and o_orderdate < date '1993-07-01' + interval '3' month 9 | and exists ( 10 | select 11 | * 12 | from 13 | lineitem 14 | where 15 | l_orderkey = o_orderkey 16 | and l_commitdate < l_receiptdate 17 | ) 18 | group by 19 | o_orderpriority 20 | order by 21 | o_orderpriority; -------------------------------------------------------------------------------- /datafusion/tests/aggregate_simple.csv: -------------------------------------------------------------------------------- 1 | c1,c2,c3 2 | 0.00001,0.000000000001,true 3 | 0.00002,0.000000000002,false 4 | 0.00002,0.000000000002,false 5 | 0.00003,0.000000000003,true 6 | 0.00003,0.000000000003,true 7 | 0.00003,0.000000000003,true 8 | 0.00004,0.000000000004,false 9 | 0.00004,0.000000000004,false 10 | 0.00004,0.000000000004,false 11 | 0.00004,0.000000000004,false 12 | 0.00005,0.000000000005,true 13 | 0.00005,0.000000000005,true 14 | 0.00005,0.000000000005,true 15 | 0.00005,0.000000000005,true 16 | 0.00005,0.000000000005,true -------------------------------------------------------------------------------- /datafusion/tests/jsons/2.json: -------------------------------------------------------------------------------- 1 | {"a":1, "b":2.0, "c":false, "d":"4"} 2 | {"a":-10, "b":-3.5, "c":true, "d":"4"} 3 | {"a":2, "b":0.6, "c":false, "d":"text"} 4 | {"a":1, "b":2.0, "c":false, "d":"4"} 5 | {"a":7, "b":-3.5, "c":true, "d":"4"} 6 | {"a":1, "b":0.6, "c":false, "d":"text"} 7 | {"a":1, "b":2.0, "c":false, "d":"4"} 8 | {"a":5, "b":-3.5, "c":true, "d":"4"} 9 | {"a":1, "b":0.6, "c":false, "d":"text"} 10 | {"a":1, "b":2.0, "c":false, "d":"4"} 11 | {"a":1, "b":-3.5, "c":true, "d":"4"} 12 | {"a":100000000000000, "b":0.6, "c":false, "d":"text"} -------------------------------------------------------------------------------- /benchmarks/queries/q13.sql: -------------------------------------------------------------------------------- 1 | select 2 | c_count, 3 | count(*) as custdist 4 | from 5 | ( 6 | select 7 | c_custkey, 8 | count(o_orderkey) 9 | from 10 | customer left outer join orders on 11 | c_custkey = o_custkey 12 | and o_comment not like '%special%requests%' 13 | group by 14 | c_custkey 15 | ) as c_orders (c_custkey, c_count) 16 | group by 17 | c_count 18 | order by 19 | custdist desc, 20 | c_count desc; -------------------------------------------------------------------------------- /benchmarks/queries/q3.sql: -------------------------------------------------------------------------------- 1 | select 2 | l_orderkey, 3 | sum(l_extendedprice * (1 - l_discount)) as revenue, 4 | o_orderdate, 5 | o_shippriority 6 | from 7 | customer, 8 | orders, 9 | lineitem 10 | where 11 | c_mktsegment = 'BUILDING' 12 | and c_custkey = o_custkey 13 | and l_orderkey = o_orderkey 14 | and o_orderdate < date '1995-03-15' 15 | and l_shipdate > date '1995-03-15' 16 | group by 17 | l_orderkey, 18 | o_orderdate, 19 | o_shippriority 20 | order by 21 | revenue desc, 22 | o_orderdate; -------------------------------------------------------------------------------- /benchmarks/queries/q5.sql: -------------------------------------------------------------------------------- 1 | select 2 | n_name, 3 | sum(l_extendedprice * (1 - l_discount)) as revenue 4 | from 5 | customer, 6 | orders, 7 | lineitem, 8 | supplier, 9 | nation, 10 | region 11 | where 12 | c_custkey = o_custkey 13 | and l_orderkey = o_orderkey 14 | and l_suppkey = s_suppkey 15 | and c_nationkey = s_nationkey 16 | and s_nationkey = n_nationkey 17 | and n_regionkey = r_regionkey 18 | and r_name = 'ASIA' 19 | and o_orderdate >= date '1994-01-01' 20 | and o_orderdate < date '1995-01-01' 21 | group by 22 | n_name 23 | order by 24 | revenue desc; -------------------------------------------------------------------------------- /ballista/ui/scheduler/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /benchmarks/queries/q1.sql: -------------------------------------------------------------------------------- 1 | select 2 | l_returnflag, 3 | l_linestatus, 4 | sum(l_quantity) as sum_qty, 5 | sum(l_extendedprice) as sum_base_price, 6 | sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, 7 | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, 8 | avg(l_quantity) as avg_qty, 9 | avg(l_extendedprice) as avg_price, 10 | avg(l_discount) as avg_disc, 11 | count(*) as count_order 12 | from 13 | lineitem 14 | where 15 | l_shipdate <= date '1998-09-02' 16 | group by 17 | l_returnflag, 18 | l_linestatus 19 | order by 20 | l_returnflag, 21 | l_linestatus; -------------------------------------------------------------------------------- /benchmarks/queries/q15.sql: -------------------------------------------------------------------------------- 1 | create view revenue0 (supplier_no, total_revenue) as 2 | select 3 | l_suppkey, 4 | sum(l_extendedprice * (1 - l_discount)) 5 | from 6 | lineitem 7 | where 8 | l_shipdate >= date '1996-01-01' 9 | and l_shipdate < date '1996-01-01' + interval '3' month 10 | group by 11 | l_suppkey; 12 | 13 | 14 | select 15 | s_suppkey, 16 | s_name, 17 | s_address, 18 | s_phone, 19 | total_revenue 20 | from 21 | supplier, 22 | revenue0 23 | where 24 | s_suppkey = supplier_no 25 | and total_revenue = ( 26 | select 27 | max(total_revenue) 28 | from 29 | revenue0 30 | ) 31 | order by 32 | s_suppkey; 33 | 34 | drop view revenue0; -------------------------------------------------------------------------------- /benchmarks/queries/q11.sql: -------------------------------------------------------------------------------- 1 | select 2 | ps_partkey, 3 | sum(ps_supplycost * ps_availqty) as value 4 | from 5 | partsupp, 6 | supplier, 7 | nation 8 | where 9 | ps_suppkey = s_suppkey 10 | and s_nationkey = n_nationkey 11 | and n_name = 'GERMANY' 12 | group by 13 | ps_partkey having 14 | sum(ps_supplycost * ps_availqty) > ( 15 | select 16 | sum(ps_supplycost * ps_availqty) * 0.0001 17 | from 18 | partsupp, 19 | supplier, 20 | nation 21 | where 22 | ps_suppkey = s_suppkey 23 | and s_nationkey = n_nationkey 24 | and n_name = 'GERMANY' 25 | ) 26 | order by 27 | value desc; -------------------------------------------------------------------------------- /benchmarks/queries/q16.sql: -------------------------------------------------------------------------------- 1 | select 2 | p_brand, 3 | p_type, 4 | p_size, 5 | count(distinct ps_suppkey) as supplier_cnt 6 | from 7 | partsupp, 8 | part 9 | where 10 | p_partkey = ps_partkey 11 | and p_brand <> 'Brand#45' 12 | and p_type not like 'MEDIUM POLISHED%' 13 | and p_size in (49, 14, 23, 45, 19, 3, 36, 9) 14 | and ps_suppkey not in ( 15 | select 16 | s_suppkey 17 | from 18 | supplier 19 | where 20 | s_comment like '%Customer%Complaints%' 21 | ) 22 | group by 23 | p_brand, 24 | p_type, 25 | p_size 26 | order by 27 | supplier_cnt desc, 28 | p_brand, 29 | p_type, 30 | p_size; -------------------------------------------------------------------------------- /benchmarks/queries/q18.sql: -------------------------------------------------------------------------------- 1 | select 2 | c_name, 3 | c_custkey, 4 | o_orderkey, 5 | o_orderdate, 6 | o_totalprice, 7 | sum(l_quantity) 8 | from 9 | customer, 10 | orders, 11 | lineitem 12 | where 13 | o_orderkey in ( 14 | select 15 | l_orderkey 16 | from 17 | lineitem 18 | group by 19 | l_orderkey having 20 | sum(l_quantity) > 300 21 | ) 22 | and c_custkey = o_custkey 23 | and o_orderkey = l_orderkey 24 | group by 25 | c_name, 26 | c_custkey, 27 | o_orderkey, 28 | o_orderdate, 29 | o_totalprice 30 | order by 31 | o_totalprice desc, 32 | o_orderdate; -------------------------------------------------------------------------------- /benchmarks/queries/q10.sql: -------------------------------------------------------------------------------- 1 | select 2 | c_custkey, 3 | c_name, 4 | sum(l_extendedprice * (1 - l_discount)) as revenue, 5 | c_acctbal, 6 | n_name, 7 | c_address, 8 | c_phone, 9 | c_comment 10 | from 11 | customer, 12 | orders, 13 | lineitem, 14 | nation 15 | where 16 | c_custkey = o_custkey 17 | and l_orderkey = o_orderkey 18 | and o_orderdate >= date '1993-10-01' 19 | and o_orderdate < date '1994-01-01' 20 | and l_returnflag = 'R' 21 | and c_nationkey = n_nationkey 22 | group by 23 | c_custkey, 24 | c_name, 25 | c_acctbal, 26 | c_phone, 27 | n_name, 28 | c_address, 29 | c_comment 30 | order by 31 | revenue desc; -------------------------------------------------------------------------------- /ballista/ui/scheduler/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "esModuleInterop": true, 12 | "allowSyntheticDefaultImports": true, 13 | "strict": true, 14 | "forceConsistentCasingInFileNames": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "module": "esnext", 17 | "moduleResolution": "node", 18 | "resolveJsonModule": true, 19 | "isolatedModules": true, 20 | "noEmit": true, 21 | "jsx": "react-jsx" 22 | }, 23 | "include": [ 24 | "src", 25 | "index.d.ts", 26 | "react-table-config.d.ts" 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /header: -------------------------------------------------------------------------------- 1 | Licensed to the Apache Software Foundation (ASF) under one 2 | or more contributor license agreements. See the NOTICE file 3 | distributed with this work for additional information 4 | regarding copyright ownership. The ASF licenses this file 5 | to you under the Apache License, Version 2.0 (the 6 | "License"); you may not use this file except in compliance 7 | with the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | -------------------------------------------------------------------------------- /benchmarks/queries/q12.sql: -------------------------------------------------------------------------------- 1 | select 2 | l_shipmode, 3 | sum(case 4 | when o_orderpriority = '1-URGENT' 5 | or o_orderpriority = '2-HIGH' 6 | then 1 7 | else 0 8 | end) as high_line_count, 9 | sum(case 10 | when o_orderpriority <> '1-URGENT' 11 | and o_orderpriority <> '2-HIGH' 12 | then 1 13 | else 0 14 | end) as low_line_count 15 | from 16 | lineitem 17 | join 18 | orders 19 | on 20 | l_orderkey = o_orderkey 21 | where 22 | l_shipmode in ('MAIL', 'SHIP') 23 | and l_commitdate < l_receiptdate 24 | and l_shipdate < l_commitdate 25 | and l_receiptdate >= date '1994-01-01' 26 | and l_receiptdate < date '1995-01-01' 27 | group by 28 | l_shipmode 29 | order by 30 | l_shipmode; -------------------------------------------------------------------------------- /integration-tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem or challenge? Please describe what you are trying to do.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | (This section helps Arrow developers understand the context and *why* for this feature, in addition to the *what*) 13 | 14 | **Describe the solution you'd like** 15 | A clear and concise description of what you want to happen. 16 | 17 | **Describe alternatives you've considered** 18 | A clear and concise description of any alternative solutions or features you've considered. 19 | 20 | **Additional context** 21 | Add any other context or screenshots about the feature request here. 22 | -------------------------------------------------------------------------------- /ballista/ui/scheduler/src/App.css: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | */ 19 | -------------------------------------------------------------------------------- /integration-tests/requirements.in: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | pytest 17 | numpy 18 | pandas 19 | -------------------------------------------------------------------------------- /ci/conda_env_gandiva.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | clang=11 19 | llvmdev=11 20 | -------------------------------------------------------------------------------- /benchmarks/queries/q9.sql: -------------------------------------------------------------------------------- 1 | select 2 | nation, 3 | o_year, 4 | sum(amount) as sum_profit 5 | from 6 | ( 7 | select 8 | n_name as nation, 9 | extract(year from o_orderdate) as o_year, 10 | l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount 11 | from 12 | part, 13 | supplier, 14 | lineitem, 15 | partsupp, 16 | orders, 17 | nation 18 | where 19 | s_suppkey = l_suppkey 20 | and ps_suppkey = l_suppkey 21 | and ps_partkey = l_partkey 22 | and p_partkey = l_partkey 23 | and o_orderkey = l_orderkey 24 | and s_nationkey = n_nationkey 25 | and p_name like '%green%' 26 | ) as profit 27 | group by 28 | nation, 29 | o_year 30 | order by 31 | nation, 32 | o_year desc; -------------------------------------------------------------------------------- /datafusion/tests/sql_integration.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | mod sql; 19 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | build 19 | source/python/generated 20 | venv/ 21 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_select.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT 1 as num; 18 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | conda: 19 | file: python/doc/environment.yml 20 | -------------------------------------------------------------------------------- /ballista/ui/scheduler/index.d.ts: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | declare module "@chakra-ui/icons"; 19 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_union_all.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT 1 num UNION ALL SELECT 2 num ORDER BY num; 18 | -------------------------------------------------------------------------------- /ballista/ui/scheduler/src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | /// 19 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | sphinx==2.4.4 19 | pydata-sphinx-theme 20 | myst-parser<1 21 | maturin<0.12 22 | -------------------------------------------------------------------------------- /ballista/rust/scheduler/testdata/nation/nation.tbl: -------------------------------------------------------------------------------- 1 | 0|ALGERIA|0| haggle. carefully final deposits detect slyly agai| 2 | 1|ARGENTINA|1|al foxes promise slyly according to the regular accounts. bold requests alon| 3 | 2|BRAZIL|1|y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special | 4 | 3|CANADA|1|eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold| 5 | 4|EGYPT|4|y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d| 6 | 5|ETHIOPIA|0|ven packages wake quickly. regu| 7 | 6|FRANCE|3|refully final requests. regular, ironi| 8 | 7|GERMANY|3|l platelets. regular accounts x-ray: unusual, regular acco| 9 | 8|INDIA|2|ss excuses cajole slyly across the packages. deposits print aroun| 10 | 9|INDONESIA|2| slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull| 11 | -------------------------------------------------------------------------------- /benchmarks/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | set -e 20 | cd /tpch-dbgen 21 | ./dbgen -vf -s 1 22 | mv *.tbl /data -------------------------------------------------------------------------------- /docs/source/user-guide/distributed/clients/python.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Python 21 | 22 | Coming soon. 23 | -------------------------------------------------------------------------------- /ci/conda_env_gandiva_win.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # llvmdev=9 or later require Visual Studio 2017 19 | clangdev=8 20 | llvmdev=8 21 | -------------------------------------------------------------------------------- /datafusion-common/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | mod error; 19 | 20 | pub use error::{DataFusionError, Result}; 21 | -------------------------------------------------------------------------------- /ballista/ui/scheduler/public/robots.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # https://www.robotstxt.org/robotstxt.html 19 | User-agent: * 20 | Disallow: 21 | -------------------------------------------------------------------------------- /tokomak/src/rules/mod.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | mod expr_simplification; 18 | mod plan_simplification; 19 | 20 | pub mod utils; 21 | -------------------------------------------------------------------------------- /benchmarks/queries/q2.sql: -------------------------------------------------------------------------------- 1 | select 2 | s_acctbal, 3 | s_name, 4 | n_name, 5 | p_partkey, 6 | p_mfgr, 7 | s_address, 8 | s_phone, 9 | s_comment 10 | from 11 | part, 12 | supplier, 13 | partsupp, 14 | nation, 15 | region 16 | where 17 | p_partkey = ps_partkey 18 | and s_suppkey = ps_suppkey 19 | and p_size = 15 20 | and p_type like '%BRASS' 21 | and s_nationkey = n_nationkey 22 | and n_regionkey = r_regionkey 23 | and r_name = 'EUROPE' 24 | and ps_supplycost = ( 25 | select 26 | min(ps_supplycost) 27 | from 28 | partsupp, 29 | supplier, 30 | nation, 31 | region 32 | where 33 | p_partkey = ps_partkey 34 | and s_suppkey = ps_suppkey 35 | and s_nationkey = n_nationkey 36 | and n_regionkey = r_regionkey 37 | and r_name = 'EUROPE' 38 | ) 39 | order by 40 | s_acctbal desc, 41 | n_name, 42 | s_name, 43 | p_partkey; -------------------------------------------------------------------------------- /datafusion/src/error.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //! DataFusion error types 19 | pub use datafusion_common::{DataFusionError, Result}; 20 | -------------------------------------------------------------------------------- /ci/conda_env_crossbow.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | click 19 | github3.py 20 | jinja2 21 | jira 22 | pygit2 23 | ruamel.yaml 24 | setuptools_scm 25 | toolz 26 | -------------------------------------------------------------------------------- /datafusion/tests/tpch-csv/nation.csv: -------------------------------------------------------------------------------- 1 | n_nationkey,n_name,n_regionkey,n_comment 2 | 1,ARGENTINA,1,al foxes promise slyly according to the regular accounts. bold requests alon 3 | 2,BRAZIL,1,y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special 4 | 3,CANADA,1,"eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold" 5 | 4,EGYPT,4,y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d 6 | 5,ETHIOPIA,0,ven packages wake quickly. regu 7 | 6,FRANCE,3,"refully final requests. regular, ironi" 8 | 7,GERMANY,3,"l platelets. regular accounts x-ray: unusual, regular acco" 9 | 8,INDIA,2,ss excuses cajole slyly across the packages. deposits print aroun 10 | 9,INDONESIA,2, slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull 11 | 10,IRAN,4,efully alongside of the slyly final dependencies. 12 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_sort.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT 18 | c2, 19 | c3, 20 | c10 21 | FROM test 22 | ORDER BY c2 ASC, c3 DESC, c10; 23 | -------------------------------------------------------------------------------- /benchmarks/queries/q21.sql: -------------------------------------------------------------------------------- 1 | select 2 | s_name, 3 | count(*) as numwait 4 | from 5 | supplier, 6 | lineitem l1, 7 | orders, 8 | nation 9 | where 10 | s_suppkey = l1.l_suppkey 11 | and o_orderkey = l1.l_orderkey 12 | and o_orderstatus = 'F' 13 | and l1.l_receiptdate > l1.l_commitdate 14 | and exists ( 15 | select 16 | * 17 | from 18 | lineitem l2 19 | where 20 | l2.l_orderkey = l1.l_orderkey 21 | and l2.l_suppkey <> l1.l_suppkey 22 | ) 23 | and not exists ( 24 | select 25 | * 26 | from 27 | lineitem l3 28 | where 29 | l3.l_orderkey = l1.l_orderkey 30 | and l3.l_suppkey <> l1.l_suppkey 31 | and l3.l_receiptdate > l3.l_commitdate 32 | ) 33 | and s_nationkey = n_nationkey 34 | and n_name = 'SAUDI ARABIA' 35 | group by 36 | s_name 37 | order by 38 | numwait desc, 39 | s_name; -------------------------------------------------------------------------------- /dev/build-set-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | export BALLISTA_VERSION=$(awk -F'[ ="]+' '$1 == "version" { print $2 }' ballista/rust/core/Cargo.toml) 21 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_sort_nulls_order.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT 18 | c5, 19 | c4, 20 | c10 21 | FROM test 22 | ORDER BY c5 ASC, c4 DESC, c10; -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | edition = "2021" 19 | max_width = 90 20 | 21 | # ignore generated files 22 | # ignore = [ 23 | # "arrow/src/ipc/gen", 24 | #] 25 | -------------------------------------------------------------------------------- /ci/conda_env_unix.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # conda package dependencies specific to Unix-like environments (Linux and macOS) 19 | 20 | autoconf 21 | ccache 22 | orc 23 | pkg-config 24 | -------------------------------------------------------------------------------- /ci/scripts/csharp_build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -ex 21 | 22 | source_dir=${1}/csharp 23 | 24 | pushd ${source_dir} 25 | dotnet build 26 | popd 27 | -------------------------------------------------------------------------------- /integration-tests/sqls/self_join_with_alias.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT 18 | t1.c9 result 19 | FROM test t1 20 | INNER JOIN test t2 21 | ON t1.c9 = t2.c9 22 | ORDER BY result; 23 | -------------------------------------------------------------------------------- /integration-tests/sqls/values_list.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT * FROM 18 | (VALUES (1,2.0,-3,1+1),(10,20.0,-30,2+2)) 19 | AS tbl(int_col, float_col, negative_col, summation); 20 | -------------------------------------------------------------------------------- /ballista/rust/client/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #![doc = include_str!("../README.md")] 19 | 20 | pub mod columnar_batch; 21 | pub mod context; 22 | pub mod prelude; 23 | -------------------------------------------------------------------------------- /docs/source/user-guide/distributed/clients/index.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | .. or more contributor license agreements. See the NOTICE file 3 | .. distributed with this work for additional information 4 | .. regarding copyright ownership. The ASF licenses this file 5 | .. to you under the Apache License, Version 2.0 (the 6 | .. "License"); you may not use this file except in compliance 7 | .. with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | .. software distributed under the License is distributed on an 13 | .. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | .. KIND, either express or implied. See the License for the 15 | .. specific language governing permissions and limitations 16 | .. under the License. 17 | 18 | Clients 19 | ======= 20 | 21 | .. toctree:: 22 | :maxdepth: 2 23 | 24 | rust 25 | python 26 | -------------------------------------------------------------------------------- /benchmarks/queries/q20.sql: -------------------------------------------------------------------------------- 1 | select 2 | s_name, 3 | s_address 4 | from 5 | supplier, 6 | nation 7 | where 8 | s_suppkey in ( 9 | select 10 | ps_suppkey 11 | from 12 | partsupp 13 | where 14 | ps_partkey in ( 15 | select 16 | p_partkey 17 | from 18 | part 19 | where 20 | p_name like 'forest%' 21 | ) 22 | and ps_availqty > ( 23 | select 24 | 0.5 * sum(l_quantity) 25 | from 26 | lineitem 27 | where 28 | l_partkey = ps_partkey 29 | and l_suppkey = ps_suppkey 30 | and l_shipdate >= date '1994-01-01' 31 | and l_shipdate < 'date 1994-01-01' + interval '1' year 32 | ) 33 | ) 34 | and s_nationkey = n_nationkey 35 | and n_name = 'CANADA' 36 | order by 37 | s_name; -------------------------------------------------------------------------------- /ci/scripts/csharp_pack.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -eux 21 | 22 | source_dir=${1}/csharp 23 | 24 | pushd ${source_dir} 25 | dotnet pack -c Release 26 | popd 27 | -------------------------------------------------------------------------------- /ci/scripts/js_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -ex 21 | 22 | source_dir=${1}/js 23 | 24 | pushd ${source_dir} 25 | 26 | yarn lint 27 | yarn test 28 | 29 | popd 30 | -------------------------------------------------------------------------------- /ci/scripts/release_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -eux 21 | 22 | arrow_dir=${1} 23 | 24 | pushd ${arrow_dir} 25 | 26 | dev/release/run-test.rb 27 | 28 | popd 29 | -------------------------------------------------------------------------------- /ci/conda_env_sphinx.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Requirements for building the documentation 19 | breathe 20 | doxygen 21 | ipython 22 | # Pinned per ARROW-9693 23 | sphinx=3.1.2 24 | pydata-sphinx-theme 25 | -------------------------------------------------------------------------------- /docs/source/_templates/docs-sidebar.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 |
7 | 8 | 9 |
10 | 11 | 20 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_math_expressions.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT 18 | abs(-1.1) as abs, 19 | exp(2.0) as exp, 20 | sin(3.0) as sin, 21 | cos(4.0) as cos, 22 | tan(5.0) as tan; 23 | -------------------------------------------------------------------------------- /python/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # DataFusion in Python 21 | 22 | This directory is now moved to its [dedicated repository](https://github.com/datafusion-contrib/datafusion-python). 23 | -------------------------------------------------------------------------------- /ci/vcpkg/x64-windows-static-md-debug.cmake: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | set(VCPKG_TARGET_ARCHITECTURE x64) 19 | set(VCPKG_CRT_LINKAGE dynamic) 20 | set(VCPKG_LIBRARY_LINKAGE static) 21 | 22 | set(VCPKG_BUILD_TYPE debug) 23 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Code of Conduct 21 | 22 | - [Code of Conduct for The Apache Software Foundation][1] 23 | 24 | [1]: https://www.apache.org/foundation/policies/conduct.html 25 | -------------------------------------------------------------------------------- /ci/vcpkg/x64-windows-static-md-release.cmake: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | set(VCPKG_TARGET_ARCHITECTURE x64) 19 | set(VCPKG_CRT_LINKAGE dynamic) 20 | set(VCPKG_LIBRARY_LINKAGE static) 21 | 22 | set(VCPKG_BUILD_TYPE release) 23 | -------------------------------------------------------------------------------- /ballista/rust/executor/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Ballista Executor Process 21 | 22 | This crate contains the Ballista executor process. Refer to for 23 | documentation. 24 | -------------------------------------------------------------------------------- /datafusion-common/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # DataFusion Common 21 | 22 | This is an internal module for the most fundamental types of [DataFusion][df]. 23 | 24 | [df]: https://crates.io/crates/datafusion 25 | -------------------------------------------------------------------------------- /dev/build-ui.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -e 21 | 22 | . ./dev/build-set-env.sh 23 | docker build -t ballista-scheduler-ui:$BALLISTA_VERSION -f dev/docker/ballista-scheduler-ui.dockerfile ballista/ui/scheduler 24 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_except.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT * FROM ( 18 | SELECT c2 19 | FROM test t1 20 | EXCEPT 21 | SELECT c2 22 | FROM test t2 23 | WHERE c2 IN (3, 4) 24 | ) s 25 | ORDER BY c2 26 | -------------------------------------------------------------------------------- /ballista/rust/scheduler/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Ballista Scheduler Process 21 | 22 | This crate contains the Ballista scheduler process. Refer to for 23 | documentation. 24 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_intersect.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT * FROM ( 18 | SELECT c2 19 | FROM test t1 20 | INTERSECT 21 | SELECT c2 22 | FROM test t2 23 | WHERE c2 IN (3, 4) 24 | ) s 25 | ORDER BY c2 26 | -------------------------------------------------------------------------------- /ci/docker/ubuntu-18.04-csharp.dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ARG platform=bionic 19 | ARG dotnet=3.1 20 | FROM mcr.microsoft.com/dotnet/core/sdk:${dotnet}-${platform} 21 | 22 | RUN dotnet tool install --tool-path /usr/local/bin sourcelink 23 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_aggregation.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT 18 | count(*) AS count_all, 19 | count(c3) AS count_c3, 20 | avg(c3) AS avg, 21 | sum(c3) AS sum, 22 | max(c3) AS max, 23 | min(c3) AS min 24 | FROM test; 25 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_except_all.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT * FROM ( 18 | SELECT c2 19 | FROM test t1 20 | EXCEPT ALL 21 | SELECT c2 22 | FROM test t2 23 | WHERE c2 IN (3, 4) 24 | ) s 25 | ORDER BY c2 26 | -------------------------------------------------------------------------------- /docs/source/python/api/dataframe.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | .. or more contributor license agreements. See the NOTICE file 3 | .. distributed with this work for additional information 4 | .. regarding copyright ownership. The ASF licenses this file 5 | .. to you under the Apache License, Version 2.0 (the 6 | .. "License"); you may not use this file except in compliance 7 | .. with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | .. software distributed under the License is distributed on an 13 | .. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | .. KIND, either express or implied. See the License for the 15 | .. specific language governing permissions and limitations 16 | .. under the License. 17 | 18 | .. _api.dataframe: 19 | .. currentmodule:: datafusion 20 | 21 | DataFrame 22 | ========= 23 | 24 | .. autosummary:: 25 | :toctree: ../generated/ 26 | 27 | DataFrame 28 | -------------------------------------------------------------------------------- /docs/source/python/api/functions.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | .. or more contributor license agreements. See the NOTICE file 3 | .. distributed with this work for additional information 4 | .. regarding copyright ownership. The ASF licenses this file 5 | .. to you under the Apache License, Version 2.0 (the 6 | .. "License"); you may not use this file except in compliance 7 | .. with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | .. software distributed under the License is distributed on an 13 | .. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | .. KIND, either express or implied. See the License for the 15 | .. specific language governing permissions and limitations 16 | .. under the License. 17 | 18 | .. _api.functions: 19 | .. currentmodule:: datafusion 20 | 21 | Functions 22 | ========= 23 | 24 | .. autosummary:: 25 | :toctree: ../generated/ 26 | 27 | functions 28 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_intersect_all.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT * FROM ( 18 | SELECT c2 19 | FROM test t1 20 | INTERSECT ALL 21 | SELECT c2 22 | FROM test t2 23 | WHERE c2 IN (3, 4) 24 | ) s 25 | ORDER BY c2 26 | -------------------------------------------------------------------------------- /docs/source/python/api/expression.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | .. or more contributor license agreements. See the NOTICE file 3 | .. distributed with this work for additional information 4 | .. regarding copyright ownership. The ASF licenses this file 5 | .. to you under the Apache License, Version 2.0 (the 6 | .. "License"); you may not use this file except in compliance 7 | .. with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | .. software distributed under the License is distributed on an 13 | .. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | .. KIND, either express or implied. See the License for the 15 | .. specific language governing permissions and limitations 16 | .. under the License. 17 | 18 | .. _api.expression: 19 | .. currentmodule:: datafusion 20 | 21 | Expression 22 | ========== 23 | 24 | .. autosummary:: 25 | :toctree: ../generated/ 26 | 27 | Expression 28 | -------------------------------------------------------------------------------- /docs/source/user-guide/sql/index.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | .. or more contributor license agreements. See the NOTICE file 3 | .. distributed with this work for additional information 4 | .. regarding copyright ownership. The ASF licenses this file 5 | .. to you under the Apache License, Version 2.0 (the 6 | .. "License"); you may not use this file except in compliance 7 | .. with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | .. software distributed under the License is distributed on an 13 | .. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | .. KIND, either express or implied. See the License for the 15 | .. specific language governing permissions and limitations 16 | .. under the License. 17 | 18 | SQL Reference 19 | ============= 20 | 21 | .. toctree:: 22 | :maxdepth: 2 23 | 24 | sql_status 25 | select 26 | ddl 27 | DataFusion Functions 28 | -------------------------------------------------------------------------------- /ci/vcpkg/x64-linux-static-debug.cmake: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | set(VCPKG_TARGET_ARCHITECTURE x64) 19 | set(VCPKG_CRT_LINKAGE dynamic) 20 | set(VCPKG_LIBRARY_LINKAGE static) 21 | 22 | set(VCPKG_CMAKE_SYSTEM_NAME Linux) 23 | 24 | set(VCPKG_BUILD_TYPE debug) 25 | -------------------------------------------------------------------------------- /ci/vcpkg/x64-linux-static-release.cmake: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | set(VCPKG_TARGET_ARCHITECTURE x64) 19 | set(VCPKG_CRT_LINKAGE dynamic) 20 | set(VCPKG_LIBRARY_LINKAGE static) 21 | 22 | set(VCPKG_CMAKE_SYSTEM_NAME Linux) 23 | 24 | set(VCPKG_BUILD_TYPE release) 25 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_group_by.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | 18 | select 19 | c2, 20 | sum(c3) sum_c3, 21 | avg(c3) avg_c3, 22 | max(c3) max_c3, 23 | min(c3) min_c3, 24 | count(c3) count_c3 25 | from test 26 | group by c2 27 | order by c2; 28 | -------------------------------------------------------------------------------- /benchmarks/queries/q22.sql: -------------------------------------------------------------------------------- 1 | select 2 | cntrycode, 3 | count(*) as numcust, 4 | sum(c_acctbal) as totacctbal 5 | from 6 | ( 7 | select 8 | substring(c_phone from 1 for 2) as cntrycode, 9 | c_acctbal 10 | from 11 | customer 12 | where 13 | substring(c_phone from 1 for 2) in 14 | ('13', '31', '23', '29', '30', '18', '17') 15 | and c_acctbal > ( 16 | select 17 | avg(c_acctbal) 18 | from 19 | customer 20 | where 21 | c_acctbal > 0.00 22 | and substring(c_phone from 1 for 2) in 23 | ('13', '31', '23', '29', '30', '18', '17') 24 | ) 25 | and not exists ( 26 | select 27 | * 28 | from 29 | orders 30 | where 31 | o_custkey = c_custkey 32 | ) 33 | ) as custsale 34 | group by 35 | cntrycode 36 | order by 37 | cntrycode; -------------------------------------------------------------------------------- /dev/docker/ballista-arm64.Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | FROM arm64v8/ubuntu 18 | 19 | ADD ballista-scheduler / 20 | ADD ballista-executor / 21 | 22 | # Add benchmarks 23 | ADD tpch / 24 | RUN mkdir /queries 25 | ADD queries/*.sql /queries/ 26 | 27 | ENV RUST_LOG=info -------------------------------------------------------------------------------- /docs/source/user-guide/distributed/index.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | .. or more contributor license agreements. See the NOTICE file 3 | .. distributed with this work for additional information 4 | .. regarding copyright ownership. The ASF licenses this file 5 | .. to you under the Apache License, Version 2.0 (the 6 | .. "License"); you may not use this file except in compliance 7 | .. with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | .. software distributed under the License is distributed on an 13 | .. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | .. KIND, either express or implied. See the License for the 15 | .. specific language governing permissions and limitations 16 | .. under the License. 17 | 18 | Ballista Distributed Compute 19 | ============================ 20 | 21 | .. toctree:: 22 | :maxdepth: 2 23 | 24 | introduction 25 | deployment/index 26 | clients/index 27 | -------------------------------------------------------------------------------- /.hadolint.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ignored: 19 | - DL3008 20 | - DL3013 21 | - DL3018 22 | - DL3015 # Avoid additional packages by specifying `--no-install-recommends` 23 | - DL3028 # Ruby gem version pinning 24 | - DL3007 # r-sanitizer must use latest 25 | -------------------------------------------------------------------------------- /dev/integration-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | set -e 20 | ./dev/build-ballista-docker.sh 21 | pushd benchmarks 22 | ./tpch-gen.sh 23 | 24 | docker-compose up -d 25 | docker-compose run ballista-client /run.sh 26 | docker-compose down 27 | 28 | popd 29 | -------------------------------------------------------------------------------- /ci/scripts/python_sdist_build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -eux 21 | 22 | source_dir=${1}/python 23 | 24 | pushd ${source_dir} 25 | export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-} 26 | ${PYTHON:-python} setup.py sdist 27 | popd 28 | -------------------------------------------------------------------------------- /ci/scripts/csharp_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -ex 21 | 22 | source_dir=${1}/csharp 23 | 24 | pushd ${source_dir} 25 | dotnet test 26 | for pdb in artifacts/Apache.Arrow/*/*/Apache.Arrow.pdb; do 27 | sourcelink test ${pdb} 28 | done 29 | popd 30 | -------------------------------------------------------------------------------- /datafusion/src/datasource/listing/mod.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //! A table that uses the `ObjectStore` listing capability 19 | //! to get the list of files to process. 20 | 21 | mod helpers; 22 | mod table; 23 | 24 | pub use table::{ListingOptions, ListingTable}; 25 | -------------------------------------------------------------------------------- /docs/source/python/api.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | .. or more contributor license agreements. See the NOTICE file 3 | .. distributed with this work for additional information 4 | .. regarding copyright ownership. The ASF licenses this file 5 | .. to you under the Apache License, Version 2.0 (the 6 | .. "License"); you may not use this file except in compliance 7 | .. with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | .. software distributed under the License is distributed on an 13 | .. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | .. KIND, either express or implied. See the License for the 15 | .. specific language governing permissions and limitations 16 | .. under the License. 17 | 18 | .. _api: 19 | 20 | ************* 21 | API Reference 22 | ************* 23 | 24 | .. toctree:: 25 | :maxdepth: 2 26 | 27 | api/dataframe 28 | api/execution_context 29 | api/expression 30 | api/functions 31 | -------------------------------------------------------------------------------- /docs/source/python/api/execution_context.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | .. or more contributor license agreements. See the NOTICE file 3 | .. distributed with this work for additional information 4 | .. regarding copyright ownership. The ASF licenses this file 5 | .. to you under the Apache License, Version 2.0 (the 6 | .. "License"); you may not use this file except in compliance 7 | .. with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | .. software distributed under the License is distributed on an 13 | .. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | .. KIND, either express or implied. See the License for the 15 | .. specific language governing permissions and limitations 16 | .. under the License. 17 | 18 | .. _api.execution_context: 19 | .. currentmodule:: datafusion 20 | 21 | ExecutionContext 22 | ================ 23 | 24 | .. autosummary:: 25 | :toctree: ../generated/ 26 | 27 | ExecutionContext 28 | -------------------------------------------------------------------------------- /ci/docker/conda-python-dask.dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ARG repo 19 | ARG arch=amd64 20 | ARG python=3.6 21 | FROM ${repo}:${arch}-conda-python-${python} 22 | 23 | ARG dask=latest 24 | COPY ci/scripts/install_dask.sh /arrow/ci/scripts/ 25 | RUN /arrow/ci/scripts/install_dask.sh ${dask} -------------------------------------------------------------------------------- /datafusion/src/physical_plan/coercion_rule/mod.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //! Define coercion rules for different Expr type. 19 | //! 20 | //! Aggregate function rule 21 | //! Binary operation rule 22 | 23 | pub(crate) mod aggregate_rule; 24 | pub(crate) mod binary_rule; 25 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ;;; Licensed to the Apache Software Foundation (ASF) under one 2 | ;;; or more contributor license agreements. See the NOTICE file 3 | ;;; distributed with this work for additional information 4 | ;;; regarding copyright ownership. The ASF licenses this file 5 | ;;; to you under the Apache License, Version 2.0 (the 6 | ;;; "License"); you may not use this file except in compliance 7 | ;;; with the License. You may obtain a copy of the License at 8 | ;;; 9 | ;;; http://www.apache.org/licenses/LICENSE-2.0 10 | ;;; 11 | ;;; Unless required by applicable law or agreed to in writing, 12 | ;;; software distributed under the License is distributed on an 13 | ;;; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | ;;; KIND, either express or implied. See the License for the 15 | ;;; specific language governing permissions and limitations 16 | ;;; under the License. 17 | 18 | ((sh-mode . ((indent-tabs-mode . nil) 19 | (sh-indentation . 2) 20 | (sh-basic-offset . 2))) 21 | (cmake-mode . ((indent-tabs-mode . nil))) 22 | (powershell-mode . ((indent-tabs-mode . nil)))) 23 | -------------------------------------------------------------------------------- /ci/vcpkg/x64-osx-static-debug.cmake: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | set(VCPKG_TARGET_ARCHITECTURE x64) 19 | set(VCPKG_CRT_LINKAGE dynamic) 20 | set(VCPKG_LIBRARY_LINKAGE static) 21 | 22 | set(VCPKG_CMAKE_SYSTEM_NAME Darwin) 23 | set(VCPKG_OSX_ARCHITECTURES x86_64) 24 | 25 | set(VCPKG_BUILD_TYPE debug) 26 | -------------------------------------------------------------------------------- /ci/vcpkg/x64-osx-static-release.cmake: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | set(VCPKG_TARGET_ARCHITECTURE x64) 19 | set(VCPKG_CRT_LINKAGE dynamic) 20 | set(VCPKG_LIBRARY_LINKAGE static) 21 | 22 | set(VCPKG_CMAKE_SYSTEM_NAME Darwin) 23 | set(VCPKG_OSX_ARCHITECTURES x86_64) 24 | 25 | set(VCPKG_BUILD_TYPE release) 26 | -------------------------------------------------------------------------------- /dev/build-ballista-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -e 21 | 22 | . ./dev/build-set-env.sh 23 | docker build -t ballista-base:$BALLISTA_VERSION -f dev/docker/ballista-base.dockerfile . 24 | docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile . 25 | -------------------------------------------------------------------------------- /datafusion/src/sql/mod.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //! This module provides a SQL parser that translates SQL queries into an abstract syntax 19 | //! tree (AST), and a SQL query planner that creates a logical plan from the AST. 20 | 21 | pub mod parser; 22 | pub mod planner; 23 | pub(crate) mod utils; 24 | -------------------------------------------------------------------------------- /docs/source/user-guide/distributed/deployment/index.rst: -------------------------------------------------------------------------------- 1 | .. Licensed to the Apache Software Foundation (ASF) under one 2 | .. or more contributor license agreements. See the NOTICE file 3 | .. distributed with this work for additional information 4 | .. regarding copyright ownership. The ASF licenses this file 5 | .. to you under the Apache License, Version 2.0 (the 6 | .. "License"); you may not use this file except in compliance 7 | .. with the License. You may obtain a copy of the License at 8 | 9 | .. http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | .. Unless required by applicable law or agreed to in writing, 12 | .. software distributed under the License is distributed on an 13 | .. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | .. KIND, either express or implied. See the License for the 15 | .. specific language governing permissions and limitations 16 | .. under the License. 17 | 18 | Start a Ballista Cluster 19 | ======================== 20 | 21 | .. toctree:: 22 | :maxdepth: 2 23 | 24 | cargo-install 25 | docker 26 | docker-compose 27 | kubernetes 28 | raspberrypi 29 | configuration 30 | -------------------------------------------------------------------------------- /datafusion/src/physical_plan/expressions/stats.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | /// Enum used for differenciating population and sample for statistical functions 19 | #[derive(Debug, Clone, Copy)] 20 | pub enum StatsType { 21 | /// Population 22 | Population, 23 | /// Sample 24 | Sample, 25 | } 26 | -------------------------------------------------------------------------------- /ci/docker/debian-10-go.dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ARG arch=amd64 19 | ARG go=1.15 20 | FROM ${arch}/golang:${go} 21 | 22 | # TODO(kszucs): 23 | # 1. add the files required to install the dependencies to .dockerignore 24 | # 2. copy these files to their appropriate path 25 | # 3. download and compile the dependencies 26 | -------------------------------------------------------------------------------- /ci/docker/linux-apt-ruby.dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # depends on a C GLib image 19 | ARG base 20 | FROM ${base} 21 | 22 | COPY ruby/ /arrow/ruby/ 23 | RUN bundle install --gemfile /arrow/ruby/Gemfile 24 | RUN \ 25 | for package in /arrow/ruby/*; do \ 26 | bundle install --gemfile ${package}/Gemfile; \ 27 | done 28 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_window_full_aggregation.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT 18 | row_number() OVER () AS row_number, 19 | count(c3) OVER () AS count_c3, 20 | avg(c3) OVER () AS avg, 21 | sum(c3) OVER () AS sum, 22 | max(c3) OVER () AS max, 23 | min(c3) OVER () AS min 24 | FROM test 25 | ORDER BY row_number; 26 | -------------------------------------------------------------------------------- /ballista/ui/scheduler/src/setupTests.ts: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 19 | // allows you to do things like: 20 | // expect(element).toHaveTextContent(/react/i) 21 | // learn more: https://github.com/testing-library/jest-dom 22 | import "@testing-library/jest-dom"; 23 | -------------------------------------------------------------------------------- /ci/scripts/go_build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -ex 21 | 22 | source_dir=${1}/go 23 | 24 | pushd ${source_dir}/arrow 25 | 26 | go get -d -t -v ./... 27 | go install -v ./... 28 | 29 | popd 30 | 31 | pushd ${source_dir}/parquet 32 | 33 | go get -d -t -v ./... 34 | go install -v ./... 35 | 36 | popd 37 | -------------------------------------------------------------------------------- /ballista/rust/core/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Ballista Core Library 21 | 22 | This crate contains the Ballista core library which is used as a dependency by the `ballista`, 23 | `ballista-scheduler`, and `ballista-executor` crates. Refer to for 24 | general Ballista documentation. 25 | -------------------------------------------------------------------------------- /ci/conda_env_r.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | r-assertthat 19 | r-base 20 | r-bit64 21 | r-dplyr 22 | r-purrr 23 | r-r6 24 | r-cpp11 25 | r-rlang 26 | r-tidyselect 27 | r-vctrs 28 | # Test/"Suggests" dependencies 29 | pandoc 30 | r-covr 31 | r-hms 32 | r-lubridate 33 | r-rcmdcheck 34 | r-reticulate 35 | r-rmarkdown 36 | r-testthat 37 | r-tibble 38 | -------------------------------------------------------------------------------- /ci/docker/conda-python-jpype.dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ARG repo 19 | ARG arch=amd64 20 | ARG python=3.6 21 | FROM ${repo}:${arch}-conda-python-${python} 22 | 23 | ARG jdk=11 24 | ARG maven=3.6 25 | RUN conda install -q \ 26 | maven=${maven} \ 27 | openjdk=${jdk} \ 28 | jpype1 && \ 29 | conda clean --all 30 | -------------------------------------------------------------------------------- /ci/scripts/ccache_setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -eux 21 | 22 | echo "ARROW_USE_CCACHE=ON" >> $GITHUB_ENV 23 | echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV 24 | echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV 25 | echo "CCACHE_COMPRESSLEVEL=6" >> $GITHUB_ENV 26 | echo "CCACHE_MAXSIZE=500M" >> $GITHUB_ENV 27 | -------------------------------------------------------------------------------- /ci/scripts/r_build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | set -ex 20 | 21 | : ${R_BIN:=R} 22 | source_dir=${1}/r 23 | with_docs=${2:-false} 24 | 25 | pushd ${source_dir} 26 | 27 | ${R_BIN} CMD INSTALL . 28 | 29 | if [ "${with_docs}" == "true" ]; then 30 | ${R_BIN} -e "pkgdown::build_site(install = FALSE)" 31 | fi 32 | 33 | popd -------------------------------------------------------------------------------- /benchmarks/queries/q8.sql: -------------------------------------------------------------------------------- 1 | select 2 | o_year, 3 | sum(case 4 | when nation = 'BRAZIL' then volume 5 | else 0 6 | end) / sum(volume) as mkt_share 7 | from 8 | ( 9 | select 10 | extract(year from o_orderdate) as o_year, 11 | l_extendedprice * (1 - l_discount) as volume, 12 | n2.n_name as nation 13 | from 14 | part, 15 | supplier, 16 | lineitem, 17 | orders, 18 | customer, 19 | nation n1, 20 | nation n2, 21 | region 22 | where 23 | p_partkey = l_partkey 24 | and s_suppkey = l_suppkey 25 | and l_orderkey = o_orderkey 26 | and o_custkey = c_custkey 27 | and c_nationkey = n1.n_nationkey 28 | and n1.n_regionkey = r_regionkey 29 | and r_name = 'AMERICA' 30 | and s_nationkey = n2.n_nationkey 31 | and o_orderdate between date '1995-01-01' and date '1996-12-31' 32 | and p_type = 'ECONOMY ANODIZED STEEL' 33 | ) as all_nations 34 | group by 35 | o_year 36 | order by 37 | o_year; -------------------------------------------------------------------------------- /ci/scripts/install_spark.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -e 21 | 22 | if [ "$#" -ne 2 ]; then 23 | echo "Usage: $0 " 24 | exit 1 25 | fi 26 | 27 | spark=$1 28 | target=$2 29 | 30 | git clone https://github.com/apache/spark "${target}" 31 | git -C "${target}" checkout "${spark}" 32 | -------------------------------------------------------------------------------- /datafusion-cli/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #![doc = include_str!("../README.md")] 19 | pub const DATAFUSION_CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); 20 | 21 | pub mod command; 22 | pub mod context; 23 | pub mod exec; 24 | pub mod functions; 25 | pub mod helper; 26 | pub mod print_format; 27 | pub mod print_options; 28 | -------------------------------------------------------------------------------- /ci/docker/debian-10-js.dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ARG arch=amd64 19 | ARG node=14 20 | FROM ${arch}/node:${node} 21 | 22 | ENV NODE_NO_WARNINGS=1 23 | 24 | # TODO(kszucs): 25 | # 1. add the files required to install the dependencies to .dockerignore 26 | # 2. copy these files to their appropriate path 27 | # 3. download and compile the dependencies 28 | -------------------------------------------------------------------------------- /ballista/rust/executor/build.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | extern crate configure_me_codegen; 19 | 20 | fn main() -> Result<(), String> { 21 | println!("cargo:rerun-if-changed=executor_config_spec.toml"); 22 | configure_me_codegen::build_script_auto() 23 | .map_err(|e| format!("configure_me code generation failed: {}", e)) 24 | } 25 | -------------------------------------------------------------------------------- /ballista/rust/scheduler/testdata/orders/orders.tbl: -------------------------------------------------------------------------------- 1 | 1|36901|O|173665.47|1996-01-02|5-LOW|Clerk#000000951|0|nstructions sleep furiously among | 2 | 2|78002|O|46929.18|1996-12-01|1-URGENT|Clerk#000000880|0| foxes. pending accounts at the pending, silent asymptot| 3 | 3|123314|F|193846.25|1993-10-14|5-LOW|Clerk#000000955|0|sly final accounts boost. carefully regular ideas cajole carefully. depos| 4 | 4|136777|O|32151.78|1995-10-11|5-LOW|Clerk#000000124|0|sits. slyly regular warthogs cajole. regular, regular theodolites acro| 5 | 5|44485|F|144659.20|1994-07-30|5-LOW|Clerk#000000925|0|quickly. bold deposits sleep slyly. packages use slyly| 6 | 6|55624|F|58749.59|1992-02-21|4-NOT SPECIFIED|Clerk#000000058|0|ggle. special, final requests are against the furiously specia| 7 | 7|39136|O|252004.18|1996-01-10|2-HIGH|Clerk#000000470|0|ly special requests | 8 | 32|130057|O|208660.75|1995-07-16|2-HIGH|Clerk#000000616|0|ise blithely bold, regular requests. quickly unusual dep| 9 | 33|66958|F|163243.98|1993-10-27|3-MEDIUM|Clerk#000000409|0|uriously. furiously final request| 10 | 34|61001|O|58949.67|1998-07-21|3-MEDIUM|Clerk#000000223|0|ly final packages. fluffily final deposits wake blithely ideas. spe| 11 | -------------------------------------------------------------------------------- /ci/conda_env_archery.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # cli 19 | click 20 | 21 | # bot, crossbow 22 | github3.py 23 | jinja2 24 | jira 25 | pygit2 26 | pygithub 27 | ruamel.yaml 28 | setuptools_scm 29 | toolz 30 | 31 | # benchmark 32 | pandas 33 | 34 | # docker 35 | python-dotenv 36 | #ruamel.yaml 37 | 38 | # release 39 | gitpython 40 | #jinja2 41 | #jira 42 | semver 43 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Which issue does this PR close? 2 | 3 | 6 | 7 | Closes #. 8 | 9 | # Rationale for this change 10 | 14 | 15 | # What changes are included in this PR? 16 | 19 | 20 | # Are there any user-facing changes? 21 | 24 | 25 | 28 | -------------------------------------------------------------------------------- /benchmarks/queries/q7.sql: -------------------------------------------------------------------------------- 1 | select 2 | supp_nation, 3 | cust_nation, 4 | l_year, 5 | sum(volume) as revenue 6 | from 7 | ( 8 | select 9 | n1.n_name as supp_nation, 10 | n2.n_name as cust_nation, 11 | extract(year from l_shipdate) as l_year, 12 | l_extendedprice * (1 - l_discount) as volume 13 | from 14 | supplier, 15 | lineitem, 16 | orders, 17 | customer, 18 | nation n1, 19 | nation n2 20 | where 21 | s_suppkey = l_suppkey 22 | and o_orderkey = l_orderkey 23 | and c_custkey = o_custkey 24 | and s_nationkey = n1.n_nationkey 25 | and c_nationkey = n2.n_nationkey 26 | and ( 27 | (n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY') 28 | or (n1.n_name = 'GERMANY' and n2.n_name = 'FRANCE') 29 | ) 30 | and l_shipdate between date '1995-01-01' and date '1996-12-31' 31 | ) as shipping 32 | group by 33 | supp_nation, 34 | cust_nation, 35 | l_year 36 | order by 37 | supp_nation, 38 | cust_nation, 39 | l_year; 40 | -------------------------------------------------------------------------------- /ci/docker/conda-python-pandas.dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ARG repo 19 | ARG arch=amd64 20 | ARG python=3.6 21 | FROM ${repo}:${arch}-conda-python-${python} 22 | 23 | ARG pandas=latest 24 | ARG numpy=latest 25 | COPY ci/scripts/install_pandas.sh /arrow/ci/scripts/ 26 | RUN conda uninstall -q -y numpy && \ 27 | /arrow/ci/scripts/install_pandas.sh ${pandas} ${numpy} 28 | -------------------------------------------------------------------------------- /ci/scripts/integration_hiveserver2.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | set -e 20 | 21 | arrow_dir=${1} 22 | source_dir=${1}/cpp 23 | build_dir=${2}/cpp 24 | 25 | ${arrow_dir}/ci/scripts/util_wait_for_it.sh impala:21050 -t 300 -s -- echo "impala is up" 26 | 27 | pushd ${build_dir} 28 | 29 | # ninja hiveserver2-test 30 | debug/hiveserver2-test 31 | 32 | popd 33 | -------------------------------------------------------------------------------- /datafusion/fuzz-utils/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | [package] 19 | name = "fuzz-utils" 20 | version = "0.1.0" 21 | edition = "2021" 22 | 23 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 24 | 25 | [dependencies] 26 | arrow = { version = "8.0.0", features = ["prettyprint"] } 27 | rand = "0.8" 28 | env_logger = "0.9.0" 29 | -------------------------------------------------------------------------------- /ballista/rust/executor/examples/example_executor_config.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # the default configuration location is "/etc/ballista/scheduler.toml" 19 | # if you include a specifc conf file using "--config-file = my_config_file.toml" 20 | # then that file will override environment variables, but not command line arguments 21 | namespace = "my_name_space" 22 | bind_host = "1.2.3.4" -------------------------------------------------------------------------------- /ci/scripts/r_deps.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | set -ex 20 | 21 | : ${R_BIN:=R} 22 | 23 | source_dir=${1}/r 24 | 25 | pushd ${source_dir} 26 | 27 | # Install R package dependencies 28 | ${R_BIN} -e "install.packages('remotes'); remotes::install_cran(c('glue', 'rcmdcheck', 'sys'))" 29 | ${R_BIN} -e "remotes::install_deps(dependencies = TRUE)" 30 | 31 | popd 32 | -------------------------------------------------------------------------------- /ballista/rust/client/src/prelude.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //! Ballista Prelude (common imports) 19 | 20 | pub use crate::context::BallistaContext; 21 | pub use ballista_core::config::BallistaConfig; 22 | pub use ballista_core::config::BALLISTA_DEFAULT_SHUFFLE_PARTITIONS; 23 | pub use ballista_core::error::{BallistaError, Result}; 24 | 25 | pub use futures::StreamExt; 26 | -------------------------------------------------------------------------------- /benchmarks/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | set -e 19 | 20 | # This bash script is meant to be run inside the docker-compose environment. Check the README for instructions 21 | 22 | cd / 23 | for query in 1 3 5 6 7 10 12 13 24 | do 25 | /tpch benchmark ballista --host ballista-scheduler --port 50050 --query $query --path /data --format tbl --iterations 1 --debug 26 | done 27 | -------------------------------------------------------------------------------- /ci/scripts/js_build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -ex 21 | 22 | source_dir=${1}/js 23 | with_docs=${2:-false} 24 | 25 | pushd ${source_dir} 26 | 27 | yarn --frozen-lockfile 28 | # TODO(kszucs): linting should be moved to archery 29 | yarn lint:ci 30 | yarn build 31 | 32 | if [ "${with_docs}" == "true" ]; then 33 | yarn doc 34 | fi 35 | 36 | popd 37 | -------------------------------------------------------------------------------- /ballista/rust/scheduler/testdata/part/part.tbl: -------------------------------------------------------------------------------- 1 | 1|goldenrod lavender spring chocolate lace|Manufacturer#1|Brand#13|PROMO BURNISHED COPPER|7|JUMBO PKG|901.00|ly. slyly ironi| 2 | 2|blush thistle blue yellow saddle|Manufacturer#1|Brand#13|LARGE BRUSHED BRASS|1|LG CASE|902.00|lar accounts amo| 3 | 3|spring green yellow purple cornsilk|Manufacturer#4|Brand#42|STANDARD POLISHED BRASS|21|WRAP CASE|903.00|egular deposits hag| 4 | 4|cornflower chocolate smoke green pink|Manufacturer#3|Brand#34|SMALL PLATED BRASS|14|MED DRUM|904.00|p furiously r| 5 | 5|forest brown coral puff cream|Manufacturer#3|Brand#32|STANDARD POLISHED TIN|15|SM PKG|905.00| wake carefully | 6 | 6|bisque cornflower lawn forest magenta|Manufacturer#2|Brand#24|PROMO PLATED STEEL|4|MED BAG|906.00|sual a| 7 | 7|moccasin green thistle khaki floral|Manufacturer#1|Brand#11|SMALL PLATED COPPER|45|SM BAG|907.00|lyly. ex| 8 | 8|misty lace thistle snow royal|Manufacturer#4|Brand#44|PROMO BURNISHED TIN|41|LG DRUM|908.00|eposi| 9 | 9|thistle dim navajo dark gainsboro|Manufacturer#4|Brand#43|SMALL BURNISHED STEEL|12|WRAP CASE|909.00|ironic foxe| 10 | 10|linen pink saddle puff powder|Manufacturer#5|Brand#54|LARGE BURNISHED STEEL|44|LG CAN|910.01|ithely final deposit| 11 | -------------------------------------------------------------------------------- /ballista/ui/scheduler/src/App.test.tsx: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | import React from "react"; 19 | import { render, screen } from "@testing-library/react"; 20 | import App from "./App"; 21 | 22 | test("renders learn react link", () => { 23 | render(); 24 | const linkElement = screen.getByText(/learn react/i); 25 | expect(linkElement).toBeInTheDocument(); 26 | }); 27 | -------------------------------------------------------------------------------- /ci/scripts/go_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -ex 21 | 22 | source_dir=${1}/go 23 | 24 | pushd ${source_dir}/arrow 25 | 26 | for d in $(go list ./... | grep -v vendor); do 27 | go test $d 28 | done 29 | 30 | popd 31 | 32 | pushd ${source_dir}/parquet 33 | 34 | for d in $(go list ./... | grep -v vendor); do 35 | go test $d 36 | done 37 | 38 | popd 39 | -------------------------------------------------------------------------------- /ci/conda_env_cpp.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | aws-sdk-cpp 19 | benchmark=1.5.2 20 | boost-cpp>=1.68.0 21 | brotli 22 | bzip2 23 | c-ares 24 | cmake 25 | gflags 26 | glog 27 | gmock>=1.10.0 28 | grpc-cpp>=1.27.3 29 | gtest=1.10.0 30 | libprotobuf 31 | libutf8proc 32 | lz4-c 33 | make 34 | ninja 35 | pkg-config 36 | python 37 | rapidjson 38 | re2 39 | snappy 40 | thrift-cpp>=0.11.0 41 | zlib 42 | zstd 43 | -------------------------------------------------------------------------------- /ci/docker/debian-9-java.dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ARG arch=amd64 19 | ARG jdk=8 20 | ARG maven=3.5.4 21 | FROM ${arch}/maven:${maven}-jdk-${jdk} 22 | 23 | ENV ARROW_JAVA_SHADE_FLATBUFS=ON 24 | 25 | # TODO(kszucs): 26 | # 1. add the files required to install the dependencies to .dockerignore 27 | # 2. copy these files to their appropriate path 28 | # 3. download and compile the dependencies 29 | -------------------------------------------------------------------------------- /ballista/rust/.dockerignore: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Turn .dockerignore to .dockerallow by excluding everything and explicitly 19 | # allowing specific files and directories. This enables us to quickly add 20 | # dependency files to the docker content without scanning the whole directory. 21 | # This setup requires to all of our docker containers have arrow's source 22 | # as a mounted directory. 23 | target -------------------------------------------------------------------------------- /benchmarks/tpchgen.dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | FROM ubuntu 19 | 20 | RUN apt-get update && \ 21 | apt-get install -y git build-essential 22 | 23 | RUN git clone https://github.com/databricks/tpch-dbgen.git && \ 24 | cd tpch-dbgen && \ 25 | make 26 | 27 | WORKDIR /tpch-dbgen 28 | ADD entrypoint.sh /tpch-dbgen/ 29 | 30 | VOLUME /data 31 | 32 | ENTRYPOINT [ "bash", "./entrypoint.sh" ] 33 | -------------------------------------------------------------------------------- /ci/conda_env_python.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # don't add pandas here, because it is not a mandatory test dependency 19 | boto3 # not a direct dependency of s3fs, but needed for our s3fs fixture 20 | cffi 21 | cython 22 | cloudpickle 23 | fsspec 24 | hypothesis 25 | numpy>=1.16.6 26 | pytest 27 | pytest-faulthandler 28 | pytest-lazy-fixture 29 | pytz 30 | s3fs>=0.4 31 | setuptools 32 | setuptools_scm 33 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_window_lead_built_in_functions.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT 18 | c8, 19 | LEAD(c8) OVER () next_c8, 20 | LEAD(c8, 10, 10) OVER() next_10_c8, 21 | LEAD(c8, 100, 10) OVER() next_out_of_bounds_c8, 22 | LAG(c8) OVER() prev_c8, 23 | LAG(c8, -2, 0) OVER() AS prev_2_c8, 24 | LAG(c8, -200, 10) OVER() AS prev_out_of_bounds_c8 25 | 26 | FROM test 27 | ORDER BY c8; 28 | -------------------------------------------------------------------------------- /ci/scripts/ruby_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -ex 21 | 22 | source_dir=${1}/ruby 23 | build_dir=${2}/ruby 24 | 25 | export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} 26 | export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig 27 | export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0 28 | 29 | rake -f ${source_dir}/Rakefile BUILD_DIR=${build_dir} USE_BUNDLER=yes 30 | -------------------------------------------------------------------------------- /benchmarks/.dockerignore: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Turn .dockerignore to .dockerallow by excluding everything and explicitly 19 | # allowing specific files and directories. This enables us to quickly add 20 | # dependency files to the docker content without scanning the whole directory. 21 | # This setup requires to all of our docker containers have arrow's source 22 | # as a mounted directory. 23 | 24 | data 25 | target -------------------------------------------------------------------------------- /ci/docker/python-wheel-manylinux-test.dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ARG arch 19 | ARG python 20 | FROM ${arch}/python:${python} 21 | 22 | # RUN pip install --upgrade pip 23 | 24 | # pandas doesn't provide wheel for aarch64 yet, so cache the compiled 25 | # test dependencies in a docker image 26 | COPY python/requirements-wheel-test.txt /arrow/python/ 27 | RUN pip install -r /arrow/python/requirements-wheel-test.txt 28 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_window_ranked_built_in_functions.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | select 18 | c9, 19 | cume_dist() OVER (PARTITION BY c2 ORDER BY c3) cume_dist_by_c3, 20 | rank() OVER (PARTITION BY c2 ORDER BY c3) rank_by_c3, 21 | dense_rank() OVER (PARTITION BY c2 ORDER BY c3) dense_rank_by_c3, 22 | percent_rank() OVER (PARTITION BY c2 ORDER BY c3) percent_rank_by_c3 23 | FROM test 24 | ORDER BY c9; 25 | -------------------------------------------------------------------------------- /ballista/docs/integration-testing.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Integration Testing 21 | 22 | We use the [DataFusion Benchmarks](https://github.com/apache/arrow-datafusion/tree/master/benchmarks) for integration 23 | testing. 24 | 25 | The integration tests can be executed by running the following command from the root of the DataFusion repository. 26 | 27 | ```bash 28 | ./dev/integration-tests.sh 29 | ``` 30 | -------------------------------------------------------------------------------- /datafusion/src/execution/mod.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //! DataFusion query execution 19 | 20 | pub mod context; 21 | pub mod dataframe_impl; 22 | pub(crate) mod disk_manager; 23 | pub mod memory_manager; 24 | pub mod options; 25 | pub mod runtime_env; 26 | 27 | pub use disk_manager::DiskManager; 28 | pub use memory_manager::{ 29 | human_readable_size, MemoryConsumer, MemoryConsumerId, MemoryManager, 30 | }; 31 | -------------------------------------------------------------------------------- /ballista/rust/scheduler/testdata/lineitem/partition0.tbl: -------------------------------------------------------------------------------- 1 | 1|155190|7706|1|17|21168.23|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER IN PERSON|TRUCK|egular courts above the| 2 | 1|67310|7311|2|36|45983.16|0.09|0.06|N|O|1996-04-12|1996-02-28|1996-04-20|TAKE BACK RETURN|MAIL|ly final dependencies: slyly bold | 3 | 1|63700|3701|3|8|13309.60|0.10|0.02|N|O|1996-01-29|1996-03-05|1996-01-31|TAKE BACK RETURN|REG AIR|riously. regular, express dep| 4 | 1|2132|4633|4|28|28955.64|0.09|0.06|N|O|1996-04-21|1996-03-30|1996-05-16|NONE|AIR|lites. fluffily even de| 5 | 1|24027|1534|5|24|22824.48|0.10|0.04|N|O|1996-03-30|1996-03-14|1996-04-01|NONE|FOB| pending foxes. slyly re| 6 | 1|15635|638|6|32|49620.16|0.07|0.02|N|O|1996-01-30|1996-02-07|1996-02-03|DELIVER IN PERSON|MAIL|arefully slyly ex| 7 | 2|106170|1191|1|38|44694.46|0.00|0.05|N|O|1997-01-28|1997-01-14|1997-02-02|TAKE BACK RETURN|RAIL|ven requests. deposits breach a| 8 | 3|4297|1798|1|45|54058.05|0.06|0.00|R|F|1994-02-02|1994-01-04|1994-02-23|NONE|AIR|ongside of the furiously brave acco| 9 | 3|19036|6540|2|49|46796.47|0.10|0.00|R|F|1993-11-09|1993-12-20|1993-11-24|TAKE BACK RETURN|RAIL| unusual accounts. eve| 10 | 3|128449|3474|3|27|39890.88|0.06|0.07|A|F|1994-01-16|1993-11-22|1994-01-23|DELIVER IN PERSON|SHIP|nal foxes wake. | 11 | -------------------------------------------------------------------------------- /ballista/rust/scheduler/testdata/lineitem/partition1.tbl: -------------------------------------------------------------------------------- 1 | 1|155190|7706|1|17|21168.23|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER IN PERSON|TRUCK|egular courts above the| 2 | 1|67310|7311|2|36|45983.16|0.09|0.06|N|O|1996-04-12|1996-02-28|1996-04-20|TAKE BACK RETURN|MAIL|ly final dependencies: slyly bold | 3 | 1|63700|3701|3|8|13309.60|0.10|0.02|N|O|1996-01-29|1996-03-05|1996-01-31|TAKE BACK RETURN|REG AIR|riously. regular, express dep| 4 | 1|2132|4633|4|28|28955.64|0.09|0.06|N|O|1996-04-21|1996-03-30|1996-05-16|NONE|AIR|lites. fluffily even de| 5 | 1|24027|1534|5|24|22824.48|0.10|0.04|N|O|1996-03-30|1996-03-14|1996-04-01|NONE|FOB| pending foxes. slyly re| 6 | 1|15635|638|6|32|49620.16|0.07|0.02|N|O|1996-01-30|1996-02-07|1996-02-03|DELIVER IN PERSON|MAIL|arefully slyly ex| 7 | 2|106170|1191|1|38|44694.46|0.00|0.05|N|O|1997-01-28|1997-01-14|1997-02-02|TAKE BACK RETURN|RAIL|ven requests. deposits breach a| 8 | 3|4297|1798|1|45|54058.05|0.06|0.00|R|F|1994-02-02|1994-01-04|1994-02-23|NONE|AIR|ongside of the furiously brave acco| 9 | 3|19036|6540|2|49|46796.47|0.10|0.00|R|F|1993-11-09|1993-12-20|1993-11-24|TAKE BACK RETURN|RAIL| unusual accounts. eve| 10 | 3|128449|3474|3|27|39890.88|0.06|0.07|A|F|1994-01-16|1993-11-22|1994-01-23|DELIVER IN PERSON|SHIP|nal foxes wake. | 11 | -------------------------------------------------------------------------------- /datafusion/src/physical_optimizer/mod.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //! This module contains a query optimizer that operates against a physical plan and applies 19 | //! rules to a physical plan, such as "Repartition". 20 | 21 | pub mod aggregate_statistics; 22 | pub mod coalesce_batches; 23 | pub mod hash_build_probe_order; 24 | pub mod merge_exec; 25 | pub mod optimizer; 26 | pub mod pruning; 27 | pub mod repartition; 28 | mod utils; 29 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_window_ordered_aggregation.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT 18 | c9, 19 | row_number() OVER (ORDER BY c2, c9) AS row_number, 20 | count(c3) OVER (ORDER BY c9) AS count_c3, 21 | avg(c3) OVER (ORDER BY c2) AS avg_c3_by_c2, 22 | sum(c3) OVER (ORDER BY c2) AS sum_c3_by_c2, 23 | max(c3) OVER (ORDER BY c2) AS max_c3_by_c2, 24 | min(c3) OVER (ORDER BY c2) AS min_c3_by_c2 25 | FROM test 26 | ORDER BY row_number; 27 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | [workspace] 19 | members = [ 20 | "datafusion", 21 | "datafusion-common", 22 | "datafusion-cli", 23 | "datafusion-examples", 24 | "benchmarks", 25 | "ballista/rust/client", 26 | "ballista/rust/core", 27 | "ballista/rust/executor", 28 | "ballista/rust/scheduler", 29 | "ballista-examples", 30 | "tokomak" 31 | ] 32 | 33 | [profile.release] 34 | lto = true 35 | codegen-units = 1 36 | -------------------------------------------------------------------------------- /ballista/ui/scheduler/src/index.css: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | */ 19 | 20 | @import url("https://fonts.googleapis.com/css2?family=Poppins:wght@100;400;600&display=swap"); 21 | 22 | body { 23 | margin: 0; 24 | font-family: "Poppins", sans-serif; 25 | -webkit-font-smoothing: antialiased; 26 | -moz-osx-font-smoothing: grayscale; 27 | } 28 | 29 | code { 30 | font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New", 31 | monospace; 32 | } 33 | -------------------------------------------------------------------------------- /ballista/rust/core/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #![doc = include_str!("../README.md")] 19 | #![allow(unused_imports)] 20 | pub const BALLISTA_VERSION: &str = env!("CARGO_PKG_VERSION"); 21 | 22 | pub fn print_version() { 23 | println!("Ballista version: {}", BALLISTA_VERSION) 24 | } 25 | 26 | pub mod client; 27 | pub mod config; 28 | pub mod error; 29 | pub mod execution_plans; 30 | pub mod utils; 31 | 32 | #[macro_use] 33 | pub mod serde; 34 | -------------------------------------------------------------------------------- /dev/release/update_change_log-all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | set -e 22 | 23 | # Usage: 24 | # CHANGELOG_GITHUB_TOKEN= ./update_change_log-datafusion.sh 25 | 26 | SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 27 | SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" 28 | 29 | ${SOURCE_DIR}/update_change_log-datafusion.sh 30 | ${SOURCE_DIR}/update_change_log-ballista.sh 31 | ${SOURCE_DIR}/update_change_log-python.sh 32 | -------------------------------------------------------------------------------- /integration-tests/sqls/simple_window_partition_aggregation.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT 18 | c9, 19 | row_number() OVER (PARTITION BY c2, c9) AS row_number, 20 | count(c3) OVER (PARTITION BY c2) AS count_c3, 21 | avg(c3) OVER (PARTITION BY c2) AS avg_c3_by_c2, 22 | sum(c3) OVER (PARTITION BY c2) AS sum_c3_by_c2, 23 | max(c3) OVER (PARTITION BY c2) AS max_c3_by_c2, 24 | min(c3) OVER (PARTITION BY c2) AS min_c3_by_c2 25 | FROM test 26 | ORDER BY c9; 27 | -------------------------------------------------------------------------------- /ballista/rust/core/build.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | fn main() -> Result<(), String> { 19 | // for use in docker build where file changes can be wonky 20 | println!("cargo:rerun-if-env-changed=FORCE_REBUILD"); 21 | 22 | println!("cargo:rerun-if-changed=proto/ballista.proto"); 23 | tonic_build::configure() 24 | .compile(&["proto/ballista.proto"], &["proto"]) 25 | .map_err(|e| format!("protobuf compilation failed: {}", e)) 26 | } 27 | -------------------------------------------------------------------------------- /ballista/rust/scheduler/testdata/supplier/supplier.tbl: -------------------------------------------------------------------------------- 1 | 1|Supplier#000000001| N kD4on9OM Ipw3,gf0JBoQDd7tgrzrddZ|17|27-918-335-1736|5755.94|each slyly above the careful| 2 | 2|Supplier#000000002|89eJ5ksX3ImxJQBvxObC,|5|15-679-861-2259|4032.68| slyly bold instructions. idle dependen| 3 | 3|Supplier#000000003|q1,G3Pj6OjIuUYfUoH18BFTKP5aU9bEV3|1|11-383-516-1199|4192.40|blithely silent requests after the express dependencies are sl| 4 | 4|Supplier#000000004|Bk7ah4CK8SYQTepEmvMkkgMwg|15|25-843-787-7479|4641.08|riously even requests above the exp| 5 | 5|Supplier#000000005|Gcdm2rJRzl5qlTVzc|11|21-151-690-3663|-283.84|. slyly regular pinto bea| 6 | 6|Supplier#000000006|tQxuVm7s7CnK|14|24-696-997-4969|1365.79|final accounts. regular dolphins use against the furiously ironic decoys. | 7 | 7|Supplier#000000007|s,4TicNGB4uO6PaSqNBUq|23|33-990-965-2201|6820.35|s unwind silently furiously regular courts. final requests are deposits. requests wake quietly blit| 8 | 8|Supplier#000000008|9Sq4bBH2FQEmaFOocY45sRTxo6yuoG|17|27-498-742-3860|7627.85|al pinto beans. asymptotes haggl| 9 | 9|Supplier#000000009|1KhUgZegwM3ua7dsYmekYBsK|10|20-403-398-8662|5302.37|s. unusual, even requests along the furiously regular pac| 10 | 10|Supplier#000000010|Saygah3gYWMp72i PY|24|34-852-489-8585|3891.91|ing waters. regular requests ar| 11 | -------------------------------------------------------------------------------- /ci/vcpkg/arm64-linux-static-debug.cmake: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | set(VCPKG_TARGET_ARCHITECTURE arm64) 19 | set(VCPKG_CRT_LINKAGE dynamic) 20 | set(VCPKG_LIBRARY_LINKAGE static) 21 | set(VCPKG_CMAKE_SYSTEM_NAME Linux) 22 | set(VCPKG_BUILD_TYPE debug) 23 | 24 | if(NOT CMAKE_HOST_SYSTEM_PROCESSOR) 25 | execute_process(COMMAND "uname" "-m" 26 | OUTPUT_VARIABLE CMAKE_HOST_SYSTEM_PROCESSOR 27 | OUTPUT_STRIP_TRAILING_WHITESPACE) 28 | endif() 29 | -------------------------------------------------------------------------------- /ci/vcpkg/arm64-linux-static-release.cmake: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | set(VCPKG_TARGET_ARCHITECTURE arm64) 19 | set(VCPKG_CRT_LINKAGE dynamic) 20 | set(VCPKG_LIBRARY_LINKAGE static) 21 | set(VCPKG_CMAKE_SYSTEM_NAME Linux) 22 | set(VCPKG_BUILD_TYPE release) 23 | 24 | if(NOT CMAKE_HOST_SYSTEM_PROCESSOR) 25 | execute_process(COMMAND "uname" "-m" 26 | OUTPUT_VARIABLE CMAKE_HOST_SYSTEM_PROCESSOR 27 | OUTPUT_STRIP_TRAILING_WHITESPACE) 28 | endif() 29 | -------------------------------------------------------------------------------- /ci/scripts/install_dask.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -e 21 | 22 | if [ "$#" -ne 1 ]; then 23 | echo "Usage: $0 " 24 | exit 1 25 | fi 26 | 27 | dask=$1 28 | 29 | if [ "${dask}" = "master" ]; then 30 | pip install https://github.com/dask/dask/archive/main.tar.gz#egg=dask[dataframe] 31 | elif [ "${dask}" = "latest" ]; then 32 | conda install -q dask 33 | else 34 | conda install -q dask=${dask} 35 | fi 36 | conda clean --all 37 | -------------------------------------------------------------------------------- /ci/scripts/msys2_system_clean.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -eux 21 | 22 | pacman \ 23 | --cascade \ 24 | --noconfirm \ 25 | --nosave \ 26 | --recursive \ 27 | --remove \ 28 | ${MINGW_PACKAGE_PREFIX}-clang-tools-extra \ 29 | ${MINGW_PACKAGE_PREFIX}-gcc-ada \ 30 | ${MINGW_PACKAGE_PREFIX}-gcc-fortran \ 31 | ${MINGW_PACKAGE_PREFIX}-gcc-libgfortran \ 32 | ${MINGW_PACKAGE_PREFIX}-gcc-objc \ 33 | ${MINGW_PACKAGE_PREFIX}-libgccjit 34 | -------------------------------------------------------------------------------- /datafusion/tests/tpch-csv/orders.csv: -------------------------------------------------------------------------------- 1 | o_orderkey,o_custkey,o_orderstatus,o_totalprice,o_orderdate,o_orderpriority,o_clerk,o_shippriority,o_comment 2 | 2,78002,O,46929.18,1996-12-01,1-URGENT,Clerk#000000880,0," foxes. pending accounts at the pending, silent asymptot" 3 | 3,123314,F,193846.25,1993-10-14,5-LOW,Clerk#000000955,0,sly final accounts boost. carefully regular ideas cajole carefully. depos 4 | 4,136777,O,32151.78,1995-10-11,5-LOW,Clerk#000000124,0,"sits. slyly regular warthogs cajole. regular, regular theodolites acro" 5 | 5,44485,F,144659.2,1994-07-30,5-LOW,Clerk#000000925,0,quickly. bold deposits sleep slyly. packages use slyly 6 | 6,55624,F,58749.59,1992-02-21,4-NOT SPECIFIED,Clerk#000000058,0,"ggle. special, final requests are against the furiously specia" 7 | 7,39136,O,252004.18,1996-01-10,2-HIGH,Clerk#000000470,0,ly special requests 8 | 32,130057,O,208660.75,1995-07-16,2-HIGH,Clerk#000000616,0,"ise blithely bold, regular requests. quickly unusual dep" 9 | 33,66958,F,163243.98,1993-10-27,3-MEDIUM,Clerk#000000409,0,uriously. furiously final request 10 | 34,61001,O,58949.67,1998-07-21,3-MEDIUM,Clerk#000000223,0,ly final packages. fluffily final deposits wake blithely ideas. spe 11 | 35,127588,O,253724.56,1995-10-23,4-NOT SPECIFIED,Clerk#000000259,0,zzle. carefully enticing deposits nag furio 12 | -------------------------------------------------------------------------------- /ci/scripts/python_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -ex 21 | 22 | arrow_dir=${1} 23 | 24 | export ARROW_SOURCE_DIR=${arrow_dir} 25 | export ARROW_TEST_DATA=${arrow_dir}/testing/data 26 | export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data 27 | export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} 28 | 29 | # Enable some checks inside Python itself 30 | export PYTHONDEVMODE=1 31 | 32 | pytest -r s ${PYTEST_ARGS} --pyargs pyarrow 33 | -------------------------------------------------------------------------------- /datafusion/tests/tpch-csv/lineitem.csv: -------------------------------------------------------------------------------- 1 | l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment 2 | 1,67310,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold 3 | 1,63700,3701,3,8.0,13309.6,0.1,0.02,N,O,1996-01-29,1996-03-05,1996-01-31,TAKE BACK RETURN,REG AIR,"riously. regular, express dep" 4 | 1,2132,4633,4,28.0,28955.64,0.09,0.06,N,O,1996-04-21,1996-03-30,1996-05-16,NONE,AIR,lites. fluffily even de 5 | 1,24027,1534,5,24.0,22824.48,0.1,0.04,N,O,1996-03-30,1996-03-14,1996-04-01,NONE,FOB, pending foxes. slyly re 6 | 1,15635,638,6,32.0,49620.16,0.07,0.02,N,O,1996-01-30,1996-02-07,1996-02-03,DELIVER IN PERSON,MAIL,arefully slyly ex 7 | 2,106170,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a 8 | 3,4297,1798,1,45.0,54058.05,0.06,0.0,R,F,1994-02-02,1994-01-04,1994-02-23,NONE,AIR,ongside of the furiously brave acco 9 | 3,19036,6540,2,49.0,46796.47,0.1,0.0,R,F,1993-11-09,1993-12-20,1993-11-24,TAKE BACK RETURN,RAIL, unusual accounts. eve 10 | 3,128449,3474,3,27.0,39890.88,0.06,0.07,A,F,1994-01-16,1993-11-22,1994-01-23,DELIVER IN PERSON,SHIP,nal foxes wake. 11 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | Changelogs are maintained separately for each subproject. Please check out the 21 | changelog file within each subproject folder for more details: 22 | 23 | * [Datafusion CHANGELOG](./datafusion/CHANGELOG.md) 24 | * [Datafusion Python Binding CHANGELOG](./python/CHANGELOG.md) 25 | * [Ballista CHANGELOG](./ballista/CHANGELOG.md) 26 | 27 | For older versions, see [apache/arrow/CHANGELOG.md](https://github.com/apache/arrow/blob/master/CHANGELOG.md). 28 | -------------------------------------------------------------------------------- /benchmarks/tpch-gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | #set -e 20 | 21 | pushd .. 22 | . ./dev/build-set-env.sh 23 | popd 24 | docker build -t ballista-tpchgen:$BALLISTA_VERSION -f tpchgen.dockerfile . 25 | 26 | # Generate data into the ./data directory if it does not already exist 27 | FILE=./data/supplier.tbl 28 | if test -f "$FILE"; then 29 | echo "$FILE exists." 30 | else 31 | mkdir data 2>/dev/null 32 | docker run -v `pwd`/data:/data -it --rm ballista-tpchgen:$BALLISTA_VERSION 33 | ls -l data 34 | fi -------------------------------------------------------------------------------- /ci/scripts/r_sanitize.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | set -ex 20 | 21 | : ${R_BIN:=RDsan} 22 | 23 | source_dir=${1}/r 24 | 25 | ${R_BIN} CMD INSTALL ${source_dir} 26 | pushd ${source_dir}/tests 27 | 28 | export TEST_R_WITH_ARROW=TRUE 29 | export UBSAN_OPTIONS="print_stacktrace=1,suppressions=/arrow/r/tools/ubsan.supp" 30 | ${R_BIN} < testthat.R > testthat.out 2>&1 || { cat testthat.out; exit 1; } 31 | 32 | cat testthat.out 33 | if grep -q "runtime error" testthat.out; then 34 | exit 1 35 | fi 36 | popd 37 | -------------------------------------------------------------------------------- /docs/source/specification/rfcs/template.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | Feature Name: 21 | 22 | Status: draft/in-progress/completed/ 23 | 24 | Start Date: YYYY-MM-DD 25 | 26 | Authors: 27 | 28 | RFC PR: # 29 | 30 | Datafusion Issue: # 31 | 32 | --- 33 | 34 | ### Background 35 | 36 | --- 37 | 38 | ### Goals 39 | 40 | --- 41 | 42 | ### Non-Goals 43 | 44 | --- 45 | 46 | ### Survey 47 | 48 | --- 49 | 50 | ### General design 51 | 52 | --- 53 | 54 | ### Detailed design 55 | 56 | --- 57 | 58 | ### Others 59 | -------------------------------------------------------------------------------- /datafusion-examples/examples/README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # DataFusion Examples 21 | 22 | ## Single Process 23 | 24 | The examples `csv_sql.rs` and `parquet_sql.rs` demonstrate building a query plan from a SQL statement and then executing the query plan against local CSV and Parquet files, respectively. 25 | 26 | ## Distributed 27 | 28 | The `flight-client.rs` and `flight-server.rs` examples demonstrate how to run DataFusion as a standalone process and execute SQL queries from a client using the Flight protocol. 29 | -------------------------------------------------------------------------------- /datafusion/benches/scalar.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | use criterion::{criterion_group, criterion_main, Criterion}; 19 | use datafusion::scalar::ScalarValue; 20 | 21 | fn criterion_benchmark(c: &mut Criterion) { 22 | c.bench_function("to_array_of_size 100000", |b| { 23 | let scalar = ScalarValue::Int32(Some(100)); 24 | 25 | b.iter(|| assert_eq!(scalar.to_array_of_size(100000).null_count(), 0)) 26 | }); 27 | } 28 | 29 | criterion_group!(benches, criterion_benchmark); 30 | criterion_main!(benches); 31 | -------------------------------------------------------------------------------- /tokomak/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | [package] 19 | name = "tokomak" 20 | version = "0.0.1" 21 | authors = ["Apache Arrow "] 22 | edition = "2021" 23 | license = "Apache-2.0" 24 | homepage = "https://github.com/apache/arrow-datafusion" 25 | repository = "https://github.com/apache/arrow-datafusion" 26 | 27 | 28 | [dependencies] 29 | datafusion = { path = "../datafusion", version = "6.0.0" } 30 | ordered-float="2.0" 31 | log="^0.4" 32 | egg = "0.7.1" 33 | tokio="^1" 34 | indexmap = "1" 35 | fxhash = "0.2" 36 | 37 | -------------------------------------------------------------------------------- /ballista/ui/scheduler/src/components/Empty.tsx: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | import React from "react"; 19 | import { Flex, Text } from "@chakra-ui/react"; 20 | interface EmptyProps { 21 | text: string; 22 | } 23 | 24 | export const Empty: React.FunctionComponent = ({ text }) => { 25 | return ( 26 | 33 | {text} 34 | 35 | ); 36 | }; 37 | -------------------------------------------------------------------------------- /benchmarks/queries/q19.sql: -------------------------------------------------------------------------------- 1 | select 2 | sum(l_extendedprice* (1 - l_discount)) as revenue 3 | from 4 | lineitem, 5 | part 6 | where 7 | ( 8 | p_partkey = l_partkey 9 | and p_brand = 'Brand#12' 10 | and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') 11 | and l_quantity >= 1 and l_quantity <= 1 + 10 12 | and p_size between 1 and 5 13 | and l_shipmode in ('AIR', 'AIR REG') 14 | and l_shipinstruct = 'DELIVER IN PERSON' 15 | ) 16 | or 17 | ( 18 | p_partkey = l_partkey 19 | and p_brand = 'Brand#23' 20 | and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') 21 | and l_quantity >= 10 and l_quantity <= 10 + 10 22 | and p_size between 1 and 10 23 | and l_shipmode in ('AIR', 'AIR REG') 24 | and l_shipinstruct = 'DELIVER IN PERSON' 25 | ) 26 | or 27 | ( 28 | p_partkey = l_partkey 29 | and p_brand = 'Brand#34' 30 | and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') 31 | and l_quantity >= 20 and l_quantity <= 20 + 10 32 | and p_size between 1 and 15 33 | and l_shipmode in ('AIR', 'AIR REG') 34 | and l_shipinstruct = 'DELIVER IN PERSON' 35 | ); -------------------------------------------------------------------------------- /integration-tests/sqls/simple_window_partition_order_aggregation.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | SELECT 18 | c9, 19 | row_number() OVER (PARTITION BY c2 ORDER BY c9) AS row_number, 20 | count(c3) OVER (PARTITION BY c2 ORDER BY c9) AS count_c3, 21 | avg(c3) OVER (PARTITION BY c2 ORDER BY c9) AS avg_c3_by_c2, 22 | sum(c3) OVER (PARTITION BY c2 ORDER BY c9) AS sum_c3_by_c2, 23 | max(c3) OVER (PARTITION BY c2 ORDER BY c9) AS max_c3_by_c2, 24 | min(c3) OVER (PARTITION BY c2 ORDER BY c9) AS min_c3_by_c2 25 | FROM test 26 | ORDER BY c9; 27 | -------------------------------------------------------------------------------- /ballista/rust/core/src/execution_plans/mod.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //! This module contains execution plans that are needed to distribute Datafusion's execution plans into 19 | //! several Ballista executors. 20 | 21 | mod distributed_query; 22 | mod shuffle_reader; 23 | mod shuffle_writer; 24 | mod unresolved_shuffle; 25 | 26 | pub use distributed_query::DistributedQueryExec; 27 | pub use shuffle_reader::ShuffleReaderExec; 28 | pub use shuffle_writer::ShuffleWriterExec; 29 | pub use unresolved_shuffle::UnresolvedShuffleExec; 30 | -------------------------------------------------------------------------------- /.github_changelog_generator: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | # some issues are just documentation 22 | add-sections={"documentation":{"prefix":"**Documentation updates:**","labels":["documentation"]},"performance":{"prefix":"**Performance improvements:**","labels":["performance"]}} 23 | # uncomment to not show PRs. TBD if we shown them or not. 24 | #pull-requests=false 25 | # so that the component is shown associated with the issue 26 | issue-line-labels=sql 27 | exclude-labels=development-process,invalid 28 | breaking-labels=api change 29 | -------------------------------------------------------------------------------- /ballista/ui/scheduler/src/components/Footer.tsx: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | import React from "react"; 19 | import { Flex, Text } from "@chakra-ui/react"; 20 | 21 | export const Footer: React.FunctionComponent = () => { 22 | return ( 23 | 29 | 30 | Licensed under the Apache License, Version 2.0. 31 | 32 | 33 | ); 34 | }; 35 | -------------------------------------------------------------------------------- /datafusion-cli/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | FROM rust:1.58 as builder 19 | 20 | COPY ./datafusion /usr/src/datafusion 21 | 22 | COPY ./datafusion-cli /usr/src/datafusion-cli 23 | 24 | COPY ./ballista /usr/src/ballista 25 | 26 | WORKDIR /usr/src/datafusion-cli 27 | 28 | RUN rustup component add rustfmt 29 | 30 | RUN cargo build --release 31 | 32 | FROM debian:bullseye-slim 33 | 34 | COPY --from=builder /usr/src/datafusion-cli/target/release/datafusion-cli /usr/local/bin 35 | 36 | ENTRYPOINT ["datafusion-cli"] 37 | 38 | CMD ["--data-path", "/data"] 39 | -------------------------------------------------------------------------------- /integration-tests/create_test_table.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | CREATE EXTERNAL TABLE test ( 18 | c1 VARCHAR NOT NULL, 19 | c2 INT NOT NULL, 20 | c3 SMALLINT NOT NULL, 21 | c4 SMALLINT, 22 | c5 INT, 23 | c6 BIGINT NOT NULL, 24 | c7 SMALLINT NOT NULL, 25 | c8 INT NOT NULL, 26 | c9 BIGINT NOT NULL, 27 | c10 VARCHAR NOT NULL, 28 | c11 FLOAT NOT NULL, 29 | c12 DOUBLE NOT NULL, 30 | c13 VARCHAR NOT NULL 31 | ) 32 | STORED AS CSV 33 | WITH HEADER ROW 34 | LOCATION 'testing/data/csv/aggregate_test_100.csv'; 35 | -------------------------------------------------------------------------------- /ballista/ui/scheduler/src/reportWebVitals.ts: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | import { ReportHandler } from "web-vitals"; 19 | 20 | const reportWebVitals = (onPerfEntry?: ReportHandler) => { 21 | if (onPerfEntry && onPerfEntry instanceof Function) { 22 | import("web-vitals").then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 23 | getCLS(onPerfEntry); 24 | getFID(onPerfEntry); 25 | getFCP(onPerfEntry); 26 | getLCP(onPerfEntry); 27 | getTTFB(onPerfEntry); 28 | }); 29 | } 30 | }; 31 | 32 | export default reportWebVitals; 33 | -------------------------------------------------------------------------------- /ci/scripts/util_checkout.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | # this script is github actions specific to check out the submodules and tags 21 | 22 | # TODO(kszucs): remove it once the "submodules: recursive" feature is released 23 | auth_header="$(git config --local --get http.https://github.com/.extraheader)" 24 | git submodule sync --recursive 25 | git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 26 | 27 | # fetch all the tags 28 | git fetch --depth=1 origin +refs/tags/*:refs/tags/* 29 | -------------------------------------------------------------------------------- /ci/scripts/integration_kartothek.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -e 21 | 22 | # check that optional pyarrow modules are available 23 | # because pytest would just skip the pyarrow tests 24 | python -c "import pyarrow.parquet" 25 | 26 | # check that kartothek is correctly installed 27 | python -c "import kartothek" 28 | 29 | pushd /kartothek 30 | # See ARROW-12314, test_load_dataframes_columns_raises_missing skipped because of changed error message 31 | pytest -n0 --ignore tests/cli/test_query.py -k "not test_load_dataframes_columns_raises_missing" 32 | -------------------------------------------------------------------------------- /datafusion/src/optimizer/mod.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | //! This module contains a query optimizer that operates against a logical plan and applies 19 | //! some simple rules to a logical plan, such as "Projection Push Down" and "Type Coercion". 20 | 21 | #![allow(clippy::module_inception)] 22 | pub mod common_subexpr_eliminate; 23 | pub mod eliminate_limit; 24 | pub mod filter_push_down; 25 | pub mod limit_push_down; 26 | pub mod optimizer; 27 | pub mod projection_push_down; 28 | pub mod simplify_expressions; 29 | pub mod single_distinct_to_groupby; 30 | pub mod utils; 31 | -------------------------------------------------------------------------------- /ballista/rust/scheduler/build.rs: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | extern crate configure_me_codegen; 19 | 20 | fn main() -> Result<(), String> { 21 | println!("cargo:rerun-if-changed=scheduler_config_spec.toml"); 22 | configure_me_codegen::build_script_auto() 23 | .map_err(|e| format!("configure_me code generation failed: {}", e))?; 24 | 25 | println!("cargo:rerun-if-changed=proto/keda.proto"); 26 | tonic_build::configure() 27 | .compile(&["proto/keda.proto"], &["proto"]) 28 | .map_err(|e| format!("protobuf compilation failed: {}", e)) 29 | } 30 | -------------------------------------------------------------------------------- /ci/scripts/install_turbodbc.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | set -e 21 | 22 | if [ "$#" -ne 2 ]; then 23 | echo "Usage: $0 " 24 | exit 1 25 | fi 26 | 27 | turbodbc=$1 28 | target=$2 29 | 30 | git clone --recurse-submodules https://github.com/blue-yonder/turbodbc "${target}" 31 | if [ "${turbodbc}" = "master" ]; then 32 | git -C "${target}" checkout master; 33 | elif [ "${turbodbc}" = "latest" ]; then 34 | git -C "${target}" checkout $(git describe --tags); 35 | else 36 | git -C "${target}" checkout ${turbodbc}; 37 | fi 38 | -------------------------------------------------------------------------------- /dev/release/update_change_log-python.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | # Usage: 22 | # CHANGELOG_GITHUB_TOKEN= ./update_change_log-python.sh 23 | 24 | SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 25 | SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" 26 | 27 | CURRENT_VER=$(grep version "${SOURCE_TOP_DIR}/python/Cargo.toml" | head -n 1 | awk '{print $3}' | tr -d '"') 28 | ${SOURCE_DIR}/update_change_log.sh \ 29 | python \ 30 | python-0.3.0 \ 31 | --exclude-tags-regex "ballista-.+" \ 32 | --future-release "python-${CURRENT_VER}" 33 | -------------------------------------------------------------------------------- /.github/workflows/dev_pr/labeler.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | datafusion: 19 | - datafusion/**/* 20 | - datafusion-cli/**/* 21 | - datafusion-examples/**/* 22 | 23 | ballista: 24 | - ballista/**/* 25 | - ballista-examples/**/* 26 | 27 | python: 28 | - python/**/* 29 | 30 | sql: 31 | - datafusion/src/sql/**/* 32 | 33 | development-process: 34 | - dev/**.* 35 | - .github/**.* 36 | - ci/**.* 37 | - .asf.yaml 38 | 39 | documentation: 40 | - docs/**.* 41 | - README.md 42 | - ./**/README.md 43 | - DEVELOPERS.md 44 | - ballista/docs/**.* 45 | - datafusion/docs/**.* 46 | -------------------------------------------------------------------------------- /dev/release/update_change_log-datafusion.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | # Usage: 22 | # CHANGELOG_GITHUB_TOKEN= ./update_change_log-datafusion.sh 23 | 24 | SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 25 | SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" 26 | 27 | CURRENT_VER=$(grep version "${SOURCE_TOP_DIR}/datafusion/Cargo.toml" | head -n 1 | awk '{print $3}' | tr -d '"') 28 | ${SOURCE_DIR}/update_change_log.sh \ 29 | datafusion \ 30 | 5.0.0 \ 31 | --exclude-tags-regex "(python|ballista)-.+" \ 32 | --future-release "${CURRENT_VER}" 33 | --------------------------------------------------------------------------------