├── CMakeLists.txt ├── README.md ├── data ├── TPC-H-small.db ├── TPC-H-small.duckdb ├── duckdb └── duckdb_cli-linux-amd64.zip ├── queries ├── h01.sql ├── h02.sql ├── h03.sql ├── h04.sql ├── h05.sql ├── h06.sql ├── h07.sql ├── h08.sql ├── h09.sql ├── h10.sql ├── h11.sql ├── h12.sql ├── h13.sql ├── h14.sql ├── h15.sql ├── h16.sql ├── h17.sql ├── h18.sql ├── h19.sql ├── h20.sql ├── h21.sql └── h22.sql ├── scripts ├── get_duckdb_database.sh ├── install_arrow.sh ├── install_duckdb.sh └── move_data_to_duckdb.py └── src ├── duckdb ├── duckdb_server.cpp ├── duckdb_server.h ├── duckdb_sql_info.cpp ├── duckdb_sql_info.h ├── duckdb_statement.cpp ├── duckdb_statement.h ├── duckdb_statement_batch_reader.cpp └── duckdb_statement_batch_reader.h ├── flight_sql.cpp └── sqlite ├── sqlite_server.cc ├── sqlite_server.h ├── sqlite_sql_info.cc ├── sqlite_sql_info.h ├── sqlite_statement.cc ├── sqlite_statement.h ├── sqlite_statement_batch_reader.cc ├── sqlite_statement_batch_reader.h ├── sqlite_tables_schema_batch_reader.cc ├── sqlite_tables_schema_batch_reader.h ├── sqlite_type_info.cc └── sqlite_type_info.h /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(flight_sql) 3 | 4 | set(CMAKE_CXX_STANDARD 17) 5 | 6 | find_package(Threads REQUIRED) 7 | find_package(Arrow REQUIRED) 8 | find_package(ArrowFlight REQUIRED) 9 | find_package(ArrowFlightSql REQUIRED) 10 | find_package(SQLite3 REQUIRED) 11 | find_package(Boost COMPONENTS program_options REQUIRED) 12 | 13 | # find duckDB 14 | find_library(DUCKDB_LIB NAMES duckdb PATHS $ENV{CONDA_PREFIX}/lib/) 15 | set(DUCKDB ${DUCKDB_LIB}) 16 | 17 | set(ARROW_FLIGHT_SQL_SQLITE_SERVER_SRCS 18 | src/sqlite/sqlite_sql_info.cc 19 | src/sqlite/sqlite_type_info.cc 20 | src/sqlite/sqlite_statement.cc 21 | src/sqlite/sqlite_statement_batch_reader.cc 22 | src/sqlite/sqlite_server.cc 23 | src/sqlite/sqlite_tables_schema_batch_reader.cc) 24 | 25 | set(ARROW_FLIGHT_DUCKDB_SQLITE_SERVER_SRCS 26 | src/duckdb/duckdb_sql_info.cpp 27 | src/duckdb/duckdb_statement.cpp 28 | src/duckdb/duckdb_statement_batch_reader.cpp 29 | src/duckdb/duckdb_server.cpp 30 | ) 31 | 32 | add_executable(flight_sql 33 | src/flight_sql.cpp 34 | ${ARROW_FLIGHT_SQL_SQLITE_SERVER_SRCS} 35 | ${ARROW_FLIGHT_DUCKDB_SQLITE_SERVER_SRCS} 36 | ) 37 | 38 | target_include_directories(flight_sql PRIVATE src/sqlite src/duckdb) 39 | # target_include_directories(flight_sql PRIVATE src/duckdb) 40 | target_link_libraries(flight_sql PRIVATE 41 | arrow_shared 42 | arrow_flight_shared 43 | arrow_flight_sql_shared 44 | Threads::Threads 45 | ${SQLite3_LIBRARIES} 46 | ${DUCKDB} 47 | ${Boost_LIBRARIES} 48 | ) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Arrow FlightSQL example 2 | 3 | ## Setup 4 | 5 | In order to run this example you need to set up a new environment on your machine. 6 | Follow these steps to do so (thanks to David Li!). 7 | 8 | 1. Make sure you have [miniconda](https://docs.conda.io/en/latest/miniconda.html) installed on your machine. 9 | 2. Install Mamba: `conda install -c conda-forge mamba`. 10 | 3. Now, create a new dev environment 11 | ```bash 12 | $ mamba create -n flight-sql -c conda-forge python=3.9 13 | $ conda activate flight-sql 14 | $ 15 | ``` 16 | 4. Build and install Arrow 17 | ```bash 18 | $ cd scripts && ./install_arrow.sh 19 | ``` 20 | 7. Build and install `duckdb`. This is sometimes necessary as conda `compilers` 21 | seem to be including incompatible GlibC library with the compiled binaries 22 | of `duckdb`. 23 | ```bash 24 | $ ./install_duckdb.sh 25 | ``` 26 | 6. Get the data. 27 | ```bash 28 | $ mkdir ../data 29 | $ wget https://github.com/lovasoa/TPCH-sqlite/releases/download/v1.0/TPC-H-small.db -O ../data/TPC-H-small.db 30 | ``` 31 | 7. Create duckdb database. 32 | ```bash 33 | $ ./get_duckdb_database.sh 34 | ``` 35 | 9. Build the example. 36 | ```bash 37 | $ mkdir build && cd build 38 | $ cmake .. -GNinja -DCMAKE_PREFIX_PATH=$CONDA_PREFIX/lib/cmake/arrow 39 | $ ninja && ./flight_sql 40 | ``` 41 | 42 | ## Selecting different backends 43 | This example allows chosing from two backends: SQLite and DuckDB. It defaults to DuckDB. 44 | 45 | ```bash 46 | $ ./flight_sql 47 | > duckdb server listening on localhost:31337 48 | > Connected to server: localhost:31337 49 | > Client created. 50 | > ... 51 | ``` 52 | 53 | The above call is equivalent to running `./flight_sql -B duckdb` or `./flight_sql --backend duckdb`. To select SQLite run 54 | 55 | ```bash 56 | $ ./flight_sql -B sqlite 57 | ``` 58 | or 59 | ```bash 60 | $ ./flight_sql --backend sqlite 61 | ``` 62 | The above will produce the following: 63 | 64 | ```bash 65 | > sqlite server listening on localhost:31337 66 | > Connected to server: localhost:31337 67 | > Client created. 68 | > ... 69 | ``` 70 | 71 | ## Printing results of running query 72 | This example defaults to running the queries but does not print out 73 | the query itself nor does it print the results. To switch to printing 74 | results and queries set the `print` flag to `true`. 75 | 76 | ```bash 77 | $ ./flight_sql --print true 78 | ``` 79 | 80 | ## Print help 81 | To see all the available options run `./flight.sql --help`. 82 | 83 | ```bash 84 | > Allowed options: 85 | > --help produce this help message 86 | > -B [ --backend ] arg (=duckdb) Specify the database backend. Allowed options: 87 | > duckdb, sqlite. 88 | > --print arg (=false) Print the results of running queries. Allowed 89 | > options: false, true. 90 | ``` -------------------------------------------------------------------------------- /data/TPC-H-small.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/voltrondata/flight-duckdb-example/8bbcfaa664e3557f78e4e6817ea4c7a554179b73/data/TPC-H-small.db -------------------------------------------------------------------------------- /data/TPC-H-small.duckdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/voltrondata/flight-duckdb-example/8bbcfaa664e3557f78e4e6817ea4c7a554179b73/data/TPC-H-small.duckdb -------------------------------------------------------------------------------- /data/duckdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/voltrondata/flight-duckdb-example/8bbcfaa664e3557f78e4e6817ea4c7a554179b73/data/duckdb -------------------------------------------------------------------------------- /data/duckdb_cli-linux-amd64.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/voltrondata/flight-duckdb-example/8bbcfaa664e3557f78e4e6817ea4c7a554179b73/data/duckdb_cli-linux-amd64.zip -------------------------------------------------------------------------------- /queries/h01.sql: -------------------------------------------------------------------------------- 1 | SELECT l_returnflag, 2 | l_linestatus, 3 | sum(l_quantity) AS sum_qty, 4 | sum(l_extendedprice) AS sum_base_price, 5 | sum(l_extendedprice*(1-l_discount)) AS sum_disc_price, 6 | sum(l_extendedprice*(1-l_discount)*(1+l_tax)) AS sum_charge, 7 | avg(l_quantity) AS avg_qty, 8 | avg(l_extendedprice) AS avg_price, 9 | avg(l_discount) AS avg_disc, 10 | count(*) AS count_order 11 | FROM lineitem 12 | WHERE l_shipdate <= '1998-09-01' -- date '1998-12-01' - interval '[DELTA=90]' DAY 13 | GROUP BY l_returnflag, 14 | l_linestatus 15 | ORDER BY l_returnflag, 16 | l_linestatus; 17 | -------------------------------------------------------------------------------- /queries/h02.sql: -------------------------------------------------------------------------------- 1 | SELECT s_acctbal, 2 | s_name, 3 | n_name, 4 | p_partkey, 5 | p_mfgr, 6 | s_address, 7 | s_phone, 8 | s_comment 9 | FROM part, 10 | supplier, 11 | partsupp, 12 | nation, 13 | region 14 | WHERE p_partkey = ps_partkey 15 | AND s_suppkey = ps_suppkey 16 | AND p_size = 25 -- [SIZE] 17 | AND p_type like '%BRASS' -- '%[TYPE]' 18 | AND s_nationkey = n_nationkey 19 | AND n_regionkey = r_regionkey 20 | AND r_name = 'EUROPE' -- '[REGION]' 21 | AND ps_supplycost = 22 | (SELECT min(ps_supplycost) 23 | FROM partsupp, 24 | supplier, 25 | nation, 26 | region 27 | WHERE p_partkey = ps_partkey 28 | AND s_suppkey = ps_suppkey 29 | AND s_nationkey = n_nationkey 30 | AND n_regionkey = r_regionkey 31 | AND r_name = 'EUROPE') -- '[REGION]' ) 32 | ORDER BY s_acctbal DESC, 33 | n_name, 34 | s_name, 35 | p_partkey 36 | LIMIT 100 37 | ; 38 | -------------------------------------------------------------------------------- /queries/h03.sql: -------------------------------------------------------------------------------- 1 | SELECT l_orderkey, 2 | sum(l_extendedprice * (1 - l_discount)) AS revenue, 3 | o_orderdate, 4 | o_shippriority 5 | FROM customer, 6 | orders, 7 | lineitem 8 | WHERE c_mktsegment = 'BUILDING' 9 | AND c_custkey = o_custkey 10 | AND l_orderkey = o_orderkey 11 | AND o_orderdate < '1995-03-15' 12 | AND l_shipdate > '1995-03-15' 13 | GROUP BY l_orderkey, 14 | o_orderdate, 15 | o_shippriority 16 | ORDER BY revenue DESC, 17 | o_orderdate 18 | LIMIT 10 19 | ; 20 | -------------------------------------------------------------------------------- /queries/h04.sql: -------------------------------------------------------------------------------- 1 | SELECT o_orderpriority, 2 | count(*) AS order_count 3 | FROM orders 4 | WHERE o_orderdate >= '1993-07-01' 5 | AND o_orderdate < '1993-10-01' 6 | AND EXISTS 7 | (SELECT * 8 | FROM lineitem 9 | WHERE l_orderkey = o_orderkey 10 | AND l_commitdate < l_receiptdate ) 11 | GROUP BY o_orderpriority 12 | ORDER BY o_orderpriority 13 | ; 14 | -------------------------------------------------------------------------------- /queries/h05.sql: -------------------------------------------------------------------------------- 1 | SELECT n_name, 2 | sum(l_extendedprice * (1 - l_discount)) AS revenue 3 | FROM customer, 4 | orders, 5 | lineitem, 6 | supplier, 7 | nation, 8 | region 9 | WHERE c_custkey = o_custkey 10 | AND l_orderkey = o_orderkey 11 | AND l_suppkey = s_suppkey 12 | AND c_nationkey = s_nationkey 13 | AND s_nationkey = n_nationkey 14 | AND n_regionkey = r_regionkey 15 | AND r_name = 'ASIA' 16 | AND o_orderdate >= '1994-01-01' 17 | AND o_orderdate < '1995-01-01' 18 | GROUP BY n_name 19 | ORDER BY revenue DESC 20 | ; 21 | -------------------------------------------------------------------------------- /queries/h06.sql: -------------------------------------------------------------------------------- 1 | SELECT sum(l_extendedprice * l_discount) AS revenue 2 | FROM lineitem 3 | WHERE l_shipdate >= '1994-01-01' 4 | AND l_shipdate < '1995-01-01' 5 | AND l_discount BETWEEN 0.05 AND 0.07 6 | AND l_quantity < 24 7 | ; 8 | -------------------------------------------------------------------------------- /queries/h07.sql: -------------------------------------------------------------------------------- 1 | SELECT supp_nation, 2 | cust_nation, 3 | l_year, 4 | sum(volume) AS revenue 5 | FROM 6 | (SELECT n1.n_name AS supp_nation, 7 | n2.n_name AS cust_nation, 8 | strftime('%Y', l_shipdate) AS l_year, 9 | l_extendedprice * (1 - l_discount) AS volume 10 | FROM supplier, 11 | lineitem, 12 | orders, 13 | customer, 14 | nation n1, 15 | nation n2 16 | WHERE s_suppkey = l_suppkey 17 | AND o_orderkey = l_orderkey 18 | AND c_custkey = o_custkey 19 | AND s_nationkey = n1.n_nationkey 20 | AND c_nationkey = n2.n_nationkey 21 | AND ((n1.n_name = 'FRANCE' 22 | AND n2.n_name = 'GERMANY') 23 | OR (n1.n_name = 'GERMANY' 24 | AND n2.n_name = 'FRANCE')) 25 | AND l_shipdate BETWEEN '1995-01-01' AND '1996-12-31' ) AS shipping 26 | GROUP BY supp_nation, 27 | cust_nation, 28 | l_year 29 | ORDER BY supp_nation, 30 | cust_nation, 31 | l_year 32 | ; 33 | -------------------------------------------------------------------------------- /queries/h08.sql: -------------------------------------------------------------------------------- 1 | SELECT o_year, 2 | sum(CASE 3 | WHEN nation = 'BRAZIL' THEN volume 4 | ELSE 0 5 | END) / sum(volume) AS mkt_share 6 | FROM 7 | (SELECT strftime('%Y', o_orderdate) AS o_year, 8 | l_extendedprice * (1 - l_discount) AS volume, 9 | n2.n_name AS nation 10 | FROM part, 11 | supplier, 12 | lineitem, 13 | orders, 14 | customer, 15 | nation n1, 16 | nation n2, 17 | region 18 | WHERE p_partkey = l_partkey 19 | AND s_suppkey = l_suppkey 20 | AND l_orderkey = o_orderkey 21 | AND o_custkey = c_custkey 22 | AND c_nationkey = n1.n_nationkey 23 | AND n1.n_regionkey = r_regionkey 24 | AND r_name = 'AMERICA' 25 | AND s_nationkey = n2.n_nationkey 26 | AND o_orderdate BETWEEN '1995-01-01' AND '1996-12-31' 27 | AND p_type = 'ECONOMY ANODIZED STEEL' ) AS all_nations 28 | GROUP BY o_year 29 | ORDER BY o_year 30 | ; 31 | -------------------------------------------------------------------------------- /queries/h09.sql: -------------------------------------------------------------------------------- 1 | SELECT nation, 2 | o_year, 3 | sum(amount) AS sum_profit 4 | FROM 5 | (SELECT n_name AS nation, 6 | strftime('%Y', o_orderdate) AS o_year, 7 | l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity AS amount 8 | FROM part, 9 | supplier, 10 | lineitem, 11 | partsupp, 12 | orders, 13 | nation 14 | WHERE s_suppkey = l_suppkey 15 | AND ps_suppkey = l_suppkey 16 | AND ps_partkey = l_partkey 17 | AND p_partkey = l_partkey 18 | AND o_orderkey = l_orderkey 19 | AND s_nationkey = n_nationkey 20 | AND p_name like '%green%' ) AS profit 21 | GROUP BY nation, 22 | o_year 23 | ORDER BY nation, 24 | o_year DESC ; 25 | -------------------------------------------------------------------------------- /queries/h10.sql: -------------------------------------------------------------------------------- 1 | SELECT c_custkey, 2 | c_name, 3 | sum(l_extendedprice * (1 - l_discount)) AS revenue, 4 | c_acctbal, 5 | n_name, 6 | c_address, 7 | c_phone, 8 | c_comment 9 | FROM customer, 10 | orders, 11 | lineitem, 12 | nation 13 | WHERE c_custkey = o_custkey 14 | AND l_orderkey = o_orderkey 15 | AND o_orderdate >= '1993-10-01' 16 | AND o_orderdate < '1994-01-01' 17 | AND l_returnflag = 'R' 18 | AND c_nationkey = n_nationkey 19 | GROUP BY c_custkey, 20 | c_name, 21 | c_acctbal, 22 | c_phone, 23 | n_name, 24 | c_address, 25 | c_comment 26 | ORDER BY revenue DESC 27 | LIMIT 20 ; 28 | -------------------------------------------------------------------------------- /queries/h11.sql: -------------------------------------------------------------------------------- 1 | SELECT ps_partkey, 2 | sum(ps_supplycost * ps_availqty) AS value 3 | FROM partsupp, 4 | supplier, 5 | nation 6 | WHERE ps_suppkey = s_suppkey 7 | AND s_nationkey = n_nationkey 8 | AND n_name = 'GERMANY' 9 | GROUP BY ps_partkey 10 | HAVING sum(ps_supplycost * ps_availqty) > 11 | (SELECT sum(ps_supplycost * ps_availqty) * .0001 -- FRACTION = .0001/SF 12 | FROM partsupp, 13 | supplier, 14 | nation 15 | WHERE ps_suppkey = s_suppkey 16 | AND s_nationkey = n_nationkey 17 | AND n_name = 'GERMANY' ) 18 | ORDER BY value DESC 19 | ; 20 | -------------------------------------------------------------------------------- /queries/h12.sql: -------------------------------------------------------------------------------- 1 | SELECT l_shipmode, 2 | sum(CASE 3 | WHEN o_orderpriority ='1-URGENT' 4 | OR o_orderpriority ='2-HIGH' THEN 1 5 | ELSE 0 6 | END) AS high_line_count, 7 | sum(CASE 8 | WHEN o_orderpriority <> '1-URGENT' 9 | AND o_orderpriority <> '2-HIGH' THEN 1 10 | ELSE 0 11 | END) AS low_line_count 12 | FROM orders, 13 | lineitem 14 | WHERE o_orderkey = l_orderkey 15 | AND l_shipmode in ('MAIL', 16 | 'SHIP') 17 | AND l_commitdate < l_receiptdate 18 | AND l_shipdate < l_commitdate 19 | AND l_receiptdate >= '1994-01-01' 20 | AND l_receiptdate < '1995-01-01' 21 | GROUP BY l_shipmode 22 | ORDER BY l_shipmode 23 | ; 24 | -------------------------------------------------------------------------------- /queries/h13.sql: -------------------------------------------------------------------------------- 1 | SELECT c_count, 2 | count(*) AS custdist 3 | FROM 4 | (SELECT c_custkey, 5 | count(o_orderkey) AS c_count 6 | FROM customer 7 | LEFT OUTER JOIN orders ON c_custkey = o_custkey 8 | AND o_comment NOT LIKE '%special%requests%' 9 | GROUP BY c_custkey) 10 | GROUP BY c_count 11 | ORDER BY custdist DESC, 12 | c_count DESC 13 | ; 14 | -------------------------------------------------------------------------------- /queries/h14.sql: -------------------------------------------------------------------------------- 1 | SELECT 100.00 * sum(CASE 2 | WHEN p_type like 'PROMO%' THEN l_extendedprice*(1-l_discount) 3 | ELSE 0 4 | END) / sum(l_extendedprice * (1 - l_discount)) AS promo_revenue 5 | FROM lineitem, 6 | part 7 | WHERE l_partkey = p_partkey 8 | AND l_shipdate >= '1995-09-01' 9 | AND l_shipdate < '1995-10-01' 10 | ; 11 | -------------------------------------------------------------------------------- /queries/h15.sql: -------------------------------------------------------------------------------- 1 | SELECT s_suppkey, 2 | s_name, 3 | s_address, 4 | s_phone, 5 | total_revenue 6 | FROM supplier, 7 | 8 | (SELECT l_suppkey AS supplier_no, 9 | sum(l_extendedprice * (1 - l_discount)) AS total_revenue 10 | FROM lineitem 11 | WHERE l_shipdate >= '1996-01-01' 12 | AND l_shipdate < '1996-04-01' 13 | GROUP BY supplier_no) revenue0 14 | WHERE s_suppkey = supplier_no 15 | AND total_revenue = 16 | (SELECT max(total_revenue) 17 | FROM 18 | (SELECT l_suppkey AS supplier_no, 19 | sum(l_extendedprice * (1 - l_discount)) AS total_revenue 20 | FROM lineitem 21 | WHERE l_shipdate >= '1996-01-01' 22 | AND l_shipdate < '1996-04-01' 23 | GROUP BY supplier_no) revenue1) 24 | ORDER BY s_suppkey 25 | ; 26 | -------------------------------------------------------------------------------- /queries/h16.sql: -------------------------------------------------------------------------------- 1 | SELECT p_brand, 2 | p_type, 3 | p_size, 4 | count(DISTINCT ps_suppkey) AS supplier_cnt 5 | FROM partsupp, 6 | part 7 | WHERE p_partkey = ps_partkey 8 | AND p_brand <> 'Brand#45' 9 | AND p_type not like 'MEDIUM POLISHED%' 10 | AND p_size in (49, 14, 23, 45, 19, 3, 36, 9) 11 | AND ps_suppkey not in 12 | (SELECT s_suppkey 13 | FROM supplier 14 | WHERE s_comment like '%Customer%Complaints%' ) 15 | GROUP BY p_brand, 16 | p_type, 17 | p_size 18 | ORDER BY supplier_cnt DESC, 19 | p_brand, 20 | p_type, 21 | p_size 22 | ; 23 | -------------------------------------------------------------------------------- /queries/h17.sql: -------------------------------------------------------------------------------- 1 | SELECT sum(l_extendedprice) / 7.0 AS avg_yearly 2 | FROM lineitem, 3 | part 4 | WHERE p_partkey = l_partkey 5 | AND p_brand = 'Brand#23' 6 | AND p_container = 'MED BOX' 7 | AND l_quantity < 8 | (SELECT 0.2 * avg(l_quantity) 9 | FROM lineitem 10 | WHERE l_partkey = p_partkey ) 11 | ; 12 | -------------------------------------------------------------------------------- /queries/h18.sql: -------------------------------------------------------------------------------- 1 | SELECT c_name, 2 | c_custkey, 3 | o_orderkey, 4 | o_orderdate, 5 | o_totalprice, 6 | sum(l_quantity) AS sum_qty 7 | FROM customer, 8 | orders, 9 | lineitem 10 | WHERE o_orderkey in 11 | (SELECT l_orderkey 12 | FROM lineitem 13 | GROUP BY l_orderkey 14 | HAVING sum(l_quantity) > 300) 15 | AND c_custkey = o_custkey 16 | AND o_orderkey = l_orderkey 17 | GROUP BY c_name, 18 | c_custkey, 19 | o_orderkey, 20 | o_orderdate, 21 | o_totalprice 22 | ORDER BY o_totalprice DESC, 23 | o_orderdate 24 | LIMIT 100 25 | ; 26 | -------------------------------------------------------------------------------- /queries/h19.sql: -------------------------------------------------------------------------------- 1 | SELECT sum(l_extendedprice * (1 - l_discount)) AS revenue 2 | FROM lineitem, 3 | part 4 | WHERE (p_partkey = l_partkey 5 | AND p_brand = 'Brand#12' 6 | AND p_container in ('SM CASE', 7 | 'SM BOX', 8 | 'SM PACK', 9 | 'SM PKG') 10 | AND l_quantity >= 1 11 | AND l_quantity <= 11 12 | AND p_size BETWEEN 1 AND 5 13 | AND l_shipmode in ('AIR', 14 | 'AIR REG') 15 | AND l_shipinstruct = 'DELIVER IN PERSON') 16 | OR (p_partkey = l_partkey 17 | AND p_brand = 'Brand#23' 18 | AND p_container in ('MED BAG', 19 | 'MED BOX', 20 | 'MED PKG', 21 | 'MED PACK') 22 | AND l_quantity >= 10 23 | AND l_quantity <= 20 24 | AND p_size BETWEEN 1 AND 10 25 | AND l_shipmode in ('AIR', 26 | 'AIR REG') 27 | AND l_shipinstruct = 'DELIVER IN PERSON') 28 | OR (p_partkey = l_partkey 29 | AND p_brand = 'Brand#34' 30 | AND p_container in ('LG CASE', 31 | 'LG BOX', 32 | 'LG PACK', 33 | 'LG PKG') 34 | AND l_quantity >= 20 35 | AND l_quantity <= 30 36 | AND p_size BETWEEN 1 AND 15 37 | AND l_shipmode in ('AIR', 38 | 'AIR REG') 39 | AND l_shipinstruct = 'DELIVER IN PERSON') 40 | ; 41 | -------------------------------------------------------------------------------- /queries/h20.sql: -------------------------------------------------------------------------------- 1 | SELECT s_name, 2 | s_address 3 | FROM supplier, 4 | nation 5 | WHERE s_suppkey in 6 | (SELECT ps_suppkey 7 | FROM partsupp 8 | WHERE ps_partkey in 9 | (SELECT p_partkey 10 | FROM part 11 | WHERE p_name like 'forest%' ) 12 | AND ps_availqty > 13 | (SELECT 0.5 * sum(l_quantity) 14 | FROM lineitem 15 | WHERE l_partkey = ps_partkey 16 | AND l_suppkey = ps_suppkey 17 | AND l_shipdate >= '1994-01-01' 18 | AND l_shipdate < '1995-01-01' ) ) 19 | AND s_nationkey = n_nationkey 20 | AND n_name = 'CANADA' 21 | ORDER BY s_name 22 | ; 23 | -------------------------------------------------------------------------------- /queries/h21.sql: -------------------------------------------------------------------------------- 1 | SELECT s_name, 2 | count(*) AS numwait 3 | FROM supplier, 4 | lineitem l1, 5 | orders, 6 | nation 7 | WHERE s_suppkey = l1.l_suppkey 8 | AND o_orderkey = l1.l_orderkey 9 | AND o_orderstatus = 'F' 10 | AND l1.l_receiptdate > l1.l_commitdate 11 | AND EXISTS 12 | (SELECT * 13 | FROM lineitem l2 14 | WHERE l2.l_orderkey = l1.l_orderkey 15 | AND l2.l_suppkey <> l1.l_suppkey ) 16 | AND NOT EXISTS 17 | (SELECT * 18 | FROM lineitem l3 19 | WHERE l3.l_orderkey = l1.l_orderkey 20 | AND l3.l_suppkey <> l1.l_suppkey 21 | AND l3.l_receiptdate > l3.l_commitdate ) 22 | AND s_nationkey = n_nationkey 23 | AND n_name = 'SAUDI ARABIA' 24 | GROUP BY s_name 25 | ORDER BY numwait DESC, 26 | s_name 27 | LIMIT 100 28 | ; 29 | -------------------------------------------------------------------------------- /queries/h22.sql: -------------------------------------------------------------------------------- 1 | SELECT cntrycode, 2 | COUNT(*) AS numcust, 3 | SUM(c_acctbal) AS totacctbal 4 | FROM 5 | (SELECT SUBSTR(c_phone, 1, 2) AS cntrycode, 6 | c_acctbal 7 | FROM customer 8 | WHERE SUBSTR(c_phone, 1, 2) 9 | IN ('13', '31', '23', '29', '30', '18', '17') 10 | AND c_acctbal > 11 | (SELECT AVG(c_acctbal) 12 | FROM customer 13 | WHERE c_acctbal > 0.00 14 | AND SUBSTR(c_phone, 1, 2) 15 | IN ('13', '31', '23', '29', '30', '18', '17') ) 16 | AND NOT EXISTS 17 | (SELECT * 18 | FROM orders 19 | WHERE o_custkey = c_custkey ) ) AS custsale 20 | GROUP BY cntrycode 21 | ORDER BY cntrycode 22 | ; 23 | -------------------------------------------------------------------------------- /scripts/get_duckdb_database.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | TEMP_DIR="../temp" 3 | 4 | # check if temp directory exists 5 | if [ ! -d "$TEMP_DIR" ]; then 6 | echo "$TEMP_DIR doesn't exist. Creating." 7 | fi 8 | 9 | # move the database to temp 10 | cd $TEMP_DIR 11 | 12 | # check if duckdb database already exists 13 | if [ -f "TPC-H-small.duckdb" ]; then 14 | rm TPC-H-small.duckdb 15 | fi 16 | 17 | # copy the SQLite db 18 | cp ../data/TPC-H-small.db . 19 | 20 | # move the data to DuckDB 21 | $CONDA_PREFIX/bin/python ../scripts/move_data_to_duckdb.py 22 | 23 | # and push the new DB to data folder 24 | cp TPC-H-small.duckdb ../data/ -------------------------------------------------------------------------------- /scripts/install_arrow.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | TEMP_DIR="../temp" 3 | 4 | # check if temp directory exists 5 | if [ ! -d "$TEMP_DIR" ]; then 6 | echo "$TEMP_DIR doesn't exist. Creating." 7 | fi 8 | 9 | # clone the repository 10 | cd $TEMP_DIR 11 | if [ ! -d "arrow" ]; then 12 | echo "Cloning Arrow." 13 | git clone https://github.com/apache/arrow.git 14 | fi 15 | 16 | cd arrow 17 | mamba install -c conda-forge -y --file ci/conda_env_cpp.txt 18 | mamba install -c conda-forge -y compilers 19 | cd cpp && mkdir -p build && cd build && rm -rf * 20 | cmake .. -GNinja -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \ 21 | -DARROW_FLIGHT=ON \ 22 | -DARROW_FLIGHT_SQL=ON \ 23 | -DARROW_JSON=ON \ 24 | -DARROW_PARQUET=ON \ 25 | -DARROW_CSV=ON \ 26 | -DARROW_WITH_SNAPPY=ON 27 | ninja install # Will install into your Conda prefix -------------------------------------------------------------------------------- /scripts/install_duckdb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | TEMP_DIR="../temp" 3 | 4 | # check if temp directory exists 5 | if [ ! -d "$TEMP_DIR" ]; then 6 | echo "$TEMP_DIR doesn't exist. Creating." 7 | fi 8 | 9 | # clone the repository 10 | cd $TEMP_DIR 11 | if [ ! -d "duckdb" ]; then 12 | echo "Cloning DuckDB." 13 | git clone https://github.com/duckdb/duckdb.git 14 | fi 15 | 16 | cd duckdb 17 | if [ ! -d "build/release" ]; then 18 | echo "Building DuckDB" 19 | make 20 | fi 21 | 22 | # copy libraries to include and lib paths 23 | cp build/release/src/libduckdb.so $CONDA_PREFIX/lib/ 24 | cp src/include/duckdb.h $CONDA_PREFIX/include/ 25 | cp src/include/duckdb.hpp $CONDA_PREFIX/include/ 26 | cp -R src/include/duckdb $CONDA_PREFIX/include/ -------------------------------------------------------------------------------- /scripts/move_data_to_duckdb.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import sqlite3 3 | import duckdb 4 | 5 | # establish all connections to databases 6 | con = sqlite3.connect('TPC-H-small.db') 7 | con_duck = duckdb.connect(database='TPC-H-small.duckdb', read_only=False) 8 | 9 | cur = con.cursor() 10 | 11 | # get a list of all the tables 12 | cur.execute('SELECT name from sqlite_master where type= "table"') 13 | tables = cur.fetchall() 14 | 15 | # get all the data 16 | sqlite_data = {} 17 | 18 | for table in tables: 19 | tname = table[0] 20 | cur.execute(f'PRAGMA table_info({tname});') 21 | sqlite_data[tname] = (pd.read_sql(f'SELECT * FROM {tname}', con=con), cur.fetchall()) 22 | 23 | # insert the data into duckdb database 24 | for table in tables: 25 | tname = table[0] 26 | 27 | temp_table = sqlite_data[tname][0] 28 | date_cols = [e[1] for e in sqlite_data[tname][1] if e[2] == 'DATE'] 29 | 30 | for col in date_cols: 31 | temp_table[col] = pd.to_datetime(temp_table[col]) 32 | 33 | con_duck.execute(f'CREATE TABLE {tname} AS SELECT * FROM temp_table') 34 | con_duck.execute(f'INSERT INTO {tname} SELECT * FROM temp_table') 35 | 36 | # close the connections 37 | con_duck.close() 38 | con.close() -------------------------------------------------------------------------------- /src/duckdb/duckdb_server.cpp: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #include "duckdb_server.h" 19 | 20 | #include 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | #include 30 | 31 | #include "duckdb_sql_info.h" 32 | #include "duckdb_statement.h" 33 | #include "duckdb_statement_batch_reader.h" 34 | 35 | namespace arrow { 36 | namespace flight { 37 | namespace sql { 38 | namespace duckdbflight { 39 | 40 | namespace { 41 | 42 | 43 | std::string PrepareQueryForGetTables(const GetTables& command) { 44 | std::stringstream table_query; 45 | 46 | table_query << "SELECT 'NOT_IMPLEMENTED' as catalog_name, table_schema as schema_name, table_name," 47 | "table_type FROM information_schema.tables where 1=1"; 48 | 49 | if (command.catalog.has_value()) { 50 | table_query << " and table_catalog='" << command.catalog.value() << "'"; 51 | } 52 | 53 | if (command.db_schema_filter_pattern.has_value()) { 54 | table_query << " and table_schame LIKE '" << command.db_schema_filter_pattern.value() 55 | << "'"; 56 | } 57 | 58 | if (command.table_name_filter_pattern.has_value()) { 59 | table_query << " and table_name LIKE '" << command.table_name_filter_pattern.value() 60 | << "'"; 61 | } 62 | 63 | if (!command.table_types.empty()) { 64 | table_query << " and table_type IN ("; 65 | size_t size = command.table_types.size(); 66 | for (size_t i = 0; i < size; i++) { 67 | table_query << "'" << command.table_types[i] << "'"; 68 | if (size - 1 != i) { 69 | table_query << ","; 70 | } 71 | } 72 | 73 | table_query << ")"; 74 | } 75 | 76 | table_query << " order by table_name"; 77 | return table_query.str(); 78 | } 79 | } // namespace 80 | 81 | class DuckDBFlightSqlServer::Impl { 82 | private: 83 | std::shared_ptr db_instance_; 84 | std::shared_ptr db_conn_; 85 | 86 | public: 87 | explicit Impl( 88 | std::shared_ptr db_instance, 89 | std::shared_ptr db_connection 90 | ) : db_instance_(std::move(db_instance)), db_conn_(std::move(db_connection)) { 91 | } 92 | 93 | ~Impl() { 94 | } 95 | 96 | arrow::Result> GetFlightInfoStatement( 97 | const ServerCallContext& context, const StatementQuery& command, 98 | const FlightDescriptor& descriptor) { 99 | const std::string& query = command.query; 100 | 101 | ARROW_ASSIGN_OR_RAISE(auto statement, DuckDBStatement::Create(db_conn_, query)); 102 | ARROW_ASSIGN_OR_RAISE(auto schema, statement->GetSchema()); 103 | ARROW_ASSIGN_OR_RAISE(auto ticket_string, CreateStatementQueryTicket(query)); 104 | std::vector endpoints{FlightEndpoint{{ticket_string}, {}}}; 105 | ARROW_ASSIGN_OR_RAISE(auto result, 106 | FlightInfo::Make(*schema, descriptor, endpoints, -1, -1)) 107 | return std::unique_ptr(new FlightInfo(result)); 108 | } 109 | 110 | arrow::Result> DoGetStatement( 111 | const ServerCallContext& context, const StatementQueryTicket& command) { 112 | const std::string& sql = command.statement_handle; 113 | 114 | ARROW_ASSIGN_OR_RAISE(auto statement, DuckDBStatement::Create(db_conn_, sql)); 115 | 116 | ARROW_ASSIGN_OR_RAISE(std::shared_ptr reader, 117 | DuckDBStatementBatchReader::Create(statement)); 118 | 119 | return std::unique_ptr(new RecordBatchStream(reader)); 120 | } 121 | 122 | arrow::Result> DoGetTables( 123 | const ServerCallContext& context, const GetTables& command) { 124 | std::string query = PrepareQueryForGetTables(command); 125 | std::shared_ptr statement; 126 | ARROW_ASSIGN_OR_RAISE(statement, DuckDBStatement::Create(db_conn_, query)); 127 | 128 | std::shared_ptr reader; 129 | ARROW_ASSIGN_OR_RAISE(reader, DuckDBStatementBatchReader::Create( 130 | statement, SqlSchema::GetTablesSchema())); 131 | return std::unique_ptr(new RecordBatchStream(reader)); 132 | } 133 | 134 | 135 | arrow::Result> GetFlightInfoTables( 136 | const ServerCallContext& context, const GetTables& command, 137 | const FlightDescriptor& descriptor) { 138 | std::vector endpoints{FlightEndpoint{{descriptor.cmd}, {}}}; 139 | 140 | bool include_schema = command.include_schema; 141 | 142 | ARROW_ASSIGN_OR_RAISE( 143 | auto result, 144 | FlightInfo::Make(include_schema ? *SqlSchema::GetTablesSchemaWithIncludedSchema() 145 | : *SqlSchema::GetTablesSchema(), 146 | descriptor, endpoints, -1, -1)) 147 | 148 | return std::unique_ptr(new FlightInfo(result)); 149 | } 150 | 151 | }; 152 | 153 | DuckDBFlightSqlServer::DuckDBFlightSqlServer(std::shared_ptr impl) 154 | : impl_(std::move(impl)) {} 155 | 156 | arrow::Result> DuckDBFlightSqlServer::Create( 157 | const std::string &path, 158 | const duckdb::DBConfig &config 159 | ) { 160 | std::shared_ptr db; 161 | std::shared_ptr con; 162 | 163 | db = std::make_shared(path); 164 | con = std::make_shared(*db); 165 | 166 | std::shared_ptr impl = std::make_shared(db, con); 167 | std::shared_ptr result(new DuckDBFlightSqlServer(impl)); 168 | 169 | for (const auto& id_to_result : GetSqlInfoResultMap()) { 170 | result->RegisterSqlInfo(id_to_result.first, id_to_result.second); 171 | } 172 | return result; 173 | } 174 | 175 | DuckDBFlightSqlServer::~DuckDBFlightSqlServer() = default; 176 | 177 | arrow::Result> DuckDBFlightSqlServer::GetFlightInfoStatement( 178 | const ServerCallContext& context, const StatementQuery& command, 179 | const FlightDescriptor& descriptor) { 180 | return impl_->GetFlightInfoStatement(context, command, descriptor); 181 | } 182 | 183 | arrow::Result> DuckDBFlightSqlServer::DoGetStatement( 184 | const ServerCallContext& context, const StatementQueryTicket& command) { 185 | return impl_->DoGetStatement(context, command); 186 | } 187 | 188 | arrow::Result> DuckDBFlightSqlServer::GetFlightInfoTables( 189 | const ServerCallContext& context, const GetTables& command, 190 | const FlightDescriptor& descriptor) { 191 | 192 | return impl_->GetFlightInfoTables(context, command, descriptor); 193 | } 194 | 195 | arrow::Result> DuckDBFlightSqlServer::DoGetTables( 196 | const ServerCallContext& context, const GetTables& command) { 197 | 198 | return impl_->DoGetTables(context, command); 199 | } 200 | 201 | } // namespace sqlite 202 | } // namespace sql 203 | } // namespace flight 204 | } // namespace arrow 205 | -------------------------------------------------------------------------------- /src/duckdb/duckdb_server.h: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | #include 23 | #include 24 | 25 | #include 26 | #include 27 | 28 | namespace arrow { 29 | namespace flight { 30 | namespace sql { 31 | namespace duckdbflight { 32 | 33 | /// \brief Convert a column type to a ArrowType. 34 | /// \param duckdb_type the duckdb type. 35 | /// \return The equivalent ArrowType. 36 | std::shared_ptr GetArrowType(const char* duckdb_type); 37 | 38 | /// \brief Example implementation of FlightSqlServerBase backed by an in-memory DuckDB 39 | /// database. 40 | class DuckDBFlightSqlServer : public FlightSqlServerBase { 41 | public: 42 | ~DuckDBFlightSqlServer() override; 43 | 44 | static arrow::Result> Create(const std::string &path, 45 | const duckdb::DBConfig &config); 46 | 47 | /// \brief Auxiliary method used to execute an arbitrary SQL statement on the underlying 48 | /// SQLite database. 49 | Status ExecuteSql(const std::string& sql); 50 | 51 | arrow::Result> GetFlightInfoStatement( 52 | const ServerCallContext& context, const StatementQuery& command, 53 | const FlightDescriptor& descriptor) override; 54 | 55 | arrow::Result> DoGetStatement( 56 | const ServerCallContext& context, const StatementQueryTicket& command) override; 57 | 58 | arrow::Result> GetFlightInfoTables( 59 | const ServerCallContext& context, const GetTables& command, 60 | const FlightDescriptor& descriptor) override; 61 | 62 | arrow::Result> DoGetTables( 63 | const ServerCallContext& context, const GetTables& command) override; 64 | 65 | private: 66 | class Impl; 67 | std::shared_ptr impl_; 68 | 69 | explicit DuckDBFlightSqlServer(std::shared_ptr impl); 70 | }; 71 | 72 | } // namespace duckdbflight 73 | } // namespace sql 74 | } // namespace flight 75 | } // namespace arrow 76 | -------------------------------------------------------------------------------- /src/duckdb/duckdb_sql_info.cpp: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #include "duckdb_sql_info.h" 19 | 20 | #include 21 | 22 | namespace arrow { 23 | namespace flight { 24 | namespace sql { 25 | namespace duckdbflight { 26 | 27 | /// \brief Gets the mapping from SQL info ids to SqlInfoResult instances. 28 | /// \return the cache. 29 | SqlInfoResultMap GetSqlInfoResultMap() { 30 | return { 31 | {SqlInfoOptions::SqlInfo::FLIGHT_SQL_SERVER_NAME, 32 | SqlInfoResult(std::string("db_name"))}, 33 | {SqlInfoOptions::SqlInfo::FLIGHT_SQL_SERVER_VERSION, 34 | SqlInfoResult(std::string("duckdb"))}, 35 | {SqlInfoOptions::SqlInfo::FLIGHT_SQL_SERVER_ARROW_VERSION, 36 | SqlInfoResult(std::string("8.0.0-SNAPSHOT" /* Only an example */))}, 37 | {SqlInfoOptions::SqlInfo::FLIGHT_SQL_SERVER_READ_ONLY, SqlInfoResult(false)}, 38 | {SqlInfoOptions::SqlInfo::SQL_DDL_CATALOG, 39 | SqlInfoResult(false)}, 40 | {SqlInfoOptions::SqlInfo::SQL_DDL_SCHEMA, 41 | SqlInfoResult(false)}, 42 | {SqlInfoOptions::SqlInfo::SQL_DDL_TABLE, SqlInfoResult(true)}, 43 | {SqlInfoOptions::SqlInfo::SQL_IDENTIFIER_CASE, 44 | SqlInfoResult(int64_t(SqlInfoOptions::SqlSupportedCaseSensitivity:: 45 | SQL_CASE_SENSITIVITY_CASE_INSENSITIVE))}, 46 | {SqlInfoOptions::SqlInfo::SQL_IDENTIFIER_QUOTE_CHAR, 47 | SqlInfoResult(std::string("\""))}, 48 | {SqlInfoOptions::SqlInfo::SQL_QUOTED_IDENTIFIER_CASE, 49 | SqlInfoResult(int64_t(SqlInfoOptions::SqlSupportedCaseSensitivity:: 50 | SQL_CASE_SENSITIVITY_CASE_INSENSITIVE))}, 51 | {SqlInfoOptions::SqlInfo::SQL_ALL_TABLES_ARE_SELECTABLE, SqlInfoResult(true)}, 52 | {SqlInfoOptions::SqlInfo::SQL_NULL_ORDERING, 53 | SqlInfoResult( 54 | int64_t(SqlInfoOptions::SqlNullOrdering::SQL_NULLS_SORTED_AT_START))}, 55 | {SqlInfoOptions::SqlInfo::SQL_KEYWORDS, 56 | SqlInfoResult(std::vector({"ABORT_P" 57 | "ABSOLUTE_P" 58 | "ACCESS" 59 | "ACTION" 60 | "ADD_P" 61 | "ADMIN" 62 | "AFTER" 63 | "AGGREGATE" 64 | "ALL" 65 | "ALSO" 66 | "ALTER" 67 | "ALWAYS" 68 | "ANALYSE" 69 | "ANALYZE" 70 | "AND" 71 | "ANY" 72 | "ARRAY" 73 | "AS" 74 | "ASC_P" 75 | "ASSERTION" 76 | "ASSIGNMENT" 77 | "ASYMMETRIC" 78 | "AT" 79 | "ATTACH" 80 | "ATTRIBUTE" 81 | "AUTHORIZATION" 82 | "AUTHORIZATION" 83 | "BACKWARD" 84 | "BEFORE" 85 | "BEGIN_P" 86 | "BETWEEN" 87 | "BIGINT" 88 | "BINARY" 89 | "BINARY" 90 | "BIT" 91 | "BOOLEAN_P" 92 | "BOTH" 93 | "BY" 94 | "CACHE" 95 | "CALL_P" 96 | "CALLED" 97 | "CASCADE" 98 | "CASCADED" 99 | "CASE" 100 | "CAST" 101 | "CATALOG_P" 102 | "CHAIN" 103 | "CHAR_P" 104 | "CHARACTER" 105 | "CHARACTERISTICS" 106 | "CHECK_P" 107 | "CHECKPOINT" 108 | "CLASS" 109 | "CLOSE" 110 | "CLUSTER" 111 | "COALESCE" 112 | "COLLATE" 113 | "COLLATION" 114 | "COLLATION" 115 | "COLUMN" 116 | "COLUMNS" 117 | "COMMENT" 118 | "COMMENTS" 119 | "COMMIT" 120 | "COMMITTED" 121 | "CONCURRENTLY" 122 | "CONCURRENTLY" 123 | "CONFIGURATION" 124 | "CONFLICT" 125 | "CONNECTION" 126 | "CONSTRAINT" 127 | "CONSTRAINTS" 128 | "CONTENT_P" 129 | "CONTINUE_P" 130 | "CONVERSION_P" 131 | "COPY" 132 | "COST" 133 | "CREATE_P" 134 | "CROSS" 135 | "CROSS" 136 | "CSV" 137 | "CUBE" 138 | "CURRENT_CATALOG" 139 | "CURRENT_DATE" 140 | "CURRENT_P" 141 | "CURRENT_ROLE" 142 | "CURRENT_SCHEMA" 143 | "CURRENT_SCHEMA" 144 | "CURRENT_TIME" 145 | "CURRENT_TIMESTAMP" 146 | "CURRENT_USER" 147 | "CURSOR" 148 | "CYCLE" 149 | "DATA_P" 150 | "DATABASE" 151 | "DAY_P" 152 | "DAYS_P" 153 | "DEALLOCATE" 154 | "DEC" 155 | "DECIMAL_P" 156 | "DECLARE" 157 | "DEFAULT" 158 | "DEFAULTS" 159 | "DEFERRABLE" 160 | "DEFERRED" 161 | "DEFINER" 162 | "DELETE_P" 163 | "DELIMITER" 164 | "DELIMITERS" 165 | "DEPENDS" 166 | "DESC_P" 167 | "DESCRIBE" 168 | "DETACH" 169 | "DICTIONARY" 170 | "DISABLE_P" 171 | "DISCARD" 172 | "DISTINCT" 173 | "DO" 174 | "DOCUMENT_P" 175 | "DOMAIN_P" 176 | "DOUBLE_P" 177 | "DROP" 178 | "EACH" 179 | "ELSE" 180 | "ENABLE_P" 181 | "ENCODING" 182 | "ENCRYPTED" 183 | "END_P" 184 | "ENUM_P" 185 | "ESCAPE" 186 | "EVENT" 187 | "EXCEPT" 188 | "EXCLUDE" 189 | "EXCLUDING" 190 | "EXCLUSIVE" 191 | "EXECUTE" 192 | "EXISTS" 193 | "EXPLAIN" 194 | "EXPORT_P" 195 | "EXTENSION" 196 | "EXTERNAL" 197 | "EXTRACT" 198 | "FALSE_P" 199 | "FAMILY" 200 | "FETCH" 201 | "FILTER" 202 | "FIRST_P" 203 | "FLOAT_P" 204 | "FOLLOWING" 205 | "FOR" 206 | "FORCE" 207 | "FOREIGN" 208 | "FORWARD" 209 | "FREEZE" 210 | "FREEZE" 211 | "FROM" 212 | "FULL" 213 | "FULL" 214 | "FUNCTION" 215 | "FUNCTIONS" 216 | "GENERATED" 217 | "GLOB" 218 | "GLOB" 219 | "GLOBAL" 220 | "GRANT" 221 | "GRANTED" 222 | "GROUP_P" 223 | "GROUPING" 224 | "HANDLER" 225 | "HAVING" 226 | "HEADER_P" 227 | "HOLD" 228 | "HOUR_P" 229 | "HOURS_P" 230 | "IDENTITY_P" 231 | "IF_P" 232 | "ILIKE" 233 | "ILIKE" 234 | "IMMEDIATE" 235 | "IMMUTABLE" 236 | "IMPLICIT_P" 237 | "IMPORT_P" 238 | "IN_P" 239 | "INCLUDING" 240 | "INCREMENT" 241 | "INDEX" 242 | "INDEXES" 243 | "INHERIT" 244 | "INHERITS" 245 | "INITIALLY" 246 | "INLINE_P" 247 | "INNER_P" 248 | "INNER_P" 249 | "INOUT" 250 | "INPUT_P" 251 | "INSENSITIVE" 252 | "INSERT" 253 | "INSTEAD" 254 | "INT_P" 255 | "INTEGER" 256 | "INTERSECT" 257 | "INTERVAL" 258 | "INTO" 259 | "INVOKER" 260 | "IS" 261 | "IS" 262 | "ISNULL" 263 | "ISNULL" 264 | "ISOLATION" 265 | "JOIN" 266 | "JOIN" 267 | "KEY" 268 | "LABEL" 269 | "LANGUAGE" 270 | "LARGE_P" 271 | "LAST_P" 272 | "LATERAL_P" 273 | "LEADING" 274 | "LEAKPROOF" 275 | "LEFT" 276 | "LEFT" 277 | "LEVEL" 278 | "LIKE" 279 | "LIKE" 280 | "LIMIT" 281 | "LISTEN" 282 | "LOAD" 283 | "LOCAL" 284 | "LOCALTIME" 285 | "LOCALTIMESTAMP" 286 | "LOCATION" 287 | "LOCK_P" 288 | "LOCKED" 289 | "LOGGED" 290 | "MACRO" 291 | "MAP" 292 | "MAP" 293 | "MAPPING" 294 | "MATCH" 295 | "MATERIALIZED" 296 | "MAXVALUE" 297 | "METHOD" 298 | "MICROSECOND_P" 299 | "MICROSECONDS_P" 300 | "MILLISECOND_P" 301 | "MILLISECONDS_P" 302 | "MINUTE_P" 303 | "MINUTES_P" 304 | "MINVALUE" 305 | "MODE" 306 | "MONTH_P" 307 | "MONTHS_P" 308 | "MOVE" 309 | "NAME_P" 310 | "NAMES" 311 | "NATIONAL" 312 | "NATURAL" 313 | "NATURAL" 314 | "NCHAR" 315 | "NEW" 316 | "NEXT" 317 | "NO" 318 | "NONE" 319 | "NOT" 320 | "NOTHING" 321 | "NOTIFY" 322 | "NOTNULL" 323 | "NOTNULL" 324 | "NOWAIT" 325 | "NULL_P" 326 | "NULLIF" 327 | "NULLS_P" 328 | "NUMERIC" 329 | "OBJECT_P" 330 | "OF" 331 | "OFF" 332 | "OFFSET" 333 | "OIDS" 334 | "OLD" 335 | "ON" 336 | "ONLY" 337 | "OPERATOR" 338 | "OPTION" 339 | "OPTIONS" 340 | "OR" 341 | "ORDER" 342 | "ORDINALITY" 343 | "OUT_P" 344 | "OUTER_P" 345 | "OUTER_P" 346 | "OVER" 347 | "OVERLAPS" 348 | "OVERLAPS" 349 | "OVERLAY" 350 | "OVERRIDING" 351 | "OWNED" 352 | "OWNER" 353 | "PARALLEL" 354 | "PARSER" 355 | "PARTIAL" 356 | "PARTITION" 357 | "PASSING" 358 | "PASSWORD" 359 | "PERCENT" 360 | "PLACING" 361 | "PLANS" 362 | "POLICY" 363 | "POSITION" 364 | "PRAGMA_P" 365 | "PRECEDING" 366 | "PRECISION" 367 | "PREPARE" 368 | "PREPARED" 369 | "PRESERVE" 370 | "PRIMARY" 371 | "PRIOR" 372 | "PRIVILEGES" 373 | "PROCEDURAL" 374 | "PROCEDURE" 375 | "PROGRAM" 376 | "PUBLICATION" 377 | "QUOTE" 378 | "RANGE" 379 | "READ_P" 380 | "REAL" 381 | "REASSIGN" 382 | "RECHECK" 383 | "RECURSIVE" 384 | "REF" 385 | "REFERENCES" 386 | "REFERENCING" 387 | "REFRESH" 388 | "REINDEX" 389 | "RELATIVE_P" 390 | "RELEASE" 391 | "RENAME" 392 | "REPEATABLE" 393 | "REPLACE" 394 | "REPLICA" 395 | "RESET" 396 | "RESTART" 397 | "RESTRICT" 398 | "RETURNING" 399 | "RETURNS" 400 | "REVOKE" 401 | "RIGHT" 402 | "RIGHT" 403 | "ROLE" 404 | "ROLLBACK" 405 | "ROLLUP" 406 | "ROW" 407 | "ROWS" 408 | "RULE" 409 | "SAMPLE" 410 | "SAVEPOINT" 411 | "SCHEMA" 412 | "SCHEMAS" 413 | "SCROLL" 414 | "SEARCH" 415 | "SECOND_P" 416 | "SECONDS_P" 417 | "SECURITY" 418 | "SELECT" 419 | "SEQUENCE" 420 | "SEQUENCES" 421 | "SERIALIZABLE" 422 | "SERVER" 423 | "SESSION" 424 | "SESSION_USER" 425 | "SET" 426 | "SETOF" 427 | "SETS" 428 | "SHARE" 429 | "SHOW" 430 | "SIMILAR" 431 | "SIMILAR" 432 | "SIMPLE" 433 | "SKIP" 434 | "SMALLINT" 435 | "SNAPSHOT" 436 | "SOME" 437 | "SQL_P" 438 | "STABLE" 439 | "STANDALONE_P" 440 | "START" 441 | "STATEMENT" 442 | "STATISTICS" 443 | "STDIN" 444 | "STDOUT" 445 | "STORAGE" 446 | "STRICT_P" 447 | "STRIP_P" 448 | "STRUCT" 449 | "STRUCT" 450 | "SUBSCRIPTION" 451 | "SUBSTRING" 452 | "SYMMETRIC" 453 | "SYSID" 454 | "SYSTEM_P" 455 | "TABLE" 456 | "TABLES" 457 | "TABLESAMPLE" 458 | "TABLESAMPLE" 459 | "TABLESPACE" 460 | "TEMP" 461 | "TEMPLATE" 462 | "TEMPORARY" 463 | "TEXT_P" 464 | "THEN" 465 | "TIME" 466 | "TIMESTAMP" 467 | "TO" 468 | "TRAILING" 469 | "TRANSACTION" 470 | "TRANSFORM" 471 | "TREAT" 472 | "TRIGGER" 473 | "TRIM" 474 | "TRUE_P" 475 | "TRUNCATE" 476 | "TRUSTED" 477 | "TRY_CAST" 478 | "TRY_CAST" 479 | "TYPE_P" 480 | "TYPES_P" 481 | "UNBOUNDED" 482 | "UNCOMMITTED" 483 | "UNENCRYPTED" 484 | "UNION" 485 | "UNIQUE" 486 | "UNKNOWN" 487 | "UNLISTEN" 488 | "UNLOGGED" 489 | "UNTIL" 490 | "UPDATE" 491 | "USER" 492 | "USING" 493 | "VACUUM" 494 | "VALID" 495 | "VALIDATE" 496 | "VALIDATOR" 497 | "VALUE_P" 498 | "VALUES" 499 | "VARCHAR" 500 | "VARIADIC" 501 | "VARYING" 502 | "VERBOSE" 503 | "VERBOSE" 504 | "VERSION_P" 505 | "VIEW" 506 | "VIEWS" 507 | "VOLATILE" 508 | "WHEN" 509 | "WHERE" 510 | "WHITESPACE_P" 511 | "WINDOW" 512 | "WITH" 513 | "WITHIN" 514 | "WITHOUT" 515 | "WORK" 516 | "WRAPPER" 517 | "WRITE_P" 518 | "XML_P" 519 | "XMLATTRIBUTES" 520 | "XMLCONCAT" 521 | "XMLELEMENT" 522 | "XMLEXISTS" 523 | "XMLFOREST" 524 | "XMLNAMESPACES" 525 | "XMLPARSE" 526 | "XMLPI" 527 | "XMLROOT" 528 | "XMLSERIALIZE" 529 | "XMLTABLE" 530 | "YEAR_P" 531 | "YEARS_P" 532 | "YES_P" 533 | "ZONE"}))}, 534 | {SqlInfoOptions::SqlInfo::SQL_NUMERIC_FUNCTIONS, 535 | SqlInfoResult(std::vector( 536 | {"ABS" 537 | "ACOS" 538 | "ASIN" 539 | "ATAN2" 540 | "ATAN2" 541 | "BIT_COUNT" 542 | "CBRT" 543 | "CEIL" 544 | "CEILING" 545 | "CHR" 546 | "COS" 547 | "COT" 548 | "DEGREES" 549 | "EVEN" 550 | "FACTORIAL" 551 | "FLOOR" 552 | "GAMMA" 553 | "GREATEST" 554 | "LEAST" 555 | "LGAMMA" 556 | "LN" 557 | "LOG" 558 | "LOG2" 559 | "LOG10" 560 | "NEXTAFTER" 561 | "PI" 562 | "POW" 563 | "POWER" 564 | "RADIANS" 565 | "RANDOM" 566 | "ROUND" 567 | "SETSEED" 568 | "SIN" 569 | "SIGN" 570 | "SQRT" 571 | "XOR" 572 | "TAN" 573 | "@"}))}, 574 | {SqlInfoOptions::SqlInfo::SQL_STRING_FUNCTIONS, 575 | SqlInfoResult( 576 | std::vector({"ARRAY_EXTRACT" 577 | "ARRAY_SLICE" 578 | "ASCII" 579 | "BASE64" 580 | "BIT_LENGTH" 581 | "CONCAT" 582 | "CONCAT_WS" 583 | "CONTAINS" 584 | "FORMAT" 585 | "FROM_BASE64" 586 | "INSTR" 587 | "LCASE" 588 | "LEFT" 589 | "LENGTH" 590 | "STRING" 591 | "LIKE_ESCAPE" 592 | "LIST_ELEMENT" 593 | "LIST_EXTRACT" 594 | "LOWER" 595 | "LPAD" 596 | "LTRIM" 597 | "LTRIM" 598 | "MD5" 599 | "NFC_NORMALIZE" 600 | "NOT_LIKE_ESCAPE" 601 | "ORD" 602 | "POSITION" 603 | "PREFIX" 604 | "PRINTF" 605 | "REGEXP_FULL_MATCH" 606 | "REGEXP_MATCHES" 607 | "REGEXP_REPLACE" 608 | "REGEXP_SPLIT_TO_ARRAY" 609 | "REPEAT" 610 | "REPLACE" 611 | "REVERSE" 612 | "RIGHT" 613 | "RPAD" 614 | "RTRIM" 615 | "RTRIM" 616 | "STRING" 617 | "STRLEN" 618 | "STRPOS" 619 | "STRIP_ACCENTS" 620 | "STR_SPLIT" 621 | "STR_SPLIT_REGEX" 622 | "STRING_SPLIT" 623 | "STRING_SPLIT_REGEX" 624 | "STRING_TO_ARRAY" 625 | "SUBSTR" 626 | "SUBSTRING" 627 | "SUFFIX" 628 | "STRPOS" 629 | "TO_BASE64" 630 | "TRIM" 631 | "TRIM" 632 | "UCASE" 633 | "UNICODE" 634 | "UPPER" 635 | "EDITDIST3" 636 | "HAMMING" 637 | "JACCARD" 638 | "LEVENSHTEIN" 639 | "MISMATCHES"}))}, 640 | {SqlInfoOptions::SqlInfo::SQL_SUPPORTS_CONVERT, 641 | SqlInfoResult(std::unordered_map>( 642 | {{SqlInfoOptions::SqlSupportsConvert::SQL_CONVERT_BIGINT, 643 | std::vector( 644 | {SqlInfoOptions::SqlSupportsConvert::SQL_CONVERT_INTEGER})}}))}}; 645 | } 646 | 647 | } // namespace sqlite 648 | } // namespace sql 649 | } // namespace flight 650 | } // namespace arrow 651 | -------------------------------------------------------------------------------- /src/duckdb/duckdb_sql_info.h: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | namespace arrow { 23 | namespace flight { 24 | namespace sql { 25 | namespace duckdbflight { 26 | 27 | /// \brief Gets the mapping from SQL info ids to SqlInfoResult instances. 28 | /// \return the cache. 29 | SqlInfoResultMap GetSqlInfoResultMap(); 30 | 31 | } // namespace sqlite 32 | } // namespace sql 33 | } // namespace flight 34 | } // namespace arrow 35 | -------------------------------------------------------------------------------- /src/duckdb/duckdb_statement.cpp: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #include "duckdb_statement.h" 19 | 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | #include 27 | #include "duckdb_server.h" 28 | 29 | using duckdb::QueryResult; 30 | 31 | namespace arrow { 32 | namespace flight { 33 | namespace sql { 34 | namespace duckdbflight { 35 | 36 | std::shared_ptr GetDataTypeFromDuckDbType( 37 | const duckdb::LogicalTypeId column_type, 38 | const duckdb::LogicalType column 39 | ) { 40 | switch (column_type) { 41 | case duckdb::LogicalTypeId::INTEGER: 42 | return int32(); 43 | case duckdb::LogicalTypeId::DECIMAL: { 44 | uint8_t width = 0; 45 | uint8_t scale = 0; 46 | bool dec_properties = column.GetDecimalProperties(width, scale); 47 | return decimal(scale, width); 48 | } 49 | case duckdb::LogicalTypeId::FLOAT: 50 | return float32(); 51 | case duckdb::LogicalTypeId::DOUBLE: 52 | return float64(); 53 | case duckdb::LogicalTypeId::CHAR: 54 | case duckdb::LogicalTypeId::VARCHAR: 55 | return utf8(); 56 | case duckdb::LogicalTypeId::BLOB: 57 | return binary(); 58 | case duckdb::LogicalTypeId::TINYINT: 59 | return int8(); 60 | case duckdb::LogicalTypeId::SMALLINT: 61 | return int16(); 62 | case duckdb::LogicalTypeId::BIGINT: 63 | return int64(); 64 | case duckdb::LogicalTypeId::BOOLEAN: 65 | return boolean(); 66 | case duckdb::LogicalTypeId::DATE: 67 | return date64(); 68 | case duckdb::LogicalTypeId::TIME: 69 | case duckdb::LogicalTypeId::TIMESTAMP_MS: 70 | return timestamp(arrow::TimeUnit::MILLI); 71 | case duckdb::LogicalTypeId::TIMESTAMP: 72 | return timestamp(arrow::TimeUnit::MICRO); 73 | case duckdb::LogicalTypeId::TIMESTAMP_SEC: 74 | return timestamp(arrow::TimeUnit::SECOND); 75 | case duckdb::LogicalTypeId::TIMESTAMP_NS: 76 | return timestamp(arrow::TimeUnit::NANO); 77 | case duckdb::LogicalTypeId::INTERVAL: 78 | return duration(arrow::TimeUnit::MICRO); // ASSUMING MICRO AS DUCKDB's DOCS DOES NOT SPECIFY 79 | case duckdb::LogicalTypeId::UTINYINT: 80 | return uint8(); 81 | case duckdb::LogicalTypeId::USMALLINT: 82 | return uint16(); 83 | case duckdb::LogicalTypeId::UINTEGER: 84 | return uint32(); 85 | case duckdb::LogicalTypeId::UBIGINT: 86 | return uint64(); 87 | case duckdb::LogicalTypeId::INVALID: 88 | case duckdb::LogicalTypeId::SQLNULL: 89 | case duckdb::LogicalTypeId::UNKNOWN: 90 | case duckdb::LogicalTypeId::ANY: 91 | case duckdb::LogicalTypeId::USER: 92 | case duckdb::LogicalTypeId::TIMESTAMP_TZ: 93 | case duckdb::LogicalTypeId::TIME_TZ: 94 | case duckdb::LogicalTypeId::HUGEINT: 95 | case duckdb::LogicalTypeId::POINTER: 96 | case duckdb::LogicalTypeId::HASH: 97 | case duckdb::LogicalTypeId::VALIDITY: 98 | case duckdb::LogicalTypeId::UUID: 99 | case duckdb::LogicalTypeId::STRUCT: 100 | case duckdb::LogicalTypeId::LIST: 101 | case duckdb::LogicalTypeId::MAP: 102 | case duckdb::LogicalTypeId::TABLE: 103 | case duckdb::LogicalTypeId::ENUM: 104 | default: 105 | return null(); 106 | } 107 | } 108 | 109 | arrow::Result> DuckDBStatement::Create( 110 | std::shared_ptr con, const std::string& sql) { 111 | std::shared_ptr stmt = con->Prepare(sql); 112 | std::shared_ptr result(new DuckDBStatement(con, stmt)); 113 | 114 | return result; 115 | } 116 | 117 | DuckDBStatement::~DuckDBStatement() { 118 | } 119 | 120 | arrow::Result DuckDBStatement::Execute() { 121 | auto res = stmt_->Execute(); 122 | 123 | ArrowArray res_arr; 124 | ArrowSchema res_schema; 125 | 126 | QueryResult::ToArrowSchema(&res_schema, res->types, res->names); 127 | res->Fetch()->ToArrowArray(&res_arr); 128 | ARROW_ASSIGN_OR_RAISE(result_, arrow::ImportRecordBatch(&res_arr, &res_schema)); 129 | schema_ = result_->schema(); 130 | 131 | return 0; 132 | } 133 | 134 | arrow::Result> DuckDBStatement::GetResult() { 135 | return result_; 136 | } 137 | 138 | arrow::Result> DuckDBStatement::GetSchema() const { 139 | std::vector> fields; 140 | 141 | int column_count = stmt_->ColumnCount(); 142 | auto column_names = stmt_->GetNames(); 143 | auto column_types = stmt_->GetTypes(); 144 | 145 | for (int i = 0; i < column_count; i++) { 146 | std::string column_name = column_names[i]; 147 | std::shared_ptr data_type = GetDataTypeFromDuckDbType(column_types[i].id(), column_types[i]); 148 | ColumnMetadata column_metadata = ColumnMetadata::Builder().Build(); 149 | 150 | fields.push_back( 151 | arrow::field(column_name, data_type, column_metadata.metadata_map())); 152 | } 153 | return arrow::schema(fields); 154 | } 155 | 156 | } // namespace sqlite 157 | } // namespace sql 158 | } // namespace flight 159 | } // namespace arrow 160 | -------------------------------------------------------------------------------- /src/duckdb/duckdb_statement.h: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | #include 23 | #include 24 | 25 | #include 26 | #include 27 | 28 | namespace arrow { 29 | namespace flight { 30 | namespace sql { 31 | namespace duckdbflight { 32 | 33 | /// \brief Create an object ColumnMetadata using the column type and 34 | /// table name. 35 | /// \param column_type The DuckDB type. 36 | /// \param table The table name. 37 | /// \return A Column Metadata object. 38 | ColumnMetadata GetColumnMetadata(int column_type, const char* table); 39 | 40 | class DuckDBStatement { 41 | public: 42 | /// \brief Creates a duckdb statement. 43 | /// \param[in] db duckdb database instance. 44 | /// \param[in] sql SQL statement. 45 | /// \return A DuckDBStatement object. 46 | static arrow::Result> Create(std::shared_ptr con, 47 | const std::string& sql); 48 | 49 | ~DuckDBStatement(); 50 | 51 | /// \brief Creates an Arrow Schema based on the results of this statement. 52 | /// \return The resulting Schema. 53 | arrow::Result> GetSchema() const; 54 | 55 | arrow::Result Execute(); 56 | arrow::Result> GetResult(); 57 | // arrow::Result> GetArrowSchema(); 58 | 59 | private: 60 | std::shared_ptr con_; 61 | std::shared_ptr stmt_; 62 | std::shared_ptr result_; 63 | std::shared_ptr schema_; 64 | 65 | DuckDBStatement( 66 | std::shared_ptr con, 67 | std::shared_ptr stmt) { 68 | con_ = con; 69 | stmt_ = stmt; 70 | } 71 | }; 72 | 73 | } // namespace duckdbflight 74 | } // namespace sql 75 | } // namespace flight 76 | } // namespace arrow 77 | -------------------------------------------------------------------------------- /src/duckdb/duckdb_statement_batch_reader.cpp: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #include "duckdb_statement_batch_reader.h" 19 | 20 | #include 21 | 22 | #include 23 | 24 | #include "arrow/builder.h" 25 | #include 26 | 27 | #include "duckdb_statement.h" 28 | 29 | namespace arrow { 30 | namespace flight { 31 | namespace sql { 32 | namespace duckdbflight { 33 | 34 | // Batch size for SQLite statement results 35 | static constexpr int kMaxBatchSize = 1024; 36 | 37 | std::shared_ptr DuckDBStatementBatchReader::schema() const { return schema_; } 38 | 39 | DuckDBStatementBatchReader::DuckDBStatementBatchReader( 40 | std::shared_ptr statement, std::shared_ptr schema) 41 | : statement_(std::move(statement)), 42 | schema_(std::move(schema)), 43 | rc_(DuckDBSuccess), 44 | already_executed_(false), 45 | results_read_(false) {} 46 | 47 | arrow::Result> 48 | DuckDBStatementBatchReader::Create(const std::shared_ptr& statement_) { 49 | ARROW_ASSIGN_OR_RAISE(auto schema, statement_->GetSchema()); 50 | 51 | std::shared_ptr result( 52 | new DuckDBStatementBatchReader(statement_, schema)); 53 | 54 | return result; 55 | } 56 | 57 | arrow::Result> 58 | DuckDBStatementBatchReader::Create(const std::shared_ptr& statement, 59 | const std::shared_ptr& schema) { 60 | std::shared_ptr result( 61 | new DuckDBStatementBatchReader(statement, schema)); 62 | 63 | return result; 64 | } 65 | 66 | Status DuckDBStatementBatchReader::ReadNext(std::shared_ptr* out) { 67 | 68 | if (!already_executed_) { 69 | ARROW_ASSIGN_OR_RAISE(rc_, statement_->Execute()); 70 | already_executed_ = true; 71 | } 72 | if(!results_read_) { 73 | ARROW_ASSIGN_OR_RAISE(*out, statement_->GetResult()); 74 | results_read_ = true; 75 | 76 | } else { 77 | *out = nullptr; 78 | } 79 | 80 | return Status::OK(); 81 | } 82 | 83 | } // namespace duckdbflight 84 | } // namespace sql 85 | } // namespace flight 86 | } // namespace arrow 87 | -------------------------------------------------------------------------------- /src/duckdb/duckdb_statement_batch_reader.h: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #pragma once 19 | 20 | #include 21 | #include 22 | #include 23 | #include "duckdb_statement.h" 24 | 25 | namespace arrow { 26 | namespace flight { 27 | namespace sql { 28 | namespace duckdbflight { 29 | 30 | class DuckDBStatementBatchReader : public RecordBatchReader { 31 | public: 32 | /// \brief Creates a RecordBatchReader backed by a duckdb statement. 33 | /// \param[in] statement duckdb statement to be read. 34 | /// \return A DuckDBStatementBatchReader. 35 | static arrow::Result> Create( 36 | const std::shared_ptr& statement); 37 | 38 | /// \brief Creates a RecordBatchReader backed by a duckdb statement. 39 | /// \param[in] statement duckdb statement to be read. 40 | /// \param[in] schema Schema to be used on results. 41 | /// \return A DuckDBStatementBatchReader.. 42 | static arrow::Result> Create( 43 | const std::shared_ptr& statement, 44 | const std::shared_ptr& schema); 45 | 46 | std::shared_ptr schema() const override; 47 | 48 | Status ReadNext(std::shared_ptr* out) override; 49 | 50 | private: 51 | std::shared_ptr statement_; 52 | std::shared_ptr schema_; 53 | int rc_; 54 | bool already_executed_; 55 | bool results_read_; 56 | 57 | DuckDBStatementBatchReader(std::shared_ptr statement, 58 | std::shared_ptr schema); 59 | }; 60 | 61 | } // namespace duckdbflight 62 | } // namespace sql 63 | } // namespace flight 64 | } // namespace arrow 65 | -------------------------------------------------------------------------------- /src/flight_sql.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "sqlite/sqlite_server.h" 17 | #include "duckdb/duckdb_server.h" 18 | 19 | namespace flight = arrow::flight; 20 | namespace flightsql = arrow::flight::sql; 21 | namespace po = boost::program_options; 22 | 23 | int port = 31337; 24 | 25 | arrow::Status printResults( 26 | std::unique_ptr &results, 27 | std::unique_ptr &client, 28 | const flight::FlightCallOptions &call_options, 29 | const bool& print_results_flag) { 30 | // Fetch each partition sequentially (though this can be done in parallel) 31 | for (const flight::FlightEndpoint& endpoint : results->endpoints()) { 32 | // Here we assume each partition is on the same server we originally queried, but this 33 | // isn't true in general: the server may split the query results between multiple 34 | // other servers, which we would have to connect to. 35 | 36 | // The "ticket" in the endpoint is opaque to the client. The server uses it to 37 | // identify which part of the query results to return. 38 | ARROW_ASSIGN_OR_RAISE(auto stream, client->DoGet(call_options, endpoint.ticket)); 39 | // Read all results into an Arrow Table, though we can iteratively process record 40 | // batches as they arrive as well 41 | ARROW_ASSIGN_OR_RAISE(auto table, stream->ToTable()); 42 | 43 | if(print_results_flag) std::cout << table->ToString() << std::endl; 44 | } 45 | 46 | return arrow::Status::OK(); 47 | } 48 | 49 | std::string readFileIntoString(const std::string& path) { 50 | auto ss = std::ostringstream{}; 51 | std::ifstream input_file(path); 52 | if (!input_file.is_open()) { 53 | std::cerr << "Could not open the file - '" 54 | << path << "'" << std::endl; 55 | exit(EXIT_FAILURE); 56 | } 57 | ss << input_file.rdbuf(); 58 | return ss.str(); 59 | } 60 | 61 | bool checkIfSkip(std::string path, int query_id) { 62 | std::string query_id_str = std::to_string(query_id); 63 | 64 | if (path.find(query_id_str) != std::string::npos) { 65 | std::cout << "Skipping query: " << query_id << '\n'; 66 | return true; 67 | } 68 | return false; 69 | } 70 | 71 | arrow::Status runQueries( 72 | std::unique_ptr &client, 73 | const std::string &query_path, 74 | const std::vector &skip_queries, 75 | flight::FlightCallOptions &call_options, 76 | const bool& print_results_flag 77 | ) { 78 | int skip_vector_it = 0; 79 | for (const auto & file : std::filesystem::directory_iterator(query_path)) { 80 | std::cout << "Query: " << file.path() << std::endl; 81 | if (skip_vector_it < skip_queries.size()) { 82 | if (checkIfSkip(file.path(), skip_queries.at(skip_vector_it))) { 83 | ++skip_vector_it; 84 | continue; 85 | } 86 | } 87 | std::string kQuery = readFileIntoString(file.path()); 88 | 89 | if (print_results_flag) std::cout << "Executing query: '" << kQuery << "'" << std::endl; 90 | ARROW_ASSIGN_OR_RAISE(auto flight_info, client->Execute(call_options, kQuery)); 91 | 92 | if (flight_info != nullptr) { 93 | ARROW_RETURN_NOT_OK(printResults(flight_info, client, call_options, print_results_flag)); 94 | } 95 | } 96 | 97 | return arrow::Status::OK(); 98 | } 99 | 100 | arrow::Result> CreateServer( 101 | const std::string &db_type, 102 | const std::string &db_path 103 | ) { 104 | ARROW_ASSIGN_OR_RAISE(auto location, 105 | arrow::flight::Location::ForGrpcTcp("0.0.0.0", port)); 106 | arrow::flight::FlightServerOptions options(location); 107 | 108 | std::shared_ptr server = nullptr; 109 | 110 | if (db_type == "sqlite") { 111 | ARROW_ASSIGN_OR_RAISE(server, 112 | arrow::flight::sql::sqlite::SQLiteFlightSqlServer::Create(db_path)); 113 | } else if (db_type == "duckdb") { 114 | duckdb::DBConfig config; 115 | ARROW_ASSIGN_OR_RAISE(server, 116 | arrow::flight::sql::duckdbflight::DuckDBFlightSqlServer::Create(db_path, config)); 117 | } else { 118 | std::string err_msg = "Unknown server type: --> "; 119 | err_msg += db_type; 120 | return arrow::Status::Invalid(err_msg); 121 | } 122 | 123 | if (server != nullptr) { 124 | ARROW_CHECK_OK(server->Init(options)); 125 | // Exit with a clean error code (0) on SIGTERM 126 | ARROW_CHECK_OK(server->SetShutdownOnSignals({SIGTERM})); 127 | 128 | std::cout << db_type << " server listening on localhost:" << server->port() << std::endl; 129 | return server; 130 | } else { 131 | std::string err_msg = "Unable to start the server"; 132 | return arrow::Status::Invalid(err_msg); 133 | } 134 | } 135 | 136 | arrow::Result> CreateClient() { 137 | ARROW_ASSIGN_OR_RAISE(auto location, 138 | arrow::flight::Location::ForGrpcTcp("localhost", port)); 139 | arrow::flight::FlightServerOptions options(location); 140 | 141 | ARROW_ASSIGN_OR_RAISE(auto flight_client, flight::FlightClient::Connect(location)); 142 | std::cout << "Connected to server: localhost:" << port << std::endl; 143 | 144 | std::unique_ptr client( 145 | new flightsql::FlightSqlClient(std::move(flight_client))); 146 | std::cout << "Client created." << std::endl; 147 | 148 | return client; 149 | } 150 | 151 | arrow::Status Main(const std::string& backend, const bool& print_results_flag) { 152 | std::map databases; 153 | databases["duckdb"] = "../data/TPC-H-small.duckdb"; 154 | databases["sqlite"] = "../data/TPC-H-small.db"; 155 | 156 | ARROW_ASSIGN_OR_RAISE(auto server, CreateServer(backend, databases[backend])); 157 | 158 | std::string query_path = "../queries"; 159 | std::vector skip_queries = {17}; // the rest of the code assumes this is ORDERED vector! 160 | ARROW_ASSIGN_OR_RAISE(auto client, CreateClient()); 161 | 162 | flight::FlightCallOptions call_options; 163 | ARROW_ASSIGN_OR_RAISE(std::unique_ptr tables, client->GetTables(call_options, NULL, NULL, NULL, NULL, NULL)); 164 | 165 | if (tables != nullptr && print_results_flag) { 166 | ARROW_RETURN_NOT_OK(printResults(tables, client, call_options, print_results_flag)); 167 | } 168 | 169 | ARROW_RETURN_NOT_OK(runQueries(client, query_path, skip_queries, call_options, print_results_flag)); 170 | 171 | return arrow::Status::OK(); 172 | } 173 | 174 | bool string2bool(const std::string &v) 175 | { 176 | return !v.empty () && 177 | (strcasecmp (v.c_str (), "true") == 0 || 178 | atoi (v.c_str ()) != 0); 179 | } 180 | 181 | int main(int argc, char** argv) { 182 | 183 | // Declare the supported options. 184 | po::options_description desc("Allowed options"); 185 | desc.add_options() 186 | ("help", "produce this help message") 187 | ("backend,B", po::value()->default_value("duckdb"), "Specify the database backend. Allowed options: duckdb, sqlite.") 188 | ("print", po::value()->default_value("false"), "Print the results of running queries. Allowed options: false, true.") 189 | ; 190 | 191 | po::variables_map vm; 192 | po::store(po::parse_command_line(argc, argv, desc), vm); 193 | po::notify(vm); 194 | 195 | if (vm.count("help")) { 196 | std::cout << desc << "\n"; 197 | return 1; 198 | } 199 | 200 | std::string backend = vm["backend"].as(); 201 | bool print_results_flag = string2bool(vm["print"].as()); 202 | 203 | auto status = Main(backend, print_results_flag); 204 | if (!status.ok()) { 205 | std::cerr << status << std::endl; 206 | return 1; 207 | } 208 | 209 | return EXIT_SUCCESS; 210 | } -------------------------------------------------------------------------------- /src/sqlite/sqlite_server.cc: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #include "sqlite_server.h" 19 | 20 | #include 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | #include 30 | 31 | #include "sqlite_sql_info.h" 32 | #include "sqlite_statement.h" 33 | #include "sqlite_statement_batch_reader.h" 34 | #include "sqlite_tables_schema_batch_reader.h" 35 | #include "sqlite_type_info.h" 36 | 37 | namespace arrow { 38 | namespace flight { 39 | namespace sql { 40 | namespace sqlite { 41 | 42 | namespace { 43 | 44 | /// \brief Gets a SqliteStatement by given handle 45 | arrow::Result> GetStatementByHandle( 46 | const std::map>& prepared_statements, 47 | const std::string& handle) { 48 | auto search = prepared_statements.find(handle); 49 | if (search == prepared_statements.end()) { 50 | return Status::Invalid("Prepared statement not found"); 51 | } 52 | 53 | return search->second; 54 | } 55 | 56 | std::string PrepareQueryForGetTables(const GetTables& command) { 57 | std::stringstream table_query; 58 | 59 | table_query << "SELECT null as catalog_name, null as schema_name, name as " 60 | "table_name, type as table_type FROM sqlite_master where 1=1"; 61 | 62 | if (command.catalog.has_value()) { 63 | table_query << " and catalog_name='" << command.catalog.value() << "'"; 64 | } 65 | 66 | if (command.db_schema_filter_pattern.has_value()) { 67 | table_query << " and schema_name LIKE '" << command.db_schema_filter_pattern.value() 68 | << "'"; 69 | } 70 | 71 | if (command.table_name_filter_pattern.has_value()) { 72 | table_query << " and table_name LIKE '" << command.table_name_filter_pattern.value() 73 | << "'"; 74 | } 75 | 76 | if (!command.table_types.empty()) { 77 | table_query << " and table_type IN ("; 78 | size_t size = command.table_types.size(); 79 | for (size_t i = 0; i < size; i++) { 80 | table_query << "'" << command.table_types[i] << "'"; 81 | if (size - 1 != i) { 82 | table_query << ","; 83 | } 84 | } 85 | 86 | table_query << ")"; 87 | } 88 | 89 | table_query << " order by table_name"; 90 | return table_query.str(); 91 | } 92 | 93 | Status SetParametersOnSQLiteStatement(sqlite3_stmt* stmt, FlightMessageReader* reader) { 94 | while (true) { 95 | ARROW_ASSIGN_OR_RAISE(FlightStreamChunk chunk, reader->Next()); 96 | std::shared_ptr& record_batch = chunk.data; 97 | if (record_batch == nullptr) break; 98 | 99 | const int64_t num_rows = record_batch->num_rows(); 100 | const int& num_columns = record_batch->num_columns(); 101 | 102 | for (int i = 0; i < num_rows; ++i) { 103 | for (int c = 0; c < num_columns; ++c) { 104 | const std::shared_ptr& column = record_batch->column(c); 105 | ARROW_ASSIGN_OR_RAISE(std::shared_ptr scalar, column->GetScalar(i)); 106 | 107 | auto& holder = static_cast(*scalar).value; 108 | 109 | switch (holder->type->id()) { 110 | case Type::INT64: { 111 | int64_t value = static_cast(*holder).value; 112 | sqlite3_bind_int64(stmt, c + 1, value); 113 | break; 114 | } 115 | case Type::FLOAT: { 116 | double value = static_cast(*holder).value; 117 | sqlite3_bind_double(stmt, c + 1, value); 118 | break; 119 | } 120 | case Type::STRING: { 121 | std::shared_ptr buffer = static_cast(*holder).value; 122 | sqlite3_bind_text(stmt, c + 1, reinterpret_cast(buffer->data()), 123 | static_cast(buffer->size()), SQLITE_TRANSIENT); 124 | break; 125 | } 126 | case Type::BINARY: { 127 | std::shared_ptr buffer = static_cast(*holder).value; 128 | sqlite3_bind_blob(stmt, c + 1, buffer->data(), 129 | static_cast(buffer->size()), SQLITE_TRANSIENT); 130 | break; 131 | } 132 | default: 133 | return Status::Invalid("Received unsupported data type: ", 134 | holder->type->ToString()); 135 | } 136 | } 137 | } 138 | } 139 | 140 | return Status::OK(); 141 | } 142 | 143 | arrow::Result> DoGetSQLiteQuery( 144 | sqlite3* db, const std::string& query, const std::shared_ptr& schema) { 145 | std::shared_ptr statement; 146 | 147 | ARROW_ASSIGN_OR_RAISE(statement, SqliteStatement::Create(db, query)); 148 | 149 | std::shared_ptr reader; 150 | ARROW_ASSIGN_OR_RAISE(reader, SqliteStatementBatchReader::Create(statement, schema)); 151 | 152 | return std::unique_ptr(new RecordBatchStream(reader)); 153 | } 154 | 155 | arrow::Result> GetFlightInfoForCommand( 156 | const FlightDescriptor& descriptor, const std::shared_ptr& schema) { 157 | std::vector endpoints{FlightEndpoint{{descriptor.cmd}, {}}}; 158 | ARROW_ASSIGN_OR_RAISE(auto result, 159 | FlightInfo::Make(*schema, descriptor, endpoints, -1, -1)) 160 | 161 | return std::unique_ptr(new FlightInfo(result)); 162 | } 163 | 164 | std::string PrepareQueryForGetImportedOrExportedKeys(const std::string& filter) { 165 | return R"(SELECT * FROM (SELECT NULL AS pk_catalog_name, 166 | NULL AS pk_schema_name, 167 | p."table" AS pk_table_name, 168 | p."to" AS pk_column_name, 169 | NULL AS fk_catalog_name, 170 | NULL AS fk_schema_name, 171 | m.name AS fk_table_name, 172 | p."from" AS fk_column_name, 173 | p.seq AS key_sequence, 174 | NULL AS pk_key_name, 175 | NULL AS fk_key_name, 176 | CASE 177 | WHEN p.on_update = 'CASCADE' THEN 0 178 | WHEN p.on_update = 'RESTRICT' THEN 1 179 | WHEN p.on_update = 'SET NULL' THEN 2 180 | WHEN p.on_update = 'NO ACTION' THEN 3 181 | WHEN p.on_update = 'SET DEFAULT' THEN 4 182 | END AS update_rule, 183 | CASE 184 | WHEN p.on_delete = 'CASCADE' THEN 0 185 | WHEN p.on_delete = 'RESTRICT' THEN 1 186 | WHEN p.on_delete = 'SET NULL' THEN 2 187 | WHEN p.on_delete = 'NO ACTION' THEN 3 188 | WHEN p.on_delete = 'SET DEFAULT' THEN 4 189 | END AS delete_rule 190 | FROM sqlite_master m 191 | JOIN pragma_foreign_key_list(m.name) p ON m.name != p."table" 192 | WHERE m.type = 'table') WHERE )" + 193 | filter + R"( ORDER BY 194 | pk_catalog_name, pk_schema_name, pk_table_name, pk_key_name, key_sequence)"; 195 | } 196 | 197 | } // namespace 198 | 199 | std::shared_ptr GetArrowType(const char* sqlite_type) { 200 | if (sqlite_type == NULLPTR) { 201 | // SQLite may not know the column type yet. 202 | return null(); 203 | } 204 | 205 | if (boost::iequals(sqlite_type, "int") || boost::iequals(sqlite_type, "integer")) { 206 | return int64(); 207 | } else if (boost::iequals(sqlite_type, "REAL")) { 208 | return float64(); 209 | } else if (boost::iequals(sqlite_type, "BLOB")) { 210 | return binary(); 211 | } else if (boost::iequals(sqlite_type, "TEXT") || 212 | boost::iequals(sqlite_type, "DATE") || 213 | boost::istarts_with(sqlite_type, "char") || 214 | boost::istarts_with(sqlite_type, "varchar")) { 215 | return utf8(); 216 | } else { 217 | throw std::invalid_argument("Invalid SQLite type: " + std::string(sqlite_type)); 218 | } 219 | } 220 | 221 | int32_t GetSqlTypeFromTypeName(const char* sqlite_type) { 222 | if (sqlite_type == NULLPTR) { 223 | // SQLite may not know the column type yet. 224 | return SQLITE_NULL; 225 | } 226 | 227 | if (boost::iequals(sqlite_type, "int") || boost::iequals(sqlite_type, "integer")) { 228 | return SQLITE_INTEGER; 229 | } else if (boost::iequals(sqlite_type, "REAL")) { 230 | return SQLITE_FLOAT; 231 | } else if (boost::iequals(sqlite_type, "BLOB")) { 232 | return SQLITE_BLOB; 233 | } else if (boost::iequals(sqlite_type, "TEXT") || 234 | boost::iequals(sqlite_type, "DATE") || 235 | boost::istarts_with(sqlite_type, "char") || 236 | boost::istarts_with(sqlite_type, "varchar")) { 237 | return SQLITE_TEXT; 238 | } else { 239 | return SQLITE_NULL; 240 | } 241 | } 242 | 243 | class SQLiteFlightSqlServer::Impl { 244 | sqlite3* db_; 245 | std::map> prepared_statements_; 246 | std::default_random_engine gen_; 247 | 248 | public: 249 | explicit Impl(sqlite3* db) : db_(db) {} 250 | 251 | ~Impl() { sqlite3_close(db_); } 252 | 253 | std::string GenerateRandomString() { 254 | uint32_t length = 16; 255 | 256 | std::uniform_int_distribution dist('0', 'z'); 257 | std::string ret(length, 0); 258 | auto get_random_char = [&]() { return dist(gen_); }; 259 | std::generate_n(ret.begin(), length, get_random_char); 260 | return ret; 261 | } 262 | 263 | arrow::Result> GetFlightInfoStatement( 264 | const ServerCallContext& context, const StatementQuery& command, 265 | const FlightDescriptor& descriptor) { 266 | const std::string& query = command.query; 267 | 268 | 269 | ARROW_ASSIGN_OR_RAISE(auto statement, SqliteStatement::Create(db_, query)); 270 | 271 | ARROW_ASSIGN_OR_RAISE(auto schema, statement->GetSchema()); 272 | 273 | ARROW_ASSIGN_OR_RAISE(auto ticket_string, CreateStatementQueryTicket(query)); 274 | std::vector endpoints{FlightEndpoint{{ticket_string}, {}}}; 275 | ARROW_ASSIGN_OR_RAISE(auto result, 276 | FlightInfo::Make(*schema, descriptor, endpoints, -1, -1)) 277 | 278 | return std::unique_ptr(new FlightInfo(result)); 279 | } 280 | 281 | arrow::Result> DoGetStatement( 282 | const ServerCallContext& context, const StatementQueryTicket& command) { 283 | const std::string& sql = command.statement_handle; 284 | 285 | std::shared_ptr statement; 286 | ARROW_ASSIGN_OR_RAISE(statement, SqliteStatement::Create(db_, sql)); 287 | 288 | std::shared_ptr reader; 289 | ARROW_ASSIGN_OR_RAISE(reader, SqliteStatementBatchReader::Create(statement)); 290 | 291 | return std::unique_ptr(new RecordBatchStream(reader)); 292 | } 293 | 294 | arrow::Result> GetFlightInfoCatalogs( 295 | const ServerCallContext& context, const FlightDescriptor& descriptor) { 296 | return GetFlightInfoForCommand(descriptor, SqlSchema::GetCatalogsSchema()); 297 | } 298 | 299 | arrow::Result> DoGetCatalogs( 300 | const ServerCallContext& context) { 301 | // As SQLite doesn't support catalogs, this will return an empty record batch. 302 | 303 | const std::shared_ptr& schema = SqlSchema::GetCatalogsSchema(); 304 | 305 | StringBuilder catalog_name_builder; 306 | ARROW_ASSIGN_OR_RAISE(auto catalog_name, catalog_name_builder.Finish()); 307 | 308 | const std::shared_ptr& batch = 309 | RecordBatch::Make(schema, 0, {catalog_name}); 310 | 311 | ARROW_ASSIGN_OR_RAISE(auto reader, RecordBatchReader::Make({batch})); 312 | 313 | return std::unique_ptr(new RecordBatchStream(reader)); 314 | } 315 | 316 | arrow::Result> GetFlightInfoSchemas( 317 | const ServerCallContext& context, const GetDbSchemas& command, 318 | const FlightDescriptor& descriptor) { 319 | return GetFlightInfoForCommand(descriptor, SqlSchema::GetDbSchemasSchema()); 320 | } 321 | 322 | arrow::Result> DoGetDbSchemas( 323 | const ServerCallContext& context, const GetDbSchemas& command) { 324 | // As SQLite doesn't support schemas, this will return an empty record batch. 325 | 326 | const std::shared_ptr& schema = SqlSchema::GetDbSchemasSchema(); 327 | 328 | StringBuilder catalog_name_builder; 329 | ARROW_ASSIGN_OR_RAISE(auto catalog_name, catalog_name_builder.Finish()); 330 | StringBuilder schema_name_builder; 331 | ARROW_ASSIGN_OR_RAISE(auto schema_name, schema_name_builder.Finish()); 332 | 333 | const std::shared_ptr& batch = 334 | RecordBatch::Make(schema, 0, {catalog_name, schema_name}); 335 | 336 | ARROW_ASSIGN_OR_RAISE(auto reader, RecordBatchReader::Make({batch})); 337 | 338 | return std::unique_ptr(new RecordBatchStream(reader)); 339 | } 340 | 341 | arrow::Result> GetFlightInfoTables( 342 | const ServerCallContext& context, const GetTables& command, 343 | const FlightDescriptor& descriptor) { 344 | std::vector endpoints{FlightEndpoint{{descriptor.cmd}, {}}}; 345 | 346 | bool include_schema = command.include_schema; 347 | 348 | ARROW_ASSIGN_OR_RAISE( 349 | auto result, 350 | FlightInfo::Make(include_schema ? *SqlSchema::GetTablesSchemaWithIncludedSchema() 351 | : *SqlSchema::GetTablesSchema(), 352 | descriptor, endpoints, -1, -1)) 353 | 354 | return std::unique_ptr(new FlightInfo(result)); 355 | } 356 | 357 | arrow::Result> DoGetTables( 358 | const ServerCallContext& context, const GetTables& command) { 359 | std::string query = PrepareQueryForGetTables(command); 360 | 361 | std::shared_ptr statement; 362 | ARROW_ASSIGN_OR_RAISE(statement, SqliteStatement::Create(db_, query)); 363 | 364 | std::shared_ptr reader; 365 | ARROW_ASSIGN_OR_RAISE(reader, SqliteStatementBatchReader::Create( 366 | statement, SqlSchema::GetTablesSchema())); 367 | 368 | if (command.include_schema) { 369 | std::shared_ptr table_schema_reader = 370 | std::make_shared(reader, query, db_); 371 | return std::unique_ptr( 372 | new RecordBatchStream(table_schema_reader)); 373 | } else { 374 | return std::unique_ptr(new RecordBatchStream(reader)); 375 | } 376 | } 377 | 378 | arrow::Result DoPutCommandStatementUpdate(const ServerCallContext& context, 379 | const StatementUpdate& command) { 380 | const std::string& sql = command.query; 381 | 382 | ARROW_ASSIGN_OR_RAISE(auto statement, SqliteStatement::Create(db_, sql)); 383 | 384 | return statement->ExecuteUpdate(); 385 | } 386 | 387 | arrow::Result CreatePreparedStatement( 388 | const ServerCallContext& context, 389 | const ActionCreatePreparedStatementRequest& request) { 390 | std::shared_ptr statement; 391 | ARROW_ASSIGN_OR_RAISE(statement, SqliteStatement::Create(db_, request.query)); 392 | const std::string handle = GenerateRandomString(); 393 | prepared_statements_[handle] = statement; 394 | 395 | ARROW_ASSIGN_OR_RAISE(auto dataset_schema, statement->GetSchema()); 396 | 397 | sqlite3_stmt* stmt = statement->GetSqlite3Stmt(); 398 | const int parameter_count = sqlite3_bind_parameter_count(stmt); 399 | std::vector> parameter_fields; 400 | parameter_fields.reserve(parameter_count); 401 | 402 | // As SQLite doesn't know the parameter types before executing the query, the 403 | // example server is accepting any SQLite supported type as input by using a dense 404 | // union. 405 | const std::shared_ptr& dense_union_type = GetUnknownColumnDataType(); 406 | 407 | for (int i = 0; i < parameter_count; i++) { 408 | const char* parameter_name_chars = sqlite3_bind_parameter_name(stmt, i + 1); 409 | std::string parameter_name; 410 | if (parameter_name_chars == NULLPTR) { 411 | parameter_name = std::string("parameter_") + std::to_string(i + 1); 412 | } else { 413 | parameter_name = parameter_name_chars; 414 | } 415 | parameter_fields.push_back(field(parameter_name, dense_union_type)); 416 | } 417 | 418 | const std::shared_ptr& parameter_schema = arrow::schema(parameter_fields); 419 | 420 | ActionCreatePreparedStatementResult result{.dataset_schema = dataset_schema, 421 | .parameter_schema = parameter_schema, 422 | .prepared_statement_handle = handle}; 423 | 424 | return result; 425 | } 426 | 427 | Status ClosePreparedStatement(const ServerCallContext& context, 428 | const ActionClosePreparedStatementRequest& request) { 429 | const std::string& prepared_statement_handle = request.prepared_statement_handle; 430 | 431 | auto search = prepared_statements_.find(prepared_statement_handle); 432 | if (search != prepared_statements_.end()) { 433 | prepared_statements_.erase(prepared_statement_handle); 434 | } else { 435 | return Status::Invalid("Prepared statement not found"); 436 | } 437 | 438 | return Status::OK(); 439 | } 440 | 441 | arrow::Result> GetFlightInfoPreparedStatement( 442 | const ServerCallContext& context, const PreparedStatementQuery& command, 443 | const FlightDescriptor& descriptor) { 444 | const std::string& prepared_statement_handle = command.prepared_statement_handle; 445 | 446 | auto search = prepared_statements_.find(prepared_statement_handle); 447 | if (search == prepared_statements_.end()) { 448 | return Status::Invalid("Prepared statement not found"); 449 | } 450 | 451 | std::shared_ptr statement = search->second; 452 | 453 | ARROW_ASSIGN_OR_RAISE(auto schema, statement->GetSchema()); 454 | 455 | return GetFlightInfoForCommand(descriptor, schema); 456 | } 457 | 458 | arrow::Result> DoGetPreparedStatement( 459 | const ServerCallContext& context, const PreparedStatementQuery& command) { 460 | const std::string& prepared_statement_handle = command.prepared_statement_handle; 461 | 462 | auto search = prepared_statements_.find(prepared_statement_handle); 463 | if (search == prepared_statements_.end()) { 464 | return Status::Invalid("Prepared statement not found"); 465 | } 466 | 467 | std::shared_ptr statement = search->second; 468 | 469 | std::shared_ptr reader; 470 | ARROW_ASSIGN_OR_RAISE(reader, SqliteStatementBatchReader::Create(statement)); 471 | 472 | return std::unique_ptr(new RecordBatchStream(reader)); 473 | } 474 | 475 | Status DoPutPreparedStatementQuery(const ServerCallContext& context, 476 | const PreparedStatementQuery& command, 477 | FlightMessageReader* reader, 478 | FlightMetadataWriter* writer) { 479 | const std::string& prepared_statement_handle = command.prepared_statement_handle; 480 | ARROW_ASSIGN_OR_RAISE( 481 | auto statement, 482 | GetStatementByHandle(prepared_statements_, prepared_statement_handle)); 483 | 484 | sqlite3_stmt* stmt = statement->GetSqlite3Stmt(); 485 | ARROW_RETURN_NOT_OK(SetParametersOnSQLiteStatement(stmt, reader)); 486 | 487 | return Status::OK(); 488 | } 489 | 490 | arrow::Result DoPutPreparedStatementUpdate( 491 | const ServerCallContext& context, const PreparedStatementUpdate& command, 492 | FlightMessageReader* reader) { 493 | const std::string& prepared_statement_handle = command.prepared_statement_handle; 494 | ARROW_ASSIGN_OR_RAISE( 495 | auto statement, 496 | GetStatementByHandle(prepared_statements_, prepared_statement_handle)); 497 | 498 | sqlite3_stmt* stmt = statement->GetSqlite3Stmt(); 499 | ARROW_RETURN_NOT_OK(SetParametersOnSQLiteStatement(stmt, reader)); 500 | 501 | return statement->ExecuteUpdate(); 502 | } 503 | 504 | arrow::Result> GetFlightInfoTableTypes( 505 | const ServerCallContext& context, const FlightDescriptor& descriptor) { 506 | return GetFlightInfoForCommand(descriptor, SqlSchema::GetTableTypesSchema()); 507 | } 508 | 509 | arrow::Result> DoGetTableTypes( 510 | const ServerCallContext& context) { 511 | std::string query = "SELECT DISTINCT type as table_type FROM sqlite_master"; 512 | 513 | return DoGetSQLiteQuery(db_, query, SqlSchema::GetTableTypesSchema()); 514 | } 515 | 516 | arrow::Result> GetFlightInfoTypeInfo( 517 | const ServerCallContext& context, const GetXdbcTypeInfo& command, 518 | const FlightDescriptor& descriptor) { 519 | return GetFlightInfoForCommand(descriptor, SqlSchema::GetXdbcTypeInfoSchema()); 520 | } 521 | 522 | arrow::Result> DoGetTypeInfo( 523 | const ServerCallContext& context, const GetXdbcTypeInfo& command) { 524 | const std::shared_ptr& type_info_result = 525 | command.data_type.has_value() ? DoGetTypeInfoResult(command.data_type.value()) 526 | : DoGetTypeInfoResult(); 527 | 528 | ARROW_ASSIGN_OR_RAISE(auto reader, RecordBatchReader::Make({type_info_result})); 529 | return std::unique_ptr(new RecordBatchStream(reader)); 530 | } 531 | 532 | arrow::Result> GetFlightInfoPrimaryKeys( 533 | const ServerCallContext& context, const GetPrimaryKeys& command, 534 | const FlightDescriptor& descriptor) { 535 | return GetFlightInfoForCommand(descriptor, SqlSchema::GetPrimaryKeysSchema()); 536 | } 537 | 538 | arrow::Result> DoGetPrimaryKeys( 539 | const ServerCallContext& context, const GetPrimaryKeys& command) { 540 | std::stringstream table_query; 541 | 542 | // The field key_name can not be recovered by the sqlite, so it is being set 543 | // to null following the same pattern for catalog_name and schema_name. 544 | table_query << "SELECT null as catalog_name, null as schema_name, table_name, " 545 | "name as column_name, pk as key_sequence, null as key_name\n" 546 | "FROM pragma_table_info(table_name)\n" 547 | " JOIN (SELECT null as catalog_name, null as schema_name, name as " 548 | "table_name, type as table_type\n" 549 | "FROM sqlite_master) where 1=1 and pk != 0"; 550 | 551 | const TableRef& table_ref = command.table_ref; 552 | if (table_ref.catalog.has_value()) { 553 | table_query << " and catalog_name LIKE '" << table_ref.catalog.value() << "'"; 554 | } 555 | 556 | if (table_ref.db_schema.has_value()) { 557 | table_query << " and schema_name LIKE '" << table_ref.db_schema.value() << "'"; 558 | } 559 | 560 | table_query << " and table_name LIKE '" << table_ref.table << "'"; 561 | 562 | return DoGetSQLiteQuery(db_, table_query.str(), SqlSchema::GetPrimaryKeysSchema()); 563 | } 564 | 565 | arrow::Result> GetFlightInfoImportedKeys( 566 | const ServerCallContext& context, const GetImportedKeys& command, 567 | const FlightDescriptor& descriptor) { 568 | return GetFlightInfoForCommand(descriptor, SqlSchema::GetImportedKeysSchema()); 569 | } 570 | 571 | arrow::Result> DoGetImportedKeys( 572 | const ServerCallContext& context, const GetImportedKeys& command) { 573 | const TableRef& table_ref = command.table_ref; 574 | std::string filter = "fk_table_name = '" + table_ref.table + "'"; 575 | if (table_ref.catalog.has_value()) { 576 | filter += " AND fk_catalog_name = '" + table_ref.catalog.value() + "'"; 577 | } 578 | if (table_ref.db_schema.has_value()) { 579 | filter += " AND fk_schema_name = '" + table_ref.db_schema.value() + "'"; 580 | } 581 | std::string query = PrepareQueryForGetImportedOrExportedKeys(filter); 582 | 583 | return DoGetSQLiteQuery(db_, query, SqlSchema::GetImportedKeysSchema()); 584 | } 585 | 586 | arrow::Result> GetFlightInfoExportedKeys( 587 | const ServerCallContext& context, const GetExportedKeys& command, 588 | const FlightDescriptor& descriptor) { 589 | return GetFlightInfoForCommand(descriptor, SqlSchema::GetExportedKeysSchema()); 590 | } 591 | 592 | arrow::Result> DoGetExportedKeys( 593 | const ServerCallContext& context, const GetExportedKeys& command) { 594 | const TableRef& table_ref = command.table_ref; 595 | std::string filter = "pk_table_name = '" + table_ref.table + "'"; 596 | if (table_ref.catalog.has_value()) { 597 | filter += " AND pk_catalog_name = '" + table_ref.catalog.value() + "'"; 598 | } 599 | if (table_ref.db_schema.has_value()) { 600 | filter += " AND pk_schema_name = '" + table_ref.db_schema.value() + "'"; 601 | } 602 | std::string query = PrepareQueryForGetImportedOrExportedKeys(filter); 603 | 604 | return DoGetSQLiteQuery(db_, query, SqlSchema::GetExportedKeysSchema()); 605 | } 606 | 607 | arrow::Result> GetFlightInfoCrossReference( 608 | const ServerCallContext& context, const GetCrossReference& command, 609 | const FlightDescriptor& descriptor) { 610 | return GetFlightInfoForCommand(descriptor, SqlSchema::GetCrossReferenceSchema()); 611 | } 612 | 613 | arrow::Result> DoGetCrossReference( 614 | const ServerCallContext& context, const GetCrossReference& command) { 615 | const TableRef& pk_table_ref = command.pk_table_ref; 616 | std::string filter = "pk_table_name = '" + pk_table_ref.table + "'"; 617 | if (pk_table_ref.catalog.has_value()) { 618 | filter += " AND pk_catalog_name = '" + pk_table_ref.catalog.value() + "'"; 619 | } 620 | if (pk_table_ref.db_schema.has_value()) { 621 | filter += " AND pk_schema_name = '" + pk_table_ref.db_schema.value() + "'"; 622 | } 623 | 624 | const TableRef& fk_table_ref = command.fk_table_ref; 625 | filter += " AND fk_table_name = '" + fk_table_ref.table + "'"; 626 | if (fk_table_ref.catalog.has_value()) { 627 | filter += " AND fk_catalog_name = '" + fk_table_ref.catalog.value() + "'"; 628 | } 629 | if (fk_table_ref.db_schema.has_value()) { 630 | filter += " AND fk_schema_name = '" + fk_table_ref.db_schema.value() + "'"; 631 | } 632 | std::string query = PrepareQueryForGetImportedOrExportedKeys(filter); 633 | 634 | return DoGetSQLiteQuery(db_, query, SqlSchema::GetCrossReferenceSchema()); 635 | } 636 | 637 | Status ExecuteSql(const std::string& sql) { 638 | char* err_msg = nullptr; 639 | int rc = sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &err_msg); 640 | 641 | if (rc != SQLITE_OK) { 642 | std::string error_msg; 643 | if (err_msg != nullptr) { 644 | error_msg = err_msg; 645 | } 646 | sqlite3_free(err_msg); 647 | return Status::ExecutionError(error_msg); 648 | } 649 | return Status::OK(); 650 | } 651 | }; 652 | 653 | SQLiteFlightSqlServer::SQLiteFlightSqlServer(std::shared_ptr impl) 654 | : impl_(std::move(impl)) {} 655 | 656 | arrow::Result> SQLiteFlightSqlServer::Create(std::string path) { 657 | sqlite3* db = nullptr; 658 | char* db_location; 659 | 660 | bool in_memory = path == ""; 661 | 662 | if (in_memory) { 663 | db_location = (char*)":memory:"; 664 | } else { 665 | db_location = (char*)path.c_str(); // TODO: validate that the path exists 666 | } 667 | 668 | if (sqlite3_open(db_location, &db)) { 669 | std::string err_msg = "Can't open database: "; 670 | if (db != nullptr) { 671 | err_msg += sqlite3_errmsg(db); 672 | sqlite3_close(db); 673 | } else { 674 | err_msg += "Unable to start SQLite. Insufficient memory"; 675 | } 676 | 677 | return Status::Invalid(err_msg); 678 | } 679 | 680 | std::shared_ptr impl = std::make_shared(db); 681 | 682 | std::shared_ptr result(new SQLiteFlightSqlServer(impl)); 683 | for (const auto& id_to_result : GetSqlInfoResultMap()) { 684 | result->RegisterSqlInfo(id_to_result.first, id_to_result.second); 685 | } 686 | 687 | if (in_memory) { 688 | ARROW_RETURN_NOT_OK(result->ExecuteSql(R"( 689 | CREATE TABLE foreignTable ( 690 | id INTEGER PRIMARY KEY AUTOINCREMENT, 691 | foreignName varchar(100), 692 | value int); 693 | 694 | CREATE TABLE intTable ( 695 | id INTEGER PRIMARY KEY AUTOINCREMENT, 696 | keyName varchar(100), 697 | value int, 698 | foreignId int references foreignTable(id)); 699 | 700 | INSERT INTO foreignTable (foreignName, value) VALUES ('keyOne', 1); 701 | INSERT INTO foreignTable (foreignName, value) VALUES ('keyTwo', 0); 702 | INSERT INTO foreignTable (foreignName, value) VALUES ('keyThree', -1); 703 | INSERT INTO intTable (keyName, value, foreignId) VALUES ('one', 1, 1); 704 | INSERT INTO intTable (keyName, value, foreignId) VALUES ('zero', 0, 1); 705 | INSERT INTO intTable (keyName, value, foreignId) VALUES ('negative one', -1, 1); 706 | INSERT INTO intTable (keyName, value, foreignId) VALUES (NULL, NULL, NULL); 707 | )")); 708 | } 709 | 710 | return result; 711 | } 712 | 713 | SQLiteFlightSqlServer::~SQLiteFlightSqlServer() = default; 714 | 715 | Status SQLiteFlightSqlServer::ExecuteSql(const std::string& sql) { 716 | return impl_->ExecuteSql(sql); 717 | } 718 | 719 | arrow::Result> SQLiteFlightSqlServer::GetFlightInfoStatement( 720 | const ServerCallContext& context, const StatementQuery& command, 721 | const FlightDescriptor& descriptor) { 722 | return impl_->GetFlightInfoStatement(context, command, descriptor); 723 | } 724 | 725 | arrow::Result> SQLiteFlightSqlServer::DoGetStatement( 726 | const ServerCallContext& context, const StatementQueryTicket& command) { 727 | return impl_->DoGetStatement(context, command); 728 | } 729 | 730 | arrow::Result> SQLiteFlightSqlServer::GetFlightInfoCatalogs( 731 | const ServerCallContext& context, const FlightDescriptor& descriptor) { 732 | return impl_->GetFlightInfoCatalogs(context, descriptor); 733 | } 734 | 735 | arrow::Result> SQLiteFlightSqlServer::DoGetCatalogs( 736 | const ServerCallContext& context) { 737 | return impl_->DoGetCatalogs(context); 738 | } 739 | 740 | arrow::Result> SQLiteFlightSqlServer::GetFlightInfoSchemas( 741 | const ServerCallContext& context, const GetDbSchemas& command, 742 | const FlightDescriptor& descriptor) { 743 | return impl_->GetFlightInfoSchemas(context, command, descriptor); 744 | } 745 | 746 | arrow::Result> SQLiteFlightSqlServer::DoGetDbSchemas( 747 | const ServerCallContext& context, const GetDbSchemas& command) { 748 | return impl_->DoGetDbSchemas(context, command); 749 | } 750 | 751 | arrow::Result> SQLiteFlightSqlServer::GetFlightInfoTables( 752 | const ServerCallContext& context, const GetTables& command, 753 | const FlightDescriptor& descriptor) { 754 | return impl_->GetFlightInfoTables(context, command, descriptor); 755 | } 756 | 757 | arrow::Result> SQLiteFlightSqlServer::DoGetTables( 758 | const ServerCallContext& context, const GetTables& command) { 759 | return impl_->DoGetTables(context, command); 760 | } 761 | 762 | arrow::Result SQLiteFlightSqlServer::DoPutCommandStatementUpdate( 763 | const ServerCallContext& context, const StatementUpdate& command) { 764 | return impl_->DoPutCommandStatementUpdate(context, command); 765 | } 766 | 767 | arrow::Result 768 | SQLiteFlightSqlServer::CreatePreparedStatement( 769 | const ServerCallContext& context, 770 | const ActionCreatePreparedStatementRequest& request) { 771 | return impl_->CreatePreparedStatement(context, request); 772 | } 773 | 774 | Status SQLiteFlightSqlServer::ClosePreparedStatement( 775 | const ServerCallContext& context, 776 | const ActionClosePreparedStatementRequest& request) { 777 | return impl_->ClosePreparedStatement(context, request); 778 | } 779 | 780 | arrow::Result> 781 | SQLiteFlightSqlServer::GetFlightInfoPreparedStatement( 782 | const ServerCallContext& context, const PreparedStatementQuery& command, 783 | const FlightDescriptor& descriptor) { 784 | return impl_->GetFlightInfoPreparedStatement(context, command, descriptor); 785 | } 786 | 787 | arrow::Result> 788 | SQLiteFlightSqlServer::DoGetPreparedStatement(const ServerCallContext& context, 789 | const PreparedStatementQuery& command) { 790 | return impl_->DoGetPreparedStatement(context, command); 791 | } 792 | 793 | Status SQLiteFlightSqlServer::DoPutPreparedStatementQuery( 794 | const ServerCallContext& context, const PreparedStatementQuery& command, 795 | FlightMessageReader* reader, FlightMetadataWriter* writer) { 796 | return impl_->DoPutPreparedStatementQuery(context, command, reader, writer); 797 | } 798 | 799 | arrow::Result SQLiteFlightSqlServer::DoPutPreparedStatementUpdate( 800 | const ServerCallContext& context, const PreparedStatementUpdate& command, 801 | FlightMessageReader* reader) { 802 | return impl_->DoPutPreparedStatementUpdate(context, command, reader); 803 | } 804 | 805 | arrow::Result> SQLiteFlightSqlServer::GetFlightInfoTableTypes( 806 | const ServerCallContext& context, const FlightDescriptor& descriptor) { 807 | return impl_->GetFlightInfoTableTypes(context, descriptor); 808 | } 809 | 810 | arrow::Result> SQLiteFlightSqlServer::DoGetTableTypes( 811 | const ServerCallContext& context) { 812 | return impl_->DoGetTableTypes(context); 813 | } 814 | 815 | arrow::Result> 816 | SQLiteFlightSqlServer::GetFlightInfoXdbcTypeInfo( 817 | const ServerCallContext& context, const arrow::flight::sql::GetXdbcTypeInfo& command, 818 | const FlightDescriptor& descriptor) { 819 | return impl_->GetFlightInfoTypeInfo(context, command, descriptor); 820 | } 821 | 822 | arrow::Result> SQLiteFlightSqlServer::DoGetXdbcTypeInfo( 823 | const ServerCallContext& context, 824 | const arrow::flight::sql::GetXdbcTypeInfo& command) { 825 | return impl_->DoGetTypeInfo(context, command); 826 | } 827 | 828 | arrow::Result> 829 | SQLiteFlightSqlServer::GetFlightInfoPrimaryKeys(const ServerCallContext& context, 830 | const GetPrimaryKeys& command, 831 | const FlightDescriptor& descriptor) { 832 | return impl_->GetFlightInfoPrimaryKeys(context, command, descriptor); 833 | } 834 | 835 | arrow::Result> SQLiteFlightSqlServer::DoGetPrimaryKeys( 836 | const ServerCallContext& context, const GetPrimaryKeys& command) { 837 | return impl_->DoGetPrimaryKeys(context, command); 838 | } 839 | 840 | arrow::Result> 841 | SQLiteFlightSqlServer::GetFlightInfoImportedKeys(const ServerCallContext& context, 842 | const GetImportedKeys& command, 843 | const FlightDescriptor& descriptor) { 844 | return impl_->GetFlightInfoImportedKeys(context, command, descriptor); 845 | } 846 | 847 | arrow::Result> SQLiteFlightSqlServer::DoGetImportedKeys( 848 | const ServerCallContext& context, const GetImportedKeys& command) { 849 | return impl_->DoGetImportedKeys(context, command); 850 | } 851 | 852 | arrow::Result> 853 | SQLiteFlightSqlServer::GetFlightInfoExportedKeys(const ServerCallContext& context, 854 | const GetExportedKeys& command, 855 | const FlightDescriptor& descriptor) { 856 | return impl_->GetFlightInfoExportedKeys(context, command, descriptor); 857 | } 858 | 859 | arrow::Result> SQLiteFlightSqlServer::DoGetExportedKeys( 860 | const ServerCallContext& context, const GetExportedKeys& command) { 861 | return impl_->DoGetExportedKeys(context, command); 862 | } 863 | 864 | arrow::Result> 865 | SQLiteFlightSqlServer::GetFlightInfoCrossReference(const ServerCallContext& context, 866 | const GetCrossReference& command, 867 | const FlightDescriptor& descriptor) { 868 | return impl_->GetFlightInfoCrossReference(context, command, descriptor); 869 | } 870 | 871 | arrow::Result> 872 | SQLiteFlightSqlServer::DoGetCrossReference(const ServerCallContext& context, 873 | const GetCrossReference& command) { 874 | return impl_->DoGetCrossReference(context, command); 875 | } 876 | 877 | } // namespace sqlite 878 | } // namespace sql 879 | } // namespace flight 880 | } // namespace arrow 881 | -------------------------------------------------------------------------------- /src/sqlite/sqlite_server.h: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | #include 23 | #include 24 | 25 | #include 26 | #include 27 | 28 | #include "sqlite_statement.h" 29 | #include "sqlite_statement_batch_reader.h" 30 | 31 | namespace arrow { 32 | namespace flight { 33 | namespace sql { 34 | namespace sqlite { 35 | 36 | /// \brief Convert a column type to a ArrowType. 37 | /// \param sqlite_type the sqlite type. 38 | /// \return The equivalent ArrowType. 39 | std::shared_ptr GetArrowType(const char* sqlite_type); 40 | 41 | /// \brief Convert a column type name to SQLite type. 42 | /// \param type_name the type name. 43 | /// \return The equivalent SQLite type. 44 | int32_t GetSqlTypeFromTypeName(const char* type_name); 45 | 46 | /// \brief Get the DataType used when parameter type is not known. 47 | /// \return DataType used when parameter type is not known. 48 | inline std::shared_ptr GetUnknownColumnDataType() { 49 | return dense_union({ 50 | field("string", utf8()), 51 | field("bytes", binary()), 52 | field("bigint", int64()), 53 | field("double", float64()), 54 | }); 55 | } 56 | 57 | /// \brief Example implementation of FlightSqlServerBase backed by an in-memory SQLite3 58 | /// database. 59 | class SQLiteFlightSqlServer : public FlightSqlServerBase { 60 | public: 61 | ~SQLiteFlightSqlServer() override; 62 | 63 | static arrow::Result> Create(std::string path); 64 | 65 | /// \brief Auxiliary method used to execute an arbitrary SQL statement on the underlying 66 | /// SQLite database. 67 | Status ExecuteSql(const std::string& sql); 68 | 69 | arrow::Result> GetFlightInfoStatement( 70 | const ServerCallContext& context, const StatementQuery& command, 71 | const FlightDescriptor& descriptor) override; 72 | 73 | arrow::Result> DoGetStatement( 74 | const ServerCallContext& context, const StatementQueryTicket& command) override; 75 | arrow::Result> GetFlightInfoCatalogs( 76 | const ServerCallContext& context, const FlightDescriptor& descriptor) override; 77 | arrow::Result> DoGetCatalogs( 78 | const ServerCallContext& context) override; 79 | arrow::Result> GetFlightInfoSchemas( 80 | const ServerCallContext& context, const GetDbSchemas& command, 81 | const FlightDescriptor& descriptor) override; 82 | arrow::Result> DoGetDbSchemas( 83 | const ServerCallContext& context, const GetDbSchemas& command) override; 84 | arrow::Result DoPutCommandStatementUpdate( 85 | const ServerCallContext& context, const StatementUpdate& update) override; 86 | arrow::Result CreatePreparedStatement( 87 | const ServerCallContext& context, 88 | const ActionCreatePreparedStatementRequest& request) override; 89 | Status ClosePreparedStatement( 90 | const ServerCallContext& context, 91 | const ActionClosePreparedStatementRequest& request) override; 92 | arrow::Result> GetFlightInfoPreparedStatement( 93 | const ServerCallContext& context, const PreparedStatementQuery& command, 94 | const FlightDescriptor& descriptor) override; 95 | arrow::Result> DoGetPreparedStatement( 96 | const ServerCallContext& context, const PreparedStatementQuery& command) override; 97 | Status DoPutPreparedStatementQuery(const ServerCallContext& context, 98 | const PreparedStatementQuery& command, 99 | FlightMessageReader* reader, 100 | FlightMetadataWriter* writer) override; 101 | arrow::Result DoPutPreparedStatementUpdate( 102 | const ServerCallContext& context, const PreparedStatementUpdate& command, 103 | FlightMessageReader* reader) override; 104 | 105 | arrow::Result> GetFlightInfoTables( 106 | const ServerCallContext& context, const GetTables& command, 107 | const FlightDescriptor& descriptor) override; 108 | 109 | arrow::Result> DoGetTables( 110 | const ServerCallContext& context, const GetTables& command) override; 111 | arrow::Result> GetFlightInfoXdbcTypeInfo( 112 | const ServerCallContext& context, 113 | const arrow::flight::sql::GetXdbcTypeInfo& command, 114 | const FlightDescriptor& descriptor) override; 115 | arrow::Result> DoGetXdbcTypeInfo( 116 | const ServerCallContext& context, 117 | const arrow::flight::sql::GetXdbcTypeInfo& command) override; 118 | arrow::Result> GetFlightInfoTableTypes( 119 | const ServerCallContext& context, const FlightDescriptor& descriptor) override; 120 | arrow::Result> DoGetTableTypes( 121 | const ServerCallContext& context) override; 122 | arrow::Result> GetFlightInfoImportedKeys( 123 | const ServerCallContext& context, const GetImportedKeys& command, 124 | const FlightDescriptor& descriptor) override; 125 | arrow::Result> DoGetImportedKeys( 126 | const ServerCallContext& context, const GetImportedKeys& command) override; 127 | arrow::Result> GetFlightInfoExportedKeys( 128 | const ServerCallContext& context, const GetExportedKeys& command, 129 | const FlightDescriptor& descriptor) override; 130 | arrow::Result> DoGetExportedKeys( 131 | const ServerCallContext& context, const GetExportedKeys& command) override; 132 | arrow::Result> GetFlightInfoCrossReference( 133 | const ServerCallContext& context, const GetCrossReference& command, 134 | const FlightDescriptor& descriptor) override; 135 | arrow::Result> DoGetCrossReference( 136 | const ServerCallContext& context, const GetCrossReference& command) override; 137 | 138 | arrow::Result> GetFlightInfoPrimaryKeys( 139 | const ServerCallContext& context, const GetPrimaryKeys& command, 140 | const FlightDescriptor& descriptor) override; 141 | 142 | arrow::Result> DoGetPrimaryKeys( 143 | const ServerCallContext& context, const GetPrimaryKeys& command) override; 144 | 145 | private: 146 | class Impl; 147 | std::shared_ptr impl_; 148 | 149 | explicit SQLiteFlightSqlServer(std::shared_ptr impl); 150 | }; 151 | 152 | } // namespace sqlite 153 | } // namespace sql 154 | } // namespace flight 155 | } // namespace arrow 156 | -------------------------------------------------------------------------------- /src/sqlite/sqlite_sql_info.cc: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #include "sqlite_sql_info.h" 19 | 20 | #include 21 | 22 | namespace arrow { 23 | namespace flight { 24 | namespace sql { 25 | namespace sqlite { 26 | 27 | /// \brief Gets the mapping from SQL info ids to SqlInfoResult instances. 28 | /// \return the cache. 29 | SqlInfoResultMap GetSqlInfoResultMap() { 30 | return { 31 | {SqlInfoOptions::SqlInfo::FLIGHT_SQL_SERVER_NAME, 32 | SqlInfoResult(std::string("db_name"))}, 33 | {SqlInfoOptions::SqlInfo::FLIGHT_SQL_SERVER_VERSION, 34 | SqlInfoResult(std::string("sqlite 3"))}, 35 | {SqlInfoOptions::SqlInfo::FLIGHT_SQL_SERVER_ARROW_VERSION, 36 | SqlInfoResult(std::string("7.0.0-SNAPSHOT" /* Only an example */))}, 37 | {SqlInfoOptions::SqlInfo::FLIGHT_SQL_SERVER_READ_ONLY, SqlInfoResult(false)}, 38 | {SqlInfoOptions::SqlInfo::SQL_DDL_CATALOG, 39 | SqlInfoResult(false /* SQLite 3 does not support catalogs */)}, 40 | {SqlInfoOptions::SqlInfo::SQL_DDL_SCHEMA, 41 | SqlInfoResult(false /* SQLite 3 does not support schemas */)}, 42 | {SqlInfoOptions::SqlInfo::SQL_DDL_TABLE, SqlInfoResult(true)}, 43 | {SqlInfoOptions::SqlInfo::SQL_IDENTIFIER_CASE, 44 | SqlInfoResult(int64_t(SqlInfoOptions::SqlSupportedCaseSensitivity:: 45 | SQL_CASE_SENSITIVITY_CASE_INSENSITIVE))}, 46 | {SqlInfoOptions::SqlInfo::SQL_IDENTIFIER_QUOTE_CHAR, 47 | SqlInfoResult(std::string("\""))}, 48 | {SqlInfoOptions::SqlInfo::SQL_QUOTED_IDENTIFIER_CASE, 49 | SqlInfoResult(int64_t(SqlInfoOptions::SqlSupportedCaseSensitivity:: 50 | SQL_CASE_SENSITIVITY_CASE_INSENSITIVE))}, 51 | {SqlInfoOptions::SqlInfo::SQL_ALL_TABLES_ARE_SELECTABLE, SqlInfoResult(true)}, 52 | {SqlInfoOptions::SqlInfo::SQL_NULL_ORDERING, 53 | SqlInfoResult( 54 | int64_t(SqlInfoOptions::SqlNullOrdering::SQL_NULLS_SORTED_AT_START))}, 55 | {SqlInfoOptions::SqlInfo::SQL_KEYWORDS, 56 | SqlInfoResult(std::vector({"ABORT", 57 | "ACTION", 58 | "ADD", 59 | "AFTER", 60 | "ALL", 61 | "ALTER", 62 | "ALWAYS", 63 | "ANALYZE", 64 | "AND", 65 | "AS", 66 | "ASC", 67 | "ATTACH", 68 | "AUTOINCREMENT", 69 | "BEFORE", 70 | "BEGIN", 71 | "BETWEEN", 72 | "BY", 73 | "CASCADE", 74 | "CASE", 75 | "CAST", 76 | "CHECK", 77 | "COLLATE", 78 | "COLUMN", 79 | "COMMIT", 80 | "CONFLICT", 81 | "CONSTRAINT", 82 | "CREATE", 83 | "CROSS", 84 | "CURRENT", 85 | "CURRENT_DATE", 86 | "CURRENT_TIME", 87 | "CURRENT_TIMESTAMP", 88 | "DATABASE", 89 | "DEFAULT", 90 | "DEFERRABLE", 91 | "DEFERRED", 92 | "DELETE", 93 | "DESC", 94 | "DETACH", 95 | "DISTINCT", 96 | "DO", 97 | "DROP", 98 | "EACH", 99 | "ELSE", 100 | "END", 101 | "ESCAPE", 102 | "EXCEPT", 103 | "EXCLUDE", 104 | "EXCLUSIVE", 105 | "EXISTS", 106 | "EXPLAIN", 107 | "FAIL", 108 | "FILTER", 109 | "FIRST", 110 | "FOLLOWING", 111 | "FOR", 112 | "FOREIGN", 113 | "FROM", 114 | "FULL", 115 | "GENERATED", 116 | "GLOB", 117 | "GROUP", 118 | "GROUPS", 119 | "HAVING", 120 | "IF", 121 | "IGNORE", 122 | "IMMEDIATE", 123 | "IN", 124 | "INDEX", 125 | "INDEXED", 126 | "INITIALLY", 127 | "INNER", 128 | "INSERT", 129 | "INSTEAD", 130 | "INTERSECT", 131 | "INTO", 132 | "IS", 133 | "ISNULL", 134 | "JOIN", 135 | "KEY", 136 | "LAST", 137 | "LEFT", 138 | "LIKE", 139 | "LIMIT", 140 | "MATCH", 141 | "MATERIALIZED", 142 | "NATURAL", 143 | "NO", 144 | "NOT", 145 | "NOTHING", 146 | "NOTNULL", 147 | "NULL", 148 | "NULLS", 149 | "OF", 150 | "OFFSET", 151 | "ON", 152 | "OR", 153 | "ORDER", 154 | "OTHERS", 155 | "OUTER", 156 | "OVER", 157 | "PARTITION", 158 | "PLAN", 159 | "PRAGMA", 160 | "PRECEDING", 161 | "PRIMARY", 162 | "QUERY", 163 | "RAISE", 164 | "RANGE", 165 | "RECURSIVE", 166 | "REFERENCES", 167 | "REGEXP", 168 | "REINDEX", 169 | "RELEASE", 170 | "RENAME", 171 | "REPLACE", 172 | "RESTRICT", 173 | "RETURNING", 174 | "RIGHT", 175 | "ROLLBACK", 176 | "ROW", 177 | "ROWS", 178 | "SAVEPOINT", 179 | "SELECT", 180 | "SET", 181 | "TABLE", 182 | "TEMP", 183 | "TEMPORARY", 184 | "THEN", 185 | "TIES", 186 | "TO", 187 | "TRANSACTION", 188 | "TRIGGER", 189 | "UNBOUNDED", 190 | "UNION", 191 | "UNIQUE", 192 | "UPDATE", 193 | "USING", 194 | "VACUUM", 195 | "VALUES", 196 | "VIEW", 197 | "VIRTUAL", 198 | "WHEN", 199 | "WHERE", 200 | "WINDOW", 201 | "WITH", 202 | "WITHOUT"}))}, 203 | {SqlInfoOptions::SqlInfo::SQL_NUMERIC_FUNCTIONS, 204 | SqlInfoResult(std::vector( 205 | {"ACOS", "ACOSH", "ASIN", "ASINH", "ATAN", "ATAN2", "ATANH", "CEIL", 206 | "CEILING", "COS", "COSH", "DEGREES", "EXP", "FLOOR", "LN", "LOG", 207 | "LOG", "LOG10", "LOG2", "MOD", "PI", "POW", "POWER", "RADIANS", 208 | "SIN", "SINH", "SQRT", "TAN", "TANH", "TRUNC"}))}, 209 | {SqlInfoOptions::SqlInfo::SQL_STRING_FUNCTIONS, 210 | SqlInfoResult( 211 | std::vector({"SUBSTR", "TRIM", "LTRIM", "RTRIM", "LENGTH", 212 | "REPLACE", "UPPER", "LOWER", "INSTR"}))}, 213 | {SqlInfoOptions::SqlInfo::SQL_SUPPORTS_CONVERT, 214 | SqlInfoResult(std::unordered_map>( 215 | {{SqlInfoOptions::SqlSupportsConvert::SQL_CONVERT_BIGINT, 216 | std::vector( 217 | {SqlInfoOptions::SqlSupportsConvert::SQL_CONVERT_INTEGER})}}))}}; 218 | } 219 | 220 | } // namespace sqlite 221 | } // namespace sql 222 | } // namespace flight 223 | } // namespace arrow 224 | -------------------------------------------------------------------------------- /src/sqlite/sqlite_sql_info.h: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | namespace arrow { 23 | namespace flight { 24 | namespace sql { 25 | namespace sqlite { 26 | 27 | /// \brief Gets the mapping from SQL info ids to SqlInfoResult instances. 28 | /// \return the cache. 29 | SqlInfoResultMap GetSqlInfoResultMap(); 30 | 31 | } // namespace sqlite 32 | } // namespace sql 33 | } // namespace flight 34 | } // namespace arrow 35 | -------------------------------------------------------------------------------- /src/sqlite/sqlite_statement.cc: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #include "sqlite_statement.h" 19 | 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | #include "sqlite_server.h" 27 | 28 | namespace arrow { 29 | namespace flight { 30 | namespace sql { 31 | namespace sqlite { 32 | 33 | std::shared_ptr GetDataTypeFromSqliteType(const int column_type) { 34 | switch (column_type) { 35 | case SQLITE_INTEGER: 36 | return int64(); 37 | case SQLITE_FLOAT: 38 | return float64(); 39 | case SQLITE_BLOB: 40 | return binary(); 41 | case SQLITE_TEXT: 42 | return utf8(); 43 | case SQLITE_NULL: 44 | default: 45 | return null(); 46 | } 47 | } 48 | 49 | int32_t GetPrecisionFromColumn(int column_type) { 50 | switch (column_type) { 51 | case SQLITE_INTEGER: 52 | return 10; 53 | case SQLITE_FLOAT: 54 | return 15; 55 | case SQLITE_NULL: 56 | default: 57 | return 0; 58 | } 59 | } 60 | 61 | ColumnMetadata GetColumnMetadata(int column_type, const char* table) { 62 | ColumnMetadata::ColumnMetadataBuilder builder = ColumnMetadata::Builder(); 63 | 64 | builder.Scale(15).IsAutoIncrement(false).IsReadOnly(false); 65 | 66 | if (table == NULLPTR) { 67 | return builder.Build(); 68 | } else if (column_type == SQLITE_TEXT || column_type == SQLITE_BLOB) { 69 | std::string table_name(table); 70 | builder.TableName(table_name); 71 | } else { 72 | std::string table_name(table); 73 | builder.TableName(table_name).Precision(GetPrecisionFromColumn(column_type)); 74 | } 75 | return builder.Build(); 76 | } 77 | 78 | arrow::Result> SqliteStatement::Create( 79 | sqlite3* db, const std::string& sql) { 80 | sqlite3_stmt* stmt = nullptr; 81 | int rc = 82 | sqlite3_prepare_v2(db, sql.c_str(), static_cast(sql.size()), &stmt, NULLPTR); 83 | 84 | if (rc != SQLITE_OK) { 85 | std::string err_msg = "Can't prepare statement: " + std::string(sqlite3_errmsg(db)); 86 | if (stmt != nullptr) { 87 | rc = sqlite3_finalize(stmt); 88 | if (rc != SQLITE_OK) { 89 | err_msg += "; Failed to finalize SQLite statement: "; 90 | err_msg += std::string(sqlite3_errmsg(db)); 91 | } 92 | } 93 | return Status::Invalid(err_msg); 94 | } 95 | 96 | std::shared_ptr result(new SqliteStatement(db, stmt)); 97 | return result; 98 | } 99 | 100 | arrow::Result> SqliteStatement::GetSchema() const { 101 | std::vector> fields; 102 | 103 | int column_count = sqlite3_column_count(stmt_); 104 | 105 | for (int i = 0; i < column_count; i++) { 106 | const char* column_name = sqlite3_column_name(stmt_, i); 107 | 108 | // SQLite does not always provide column types, especially when the statement has not 109 | // been executed yet. Because of this behaviour this method tries to get the column 110 | // types in two attempts: 111 | // 1. Use sqlite3_column_type(), which return SQLITE_NULL if the statement has not 112 | // been executed yet 113 | // 2. Use sqlite3_column_decltype(), which returns correctly if given column is 114 | // declared in the table. 115 | // Because of this limitation, it is not possible to know the column types for some 116 | // prepared statements, in this case it returns a dense_union type covering any type 117 | // SQLite supports. 118 | const int column_type = sqlite3_column_type(stmt_, i); 119 | const char* table = sqlite3_column_table_name(stmt_, i); 120 | std::shared_ptr data_type = GetDataTypeFromSqliteType(column_type); 121 | 122 | if (data_type->id() == Type::NA) { 123 | // Try to retrieve column type from sqlite3_column_decltype 124 | const char* column_decltype = sqlite3_column_decltype(stmt_, i); 125 | 126 | if (column_decltype != NULLPTR) { 127 | data_type = GetArrowType(column_decltype); 128 | } else { 129 | // If it can not determine the actual column type, return a dense_union type 130 | // covering any type SQLite supports. 131 | data_type = GetUnknownColumnDataType(); 132 | } 133 | } 134 | ColumnMetadata column_metadata = GetColumnMetadata(column_type, table); 135 | 136 | fields.push_back( 137 | arrow::field(column_name, data_type, column_metadata.metadata_map())); 138 | } 139 | return arrow::schema(fields); 140 | } 141 | 142 | SqliteStatement::~SqliteStatement() { sqlite3_finalize(stmt_); } 143 | 144 | arrow::Result SqliteStatement::Step() { 145 | int rc = sqlite3_step(stmt_); 146 | if (rc == SQLITE_ERROR) { 147 | return Status::ExecutionError("A SQLite runtime error has occurred: ", 148 | sqlite3_errmsg(db_)); 149 | } 150 | 151 | return rc; 152 | } 153 | 154 | arrow::Result SqliteStatement::Reset() { 155 | int rc = sqlite3_reset(stmt_); 156 | if (rc == SQLITE_ERROR) { 157 | return Status::ExecutionError("A SQLite runtime error has occurred: ", 158 | sqlite3_errmsg(db_)); 159 | } 160 | 161 | return rc; 162 | } 163 | 164 | sqlite3_stmt* SqliteStatement::GetSqlite3Stmt() const { return stmt_; } 165 | 166 | arrow::Result SqliteStatement::ExecuteUpdate() { 167 | ARROW_RETURN_NOT_OK(Step()); 168 | return sqlite3_changes(db_); 169 | } 170 | 171 | } // namespace sqlite 172 | } // namespace sql 173 | } // namespace flight 174 | } // namespace arrow 175 | -------------------------------------------------------------------------------- /src/sqlite/sqlite_statement.h: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | #include 23 | #include 24 | 25 | #include 26 | #include 27 | 28 | namespace arrow { 29 | namespace flight { 30 | namespace sql { 31 | namespace sqlite { 32 | 33 | /// \brief Create an object ColumnMetadata using the column type and 34 | /// table name. 35 | /// \param column_type The SQLite type. 36 | /// \param table The table name. 37 | /// \return A Column Metadata object. 38 | ColumnMetadata GetColumnMetadata(int column_type, const char* table); 39 | 40 | class SqliteStatement { 41 | public: 42 | /// \brief Creates a SQLite3 statement. 43 | /// \param[in] db SQLite3 database instance. 44 | /// \param[in] sql SQL statement. 45 | /// \return A SqliteStatement object. 46 | static arrow::Result> Create(sqlite3* db, 47 | const std::string& sql); 48 | 49 | ~SqliteStatement(); 50 | 51 | /// \brief Creates an Arrow Schema based on the results of this statement. 52 | /// \return The resulting Schema. 53 | arrow::Result> GetSchema() const; 54 | 55 | /// \brief Steps on underlying sqlite3_stmt. 56 | /// \return The resulting return code from SQLite. 57 | arrow::Result Step(); 58 | 59 | /// \brief Reset the state of the sqlite3_stmt. 60 | /// \return The resulting return code from SQLite. 61 | arrow::Result Reset(); 62 | 63 | /// \brief Returns the underlying sqlite3_stmt. 64 | /// \return A sqlite statement. 65 | sqlite3_stmt* GetSqlite3Stmt() const; 66 | 67 | /// \brief Executes an UPDATE, INSERT or DELETE statement. 68 | /// \return The number of rows changed by execution. 69 | arrow::Result ExecuteUpdate(); 70 | 71 | private: 72 | sqlite3* db_; 73 | sqlite3_stmt* stmt_; 74 | 75 | SqliteStatement(sqlite3* db, sqlite3_stmt* stmt) : db_(db), stmt_(stmt) {} 76 | }; 77 | 78 | } // namespace sqlite 79 | } // namespace sql 80 | } // namespace flight 81 | } // namespace arrow 82 | -------------------------------------------------------------------------------- /src/sqlite/sqlite_statement_batch_reader.cc: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #include "sqlite_statement_batch_reader.h" 19 | 20 | #include 21 | 22 | #include "arrow/builder.h" 23 | #include "sqlite_statement.h" 24 | 25 | #define STRING_BUILDER_CASE(TYPE_CLASS, STMT, COLUMN) \ 26 | case TYPE_CLASS##Type::type_id: { \ 27 | int bytes = sqlite3_column_bytes(STMT, COLUMN); \ 28 | const unsigned char* string = sqlite3_column_text(STMT, COLUMN); \ 29 | if (string == nullptr) { \ 30 | ARROW_RETURN_NOT_OK( \ 31 | (reinterpret_cast(builder)).AppendNull()); \ 32 | break; \ 33 | } \ 34 | ARROW_RETURN_NOT_OK( \ 35 | (reinterpret_cast(builder)).Append(string, bytes)); \ 36 | break; \ 37 | } 38 | 39 | #define BINARY_BUILDER_CASE(TYPE_CLASS, STMT, COLUMN) \ 40 | case TYPE_CLASS##Type::type_id: { \ 41 | int bytes = sqlite3_column_bytes(STMT, COLUMN); \ 42 | const void* blob = sqlite3_column_blob(STMT, COLUMN); \ 43 | if (blob == nullptr) { \ 44 | ARROW_RETURN_NOT_OK( \ 45 | (reinterpret_cast(builder)).AppendNull()); \ 46 | break; \ 47 | } \ 48 | ARROW_RETURN_NOT_OK( \ 49 | (reinterpret_cast(builder)).Append((char*)blob, bytes)); \ 50 | break; \ 51 | } 52 | 53 | #define INT_BUILDER_CASE(TYPE_CLASS, STMT, COLUMN) \ 54 | case TYPE_CLASS##Type::type_id: { \ 55 | if (sqlite3_column_type(stmt_, i) == SQLITE_NULL) { \ 56 | ARROW_RETURN_NOT_OK( \ 57 | (reinterpret_cast(builder)).AppendNull()); \ 58 | break; \ 59 | } \ 60 | sqlite3_int64 value = sqlite3_column_int64(STMT, COLUMN); \ 61 | ARROW_RETURN_NOT_OK( \ 62 | (reinterpret_cast(builder)).Append(value)); \ 63 | break; \ 64 | } 65 | 66 | #define FLOAT_BUILDER_CASE(TYPE_CLASS, STMT, COLUMN) \ 67 | case TYPE_CLASS##Type::type_id: { \ 68 | if (sqlite3_column_type(stmt_, i) == SQLITE_NULL) { \ 69 | ARROW_RETURN_NOT_OK( \ 70 | (reinterpret_cast(builder)).AppendNull()); \ 71 | break; \ 72 | } \ 73 | double value = sqlite3_column_double(STMT, COLUMN); \ 74 | ARROW_RETURN_NOT_OK( \ 75 | (reinterpret_cast(builder)).Append(value)); \ 76 | break; \ 77 | } 78 | 79 | namespace arrow { 80 | namespace flight { 81 | namespace sql { 82 | namespace sqlite { 83 | 84 | // Batch size for SQLite statement results 85 | static constexpr int kMaxBatchSize = 1024; 86 | 87 | std::shared_ptr SqliteStatementBatchReader::schema() const { return schema_; } 88 | 89 | SqliteStatementBatchReader::SqliteStatementBatchReader( 90 | std::shared_ptr statement, std::shared_ptr schema) 91 | : statement_(std::move(statement)), 92 | schema_(std::move(schema)), 93 | rc_(SQLITE_OK), 94 | already_executed_(false) {} 95 | 96 | arrow::Result> 97 | SqliteStatementBatchReader::Create(const std::shared_ptr& statement_) { 98 | ARROW_RETURN_NOT_OK(statement_->Step()); 99 | 100 | ARROW_ASSIGN_OR_RAISE(auto schema, statement_->GetSchema()); 101 | 102 | std::shared_ptr result( 103 | new SqliteStatementBatchReader(statement_, schema)); 104 | 105 | return result; 106 | } 107 | 108 | arrow::Result> 109 | SqliteStatementBatchReader::Create(const std::shared_ptr& statement, 110 | const std::shared_ptr& schema) { 111 | std::shared_ptr result( 112 | new SqliteStatementBatchReader(statement, schema)); 113 | 114 | return result; 115 | } 116 | 117 | Status SqliteStatementBatchReader::ReadNext(std::shared_ptr* out) { 118 | sqlite3_stmt* stmt_ = statement_->GetSqlite3Stmt(); 119 | 120 | const int num_fields = schema_->num_fields(); 121 | std::vector> builders(num_fields); 122 | 123 | for (int i = 0; i < num_fields; i++) { 124 | const std::shared_ptr& field = schema_->field(i); 125 | const std::shared_ptr& field_type = field->type(); 126 | 127 | ARROW_RETURN_NOT_OK(MakeBuilder(default_memory_pool(), field_type, &builders[i])); 128 | } 129 | 130 | if (!already_executed_) { 131 | ARROW_ASSIGN_OR_RAISE(rc_, statement_->Reset()); 132 | ARROW_ASSIGN_OR_RAISE(rc_, statement_->Step()); 133 | already_executed_ = true; 134 | } 135 | 136 | int64_t rows = 0; 137 | while (rows < kMaxBatchSize && rc_ == SQLITE_ROW) { 138 | rows++; 139 | for (int i = 0; i < num_fields; i++) { 140 | const std::shared_ptr& field = schema_->field(i); 141 | const std::shared_ptr& field_type = field->type(); 142 | ArrayBuilder& builder = *builders[i]; 143 | 144 | // NOTE: This is not the optimal way of building Arrow vectors. 145 | // That would be to presize the builders to avoiding several resizing operations 146 | // when appending values and also to build one vector at a time. 147 | switch (field_type->id()) { 148 | INT_BUILDER_CASE(Int64, stmt_, i) 149 | INT_BUILDER_CASE(UInt64, stmt_, i) 150 | INT_BUILDER_CASE(Int32, stmt_, i) 151 | INT_BUILDER_CASE(UInt32, stmt_, i) 152 | INT_BUILDER_CASE(Int16, stmt_, i) 153 | INT_BUILDER_CASE(UInt16, stmt_, i) 154 | INT_BUILDER_CASE(Int8, stmt_, i) 155 | INT_BUILDER_CASE(UInt8, stmt_, i) 156 | FLOAT_BUILDER_CASE(Double, stmt_, i) 157 | FLOAT_BUILDER_CASE(Float, stmt_, i) 158 | FLOAT_BUILDER_CASE(HalfFloat, stmt_, i) 159 | BINARY_BUILDER_CASE(Binary, stmt_, i) 160 | BINARY_BUILDER_CASE(LargeBinary, stmt_, i) 161 | STRING_BUILDER_CASE(String, stmt_, i) 162 | STRING_BUILDER_CASE(LargeString, stmt_, i) 163 | default: 164 | return Status::NotImplemented("Not implemented SQLite data conversion to ", 165 | field_type->name()); 166 | } 167 | } 168 | 169 | ARROW_ASSIGN_OR_RAISE(rc_, statement_->Step()); 170 | } 171 | 172 | if (rows > 0) { 173 | std::vector> arrays(builders.size()); 174 | for (int i = 0; i < num_fields; i++) { 175 | ARROW_RETURN_NOT_OK(builders[i]->Finish(&arrays[i])); 176 | } 177 | 178 | *out = RecordBatch::Make(schema_, rows, arrays); 179 | } else { 180 | *out = NULLPTR; 181 | } 182 | 183 | return Status::OK(); 184 | } 185 | 186 | } // namespace sqlite 187 | } // namespace sql 188 | } // namespace flight 189 | } // namespace arrow 190 | -------------------------------------------------------------------------------- /src/sqlite/sqlite_statement_batch_reader.h: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | #include 23 | 24 | #include 25 | 26 | #include "sqlite_statement.h" 27 | 28 | namespace arrow { 29 | namespace flight { 30 | namespace sql { 31 | namespace sqlite { 32 | 33 | class SqliteStatementBatchReader : public RecordBatchReader { 34 | public: 35 | /// \brief Creates a RecordBatchReader backed by a SQLite statement. 36 | /// \param[in] statement SQLite statement to be read. 37 | /// \return A SqliteStatementBatchReader. 38 | static arrow::Result> Create( 39 | const std::shared_ptr& statement); 40 | 41 | /// \brief Creates a RecordBatchReader backed by a SQLite statement. 42 | /// \param[in] statement SQLite statement to be read. 43 | /// \param[in] schema Schema to be used on results. 44 | /// \return A SqliteStatementBatchReader.. 45 | static arrow::Result> Create( 46 | const std::shared_ptr& statement, 47 | const std::shared_ptr& schema); 48 | 49 | std::shared_ptr schema() const override; 50 | 51 | Status ReadNext(std::shared_ptr* out) override; 52 | 53 | private: 54 | std::shared_ptr statement_; 55 | std::shared_ptr schema_; 56 | int rc_; 57 | bool already_executed_; 58 | 59 | SqliteStatementBatchReader(std::shared_ptr statement, 60 | std::shared_ptr schema); 61 | }; 62 | 63 | } // namespace sqlite 64 | } // namespace sql 65 | } // namespace flight 66 | } // namespace arrow 67 | -------------------------------------------------------------------------------- /src/sqlite/sqlite_tables_schema_batch_reader.cc: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #include "sqlite_tables_schema_batch_reader.h" 19 | 20 | #include 21 | 22 | #include 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include "sqlite_server.h" 29 | #include "sqlite_statement.h" 30 | 31 | namespace arrow { 32 | namespace flight { 33 | namespace sql { 34 | namespace sqlite { 35 | 36 | std::shared_ptr SqliteTablesWithSchemaBatchReader::schema() const { 37 | return SqlSchema::GetTablesSchemaWithIncludedSchema(); 38 | } 39 | 40 | Status SqliteTablesWithSchemaBatchReader::ReadNext(std::shared_ptr* batch) { 41 | std::stringstream schema_query; 42 | 43 | schema_query 44 | << "SELECT table_name, name, type, [notnull] FROM pragma_table_info(table_name)" 45 | << "JOIN(" << main_query_ << ") order by table_name"; 46 | 47 | std::shared_ptr schema_statement; 48 | ARROW_ASSIGN_OR_RAISE(schema_statement, 49 | SqliteStatement::Create(db_, schema_query.str())) 50 | 51 | std::shared_ptr first_batch; 52 | 53 | ARROW_RETURN_NOT_OK(reader_->ReadNext(&first_batch)); 54 | 55 | if (!first_batch) { 56 | *batch = NULLPTR; 57 | return Status::OK(); 58 | } 59 | 60 | const std::shared_ptr table_name_array = 61 | first_batch->GetColumnByName("table_name"); 62 | 63 | BinaryBuilder schema_builder; 64 | 65 | auto* string_array = reinterpret_cast(table_name_array.get()); 66 | 67 | std::vector> column_fields; 68 | for (int i = 0; i < table_name_array->length(); i++) { 69 | const std::string& table_name = string_array->GetString(i); 70 | 71 | while (sqlite3_step(schema_statement->GetSqlite3Stmt()) == SQLITE_ROW) { 72 | std::string sqlite_table_name = std::string(reinterpret_cast( 73 | sqlite3_column_text(schema_statement->GetSqlite3Stmt(), 0))); 74 | if (sqlite_table_name == table_name) { 75 | const char* column_name = reinterpret_cast( 76 | sqlite3_column_text(schema_statement->GetSqlite3Stmt(), 1)); 77 | const char* column_type = reinterpret_cast( 78 | sqlite3_column_text(schema_statement->GetSqlite3Stmt(), 2)); 79 | int nullable = sqlite3_column_int(schema_statement->GetSqlite3Stmt(), 3); 80 | 81 | const ColumnMetadata& column_metadata = GetColumnMetadata( 82 | GetSqlTypeFromTypeName(column_type), sqlite_table_name.c_str()); 83 | column_fields.push_back(arrow::field(column_name, GetArrowType(column_type), 84 | nullable == 0, 85 | column_metadata.metadata_map())); 86 | } 87 | } 88 | const arrow::Result>& value = 89 | ipc::SerializeSchema(*arrow::schema(column_fields)); 90 | 91 | std::shared_ptr schema_buffer; 92 | ARROW_ASSIGN_OR_RAISE(schema_buffer, value); 93 | 94 | column_fields.clear(); 95 | ARROW_RETURN_NOT_OK( 96 | schema_builder.Append(schema_buffer->data(), schema_buffer->size())); 97 | } 98 | 99 | std::shared_ptr schema_array; 100 | ARROW_RETURN_NOT_OK(schema_builder.Finish(&schema_array)); 101 | 102 | ARROW_ASSIGN_OR_RAISE(*batch, first_batch->AddColumn(4, "table_schema", schema_array)); 103 | 104 | return Status::OK(); 105 | } 106 | 107 | } // namespace sqlite 108 | } // namespace sql 109 | } // namespace flight 110 | } // namespace arrow 111 | -------------------------------------------------------------------------------- /src/sqlite/sqlite_tables_schema_batch_reader.h: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | #include 23 | #include 24 | 25 | #include 26 | 27 | #include "sqlite_statement.h" 28 | #include "sqlite_statement_batch_reader.h" 29 | 30 | namespace arrow { 31 | namespace flight { 32 | namespace sql { 33 | namespace sqlite { 34 | 35 | class SqliteTablesWithSchemaBatchReader : public RecordBatchReader { 36 | private: 37 | std::shared_ptr reader_; 38 | std::string main_query_; 39 | sqlite3* db_; 40 | 41 | public: 42 | /// Constructor for SqliteTablesWithSchemaBatchReader class 43 | /// \param reader an shared_ptr from a SqliteStatementBatchReader. 44 | /// \param main_query SQL query that originated reader's data. 45 | /// \param db a pointer to the sqlite3 db. 46 | SqliteTablesWithSchemaBatchReader( 47 | std::shared_ptr reader, std::string main_query, 48 | sqlite3* db) 49 | : reader_(std::move(reader)), main_query_(std::move(main_query)), db_(db) {} 50 | 51 | std::shared_ptr schema() const override; 52 | 53 | Status ReadNext(std::shared_ptr* batch) override; 54 | }; 55 | 56 | } // namespace sqlite 57 | } // namespace sql 58 | } // namespace flight 59 | } // namespace arrow 60 | -------------------------------------------------------------------------------- /src/sqlite/sqlite_type_info.cc: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #include "sqlite_type_info.h" 19 | 20 | #include 21 | #include 22 | #include 23 | // #include 24 | #include 25 | 26 | 27 | namespace arrow { 28 | namespace flight { 29 | namespace sql { 30 | namespace sqlite { 31 | 32 | using arrow::ipc::internal::json::ArrayFromJSON; 33 | 34 | std::shared_ptr DoGetTypeInfoResult() { 35 | auto type_name_array = 36 | ArrayFromJSON(utf8(), R"(["bit", "tinyint", "bigint", "longvarbinary", 37 | "varbinary", "text", "longvarchar", "char", 38 | "integer", "smallint", "float", "double", 39 | "numeric", "varchar", "date", "time", 40 | "timestamp"])"); 41 | auto data_type = ArrayFromJSON( 42 | int32(), R"([-7, -6, -5, -4, -3, -1, -1, 1, 4, 5, 6, 8, 8, 12, 91, 92, 93])"); 43 | auto column_size = ArrayFromJSON( 44 | int32(), 45 | R"([1, 3, 19, 65536, 255, 65536, 65536, 255, 9, 5, 7, 15, 15, 255, 10, 8, 32])"); 46 | auto literal_prefix = ArrayFromJSON( 47 | utf8(), 48 | R"([null, null, null, null, null, "'", "'", "'", null, null, null, null, null, "'", "'", "'", "'"])"); 49 | auto literal_suffix = ArrayFromJSON( 50 | utf8(), 51 | R"([null, null, null, null, null, "'", "'", "'", null, null, null, null, null, "'", "'", "'", "'"])"); 52 | auto create_params = ArrayFromJSON( 53 | list(field("item", utf8(), false)), 54 | R"([[], [], [], [], [], ["length"], ["length"], ["length"], [], [], [], [], [], ["length"], [], [], []])"); 55 | auto nullable = 56 | ArrayFromJSON(int32(), R"([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])"); 57 | // Reference for creating a boolean() array only with zero. 58 | auto zero_bool_array = 59 | ArrayFromJSON(boolean(), R"([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"); 60 | const auto& case_sensitive = zero_bool_array; 61 | auto searchable = 62 | ArrayFromJSON(int32(), R"([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3])"); 63 | const auto& unsigned_attribute = zero_bool_array; 64 | const auto& fixed_prec_scale = zero_bool_array; 65 | const auto& auto_unique_value = zero_bool_array; 66 | auto local_type_name = 67 | ArrayFromJSON(utf8(), R"(["bit", "tinyint", "bigint", "longvarbinary", 68 | "varbinary", "text", "longvarchar", "char", 69 | "integer", "smallint", "float", "double", 70 | "numeric", "varchar", "date", "time", 71 | "timestamp"])"); 72 | // Reference for creating an int32() array only with zero. 73 | auto zero_int_array = 74 | ArrayFromJSON(int32(), R"([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"); 75 | const auto& minimal_scale = zero_int_array; 76 | const auto& maximum_scale = zero_int_array; 77 | const auto& sql_data_type = data_type; 78 | const auto& sql_datetime_sub = zero_int_array; 79 | const auto& num_prec_radix = zero_int_array; 80 | const auto& interval_precision = zero_int_array; 81 | 82 | std::vector> columns{ 83 | type_name_array.ValueOrDie(), 84 | data_type.ValueOrDie(), 85 | column_size.ValueOrDie(), 86 | literal_prefix.ValueOrDie(), 87 | literal_suffix.ValueOrDie(), 88 | create_params.ValueOrDie(), 89 | nullable.ValueOrDie(), 90 | case_sensitive.ValueOrDie(), 91 | searchable.ValueOrDie(), 92 | unsigned_attribute.ValueOrDie(), 93 | fixed_prec_scale.ValueOrDie(), 94 | auto_unique_value.ValueOrDie(), 95 | local_type_name.ValueOrDie(), 96 | minimal_scale.ValueOrDie(), 97 | maximum_scale.ValueOrDie(), 98 | sql_data_type.ValueOrDie(), 99 | sql_datetime_sub.ValueOrDie(), 100 | num_prec_radix.ValueOrDie(), 101 | interval_precision.ValueOrDie()}; 102 | 103 | return RecordBatch::Make( 104 | SqlSchema::GetXdbcTypeInfoSchema(), 17, columns); 105 | } 106 | 107 | std::shared_ptr DoGetTypeInfoResult(int data_type_filter) { 108 | auto record_batch = DoGetTypeInfoResult(); 109 | 110 | std::vector data_type_vector{-7, -6, -5, -4, -3, -1, -1, 1, 4, 111 | 5, 6, 8, 8, 12, 91, 92, 93}; 112 | 113 | // Checking if the data_type is in the vector with the sqlite3 data types 114 | // and returning a slice from the vector containing the filtered values. 115 | auto pair = std::equal_range(data_type_vector.begin(), data_type_vector.end(), 116 | data_type_filter); 117 | 118 | return record_batch->Slice(pair.first - data_type_vector.begin(), 119 | pair.second - pair.first); 120 | } 121 | } // namespace sqlite 122 | } // namespace sql 123 | } // namespace flight 124 | } // namespace arrow 125 | -------------------------------------------------------------------------------- /src/sqlite/sqlite_type_info.h: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one 2 | // or more contributor license agreements. See the NOTICE file 3 | // distributed with this work for additional information 4 | // regarding copyright ownership. The ASF licenses this file 5 | // to you under the Apache License, Version 2.0 (the 6 | // "License"); you may not use this file except in compliance 7 | // with the License. You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, 12 | // software distributed under the License is distributed on an 13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | // KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations 16 | // under the License. 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | namespace arrow { 23 | namespace flight { 24 | namespace sql { 25 | namespace sqlite { 26 | 27 | /// \brief Gets the harded-coded type info from Sqlite for all data types. 28 | /// \return A record batch. 29 | std::shared_ptr DoGetTypeInfoResult(); 30 | 31 | /// \brief Gets the harded-coded type info from Sqlite filtering 32 | /// for a specific data type. 33 | /// \return A record batch. 34 | std::shared_ptr DoGetTypeInfoResult(int data_type_filter); 35 | } // namespace sqlite 36 | } // namespace sql 37 | } // namespace flight 38 | } // namespace arrow 39 | --------------------------------------------------------------------------------