├── worker-instances ├── .gitignore ├── generateRandomQueries ├── .gitignore └── README.md ├── schema-bench └── requirements.txt ├── tpch_2_13_0 ├── answers │ ├── q14.out │ ├── q17.out │ ├── q19.out │ ├── q6.out │ ├── q8.out │ ├── q12.out │ ├── q4.out │ ├── q15.out │ ├── q5.out │ ├── q7.out │ ├── q22.out │ ├── q3.out │ ├── q1.out │ ├── q13.out │ ├── q21.out │ └── q10.out ├── release.h ├── dop.sh ├── column_split.sh ├── drop_tables.sql ├── reference │ ├── cmd_update_sf1 │ ├── cmd_update_sf100 │ ├── cmd_update_sf300 │ ├── cmd_update_sf1000 │ ├── cmd_update_sf10000 │ ├── cmd_update_sf3000 │ ├── cmd_update_sf30000 │ ├── cmd_update_sf100000 │ ├── trim_updates.sh │ ├── cmd_base_small │ ├── cmd_base_sf1 │ ├── cmd_base_sf100 │ ├── cmd_base_sf300 │ ├── cmd_base_sf1000 │ ├── cmd_base_sf3000 │ ├── cmd_base_sf10000 │ ├── cmd_base_sf30000 │ ├── cmd_base_sf100000 │ └── README.txt ├── distributed_queries │ ├── 6.sql │ ├── 3.sql │ ├── 14.sql │ ├── 5.sql │ ├── 1.sql │ ├── 10.sql │ ├── 9.sql │ ├── 12.sql │ ├── 8.sql │ ├── 7.sql │ └── 19.sql ├── queries │ ├── 6.sql │ ├── 17.sql │ ├── 14.sql │ ├── 13.sql │ ├── 4.sql │ ├── 3.sql │ ├── 5.sql │ ├── 18.sql │ ├── 16.sql │ ├── 11.sql │ ├── 1.sql │ ├── 10.sql │ ├── 15.sql │ ├── 12.sql │ ├── 20.sql │ ├── 9.sql │ ├── 22.sql │ ├── 21.sql │ ├── 2.sql │ ├── 8.sql │ ├── 7.sql │ └── 19.sql ├── check55.sh ├── check_dirs.sh ├── variants │ ├── 14a.sql │ ├── 13a.sql │ ├── 12a.sql │ ├── 15a.sql │ └── 8a.sql ├── update_release.sh ├── gen_tasks.sh ├── rng64.h ├── tpch.dsw ├── generate2.sh ├── bcd2.h ├── warm_up_cache.sql ├── tpch.sln ├── load_balance.sh ├── new55.sh ├── last_row.sh ├── shared.h ├── dss.ri ├── permute.h ├── tpch_create_tables.ddl └── tpch_create_hash_partitioned_tables.ddl ├── fabfile ├── pgbench_scripts │ ├── realtime_select.sql │ ├── copy.sql │ ├── select_all_with_limit.sql │ ├── scale_copy.sql │ ├── delete.sql │ ├── router_select.sql │ ├── update.sql │ ├── insert.sql │ ├── pgbench_select_only_no_transactions.sql │ ├── generate_series.sql │ ├── insert_select_pushdown.sql │ ├── insert_select_coordinator.sql │ ├── insert_foreign.sql │ ├── multi_shard_update.sql │ ├── single_shard_update.sql │ ├── multi_row_insert_foreign.sql │ ├── pgbench_simple_update_no_transactions.sql │ ├── multi_row_insert_prepared.sql │ ├── insert_complex_prepared.sql │ ├── insert_complex.sql │ ├── pgbench_default_no_transactions.sql │ └── scale_multi_row_insert.sql ├── requirements.txt ├── tpch_confs │ ├── tpch_q1.ini │ └── tpch_default.ini ├── cache.py ├── hammerdb_confs │ └── citus_config.ini ├── config.py ├── connection.py ├── extension_hooks.py ├── pgbench_confs │ ├── pgbench_cloud.ini │ ├── pgbench_issue_1799.ini │ ├── scale_test_foreign.ini │ ├── pgbench_custom.ini │ ├── pgbench_default.ini │ ├── pgbench_default_without_transaction.ini │ ├── scale_test_100_columns.ini │ ├── scale_test_no_index.ini │ ├── scale_test_reference.ini │ ├── scale_test.ini │ └── scale_test_prepared.ini ├── multi_connections.py ├── __init__.py ├── prefix.py ├── use.py ├── pg.py └── utils.py ├── hammerdb ├── sql │ ├── do-checkpoint.sql │ ├── vacuum-ch.sql │ ├── tables-total-size.sql │ ├── vacuum-tpcc.sql │ ├── drop-tables.sql │ ├── ch-benchmark-distribute.sql │ ├── tpcc-distribute.sql │ └── tpcc-distribute-funcs.sql ├── send_pubkey.sh ├── entry.sh ├── azuredeploy.parameters.json ├── build.tcl ├── connect-driver.sh ├── run.tcl ├── delete-resource-group.sh ├── cache_git_creds.sh ├── setup.sh ├── upload-results.sh ├── run_all.sh ├── download-hammerdb.sh ├── create.sh └── create-run.sh ├── azure ├── delete-resource-group.sh ├── delete-resource-group-job.sh ├── azuredeploy.parameters.json ├── run-all-tests.sh ├── connect.sh ├── citus-bot.sh ├── push-results.sh ├── commons.sh └── create-cluster.sh ├── jdbc ├── jdbc_config.json ├── README.md └── utils.sh ├── release ├── README.md └── update_files.sh ├── cloudformation ├── fab └── find-workers.sh ├── .github └── workflows │ ├── jdbc-tests.yml │ ├── cleanup.yml │ ├── scale-tests.yml │ ├── tpch-tests.yml │ ├── pgbench-tests.yml │ └── extension.yml ├── ci └── editorconfig.sh ├── valgrind ├── docker │ ├── Dockerfile │ └── run_valgrind_test.sh └── run.sh └── routes.ps1 /worker-instances: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .vscode/ 3 | .venv 4 | -------------------------------------------------------------------------------- /generateRandomQueries/.gitignore: -------------------------------------------------------------------------------- 1 | *.sql 2 | *.out -------------------------------------------------------------------------------- /schema-bench/requirements.txt: -------------------------------------------------------------------------------- 1 | fire 2 | psycopg[c,pool] 3 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q14.out: -------------------------------------------------------------------------------- 1 | promo_revenue 2 | 16.38 3 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q17.out: -------------------------------------------------------------------------------- 1 | avg_yearly 2 | 348406.05 3 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q19.out: -------------------------------------------------------------------------------- 1 | revenue 2 | 3083843.06 3 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q6.out: -------------------------------------------------------------------------------- 1 | revenue 2 | 123141078.23 3 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/realtime_select.sql: -------------------------------------------------------------------------------- 1 | SELECT count(*) FROM test_table; -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q8.out: -------------------------------------------------------------------------------- 1 | o_year | mkt_share 2 | 1995 | 0.03 3 | 1996 | 0.04 4 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/copy.sql: -------------------------------------------------------------------------------- 1 | COPY test_table FROM ':HOME/scale_test_data.csv'; 2 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/select_all_with_limit.sql: -------------------------------------------------------------------------------- 1 | SELECT * FROM test_table LIMIT 10; 2 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/scale_copy.sql: -------------------------------------------------------------------------------- 1 | COPY test_table FROM ':HOME/scale_test_data.csv'; 2 | -------------------------------------------------------------------------------- /fabfile/requirements.txt: -------------------------------------------------------------------------------- 1 | Fabric==2.7.1 2 | paramiko==3.4.1 3 | patchwork==1.0.1 4 | invoke==1.7.3 -------------------------------------------------------------------------------- /hammerdb/sql/do-checkpoint.sql: -------------------------------------------------------------------------------- 1 | checkpoint; 2 | select run_command_on_workers($$checkpoint$$); 3 | -------------------------------------------------------------------------------- /hammerdb/sql/vacuum-ch.sql: -------------------------------------------------------------------------------- 1 | VACUUM ANALYZE region; 2 | VACUUM ANALYZE nation; 3 | VACUUM ANALYZE supplier; 4 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/delete.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 10000000) 2 | DELETE FROM test_table WHERE a = :aid; 3 | 4 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/router_select.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000) 2 | SELECT count(*) FROM test_table WHERE key = :aid; -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/update.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 10000000) 2 | UPDATE test_table SET b = :aid WHERE a = :aid; 3 | 4 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q12.out: -------------------------------------------------------------------------------- 1 | l_shipmode | high_line_count | low_line_count 2 | MAIL | 6202 | 9324 3 | SHIP | 6200 | 9262 4 | -------------------------------------------------------------------------------- /hammerdb/sql/tables-total-size.sql: -------------------------------------------------------------------------------- 1 | SELECT pg_size_pretty(sum(citus_relation_size(logicalrelid))) FROM pg_dist_partition; 2 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/insert.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 10000000) 2 | INSERT INTO test_table VALUES (:aid, :aid, :aid, :aid); 3 | 4 | -------------------------------------------------------------------------------- /tpch_2_13_0/release.h: -------------------------------------------------------------------------------- 1 | /* 2 | * release.h 3 | */ 4 | #define VERSION 2 5 | #define RELEASE 13 6 | #define PATCH 0 7 | #define BUILD 4 8 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/pgbench_select_only_no_transactions.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000 * 100) 2 | SELECT abalance FROM pgbench_accounts WHERE aid = :aid; 3 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q4.out: -------------------------------------------------------------------------------- 1 | o_orderpriority | order_count 2 | 1-URGENT | 10594 3 | 2-HIGH | 10476 4 | 3-MEDIUM | 10410 5 | 4-NOT SPECIFIED | 10556 6 | 5-LOW | 10487 7 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/generate_series.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO test_table SELECT generate_series(1,10), generate_series(1,10), generate_series(1,10), generate_series(1,10); 2 | -------------------------------------------------------------------------------- /fabfile/tpch_confs/tpch_q1.ini: -------------------------------------------------------------------------------- 1 | [test] 2 | postgres_citus_versions: [('17.5', 'release-13.2')] 3 | tpch_tasks_executor_types: [('1.sql', 'real-time')] 4 | scale_factor: 1 5 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q15.out: -------------------------------------------------------------------------------- 1 | s_suppkey | s_name | s_address | s_phone | total_revenue 2 | 8449 | Supplier#000008449 | Wp34zim9qYFbVctdW | 20-469-856-8873 | 1772627.21 3 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q5.out: -------------------------------------------------------------------------------- 1 | n_name | revenue 2 | INDONESIA | 55502041.17 3 | VIETNAM | 55295087.00 4 | CHINA | 53724494.26 5 | INDIA | 52035512.00 6 | JAPAN | 45410175.70 7 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/insert_select_pushdown.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000) 2 | \set bid random(1, 100000) 3 | INSERT INTO test_table_target SELECT * FROM test_table WHERE :bid < 1000; 4 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/insert_select_coordinator.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000) 2 | \set bid random(1, 100000) 3 | INSERT INTO test_table_target SELECT * FROM test_table WHERE key = :aid OR key =:bid LIMIT 10; 4 | -------------------------------------------------------------------------------- /azure/delete-resource-group.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "waiting a long time to delete resource group, this might take up to 30 mins depending on your cluster size" 4 | az group delete -n ${RESOURCE_GROUP_NAME} --yes 5 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q7.out: -------------------------------------------------------------------------------- 1 | supp_nation | cust_nation | l_year | revenue 2 | FRANCE | GERMANY | 1995 | 54639732.73 3 | FRANCE | GERMANY | 1996 | 54633083.31 4 | GERMANY | FRANCE | 1995 | 52531746.67 5 | GERMANY | FRANCE | 1996 | 52520549.02 6 | -------------------------------------------------------------------------------- /jdbc/jdbc_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "pg_version": "14.3", 3 | "citus_branch": "release-11.0", 4 | "COMMENT_NOT_PARSED": "You can lookup jdbc versions at: https://jdbc.postgresql.org/download.html", 5 | "jdbc_version": "42.3.6" 6 | } 7 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q22.out: -------------------------------------------------------------------------------- 1 | cntrycode | numcust | totacctbal 2 | 13 | 888 | 6737713.99 3 | 17 | 861 | 6460573.72 4 | 18 | 964 | 7236687.40 5 | 23 | 892 | 6701457.95 6 | 29 | 948 | 7158866.63 7 | 30 | 909 | 6808436.13 8 | 31 | 922 | 6806670.18 9 | -------------------------------------------------------------------------------- /tpch_2_13_0/dop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SF=$1 3 | [ -z "$SF" ] && SF = 100 4 | 5 | base=`expr $SF \* 2` 6 | 7 | for x in 1 2 3 4 7 8 11 13 16 17 32 64 8 | do 9 | dop=`expr $base \* $x` 10 | dop=`expr $dop + $x` 11 | echo -n "$dop " 12 | done 13 | -------------------------------------------------------------------------------- /tpch_2_13_0/column_split.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | TABLE=$1 3 | IFS="|" 4 | 5 | while read line 6 | do 7 | set $line 8 | count=1 9 | while [ $# -gt 0 ] 10 | do 11 | echo $1 >> ${TABLE}$count 12 | count=`expr $count + 1` 13 | shift 14 | done 15 | done 16 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/insert_foreign.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000) 2 | \set bid random(1, 100000) 3 | \set cid random(1, 100000) 4 | 5 | INSERT INTO test_table (key) VALUES (:aid) ON CONFLICT DO NOTHING; 6 | INSERT INTO test_table_referencing VALUES (:aid, :bid, :cid + 10); -------------------------------------------------------------------------------- /hammerdb/sql/vacuum-tpcc.sql: -------------------------------------------------------------------------------- 1 | VACUUM ANALYZE customer; 2 | VACUUM ANALYZE district; 3 | VACUUM ANALYZE history; 4 | VACUUM ANALYZE warehouse; 5 | VACUUM ANALYZE stock; 6 | VACUUM ANALYZE new_order; 7 | VACUUM ANALYZE orders; 8 | VACUUM ANALYZE order_line; 9 | VACUUM ANALYZE item; 10 | -------------------------------------------------------------------------------- /tpch_2_13_0/drop_tables.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS nation; 2 | DROP TABLE IF EXISTS region; 3 | DROP TABLE IF EXISTS part; 4 | DROP TABLE IF EXISTS supplier; 5 | DROP TABLE IF EXISTS partsupp; 6 | DROP TABLE IF EXISTS customer; 7 | DROP TABLE IF EXISTS orders; 8 | DROP TABLE IF EXISTS lineitem; 9 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/cmd_update_sf1: -------------------------------------------------------------------------------- 1 | ../dbgen -s 1 -U 1 -S 1 -i 100 -d 100 -f -b ../dists.dss 2 | ./trim_updates.sh 1 100 3 | ../dbgen -s 1 -U 75 -S 75 -i 100 -d 100 -f -b ../dists.dss 4 | ./trim_updates.sh 75 100 5 | ../dbgen -s 1 -U 150 -S 150 -i 100 -d 100 -f -b ../dists.dss 6 | ./trim_updates.sh 150 100 7 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/multi_shard_update.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000) 2 | \set bid random(1, 100000) 3 | \set cid random(1, 100000) 4 | \set did random(1, 100000) 5 | UPDATE test_table SET value_1 = row_to_json(row(:aid,:bid, row(:cid+10, :did+500,:aid *7), row(:cid-10, :did-500,:aid *71))) WHERE key != :aid; -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/single_shard_update.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000) 2 | \set bid random(1, 100000) 3 | \set cid random(1, 100000) 4 | \set did random(1, 100000) 5 | UPDATE test_table SET value_1 = row_to_json(row(:aid,:bid, row(:cid+10, :did+500,:aid *7), row(:cid-10, :did-500,:aid *71))) WHERE key = :aid; 6 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/cmd_update_sf100: -------------------------------------------------------------------------------- 1 | ../dbgen -s 100 -U 1 -S 1 -i 100 -d 100 -f -b ../dists.dss 2 | ./trim_updates.sh 1 100 3 | ../dbgen -s 100 -U 75 -S 75 -i 100 -d 100 -f -b ../dists.dss 4 | ./trim_updates.sh 75 100 5 | ../dbgen -s 100 -U 150 -S 150 -i 100 -d 100 -f -b ../dists.dss 6 | ./trim_updates.sh 150 100 7 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/cmd_update_sf300: -------------------------------------------------------------------------------- 1 | ../dbgen -s 300 -U 1 -S 1 -i 100 -d 100 -f -b ../dists.dss 2 | ./trim_updates.sh 1 100 3 | ../dbgen -s 300 -U 75 -S 75 -i 100 -d 100 -f -b ../dists.dss 4 | ./trim_updates.sh 75 100 5 | ../dbgen -s 300 -U 150 -S 150 -i 100 -d 100 -f -b ../dists.dss 6 | ./trim_updates.sh 150 100 7 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/cmd_update_sf1000: -------------------------------------------------------------------------------- 1 | ../dbgen -s 1000 -U 1 -S 1 -i 100 -d 100 -f -b ../dists.dss 2 | ./trim_updates.sh 1 100 3 | ../dbgen -s 1000 -U 75 -S 75 -i 100 -d 100 -f -b ../dists.dss 4 | ./trim_updates.sh 75 100 5 | ../dbgen -s 1000 -U 150 -S 150 -i 100 -d 100 -f -b ../dists.dss 6 | ./trim_updates.sh 150 100 7 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/cmd_update_sf10000: -------------------------------------------------------------------------------- 1 | ../dbgen -s 10000 -U 1 -S 1 -i 100 -d 100 -f -b ../dists.dss 2 | ./trim_updates.sh 1 100 3 | ../dbgen -s 10000 -U 75 -S 75 -i 100 -d 100 -f -b ../dists.dss 4 | ./trim_updates.sh 75 100 5 | ../dbgen -s 10000 -U 150 -S 150 -i 100 -d 100 -f -b ../dists.dss 6 | ./trim_updates.sh 150 100 7 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/cmd_update_sf3000: -------------------------------------------------------------------------------- 1 | ../dbgen -s 3000 -U 1 -S 1 -i 100 -d 100 -f -b ../dists.dss 2 | ./trim_updates.sh 1 100 3 | ../dbgen -s 3000 -U 75 -S 75 -i 100 -d 100 -f -b ../dists.dss 4 | ./trim_updates.sh 75 100 5 | ../dbgen -s 3000 -U 150 -S 150 -i 100 -d 100 -f -b ../dists.dss 6 | ./trim_updates.sh 150 100 7 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/cmd_update_sf30000: -------------------------------------------------------------------------------- 1 | ../dbgen -s 30000 -U 1 -S 1 -i 100 -d 100 -f -b ../dists.dss 2 | ./trim_updates.sh 1 100 3 | ../dbgen -s 30000 -U 75 -S 75 -i 100 -d 100 -f -b ../dists.dss 4 | ./trim_updates.sh 75 100 5 | ../dbgen -s 30000 -U 150 -S 150 -i 100 -d 100 -f -b ../dists.dss 6 | ./trim_updates.sh 150 100 7 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/cmd_update_sf100000: -------------------------------------------------------------------------------- 1 | ../dbgen -s 100000 -U 1 -S 1 -i 100 -d 100 -f -b ../dists.dss 2 | ./trim_updates.sh 1 100 3 | ../dbgen -s 100000 -U 75 -S 75 -i 100 -d 100 -f -b ../dists.dss 4 | ./trim_updates.sh 75 100 5 | ../dbgen -s 100000 -U 150 -S 150 -i 100 -d 100 -f -b ../dists.dss 6 | ./trim_updates.sh 150 100 7 | -------------------------------------------------------------------------------- /generateRandomQueries/README.md: -------------------------------------------------------------------------------- 1 | In order to generate a sql file that contains random procedures and calls, use `generate_procedure.py`. 2 | This will generate `init.sql` and `query.sql`. `init.sql` contains table definitions, settings, procedures etc. `query.sql` contains producedure calls. 3 | 4 | The generated procedures are pushed down to workers. -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/multi_row_insert_foreign.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000) 2 | \set bid random(1, 100000) 3 | \set cid random(1, 100000) 4 | 5 | INSERT INTO test_table (key) VALUES (:aid), (:bid), (:cid) ON CONFLICT DO NOTHING; 6 | INSERT INTO test_table_referencing VALUES (:aid, :bid, :cid + 10), (:bid, :cid, :aid + 10), (:cid, :aid, :bid + 10); 7 | -------------------------------------------------------------------------------- /tpch_2_13_0/distributed_queries/6.sql: -------------------------------------------------------------------------------- 1 | -- Query #6 2 | ----------- 3 | 4 | SELECT 5 | sum(l_extendedprice * l_discount) as revenue 6 | FROM 7 | lineitem 8 | WHERE 9 | l_shipdate >= date '1994-01-01' 10 | AND l_shipdate < date '1994-01-01' + interval '1' year 11 | AND l_discount between 0.06 - 0.01 AND 0.06 + 0.01 12 | AND l_quantity < 24; 13 | -------------------------------------------------------------------------------- /hammerdb/send_pubkey.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u 5 | # exit immediately if a command fails 6 | set -e 7 | # echo commands 8 | set -x 9 | 10 | coordinator_private_ip=$1 11 | export pub_key=$(cat ~/.ssh/id_rsa.pub) 12 | ssh -o "StrictHostKeyChecking no" -A "${coordinator_private_ip}" "echo ${pub_key} >> ~/.ssh/authorized_keys" 13 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/pgbench_simple_update_no_transactions.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000 * 100) 2 | \set bid random(1, 1 * 100) 3 | \set tid random(1, 10 * 100) 4 | \set delta random(-5000, 5000) 5 | UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid; 6 | SELECT abalance FROM pgbench_accounts WHERE aid = :aid; 7 | INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP); -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/multi_row_insert_prepared.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000) 2 | \set bid random(1, 100000) 3 | \set cid random(1, 100000) 4 | \set did random(1, 100000) 5 | 6 | INSERT INTO test_table (key, value_1, value_2, value_3, value_4) VALUES 7 | ((:aid)::int, NULL, NULL, NULL, NULL), 8 | ((:bid)::int, NULL, NULL, NULL, NULL), 9 | ((:cid)::int, NULL, NULL, NULL, NULL), 10 | ((:did)::int, NULL, NULL, NULL, NULL); 11 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/trim_updates.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ $# -ne 2 ] 3 | then 4 | echo "USAGE: `basename $0` " 5 | echo " trim all files in the named update set, retaining only the first" 6 | echo " lines" 7 | exit 8 | fi 9 | if [ -z "$DSS_PATH" ] 10 | then 11 | DSS_PATH=. 12 | export DSS_PATH 13 | fi 14 | tmp=`mktemp` 15 | for f in $DSS_PATH/*u$1.* 16 | do 17 | head -$2 $f > $tmp 18 | mv $tmp $f 19 | done 20 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/6.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Forecasting Revenue Change Query (Q6) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | sum(l_extendedprice * l_discount) as revenue 9 | from 10 | lineitem 11 | where 12 | l_shipdate >= date ':1' 13 | and l_shipdate < date ':1' + interval '1' year 14 | and l_discount between :2 - 0.01 and :2 + 0.01 15 | and l_quantity < :3; 16 | :n -1 17 | -------------------------------------------------------------------------------- /fabfile/tpch_confs/tpch_default.ini: -------------------------------------------------------------------------------- 1 | [test] 2 | postgres_citus_versions: [('17.5', 'release-13.2')] 3 | tpch_tasks_executor_types: [('1.sql', 'adaptive'), ('3.sql', 'adaptive'), ('5.sql', 'adaptive'), ('6.sql', 'adaptive') 4 | ,('7.sql', 'adaptive'), ('8.sql', 'adaptive'), ('9.sql', 'adaptive'), ('10.sql', 'adaptive') 5 | , ('12.sql', 'adaptive'), ('14.sql', 'adaptive'), ('19.sql', 'adaptive')] 6 | scale_factor: 10 7 | -------------------------------------------------------------------------------- /tpch_2_13_0/check55.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SF="1 100 300 1000 3000 10000 30000 100000" 3 | case $# in 4 | 2) 5 | SF=$2; 6 | dop=$1; 7 | ;; 8 | 1) 9 | dop=$1; 10 | ;; 11 | *) 12 | echo "USAGE: $0 [sf]" 13 | echo " work through each SF value using parallel processes" 14 | echo " and the defined DOP settings for bug 55 verification" 15 | exit 16 | ;; 17 | esac 18 | for sf in $SF 19 | do 20 | ./new55.sh $sf "`./dop.sh $sf`" $dop 21 | done 22 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/insert_complex_prepared.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000) 2 | \set bid random(1, 100000) 3 | \set cid random(1, 100000) 4 | \set did random(1, 100000) 5 | 6 | INSERT INTO test_table (key, value_1, value_2, value_3, value_4) VALUES 7 | (:aid, 8 | row_to_json(row((:aid)::text, row(:cid-10, :did-500))), 9 | ARRAY[:aid::text, 'CitusData Rocks']::text[], 10 | int4range(:aid, :bid + 100000), 11 | (:aid, :did)::complex); 12 | -------------------------------------------------------------------------------- /tpch_2_13_0/check_dirs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | er=0 4 | 5 | if [ -z "$1" ]; then 6 | DIRS='1* 300*' 7 | else 8 | DIRS=$1 9 | fi 10 | 11 | for d in $DIRS;do 12 | echo Checking dir $d 13 | cd $d 14 | for f in results.*;do 15 | c=`sed 's/^[0-9]* *//' <$f |uniq |wc -l` 16 | if [ $c -ne 1 ]; then 17 | echo BAD RESULT $d/$f 18 | er=1; 19 | fi 20 | done 21 | cd .. 22 | done 23 | if [ $er -eq 0 ]; then 24 | echo ALL TESTS PASSED 25 | fi 26 | 27 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q3.out: -------------------------------------------------------------------------------- 1 | l_orderkey | revenue | o_orderdate | o_shippriority 2 | 2456423 | 406181.01 | 1995-03-05 | 0 3 | 3459808 | 405838.70 | 1995-03-04 | 0 4 | 492164 | 390324.06 | 1995-02-19 | 0 5 | 1188320 | 384537.94 | 1995-03-09 | 0 6 | 2435712 | 378673.06 | 1995-02-26 | 0 7 | 4878020 | 378376.80 | 1995-03-12 | 0 8 | 5521732 | 375153.92 | 1995-03-13 | 0 9 | 2628192 | 373133.31 | 1995-02-22 | 0 10 | 993600 | 371407.46 | 1995-03-05 | 0 11 | 2300070 | 367371.15 | 1995-03-13 | 0 12 | -------------------------------------------------------------------------------- /hammerdb/sql/drop-tables.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS customer; 2 | DROP TABLE IF EXISTS district; 3 | DROP TABLE IF EXISTS history; 4 | DROP TABLE IF EXISTS warehouse; 5 | DROP TABLE IF EXISTS stock; 6 | DROP TABLE IF EXISTS new_order; 7 | DROP TABLE IF EXISTS orders; 8 | DROP TABLE IF EXISTS order_line; 9 | DROP TABLE IF EXISTS item; 10 | 11 | -- ch benchmark tables 12 | DROP TABLE IF EXISTS supplier CASCADE; 13 | DROP TABLE IF EXISTS nation CASCADE; 14 | DROP TABLE IF EXISTS region CASCADE; 15 | -------------------------------------------------------------------------------- /hammerdb/sql/ch-benchmark-distribute.sql: -------------------------------------------------------------------------------- 1 | -- CH benchmark consists of transactional and analytical part. 2 | -- The analytical part is a modified version of TPC-H, therefore we 3 | -- separate them from TPC-C to distinguish them. TPC-C tables are created 4 | -- in hammerdb, yet these modified tables do not exist in hammerdb, so we need 5 | -- to create them with additonal scripts. 6 | SELECT create_reference_table('region'); 7 | SELECT create_reference_table('nation'); 8 | SELECT create_reference_table('supplier'); 9 | -------------------------------------------------------------------------------- /tpch_2_13_0/variants/14a.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Promotion Effect Query (Q14) 3 | -- Variant A 4 | -- Approved March 1998 5 | :x 6 | :o 7 | select 8 | 100.00 * sum(decode(substring(p_type from 1 for 5), 'PROMO', 9 | l_extendedprice * (1-l_discount), 0)) / 10 | sum(l_extendedprice * (1-l_discount)) as promo_revenue 11 | from 12 | lineitem, 13 | part 14 | where 15 | l_partkey = p_partkey 16 | and l_shipdate >= date ':1' 17 | and l_shipdate < date ':1' + interval '1' month; 18 | :n -1 19 | -------------------------------------------------------------------------------- /release/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Usage of update_files.sh 3 | 4 | If you will test a custom branch, you can update all the config files with `./update_files.sh`. Say that you 5 | will do the release testing for `release-9.2` and `release-9.3`. You can do the following: 6 | 7 | ```bash 8 | vim update_files.sh 9 | 10 | new_branch1=release-9.3 11 | new_branch2=release-9.2 12 | new_pg_version=12.2 13 | 14 | ./update_files.sh # this will update all the branches correctly 15 | git diff #make sure that everything is fine 16 | ``` 17 | -------------------------------------------------------------------------------- /hammerdb/sql/tpcc-distribute.sql: -------------------------------------------------------------------------------- 1 | SELECT create_distributed_table('customer', 'c_w_id'); 2 | SELECT create_distributed_table('district', 'd_w_id'); 3 | SELECT create_distributed_table('history', 'h_w_id'); 4 | SELECT create_distributed_table('warehouse', 'w_id'); 5 | SELECT create_distributed_table('stock', 's_w_id'); 6 | SELECT create_distributed_table('new_order', 'no_w_id'); 7 | SELECT create_distributed_table('orders', 'o_w_id'); 8 | SELECT create_distributed_table('order_line', 'ol_w_id'); 9 | SELECT create_reference_table('item'); 10 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/17.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Small-Quantity-Order Revenue Query (Q17) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | sum(l_extendedprice) / 7.0 as avg_yearly 9 | from 10 | lineitem, 11 | part 12 | where 13 | p_partkey = l_partkey 14 | and p_brand = ':1' 15 | and p_container = ':2' 16 | and l_quantity < ( 17 | select 18 | 0.2 * avg(l_quantity) 19 | from 20 | lineitem 21 | where 22 | l_partkey = p_partkey 23 | ); 24 | :n -1 25 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/14.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Promotion Effect Query (Q14) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | 100.00 * sum(case 9 | when p_type like 'PROMO%' 10 | then l_extendedprice * (1 - l_discount) 11 | else 0 12 | end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue 13 | from 14 | lineitem, 15 | part 16 | where 17 | l_partkey = p_partkey 18 | and l_shipdate >= date ':1' 19 | and l_shipdate < date ':1' + interval '1' month; 20 | :n -1 21 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/insert_complex.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000) 2 | \set bid random(1, 100000) 3 | \set cid random(1, 100000) 4 | \set did random(1, 100000) 5 | 6 | INSERT INTO test_table (key, value_1, value_2, value_3, value_4) VALUES 7 | (:aid, 8 | row_to_json(row(:aid,:bid, row(:cid+10, :did+500,:aid *7), row(:cid-10, :did-500,:aid *71))), 9 | ARRAY[:aid::text, :bid::text, :cid::text, :did::text, 'Onder Kalaci', 'CitusData Rocks']::text[], 10 | int4range(:aid, :aid + :bid), 11 | (:aid, :did)::complex); 12 | -------------------------------------------------------------------------------- /tpch_2_13_0/update_release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | PATH=.:$PATH;export PATH 3 | if [ -f UPDATE_RELEASE_NUMBER ] 4 | then 5 | cvs -q update 6 | if [ ! -f release.h ] 7 | then 8 | BUILD=1 9 | else 10 | BUILD=`grep BUILD release.h |cut -f3 -d' '` 11 | BUILD=`expr $BUILD + 1` 12 | fi 13 | cat > release.h << __EOF__ 14 | /* 15 | * $Id: update_release.sh,v 1.4 2008/03/21 17:38:39 jms Exp $ 16 | */ 17 | #define VERSION $1 18 | #define RELEASE $2 19 | #define PATCH $3 20 | #define BUILD $BUILD 21 | __EOF__ 22 | cvs commit -m "update release number" release.h 23 | fi 24 | -------------------------------------------------------------------------------- /cloudformation/fab: -------------------------------------------------------------------------------- 1 | _fab() 2 | { 3 | local cur 4 | COMPREPLY=() 5 | # Variable to hold the current word 6 | cur="${COMP_WORDS[COMP_CWORD]}" 7 | 8 | # Build a list of the available tasks using the command 'fab -l' 9 | local tags=$(fab -l 2>/dev/null | grep "^ " | awk '{print $1;}') 10 | 11 | # Generate possible matches and store them in the 12 | # array variable COMPREPLY 13 | COMPREPLY=($(compgen -W "${tags}" $cur)) 14 | } 15 | 16 | # Assign the auto-completion function _fab for our command fab. 17 | complete -F _fab fab 18 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q1.out: -------------------------------------------------------------------------------- 1 | l_returnflag | l_linestatus | sum_qty | sum_base_price | sum_disc_price | sum_charge | avg_qty | avg_price | avg_disc | count_order 2 | A | F | 37734107 | 56586554400.73 | 53758257134.87 | 55909065222.83 | 25.52 | 38273.13 | 0.05 | 1478493 3 | N | F | 991417 | 1487504710.38 | 1413082168.05 | 1469649223.19 | 25.52 | 38284.47 | 0.05 | 38854 4 | N | O | 74476040 | 111701729697.74 | 106118230307.61 | 110367043872.50 | 25.50 | 38249.12 | 0.05 | 2920374 5 | R | F | 37719753 | 56568041380.90 | 53741292684.60 | 55889619119.83 | 25.51 | 38250.85 | 0.05 | 1478870 6 | -------------------------------------------------------------------------------- /azure/delete-resource-group-job.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u 5 | # exit immediately if a command fails 6 | set -e 7 | # echo commands 8 | set -x 9 | 10 | BRANCH=${CIRCLE_BRANCH} 11 | 12 | # the resource group name will be after the prefix "delete_resource_group/id_" 13 | # example branch: delete_resource_group/1231123_ch_benchmark_resource_group 14 | resource_group_name=$(echo "$BRANCH" | sed -e "s@delete_resource_group/[0-9]\+_@@g") 15 | 16 | export RESOURCE_GROUP_NAME="$resource_group_name" 17 | ./delete-resource-group.sh 18 | 19 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/pgbench_default_no_transactions.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000 * 100) 2 | \set bid random(1, 1 * 100) 3 | \set tid random(1, 10 * 100) 4 | \set delta random(-5000, 5000) 5 | UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid; 6 | SELECT abalance FROM pgbench_accounts WHERE aid = :aid; 7 | UPDATE pgbench_tellers SET tbalance = tbalance + :delta WHERE tid = :tid; 8 | UPDATE pgbench_branches SET bbalance = bbalance + :delta WHERE bid = :bid; 9 | INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP); 10 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/13.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Customer Distribution Query (Q13) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | c_count, 9 | count(*) as custdist 10 | from 11 | ( 12 | select 13 | c_custkey, 14 | count(o_orderkey) 15 | from 16 | customer left outer join orders on 17 | c_custkey = o_custkey 18 | and o_comment not like '%:1%:2%' 19 | group by 20 | c_custkey 21 | ) as c_orders (c_custkey, c_count) 22 | group by 23 | c_count 24 | order by 25 | custdist desc, 26 | c_count desc; 27 | :n -1 28 | -------------------------------------------------------------------------------- /tpch_2_13_0/distributed_queries/3.sql: -------------------------------------------------------------------------------- 1 | -- Query #3 2 | ----------- 3 | 4 | SELECT 5 | l_orderkey, 6 | sum(l_extendedprice * (1 - l_discount)) as revenue, 7 | o_orderdate, 8 | o_shippriority 9 | FROM 10 | customer, 11 | orders, 12 | lineitem 13 | WHERE 14 | c_mktsegment = 'BUILDING' 15 | AND c_custkey = o_custkey 16 | AND l_orderkey = o_orderkey 17 | AND o_orderdate < date '1995-03-15' 18 | AND l_shipdate > date '1995-03-15' 19 | GROUP BY 20 | l_orderkey, 21 | o_orderdate, 22 | o_shippriority 23 | ORDER BY 24 | revenue desc, 25 | o_orderdate 26 | LIMIT 20; -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q13.out: -------------------------------------------------------------------------------- 1 | c_count | custdist 2 | 0 | 50005 3 | 9 | 6641 4 | 10 | 6532 5 | 11 | 6014 6 | 8 | 5937 7 | 12 | 5639 8 | 13 | 5024 9 | 19 | 4793 10 | 7 | 4687 11 | 17 | 4587 12 | 18 | 4529 13 | 20 | 4516 14 | 15 | 4505 15 | 14 | 4446 16 | 16 | 4273 17 | 21 | 4190 18 | 22 | 3623 19 | 6 | 3265 20 | 23 | 3225 21 | 24 | 2742 22 | 25 | 2086 23 | 5 | 1948 24 | 26 | 1612 25 | 27 | 1179 26 | 4 | 1007 27 | 28 | 893 28 | 29 | 593 29 | 3 | 415 30 | 30 | 376 31 | 31 | 226 32 | 32 | 148 33 | 2 | 134 34 | 33 | 75 35 | 34 | 50 36 | 35 | 37 37 | 1 | 17 38 | 36 | 14 39 | 38 | 5 40 | 37 | 5 41 | 40 | 4 42 | 41 | 2 43 | 39 | 1 44 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/4.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Order Priority Checking Query (Q4) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | o_orderpriority, 9 | count(*) as order_count 10 | from 11 | orders 12 | where 13 | o_orderdate >= date ':1' 14 | and o_orderdate < date ':1' + interval '3' month 15 | and exists ( 16 | select 17 | * 18 | from 19 | lineitem 20 | where 21 | l_orderkey = o_orderkey 22 | and l_commitdate < l_receiptdate 23 | ) 24 | group by 25 | o_orderpriority 26 | order by 27 | o_orderpriority; 28 | :n -1 29 | -------------------------------------------------------------------------------- /jdbc/README.md: -------------------------------------------------------------------------------- 1 | To trigger the test push a branch called `jdbc/{whatever-you-want}` 2 | To configure the citus branch or the jdbc version change the appropriate properties 3 | in [JDBC Config](./jdbc_config.json) 4 | 5 | Dependencies: 6 | * [citus_dev](https://github.com/citusdata/tools/tree/develop/citus_dev) 7 | * java jdk (most recent version is install by apt-get) 8 | * jdbc driver for PostgreSQL (see https://jdbc.postgresql.org/) 9 | 10 | Compare resulting output files from 6 combinations that script executes the tests [(hash, append) x (real-time, task-tracker, adaptive)]. 11 | Resulting counts should all match each other. 12 | -------------------------------------------------------------------------------- /.github/workflows/jdbc-tests.yml: -------------------------------------------------------------------------------- 1 | name: Compatibility Tests - JDBC 2 | on: 3 | push: 4 | branches: 5 | - jdbc/* 6 | env: 7 | RESULT_REPO_ACCESS_TOKEN: ${{ secrets.TEST_RESULTS_REPO_TOKEN }} 8 | jobs: 9 | jdbc: 10 | runs-on: ubuntu-latest 11 | container: 12 | image: buildpack-deps:focal 13 | options: --user root 14 | steps: 15 | - uses: actions/checkout@v3.5.0 16 | - name: Fix ownership issues 17 | run: git config --global --add safe.directory ${GITHUB_WORKSPACE} 18 | - name: Run JDBC compatibility tests 19 | run: |- 20 | cd ./jdbc 21 | ./run_jdbc_tests.sh 22 | shell: bash 23 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/3.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Shipping Priority Query (Q3) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | l_orderkey, 9 | sum(l_extendedprice * (1 - l_discount)) as revenue, 10 | o_orderdate, 11 | o_shippriority 12 | from 13 | customer, 14 | orders, 15 | lineitem 16 | where 17 | c_mktsegment = ':1' 18 | and c_custkey = o_custkey 19 | and l_orderkey = o_orderkey 20 | and o_orderdate < date ':2' 21 | and l_shipdate > date ':2' 22 | group by 23 | l_orderkey, 24 | o_orderdate, 25 | o_shippriority 26 | order by 27 | revenue desc, 28 | o_orderdate; 29 | :n 10 30 | -------------------------------------------------------------------------------- /tpch_2_13_0/variants/13a.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Customer Distribution Query (Q13) 3 | -- Variant A 4 | -- Approved March 1998 5 | :x 6 | create view orders_per_cust:s (custkey, ordercount) as 7 | select 8 | c_custkey, 9 | count(o_orderkey) 10 | from 11 | customer left outer join orders on 12 | c_custkey = o_custkey 13 | and o_comment not like '%:1%:2%' 14 | group by 15 | c_custkey; 16 | 17 | :o 18 | select 19 | ordercount, 20 | count(*) as custdist 21 | from 22 | orders_per_cust:s 23 | group by 24 | ordercount 25 | order by 26 | custdist desc, 27 | ordercount desc; 28 | 29 | drop view orders_per_cust:s; 30 | :n -1 31 | -------------------------------------------------------------------------------- /tpch_2_13_0/gen_tasks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | COMMAND="./last_row.sh" 3 | if [ -d "c:/" ] 4 | then COMMAND="./no_tail.sh" 5 | fi 6 | 7 | if [ $# -eq 0 ] 8 | then 9 | echo "USAGE: `basename $0` [...]" 10 | echo " generate the commands for the load balancer" 11 | echo " if DOP is ommitted, {1,2,3,4,7,8,11,13,16,17,32,64} is used" 12 | exit 13 | fi 14 | 15 | SF=$1 16 | DOP="1 2 3 4 7 8 11 13 16 17 32 64" 17 | shift 18 | [ $# -ge 1 ] && DOP="$*" 19 | for D in $DOP 20 | do 21 | echo "$COMMAND c $SF $D" 22 | echo "$COMMAND L $SF $D" 23 | echo "$COMMAND O $SF $D" 24 | echo "$COMMAND P $SF $D" 25 | echo "$COMMAND S $SF $D" 26 | echo "$COMMAND s $SF $D" 27 | done 28 | 29 | -------------------------------------------------------------------------------- /tpch_2_13_0/distributed_queries/14.sql: -------------------------------------------------------------------------------- 1 | -- Query #14 2 | ------------ 3 | 4 | SELECT 5 | 100.00 * sum(case 6 | when p_type like 'PROMO%' 7 | then l_extendedprice * (1 - l_discount) 8 | else 0 9 | end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue 10 | FROM 11 | lineitem, 12 | part 13 | WHERE 14 | l_partkey = p_partkey 15 | AND l_shipdate >= date '1995-09-01' 16 | AND l_shipdate < date '1995-09-01' + interval '1' month 17 | -- the following LIMIT is not part of the benchmark suite 18 | -- but required to run the tests automatically 19 | -- see https://github.com/citusdata/test-automation/issues/120 20 | LIMIT 4; 21 | -------------------------------------------------------------------------------- /tpch_2_13_0/distributed_queries/5.sql: -------------------------------------------------------------------------------- 1 | -- Query #5 2 | ----------- 3 | 4 | SELECT 5 | n_name, 6 | sum(l_extendedprice * (1 - l_discount)) as revenue 7 | FROM 8 | customer, 9 | orders, 10 | lineitem, 11 | supplier, 12 | nation, 13 | region 14 | WHERE 15 | c_custkey = o_custkey 16 | AND l_orderkey = o_orderkey 17 | AND l_suppkey = s_suppkey 18 | AND c_nationkey = s_nationkey 19 | AND s_nationkey = n_nationkey 20 | AND n_regionkey = r_regionkey 21 | AND r_name = 'ASIA' 22 | AND o_orderdate >= date '1994-01-01' 23 | AND o_orderdate < date '1994-01-01' + interval '1' year 24 | GROUP BY 25 | n_name 26 | ORDER BY 27 | revenue desc; 28 | -------------------------------------------------------------------------------- /tpch_2_13_0/distributed_queries/1.sql: -------------------------------------------------------------------------------- 1 | -- Query #1 2 | ----------- 3 | 4 | SELECT 5 | l_returnflag, 6 | l_linestatus, 7 | sum(l_quantity) as sum_qty, 8 | sum(l_extendedprice) as sum_base_price, 9 | sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, 10 | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, 11 | avg(l_quantity) as avg_qty, 12 | avg(l_extendedprice) as avg_price, 13 | avg(l_discount) as avg_disc, 14 | count(*) as count_order 15 | FROM 16 | lineitem 17 | WHERE 18 | l_shipdate <= date '1998-12-01' - interval '90' day 19 | GROUP BY 20 | l_returnflag, 21 | l_linestatus 22 | ORDER BY 23 | l_returnflag, 24 | l_linestatus; -------------------------------------------------------------------------------- /azure/azuredeploy.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "numberOfWorkers": { 6 | "value": 2 7 | }, 8 | "diskType": { 9 | "value": "Premium_LRS" 10 | }, 11 | "coordinatorVmSize": { 12 | "value": "Standard_D4s_v5" 13 | }, 14 | "workerVmSize": { 15 | "value": "Standard_E4s_v5" 16 | }, 17 | "coordinatorDiskSizeGB": { 18 | "value": 512 19 | }, 20 | "workerDiskSizeGB": { 21 | "value": 512 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /fabfile/cache.py: -------------------------------------------------------------------------------- 1 | class Singleton(type): 2 | _instances = {} 3 | def __call__(cls, *args, **kwargs): 4 | if cls not in cls._instances: 5 | cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) 6 | return cls._instances[cls] 7 | 8 | class Cache: 9 | __metaclass__ = Singleton 10 | def __init__(self): 11 | self._build_cache = {} 12 | 13 | def search_build_cache(self,package_name): 14 | if package_name in self._build_cache: 15 | # we have already installed it 16 | return True 17 | else: 18 | return False 19 | 20 | def insert_build_cache(self,package_name): 21 | self._build_cache[package_name] = True 22 | -------------------------------------------------------------------------------- /tpch_2_13_0/rng64.h: -------------------------------------------------------------------------------- 1 | /* 2 | * $Id: rng64.h,v 1.2 2005/01/03 20:08:59 jms Exp $ 3 | * 4 | * Revision History 5 | * =================== 6 | * $Log: rng64.h,v $ 7 | * Revision 1.2 2005/01/03 20:08:59 jms 8 | * change line terminations 9 | * 10 | * Revision 1.1.1.1 2004/11/24 23:31:47 jms 11 | * re-establish external server 12 | * 13 | * Revision 1.1.1.1 2003/08/08 21:57:34 jms 14 | * recreation after CVS crash 15 | * 16 | * Revision 1.1 2003/08/08 21:57:34 jms 17 | * first integration of rng64 for o_custkey and l_partkey 18 | * 19 | * 20 | */ 21 | DSS_HUGE AdvanceRand64( DSS_HUGE nSeed, DSS_HUGE nCount); 22 | void dss_random64(DSS_HUGE *tgt, DSS_HUGE nLow, DSS_HUGE nHigh, long stream); 23 | DSS_HUGE NextRand64(DSS_HUGE nSeed); 24 | -------------------------------------------------------------------------------- /hammerdb/entry.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u 5 | # exit immediately if a command fails 6 | set -e 7 | # echo commands 8 | set -x 9 | 10 | export NODE_ID=$1 11 | export NODE_COUNT=$2 12 | export AZURE_STORAGE_ACCOUNT=$3 13 | export AZURE_STORAGE_KEY=$4 14 | export BRANCH=$5 15 | export GIT_USERNAME=$6 16 | export GIT_TOKEN=$7 17 | export DATA_DISK_SIZE=$8 18 | 19 | # driver node is the last node. 20 | if [[ $(( NODE_COUNT - 1 )) == "$NODE_ID" ]]; then 21 | ./driver-init.sh "$BRANCH" "$DATA_DISK_SIZE" 22 | else 23 | ./init.sh "$NODE_ID" $(( NODE_COUNT - 1 )) "$AZURE_STORAGE_ACCOUNT" "$AZURE_STORAGE_KEY" "$BRANCH" "$DATA_DISK_SIZE" 24 | fi 25 | 26 | ./cache_git_creds.sh 27 | -------------------------------------------------------------------------------- /tpch_2_13_0/variants/12a.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Shipping Modes and Order Priority Query(Q12) 3 | -- Variant A 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | l_shipmode, 9 | sum(decode(o_orderpriority, '1-URGENT', 1, '2-HIGH', 1, 0)) as 10 | high_line_count, 11 | sum(decode(o_orderpriority, '1-URGENT', 0, '2-HIGH', 0, 1)) as 12 | low_line_count 13 | from 14 | orders, 15 | lineitem 16 | where 17 | o_orderkey = l_orderkey 18 | and l_shipmode in (':1', ':2') 19 | and l_commitdate < l_receiptdate 20 | and l_shipdate < l_commitdate 21 | and l_receiptdate >= date ':3' 22 | and l_receiptdate < date ':3' + interval '1' year 23 | group by 24 | l_shipmode 25 | order by 26 | l_shipmode; 27 | :n -1 28 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/5.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Local Supplier Volume Query (Q5) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | n_name, 9 | sum(l_extendedprice * (1 - l_discount)) as revenue 10 | from 11 | customer, 12 | orders, 13 | lineitem, 14 | supplier, 15 | nation, 16 | region 17 | where 18 | c_custkey = o_custkey 19 | and l_orderkey = o_orderkey 20 | and l_suppkey = s_suppkey 21 | and c_nationkey = s_nationkey 22 | and s_nationkey = n_nationkey 23 | and n_regionkey = r_regionkey 24 | and r_name = ':1' 25 | and o_orderdate >= date ':2' 26 | and o_orderdate < date ':2' + interval '1' year 27 | group by 28 | n_name 29 | order by 30 | revenue desc; 31 | :n -1 32 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/18.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Large Volume Customer Query (Q18) 3 | -- Function Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | c_name, 9 | c_custkey, 10 | o_orderkey, 11 | o_orderdate, 12 | o_totalprice, 13 | sum(l_quantity) 14 | from 15 | customer, 16 | orders, 17 | lineitem 18 | where 19 | o_orderkey in ( 20 | select 21 | l_orderkey 22 | from 23 | lineitem 24 | group by 25 | l_orderkey having 26 | sum(l_quantity) > :1 27 | ) 28 | and c_custkey = o_custkey 29 | and o_orderkey = l_orderkey 30 | group by 31 | c_name, 32 | c_custkey, 33 | o_orderkey, 34 | o_orderdate, 35 | o_totalprice 36 | order by 37 | o_totalprice desc, 38 | o_orderdate; 39 | :n 100 40 | -------------------------------------------------------------------------------- /ci/editorconfig.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # The file is taken from https://github.com/citusdata/citus/blob/main/ci/editorconfig.sh 4 | # We run that script to remove trailing spaces and to add a newline at file ends only for *.sh, *.py, and *azuredeploy* files 5 | 6 | set -euo pipefail 7 | 8 | for f in $(git ls-tree -r HEAD --name-only); do 9 | # only process files with extensions sh, py or azuredeploy 10 | if [ "$f" != "${f%.sh}" ] || 11 | [ "$f" != "${f%.py}" ] || 12 | [ "$f" != "${f%azuredeploy*.json}" ] 13 | then 14 | # Trim trailing whitespace 15 | sed -e 's/[[:space:]]*$//' -i "./$f" 16 | # Add final newline if not there 17 | if [ -n "$(tail -c1 "$f")" ]; then 18 | echo >> "$f" 19 | fi 20 | fi 21 | done 22 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/16.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Parts/Supplier Relationship Query (Q16) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | p_brand, 9 | p_type, 10 | p_size, 11 | count(distinct ps_suppkey) as supplier_cnt 12 | from 13 | partsupp, 14 | part 15 | where 16 | p_partkey = ps_partkey 17 | and p_brand <> ':1' 18 | and p_type not like ':2%' 19 | and p_size in (:3, :4, :5, :6, :7, :8, :9, :10) 20 | and ps_suppkey not in ( 21 | select 22 | s_suppkey 23 | from 24 | supplier 25 | where 26 | s_comment like '%Customer%Complaints%' 27 | ) 28 | group by 29 | p_brand, 30 | p_type, 31 | p_size 32 | order by 33 | supplier_cnt desc, 34 | p_brand, 35 | p_type, 36 | p_size; 37 | :n -1 38 | -------------------------------------------------------------------------------- /tpch_2_13_0/variants/15a.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Top Supplier Query (Q15) 3 | -- Variant A 4 | -- Approved February 1998 5 | :x 6 | with revenue (supplier_no, total_revenue) as ( 7 | select 8 | l_suppkey, 9 | sum(l_extendedprice * (1-l_discount)) 10 | from 11 | lineitem 12 | where 13 | l_shipdate >= date ':1' 14 | and l_shipdate < date ':1' + interval '3' month 15 | group by 16 | l_suppkey 17 | ) 18 | 19 | :o 20 | select 21 | s_suppkey, 22 | s_name, 23 | s_address, 24 | s_phone, 25 | total_revenue 26 | from 27 | supplier, 28 | revenue 29 | where 30 | s_suppkey = supplier_no 31 | and total_revenue = ( 32 | select 33 | max(total_revenue) 34 | from 35 | revenue 36 | ) 37 | order by 38 | s_suppkey; 39 | :n -1 40 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/11.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Important Stock Identification Query (Q11) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | ps_partkey, 9 | sum(ps_supplycost * ps_availqty) as value 10 | from 11 | partsupp, 12 | supplier, 13 | nation 14 | where 15 | ps_suppkey = s_suppkey 16 | and s_nationkey = n_nationkey 17 | and n_name = ':1' 18 | group by 19 | ps_partkey having 20 | sum(ps_supplycost * ps_availqty) > ( 21 | select 22 | sum(ps_supplycost * ps_availqty) * :2 23 | from 24 | partsupp, 25 | supplier, 26 | nation 27 | where 28 | ps_suppkey = s_suppkey 29 | and s_nationkey = n_nationkey 30 | and n_name = ':1' 31 | ) 32 | order by 33 | value desc; 34 | :n -1 35 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/1.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Pricing Summary Report Query (Q1) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | l_returnflag, 9 | l_linestatus, 10 | sum(l_quantity) as sum_qty, 11 | sum(l_extendedprice) as sum_base_price, 12 | sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, 13 | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, 14 | avg(l_quantity) as avg_qty, 15 | avg(l_extendedprice) as avg_price, 16 | avg(l_discount) as avg_disc, 17 | count(*) as count_order 18 | from 19 | lineitem 20 | where 21 | l_shipdate <= date '1998-12-01' - interval ':1' day (3) 22 | group by 23 | l_returnflag, 24 | l_linestatus 25 | order by 26 | l_returnflag, 27 | l_linestatus; 28 | :n -1 29 | -------------------------------------------------------------------------------- /hammerdb/azuredeploy.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "numberOfWorkers": { 6 | "value": 4 7 | }, 8 | "diskType": { 9 | "value": "Premium_LRS" 10 | }, 11 | "coordinatorVmSize": { 12 | "value": "Standard_D16s_v5" 13 | }, 14 | "workerVmSize": { 15 | "value": "Standard_E16s_v5" 16 | }, 17 | "driverVmSize": { 18 | "value": "Standard_D64s_v5" 19 | }, 20 | "coordinatorDiskSizeGB": { 21 | "value": 256 22 | }, 23 | "workerDiskSizeGB": { 24 | "value": 256 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/10.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Returned Item Reporting Query (Q10) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | c_custkey, 9 | c_name, 10 | sum(l_extendedprice * (1 - l_discount)) as revenue, 11 | c_acctbal, 12 | n_name, 13 | c_address, 14 | c_phone, 15 | c_comment 16 | from 17 | customer, 18 | orders, 19 | lineitem, 20 | nation 21 | where 22 | c_custkey = o_custkey 23 | and l_orderkey = o_orderkey 24 | and o_orderdate >= date ':1' 25 | and o_orderdate < date ':1' + interval '3' month 26 | and l_returnflag = 'R' 27 | and c_nationkey = n_nationkey 28 | group by 29 | c_custkey, 30 | c_name, 31 | c_acctbal, 32 | c_phone, 33 | n_name, 34 | c_address, 35 | c_comment 36 | order by 37 | revenue desc; 38 | :n 20 39 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/15.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Top Supplier Query (Q15) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | create view revenue:s (supplier_no, total_revenue) as 7 | select 8 | l_suppkey, 9 | sum(l_extendedprice * (1 - l_discount)) 10 | from 11 | lineitem 12 | where 13 | l_shipdate >= date ':1' 14 | and l_shipdate < date ':1' + interval '3' month 15 | group by 16 | l_suppkey; 17 | 18 | :o 19 | select 20 | s_suppkey, 21 | s_name, 22 | s_address, 23 | s_phone, 24 | total_revenue 25 | from 26 | supplier, 27 | revenue:s 28 | where 29 | s_suppkey = supplier_no 30 | and total_revenue = ( 31 | select 32 | max(total_revenue) 33 | from 34 | revenue:s 35 | ) 36 | order by 37 | s_suppkey; 38 | 39 | drop view revenue:s; 40 | :n -1 41 | -------------------------------------------------------------------------------- /cloudformation/find-workers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | if [ -z "$AWS_DEFAULT_REGION" ]; then 4 | export AWS_DEFAULT_REGION=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep region | awk -F\" '{print $4}') 5 | fi 6 | 7 | if [ -z "$WORKER_AS_GROUP" ]; then 8 | WORKER_AS_GROUP=$(cat $HOME/worker-as-group.txt) 9 | fi 10 | 11 | AS_GROUP_FILE=/home/ec2-user/group-file 12 | AS_INSTANCES_FILE=/home/ec2-user/test-automation/worker-instances 13 | 14 | aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $WORKER_AS_GROUP --output text > $AS_GROUP_FILE 15 | INSTANCES=$(grep ^INSTANCES $AS_GROUP_FILE | awk '{print $4}') 16 | 17 | if [ "$INSTANCES" != "" ]; then 18 | aws ec2 describe-instances --instance-ids $INSTANCES --output text | grep PRIVATEIPADDRESSES | cut -f4 > $AS_INSTANCES_FILE 19 | fi 20 | -------------------------------------------------------------------------------- /azure/run-all-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u 5 | # exit immediately if a command fails 6 | set -e 7 | # fail in a pipeline if any of the commands fails 8 | set -o pipefail 9 | 10 | rg_name=$1 11 | 12 | if [ "$rg_name" = "citusbot_pgbench_test_resource_group" ]; then 13 | fab run.pgbench-tests 14 | fab run.pgbench-tests --config-file=pgbench_default_without_transaction.ini 15 | fi 16 | 17 | if [ "$rg_name" = "citusbot_scale_test_resource_group" ]; then 18 | fab run.pgbench-tests --config-file=scale_test.ini 19 | fi 20 | 21 | if [ "$rg_name" = "citusbot_tpch_test_resource_group" ]; then 22 | fab run.tpch-automate 23 | fi 24 | 25 | if [ "$rg_name" = "citusbot_extension_test_resource_group" ]; then 26 | fab run.extension-tests 27 | fi 28 | 29 | sh "${HOME}"/test-automation/azure/push-results.sh "$1"; 30 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/12.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Shipping Modes and Order Priority Query (Q12) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | l_shipmode, 9 | sum(case 10 | when o_orderpriority = '1-URGENT' 11 | or o_orderpriority = '2-HIGH' 12 | then 1 13 | else 0 14 | end) as high_line_count, 15 | sum(case 16 | when o_orderpriority <> '1-URGENT' 17 | and o_orderpriority <> '2-HIGH' 18 | then 1 19 | else 0 20 | end) as low_line_count 21 | from 22 | orders, 23 | lineitem 24 | where 25 | o_orderkey = l_orderkey 26 | and l_shipmode in (':1', ':2') 27 | and l_commitdate < l_receiptdate 28 | and l_shipdate < l_commitdate 29 | and l_receiptdate >= date ':3' 30 | and l_receiptdate < date ':3' + interval '1' year 31 | group by 32 | l_shipmode 33 | order by 34 | l_shipmode; 35 | :n -1 36 | -------------------------------------------------------------------------------- /tpch_2_13_0/tpch.dsw: -------------------------------------------------------------------------------- 1 | Microsoft Developer Studio Workspace File, Format Version 6.00 2 | # WARNING: DO NOT EDIT OR DELETE THIS WORKSPACE FILE! 3 | 4 | ############################################################################### 5 | 6 | Project: "dbgen"=".\dbgen.dsp" - Package Owner=<4> 7 | 8 | Package=<5> 9 | {{{ 10 | }}} 11 | 12 | Package=<4> 13 | {{{ 14 | }}} 15 | 16 | ############################################################################### 17 | 18 | Project: "test"=".\test\test.dsp" - Package Owner=<4> 19 | 20 | Package=<5> 21 | {{{ 22 | }}} 23 | 24 | Package=<4> 25 | {{{ 26 | }}} 27 | 28 | ############################################################################### 29 | 30 | Global: 31 | 32 | Package=<5> 33 | {{{ 34 | }}} 35 | 36 | Package=<3> 37 | {{{ 38 | }}} 39 | 40 | ############################################################################### 41 | 42 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/20.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Potential Part Promotion Query (Q20) 3 | -- Function Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | s_name, 9 | s_address 10 | from 11 | supplier, 12 | nation 13 | where 14 | s_suppkey in ( 15 | select 16 | ps_suppkey 17 | from 18 | partsupp 19 | where 20 | ps_partkey in ( 21 | select 22 | p_partkey 23 | from 24 | part 25 | where 26 | p_name like ':1%' 27 | ) 28 | and ps_availqty > ( 29 | select 30 | 0.5 * sum(l_quantity) 31 | from 32 | lineitem 33 | where 34 | l_partkey = ps_partkey 35 | and l_suppkey = ps_suppkey 36 | and l_shipdate >= date ':2' 37 | and l_shipdate < date ':2' + interval '1' year 38 | ) 39 | ) 40 | and s_nationkey = n_nationkey 41 | and n_name = ':3' 42 | order by 43 | s_name; 44 | :n -1 45 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/9.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Product Type Profit Measure Query (Q9) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | nation, 9 | o_year, 10 | sum(amount) as sum_profit 11 | from 12 | ( 13 | select 14 | n_name as nation, 15 | extract(year from o_orderdate) as o_year, 16 | l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount 17 | from 18 | part, 19 | supplier, 20 | lineitem, 21 | partsupp, 22 | orders, 23 | nation 24 | where 25 | s_suppkey = l_suppkey 26 | and ps_suppkey = l_suppkey 27 | and ps_partkey = l_partkey 28 | and p_partkey = l_partkey 29 | and o_orderkey = l_orderkey 30 | and s_nationkey = n_nationkey 31 | and p_name like '%:1%' 32 | ) as profit 33 | group by 34 | nation, 35 | o_year 36 | order by 37 | nation, 38 | o_year desc; 39 | :n -1 40 | -------------------------------------------------------------------------------- /azure/connect.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u 5 | # exit immediately if a command fails 6 | set -e 7 | # echo commands 8 | set -x 9 | 10 | ## Set mydir to the directory containing the script 11 | ## The ${var%pattern} format will remove the shortest match of 12 | ## pattern from the end of the string. Here, it will remove the 13 | ## script's name,. leaving only the directory. 14 | azuredir="${0%/*}" 15 | cd ${azuredir} 16 | 17 | public_ip=$(az deployment group show -g ${RESOURCE_GROUP_NAME} -n azuredeploy --query properties.outputs.publicIP.value) 18 | ssh_port=$(az deployment group show -g ${RESOURCE_GROUP_NAME} -n azuredeploy --query properties.outputs.customSshPort.value) 19 | 20 | # remove the quotes 21 | public_ip=$(echo ${public_ip} | cut -d "\"" -f 2) 22 | ssh_port=$(echo ${ssh_port} | cut -d "\"" -f 2) 23 | 24 | ssh -o "StrictHostKeyChecking no" -A pguser@${public_ip} -p ${ssh_port} 25 | -------------------------------------------------------------------------------- /fabfile/hammerdb_confs/citus_config.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | postgres_citus_version: ('14.5', 'main') 3 | postgresql_conf: [ 4 | # the following two are necessary to prevent getting timeouts, do not change them. 5 | "tcp_keepalives_idle = 120", 6 | "tcp_keepalives_interval = 30", 7 | "autovacuum_vacuum_cost_delay = 0", 8 | "checkpoint_completion_target = 0.9", 9 | "checkpoint_timeout = '30min'", 10 | "max_connections = 500", 11 | "max_prepared_transactions = 300", 12 | "min_wal_size = 32", 13 | "max_wal_size = '50GB'", 14 | "wal_buffers = '16MB'", 15 | "wal_compression = on", 16 | "wal_level = 'logical'", 17 | "citus.replication_model = 'streaming'", 18 | "citus.enable_repartition_joins = 'on'", 19 | "effective_io_concurrency = 128", 20 | "citus.shard_count = 60", 21 | "track_io_timing = 'off'", 22 | "citus.enable_deadlock_prevention = 'off'", 23 | "log_checkpoints = 'on'" 24 | ] 25 | -------------------------------------------------------------------------------- /tpch_2_13_0/generate2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -z "$CHUNKS" ]; then 4 | echo Need to specify CHUNKS 5 | exit 1 6 | fi 7 | 8 | export NUM_WORKERS=${NUM_WORKERS:-1} 9 | export NUM_PROCS=${NUM_PROCS:-4} 10 | export SCALE_FACTOR=${SCALE_FACTOR:-1} 11 | export ROLE=${ROLE:-1} 12 | 13 | export R=$(($ROLE-1)) 14 | export W=$NUM_WORKERS 15 | export SF=$SCALE_FACTOR 16 | 17 | # D = chunks per worker 18 | # M = remaining chunks 19 | # S = first chunk 20 | # E = last chunk 21 | 22 | echo $CHUNKS \ 23 | | xargs -r -n 2 sh -c 'D=$(($1/$W)) M=$(($1%$W)) S=$(( $R>=$M ? $M+$R*$D+1 : $R*($D+1)+1 )) E=$(( $R>=$M ? $S+$D-1 : $S+$D )) E=$(( $E>$1 ? $1 : $E)); (for c in `seq $S $E`; do echo $0 $1 $c; done)' \ 24 | | xargs -r -n 3 -P $NUM_PROCS sh -c 'if [ $1 = "1" ]; then ./dbgen -q -f -T $0 -s $SF; else ./dbgen -q -f -T $0 -C $1 -S $2 -s $SF; fi' 25 | 26 | if [ "$ROLE" = "1" ] 27 | then 28 | ./dbgen -q -f -T l -s $SCALE_FACTOR 29 | fi 30 | 31 | chmod 644 *.tbl* 32 | -------------------------------------------------------------------------------- /fabfile/config.py: -------------------------------------------------------------------------------- 1 | from os.path import expanduser 2 | import os 3 | 4 | IS_SSH_KEYS_USED ='is_ssh_keys_used' 5 | HOME_DIR = expanduser("~") 6 | 7 | CODE_DIRECTORY = os.path.join(HOME_DIR, 'code') 8 | TEST_REPO = os.path.join(HOME_DIR, 'test-automation') 9 | CITUS_REPO = os.path.join(HOME_DIR, 'citus') 10 | PG_LATEST = os.path.join(HOME_DIR, 'pg-latest') 11 | PG_SOURCE_BALLS = os.path.join(HOME_DIR, 'postgres-source') 12 | HOME_DIRECTORY = HOME_DIR 13 | RESULTS_DIRECTORY = os.path.join(HOME_DIR, 'results') 14 | POSTGRES_INSTALLATION = os.path.join(HOME_DIR, 'postgres-installation') 15 | PORT = 5432 16 | RELATIVE_REGRESS_PATH = 'src/test/regress' 17 | 18 | # keys to access settings dictionary 19 | REPO_PATH = 'repo_path' 20 | BUILD_CITUS_FUNC = 'build_citus_func' 21 | 22 | REGRESSION_DIFFS_FILE = 'regression.diffs' 23 | 24 | PG_VERSION = '9.6.1' 25 | PG_CONFIGURE_FLAGS = ['--with-openssl','--enable-tap-tests'] 26 | 27 | settings = { 28 | IS_SSH_KEYS_USED: True 29 | } 30 | -------------------------------------------------------------------------------- /tpch_2_13_0/distributed_queries/10.sql: -------------------------------------------------------------------------------- 1 | -- Query #10 2 | ------------ 3 | 4 | SELECT 5 | c_custkey, 6 | c_name, 7 | sum(l_extendedprice * (1 - l_discount)) as revenue, 8 | c_acctbal, 9 | n_name, 10 | c_address, 11 | c_phone, 12 | c_comment 13 | FROM 14 | customer, 15 | orders, 16 | lineitem, 17 | nation 18 | WHERE 19 | c_custkey = o_custkey 20 | AND l_orderkey = o_orderkey 21 | AND o_orderdate >= date '1993-10-01' 22 | AND o_orderdate < date '1993-10-01' + interval '3' month 23 | AND l_returnflag = 'R' 24 | AND c_nationkey = n_nationkey 25 | GROUP BY 26 | c_custkey, 27 | c_name, 28 | c_acctbal, 29 | c_phone, 30 | n_name, 31 | c_address, 32 | c_comment 33 | ORDER BY 34 | revenue desc 35 | -- the following LIMIT is not part of the benchmark suite 36 | -- but required to run the tests automatically 37 | -- see https://github.com/citusdata/test-automation/issues/120 38 | LIMIT 4; 39 | -------------------------------------------------------------------------------- /tpch_2_13_0/bcd2.h: -------------------------------------------------------------------------------- 1 | /* 2 | * $Id: bcd2.h,v 1.2 2005/01/03 20:08:58 jms Exp $ 3 | * 4 | * Revision History 5 | * =================== 6 | * $Log: bcd2.h,v $ 7 | * Revision 1.2 2005/01/03 20:08:58 jms 8 | * change line terminations 9 | * 10 | * Revision 1.1.1.1 2004/11/24 23:31:45 jms 11 | * re-establish external server 12 | * 13 | * Revision 1.1.1.1 2003/04/03 18:54:21 jms 14 | * recreation after CVS crash 15 | * 16 | * Revision 1.1.1.1 2003/04/03 18:54:21 jms 17 | * initial checkin 18 | * 19 | * 20 | */ 21 | int bin_bcd2(long binary, long *low_res, long *high_res); 22 | int bcd2_bin(long *dest, long bcd); 23 | int bcd2_add(long *bcd_low, long *bcd_high, long addend); 24 | int bcd2_sub(long *bcd_low, long *bcd_high, long subend); 25 | int bcd2_mul(long *bcd_low, long *bcd_high, long multiplier); 26 | int bcd2_div(long *bcd_low, long *bcd_high, long divisor); 27 | long bcd2_mod(long *bcd_low, long *bcd_high, long modulo); 28 | long bcd2_cmp(long *bcd_low, long *bcd_high, long compare); 29 | -------------------------------------------------------------------------------- /tpch_2_13_0/distributed_queries/9.sql: -------------------------------------------------------------------------------- 1 | -- Query #9 2 | ----------- 3 | 4 | select 5 | nation, 6 | o_year, 7 | sum(amount) as sum_profit 8 | from 9 | ( 10 | select 11 | n_name as nation, 12 | extract(year from o_orderdate) as o_year, 13 | l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as 14 | amount 15 | from 16 | part, 17 | supplier, 18 | lineitem, 19 | partsupp, 20 | orders, 21 | nation 22 | where 23 | s_suppkey = l_suppkey 24 | and ps_suppkey = l_suppkey 25 | and ps_partkey = l_partkey 26 | and p_partkey = l_partkey 27 | and o_orderkey = l_orderkey 28 | and s_nationkey = n_nationkey 29 | and p_name like '%green%' 30 | ) as profit 31 | group by 32 | nation, 33 | o_year 34 | order by 35 | nation, 36 | o_year desc 37 | limit 20; 38 | -------------------------------------------------------------------------------- /hammerdb/build.tcl: -------------------------------------------------------------------------------- 1 | #!/bin/tclsh 2 | puts "SETTING CONFIGURATION" 3 | global complete 4 | proc wait_to_complete {} { 5 | global complete 6 | set complete [vucomplete] 7 | if {!$complete} { after 5000 wait_to_complete } else { exit } 8 | } 9 | dbset db pg 10 | loadscript 11 | diset connection pg_host replace_with_ip_address 12 | diset tpcc pg_dbase pguser 13 | diset tpcc pg_user pguser 14 | diset tpcc pg_superuser pguser 15 | diset tpcc pg_defaultdbase pguser 16 | #diset tpcc pg_pass yourpasswordhere 17 | #diset tpcc pg_superuserpass yourpasswordhere 18 | # if you change this to false, make sure to run tpcc-distribute-funcs.sql 19 | diset tpcc pg_storedprocs true 20 | diset tpcc pg_num_vu 250 21 | diset tpcc pg_count_ware 1000 22 | diset tpcc pg_timeprofile false 23 | diset tpcc pg_allwarehouse true 24 | diset tpcc pg_keyandthink false 25 | # this flag is only available in hammerdb 4.4+ 26 | diset tpcc pg_cituscompat true 27 | loadscript 28 | print dict 29 | buildschema 30 | wait_to_complete 31 | vwait forever 32 | -------------------------------------------------------------------------------- /fabfile/connection.py: -------------------------------------------------------------------------------- 1 | from fabric import Connection 2 | 3 | import config 4 | import os 5 | 6 | # init configuration params for connections 7 | forward_agent = True 8 | 9 | master_host = 'localhost' 10 | worker_hosts = [ip.strip() for ip in open(config.HOME_DIR + '/test-automation/worker-instances')] 11 | all_hosts = [master_host] + worker_hosts 12 | 13 | key_file_path = os.path.join(config.HOME_DIR, ".ssh/id_rsa") 14 | connection_settings = { 15 | "look_for_keys": True, 16 | "key_filename": key_file_path, 17 | } 18 | 19 | # create connections for master and workers 20 | master_connection = Connection(master_host, user="pguser", forward_agent=forward_agent, connect_kwargs=connection_settings,) 21 | worker_connections = [] 22 | for worker_host in worker_hosts: 23 | worker_connection = Connection(worker_host, user="pguser", forward_agent=forward_agent, connect_kwargs=connection_settings,) 24 | worker_connections.append(worker_connection) 25 | all_connections = [master_connection] + worker_connections 26 | -------------------------------------------------------------------------------- /fabfile/extension_hooks.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | import utils 4 | import config 5 | 6 | def after_citus_create(c): 7 | # it exposes a function which is normally not exposed by citus. It is only expected to be used in tests by design. 8 | # it hides citus objects from showing up in pg meta queries. e.g. 'select * from pg_class;' 9 | with c.cd(config.PG_LATEST): 10 | # write file content 11 | is_citus_depended_obj_func = ("CREATE OR REPLACE FUNCTION pg_catalog.is_citus_depended_object(oid,oid) " 12 | "RETURNS bool LANGUAGE C AS 'citus', " 13 | "$$is_citus_depended_object$$;") 14 | func_file_name = 'is_citus_depended_obj_func.sql' 15 | func_file_path = os.path.join(config.PG_LATEST, func_file_name) 16 | with open(func_file_path, 'w') as f: 17 | f.write(is_citus_depended_obj_func) 18 | 19 | # run psql command with file option 20 | utils.psql(c, '', func_file_path) 21 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/22.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Global Sales Opportunity Query (Q22) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | cntrycode, 9 | count(*) as numcust, 10 | sum(c_acctbal) as totacctbal 11 | from 12 | ( 13 | select 14 | substring(c_phone from 1 for 2) as cntrycode, 15 | c_acctbal 16 | from 17 | customer 18 | where 19 | substring(c_phone from 1 for 2) in 20 | (':1', ':2', ':3', ':4', ':5', ':6', ':7') 21 | and c_acctbal > ( 22 | select 23 | avg(c_acctbal) 24 | from 25 | customer 26 | where 27 | c_acctbal > 0.00 28 | and substring(c_phone from 1 for 2) in 29 | (':1', ':2', ':3', ':4', ':5', ':6', ':7') 30 | ) 31 | and not exists ( 32 | select 33 | * 34 | from 35 | orders 36 | where 37 | o_custkey = c_custkey 38 | ) 39 | ) as custsale 40 | group by 41 | cntrycode 42 | order by 43 | cntrycode; 44 | :n -1 45 | -------------------------------------------------------------------------------- /hammerdb/connect-driver.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u 5 | # exit immediately if a command fails 6 | set -e 7 | # echo commands 8 | set -x 9 | 10 | ## Set mydir to the directory containing the script 11 | ## The ${var%pattern} format will remove the shortest match of 12 | ## pattern from the end of the string. Here, it will remove the 13 | ## script's name,. leaving only the directory. 14 | dir="${0%/*}" 15 | cd "${dir}" 16 | 17 | # get the public ip of driver from the cluster outputs. 18 | public_ip=$(az deployment group show -g "${RESOURCE_GROUP_NAME}" -n azuredeploy --query properties.outputs.driverPublicIP.value) 19 | ssh_port=$(az deployment group show -g "${RESOURCE_GROUP_NAME}" -n azuredeploy --query properties.outputs.customSshPort.value) 20 | # remove the quotes 21 | public_ip=$(echo "${public_ip}" | cut -d "\"" -f 2) 22 | ssh_port=$(echo "${ssh_port}" | cut -d "\"" -f 2) 23 | 24 | ssh -o "StrictHostKeyChecking no" -A pguser@"${public_ip}" -p "${ssh_port}" 25 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/21.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Suppliers Who Kept Orders Waiting Query (Q21) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | s_name, 9 | count(*) as numwait 10 | from 11 | supplier, 12 | lineitem l1, 13 | orders, 14 | nation 15 | where 16 | s_suppkey = l1.l_suppkey 17 | and o_orderkey = l1.l_orderkey 18 | and o_orderstatus = 'F' 19 | and l1.l_receiptdate > l1.l_commitdate 20 | and exists ( 21 | select 22 | * 23 | from 24 | lineitem l2 25 | where 26 | l2.l_orderkey = l1.l_orderkey 27 | and l2.l_suppkey <> l1.l_suppkey 28 | ) 29 | and not exists ( 30 | select 31 | * 32 | from 33 | lineitem l3 34 | where 35 | l3.l_orderkey = l1.l_orderkey 36 | and l3.l_suppkey <> l1.l_suppkey 37 | and l3.l_receiptdate > l3.l_commitdate 38 | ) 39 | and s_nationkey = n_nationkey 40 | and n_name = ':1' 41 | group by 42 | s_name 43 | order by 44 | numwait desc, 45 | s_name; 46 | :n 100 47 | -------------------------------------------------------------------------------- /fabfile/pgbench_confs/pgbench_cloud.ini: -------------------------------------------------------------------------------- 1 | [initialization] 2 | pgbench_command: pgbench -i -s 100 3 | 4 | [citus1] 5 | sql_command: SELECT create_distributed_table('pgbench_accounts', 'aid'); 6 | 7 | [citus1a] 8 | sql_command: VACUUM ANALYZE pgbench_accounts; 9 | 10 | [citus2] 11 | sql_command: SELECT create_distributed_table('pgbench_branches', 'bid'); 12 | 13 | [citus2a] 14 | sql_command: VACUUM ANALYZE pgbench_branches; 15 | 16 | [citus3] 17 | sql_command: SELECT create_distributed_table('pgbench_history', 'tid'); 18 | 19 | [citus3a] 20 | sql_command: VACUUM ANALYZE pgbench_history; 21 | 22 | [citus4] 23 | sql_command: SELECT create_distributed_table('pgbench_tellers', 'tid'); 24 | 25 | [citus4a] 26 | sql_command: VACUUM ANALYZE pgbench_tellers; 27 | 28 | [default-pgbench] 29 | pgbench_command: pgbench -c 32 -j 16 -T 600 -P 10 -r 30 | 31 | [simple-update] 32 | pgbench_command: pgbench -c 32 -j 16 -T 600 -P 10 -r -b simple-update 33 | 34 | [select-only] 35 | pgbench_command: pgbench -c 32 -j 16 -T 600 -P 10 -r -b select-only 36 | -------------------------------------------------------------------------------- /tpch_2_13_0/distributed_queries/12.sql: -------------------------------------------------------------------------------- 1 | -- Query #12 2 | ------------ 3 | 4 | SELECT 5 | l_shipmode, 6 | sum(case 7 | when o_orderpriority = '1-URGENT' 8 | OR o_orderpriority = '2-HIGH' 9 | then 1 10 | else 0 11 | end) as high_line_count, 12 | sum(case 13 | when o_orderpriority <> '1-URGENT' 14 | AND o_orderpriority <> '2-HIGH' 15 | then 1 16 | else 0 17 | end) AS low_line_count 18 | FROM 19 | orders, 20 | lineitem 21 | WHERE 22 | o_orderkey = l_orderkey 23 | AND l_shipmode in ('MAIL', 'SHIP') 24 | AND l_commitdate < l_receiptdate 25 | AND l_shipdate < l_commitdate 26 | AND l_receiptdate >= date '1994-01-01' 27 | AND l_receiptdate < date '1994-01-01' + interval '1' year 28 | GROUP BY 29 | l_shipmode 30 | ORDER BY 31 | l_shipmode 32 | -- the following LIMIT is not part of the benchmark suite 33 | -- but required to run the tests automatically 34 | -- see https://github.com/citusdata/test-automation/issues/120 35 | LIMIT 4; 36 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/2.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Minimum Cost Supplier Query (Q2) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | s_acctbal, 9 | s_name, 10 | n_name, 11 | p_partkey, 12 | p_mfgr, 13 | s_address, 14 | s_phone, 15 | s_comment 16 | from 17 | part, 18 | supplier, 19 | partsupp, 20 | nation, 21 | region 22 | where 23 | p_partkey = ps_partkey 24 | and s_suppkey = ps_suppkey 25 | and p_size = :1 26 | and p_type like '%:2' 27 | and s_nationkey = n_nationkey 28 | and n_regionkey = r_regionkey 29 | and r_name = ':3' 30 | and ps_supplycost = ( 31 | select 32 | min(ps_supplycost) 33 | from 34 | partsupp, 35 | supplier, 36 | nation, 37 | region 38 | where 39 | p_partkey = ps_partkey 40 | and s_suppkey = ps_suppkey 41 | and s_nationkey = n_nationkey 42 | and n_regionkey = r_regionkey 43 | and r_name = ':3' 44 | ) 45 | order by 46 | s_acctbal desc, 47 | n_name, 48 | s_name, 49 | p_partkey; 50 | :n 100 51 | -------------------------------------------------------------------------------- /hammerdb/run.tcl: -------------------------------------------------------------------------------- 1 | #!/bin/tclsh 2 | puts "SETTING CONFIGURATION" 3 | global complete 4 | proc wait_to_complete {} { 5 | global complete 6 | set complete [vucomplete] 7 | if {!$complete} { after 5000 wait_to_complete } else { exit } 8 | } 9 | dbset db pg 10 | loadscript 11 | diset connection pg_host replace_with_ip_address 12 | diset tpcc pg_dbase pguser 13 | diset tpcc pg_user pguser 14 | diset tpcc pg_superuser pguser 15 | diset tpcc pg_defaultdbase pguser 16 | #diset tpcc pg_pass yourpasswordhere 17 | #diset tpcc pg_superuserpass yourpasswordhere 18 | # if you change this, make sure to change tpcc-distribute-funcs.sql 19 | diset tpcc pg_storedprocs true 20 | diset tpcc pg_driver timed 21 | diset tpcc pg_rampup 3 22 | diset tpcc pg_duration 60 23 | diset tpcc pg_timeprofile false 24 | diset tpcc pg_allwarehouse true 25 | diset tpcc pg_keyandthink false 26 | loadscript 27 | print dict 28 | vuset vu 250 29 | vuset timestamps 1 30 | vuset logtotemp 1 31 | vuset showoutput 0 32 | vuset unique 1 33 | vuset delay 100 34 | vuset repeat 1 35 | vurun 36 | wait_to_complete 37 | vwait forever 38 | -------------------------------------------------------------------------------- /tpch_2_13_0/warm_up_cache.sql: -------------------------------------------------------------------------------- 1 | VACUUM ANALYZE nation; 2 | SELECT COUNT(*) FROM nation; 3 | SELECT COUNT(*) FROM nation; 4 | SELECT COUNT(*) FROM nation; 5 | 6 | VACUUM ANALYZE region; 7 | SELECT COUNT(*) FROM region; 8 | SELECT COUNT(*) FROM region; 9 | SELECT COUNT(*) FROM region; 10 | 11 | VACUUM ANALYZE part; 12 | SELECT COUNT(*) FROM part; 13 | SELECT COUNT(*) FROM part; 14 | SELECT COUNT(*) FROM part; 15 | 16 | VACUUM ANALYZE supplier; 17 | SELECT COUNT(*) FROM supplier; 18 | SELECT COUNT(*) FROM supplier; 19 | SELECT COUNT(*) FROM supplier; 20 | 21 | VACUUM ANALYZE partsupp; 22 | SELECT COUNT(*) FROM partsupp; 23 | SELECT COUNT(*) FROM partsupp; 24 | SELECT COUNT(*) FROM partsupp; 25 | 26 | VACUUM ANALYZE customer; 27 | SELECT COUNT(*) FROM customer; 28 | SELECT COUNT(*) FROM customer; 29 | SELECT COUNT(*) FROM customer; 30 | 31 | VACUUM ANALYZE orders; 32 | SELECT COUNT(*) FROM orders; 33 | SELECT COUNT(*) FROM orders; 34 | SELECT COUNT(*) FROM orders; 35 | 36 | VACUUM ANALYZE lineitem; 37 | SELECT COUNT(*) FROM lineitem; 38 | SELECT COUNT(*) FROM lineitem; 39 | SELECT COUNT(*) FROM lineitem; 40 | -------------------------------------------------------------------------------- /hammerdb/delete-resource-group.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u 5 | # exit immediately if a command fails 6 | set -e 7 | # echo commands 8 | set -x 9 | 10 | cluster_rg=$1 11 | 12 | cd /tmp 13 | 14 | # add github to known hosts 15 | echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" >> ~/.ssh/known_hosts 16 | 17 | git clone https://github.com/citusdata/test-automation.git 18 | 19 | git config --global user.email "citus-bot@microsoft.com" 20 | git config --global user.name "citus bot" 21 | 22 | # use the current second as id 23 | id=$(date +"%s") 24 | 25 | cd test-automation 26 | git checkout -b delete_resource_group/"${id}"_"${cluster_rg}" 27 | git push origin delete_resource_group/"${id}"_"${cluster_rg}" 28 | 29 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/8.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R National Market Share Query (Q8) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | o_year, 9 | sum(case 10 | when nation = ':1' then volume 11 | else 0 12 | end) / sum(volume) as mkt_share 13 | from 14 | ( 15 | select 16 | extract(year from o_orderdate) as o_year, 17 | l_extendedprice * (1 - l_discount) as volume, 18 | n2.n_name as nation 19 | from 20 | part, 21 | supplier, 22 | lineitem, 23 | orders, 24 | customer, 25 | nation n1, 26 | nation n2, 27 | region 28 | where 29 | p_partkey = l_partkey 30 | and s_suppkey = l_suppkey 31 | and l_orderkey = o_orderkey 32 | and o_custkey = c_custkey 33 | and c_nationkey = n1.n_nationkey 34 | and n1.n_regionkey = r_regionkey 35 | and r_name = ':2' 36 | and s_nationkey = n2.n_nationkey 37 | and o_orderdate between date '1995-01-01' and date '1996-12-31' 38 | and p_type = ':3' 39 | ) as all_nations 40 | group by 41 | o_year 42 | order by 43 | o_year; 44 | :n -1 45 | -------------------------------------------------------------------------------- /azure/citus-bot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u 5 | # exit immediately if a command fails 6 | set -e 7 | # echo commands 8 | set -x 9 | 10 | source commons.sh 11 | 12 | # generate PEM to use to ssh into the Azure VM-s 13 | ssh-keygen -m PEM -b 2048 -t rsa -f ~/.ssh/vm-key -q -N "" 14 | eval `ssh-agent -s` 15 | 16 | # add key for azure VM 17 | ssh-add ~/.ssh/vm-key 18 | 19 | function cleanup { 20 | sh ./delete-resource-group.sh 21 | } 22 | 23 | trap cleanup EXIT 24 | 25 | export RESOURCE_GROUP_NAME="$1" 26 | 27 | ./create-cluster.sh 28 | 29 | ssh_port=$(rg_get_ssh_port ${RESOURCE_GROUP_NAME}) 30 | public_ip=$(rg_get_public_ip ${RESOURCE_GROUP_NAME}) 31 | 32 | echo ${public_ip} 33 | 34 | ssh_execute ${public_ip} ${ssh_port} 'echo "'$RESULT_REPO_ACCESS_TOKEN'" > /tmp/result-repo-token' 35 | 36 | echo "running tests in remote" 37 | # ssh with non-interactive mode does not source bash profile, so we will need to do it ourselves here. 38 | ssh_execute ${public_ip} ${ssh_port} "source ~/.bash_profile;/home/pguser/test-automation/azure/run-all-tests.sh ${RESOURCE_GROUP_NAME}" 39 | 40 | -------------------------------------------------------------------------------- /tpch_2_13_0/queries/7.sql: -------------------------------------------------------------------------------- 1 | -- $ID$ 2 | -- TPC-H/TPC-R Volume Shipping Query (Q7) 3 | -- Functional Query Definition 4 | -- Approved February 1998 5 | :x 6 | :o 7 | select 8 | supp_nation, 9 | cust_nation, 10 | l_year, 11 | sum(volume) as revenue 12 | from 13 | ( 14 | select 15 | n1.n_name as supp_nation, 16 | n2.n_name as cust_nation, 17 | extract(year from l_shipdate) as l_year, 18 | l_extendedprice * (1 - l_discount) as volume 19 | from 20 | supplier, 21 | lineitem, 22 | orders, 23 | customer, 24 | nation n1, 25 | nation n2 26 | where 27 | s_suppkey = l_suppkey 28 | and o_orderkey = l_orderkey 29 | and c_custkey = o_custkey 30 | and s_nationkey = n1.n_nationkey 31 | and c_nationkey = n2.n_nationkey 32 | and ( 33 | (n1.n_name = ':1' and n2.n_name = ':2') 34 | or (n1.n_name = ':2' and n2.n_name = ':1') 35 | ) 36 | and l_shipdate between date '1995-01-01' and date '1996-12-31' 37 | ) as shipping 38 | group by 39 | supp_nation, 40 | cust_nation, 41 | l_year 42 | order by 43 | supp_nation, 44 | cust_nation, 45 | l_year; 46 | :n -1 47 | -------------------------------------------------------------------------------- /tpch_2_13_0/distributed_queries/8.sql: -------------------------------------------------------------------------------- 1 | -- Query #8 2 | ----------- 3 | 4 | select 5 | o_year, 6 | sum(case 7 | when nation = 'BRAZIL' then volume 8 | else 0 9 | end) / sum(volume) as mkt_share 10 | from 11 | ( 12 | select 13 | extract(year from o_orderdate) as o_year, 14 | l_extendedprice * (1 - l_discount) as volume, 15 | n2.n_name as nation 16 | from 17 | part, 18 | supplier, 19 | lineitem, 20 | orders, 21 | customer, 22 | nation n1, 23 | nation n2, 24 | region 25 | where 26 | p_partkey = l_partkey 27 | and s_suppkey = l_suppkey 28 | and l_orderkey = o_orderkey 29 | and o_custkey = c_custkey 30 | and c_nationkey = n1.n_nationkey 31 | and n1.n_regionkey = r_regionkey 32 | and r_name = 'AMERICA' 33 | and s_nationkey = n2.n_nationkey 34 | and o_orderdate between date '1995-01-01' and date '1996-12-31' 35 | and p_type = 'ECONOMY ANODIZED STEEL' 36 | ) as all_nations 37 | group by 38 | o_year 39 | order by 40 | o_year; 41 | -------------------------------------------------------------------------------- /release/update_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u 5 | # exit immediately if a command fails 6 | set -e 7 | # echo commands 8 | set -x 9 | 10 | ## VARIABLES ## 11 | new_branch1=release-9.3 12 | new_branch2=release-9.2 13 | new_pg_version=12.2 14 | ## VARIABLES ## 15 | 16 | releasedir="${0%/*}" 17 | cd ${releasedir}/.. # topdir 18 | # replace all .ini config files: 19 | # postgres_citus_versions: [('', ''), ('', '')] 20 | # postgres_citus_versions: [('', ''), ('', '')] 21 | find . -type f -name "*.ini" | 22 | xargs sed -i "s@postgres_citus_versions: \[('[^,;]\+', '[^,;]\+'), ('[^,;]\+', '[^,;]\+')\]@postgres_citus_versions: \[('${new_pg_version}', '${new_branch1}'), ('${new_pg_version}', '${new_branch2}')\]@g" 23 | # replace all .ini config files: 24 | # postgres_citus_versions: [('', '')] 25 | # postgres_citus_versions: [('', '')] 26 | find . -type f -name "*.ini" | 27 | xargs sed -i "s@postgres_citus_versions: \[('[^,;]\+', '[^,;]\+')\]@postgres_citus_versions: \[('${new_pg_version}', '${new_branch1}')\]@g" 28 | -------------------------------------------------------------------------------- /fabfile/pgbench_confs/pgbench_issue_1799.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | postgres_citus_versions: [('12.1', 'release-9.2')] 3 | shard_counts_replication_factors: [(32, 1)] 4 | postgresql_conf: [ 5 | "max_wal_size = '50GB'", 6 | "checkpoint_completion_target = 0.9", 7 | "checkpoint_timeout = '1h'", 8 | "max_connections = 1000", 9 | "max_prepared_transactions = 1000" 10 | ] 11 | 12 | [create_type] 13 | sql_command: CREATE TYPE complex AS (r double precision, i double precision); 14 | 15 | [distribute_type] 16 | sql_command: SELECT run_command_on_workers('CREATE TYPE complex AS (r double precision, i double precision)'); 17 | 18 | [create_table] 19 | sql_command: CREATE TABLE test_table (key int, occurred_at timestamp DEFAULT now(), value_1 jsonb, value_2 text[], value_3 int4range, value_4 complex NOT NULL); 20 | distribute_table_command: SELECT create_distributed_table('test_table', 'key'); 21 | 22 | [insert_test] 23 | pgbench_command: pgbench -c128 -j128 -T 6000 -P 1 -n -r -f fabfile/pgbench_scripts/insert_complex.sql 24 | 25 | [select_test] 26 | pgbench_command: pgbench -c32 -j32 -T 60012 -P 1 -n -r -f fabfile/pgbench_scripts/select_all_with_limit.sql 27 | -------------------------------------------------------------------------------- /hammerdb/cache_git_creds.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # this file puts git username and token to a temporary file so that 4 | # a user can read them from a file and clone a private repository 5 | # with caching enabled. After that this user can safely clone private 6 | # repositories without the need for password. 7 | 8 | # fail if trying to reference a variable that is not set. 9 | set -u 10 | # exit immediately if a command fails 11 | set -e 12 | 13 | 14 | TARGET_USER=pguser 15 | 16 | echo "${GIT_USERNAME}" > /tmp/git_username 17 | echo "${GIT_TOKEN}" > /tmp/git_token 18 | 19 | su --login ${TARGET_USER} <<'EOSU' 20 | GIT_USERNAME=$(= :4 and l_quantity <= :4 + 10 18 | and p_size between 1 and 5 19 | and l_shipmode in ('AIR', 'AIR REG') 20 | and l_shipinstruct = 'DELIVER IN PERSON' 21 | ) 22 | or 23 | ( 24 | p_partkey = l_partkey 25 | and p_brand = ':2' 26 | and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') 27 | and l_quantity >= :5 and l_quantity <= :5 + 10 28 | and p_size between 1 and 10 29 | and l_shipmode in ('AIR', 'AIR REG') 30 | and l_shipinstruct = 'DELIVER IN PERSON' 31 | ) 32 | or 33 | ( 34 | p_partkey = l_partkey 35 | and p_brand = ':3' 36 | and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') 37 | and l_quantity >= :6 and l_quantity <= :6 + 10 38 | and p_size between 1 and 15 39 | and l_shipmode in ('AIR', 'AIR REG') 40 | and l_shipinstruct = 'DELIVER IN PERSON' 41 | ); 42 | :n -1 43 | -------------------------------------------------------------------------------- /fabfile/pgbench_confs/scale_test_foreign.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | postgres_citus_versions: [('12.1', 'release-9.2'), ('12.1', 'release-9.2')] 3 | shard_counts_replication_factors: [(32, 1)] 4 | postgresql_conf: [ 5 | "max_wal_size = '50GB'", 6 | "checkpoint_completion_target = 0.9", 7 | "checkpoint_timeout = '1h'", 8 | "max_connections = 1000", 9 | "max_prepared_transactions = 1000", 10 | "shared_buffers = '256MB'", 11 | "citus.multi_shard_commit_protocol = '2pc'" 12 | ] 13 | 14 | [create_table] 15 | sql_command: CREATE TABLE test_table (key int PRIMARY KEY, occurred_at timestamptz DEFAULT now()); 16 | distribute_table_command: SELECT create_distributed_table('test_table', 'key'); 17 | 18 | [create_referencing_table] 19 | sql_command: CREATE TABLE test_table_referencing (key int REFERENCES test_table (key), value_1 int DEFAULT 5, value_2 int DEFAULT 10, value_3 bigint CHECK (value_3 > 10)) 20 | distribute_table_command: SELECT create_distributed_table('test_table_referencing', 'key'); 21 | 22 | [single_insert] 23 | pgbench_command: pgbench -c128 -j16 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert_foreign.sql 24 | 25 | [muti_row_insert] 26 | pgbench_command: pgbench -c32 -j16 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/multi_row_insert_foreign.sql 27 | -------------------------------------------------------------------------------- /tpch_2_13_0/tpch.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 9.00 3 | # Visual Studio 2005 4 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tpch", "tpch.vcproj", "{659CDC00-AE59-4839-B79E-93A846C8FDA2}" 5 | EndProject 6 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "qgen", "qgen.vcproj", "{9BAF8942-2DFD-4BD1-B5ED-DBDDF0169E22}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Win32 = Debug|Win32 11 | Release|Win32 = Release|Win32 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {659CDC00-AE59-4839-B79E-93A846C8FDA2}.Debug|Win32.ActiveCfg = Debug|Win32 15 | {659CDC00-AE59-4839-B79E-93A846C8FDA2}.Debug|Win32.Build.0 = Debug|Win32 16 | {659CDC00-AE59-4839-B79E-93A846C8FDA2}.Release|Win32.ActiveCfg = Release|Win32 17 | {659CDC00-AE59-4839-B79E-93A846C8FDA2}.Release|Win32.Build.0 = Release|Win32 18 | {9BAF8942-2DFD-4BD1-B5ED-DBDDF0169E22}.Debug|Win32.ActiveCfg = Debug|Win32 19 | {9BAF8942-2DFD-4BD1-B5ED-DBDDF0169E22}.Debug|Win32.Build.0 = Debug|Win32 20 | {9BAF8942-2DFD-4BD1-B5ED-DBDDF0169E22}.Release|Win32.ActiveCfg = Release|Win32 21 | {9BAF8942-2DFD-4BD1-B5ED-DBDDF0169E22}.Release|Win32.Build.0 = Release|Win32 22 | EndGlobalSection 23 | GlobalSection(SolutionProperties) = preSolution 24 | HideSolutionNode = FALSE 25 | EndGlobalSection 26 | EndGlobal 27 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/cmd_base_small: -------------------------------------------------------------------------------- 1 | ../dbgen -s 30000 -T P -C 60000000 -S 1 -f -b ../dists.dss 2 | ../dbgen -s 30000 -T P -C 60000000 -S 19999999 -f -b ../dists.dss 3 | ../dbgen -s 30000 -T P -C 60000000 -S 39999998 -f -b ../dists.dss 4 | ../dbgen -s 30000 -T P -C 60000000 -S 59999997 -f -b ../dists.dss 5 | ../dbgen -s 30000 -T P -C 60000000 -S 60000000 -f -b ../dists.dss 6 | ../dbgen -s 30000 -T S -C 240000000 -S 1 -f -b ../dists.dss 7 | ../dbgen -s 30000 -T S -C 240000000 -S 79999999 -f -b ../dists.dss 8 | ../dbgen -s 30000 -T S -C 240000000 -S 159999998 -f -b ../dists.dss 9 | ../dbgen -s 30000 -T S -C 240000000 -S 239999997 -f -b ../dists.dss 10 | ../dbgen -s 30000 -T S -C 240000000 -S 240000000 -f -b ../dists.dss 11 | ../dbgen -s 30000 -T s -C 3000000 -S 1 -f -b ../dists.dss 12 | ../dbgen -s 30000 -T s -C 3000000 -S 999999 -f -b ../dists.dss 13 | ../dbgen -s 30000 -T s -C 3000000 -S 1999998 -f -b ../dists.dss 14 | ../dbgen -s 30000 -T s -C 3000000 -S 2999997 -f -b ../dists.dss 15 | ../dbgen -s 30000 -T s -C 3000000 -S 3000000 -f -b ../dists.dss 16 | ../dbgen -s 30000 -T c -C 45000000 -S 1 -f -b ../dists.dss 17 | ../dbgen -s 30000 -T c -C 45000000 -S 14999999 -f -b ../dists.dss 18 | ../dbgen -s 30000 -T c -C 45000000 -S 29999998 -f -b ../dists.dss 19 | ../dbgen -s 30000 -T c -C 45000000 -S 44999997 -f -b ../dists.dss 20 | ../dbgen -s 30000 -T c -C 45000000 -S 45000000 -f -b ../dists.dss 21 | ../dbgen -T n -f -b ../dists.dss 22 | ../dbgen -T r -f -b ../dists.dss 23 | -------------------------------------------------------------------------------- /fabfile/pgbench_confs/pgbench_custom.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | postgres_citus_versions: [('12.1', 'release-9.2'), ('12.1', 'release-9.2')] 3 | shard_counts_replication_factors: [(32, 1)] 4 | postgresql_conf: [ 5 | "max_wal_size = '50GB'", 6 | "checkpoint_completion_target = 0.9", 7 | "checkpoint_timeout = '1h'", 8 | "max_connections = 1000", 9 | "max_prepared_transactions = 1000", 10 | "citus.multi_shard_commit_protocol = '2pc'" 11 | ] 12 | 13 | [citus1] 14 | sql_command: CREATE TABLE test_table(a int, b int, c int, d int); 15 | distribute_table_command: SELECT create_distributed_table('test_table', 'a'); 16 | 17 | [insert_test] 18 | pgbench_command: pgbench -c 32 -j 32 -T 120 -P 5 -n -f fabfile/pgbench_scripts/insert.sql 19 | 20 | [update_test] 21 | pgbench_command: pgbench -c 32 -j 32 -T 120 -P 5 -n -f fabfile/pgbench_scripts/update.sql 22 | 23 | [delete_test] 24 | pgbench_command: pgbench -c 32 -j 32 -T 120 -P 5 -n -f fabfile/pgbench_scripts/delete.sql 25 | 26 | [copy_test] 27 | sql_command: COPY (SELECT generate_series(1,10000), (random() * 32767)::int, (random() * 32767)::int, (random() * 32767)::int) TO '${HOME}/test_data.csv'; 28 | pgbench_command: pgbench -c 32 -j 16 -T 120 -P 5 -n -f fabfile/pgbench_scripts/copy.sql-D HOME=${HOME} 29 | 30 | [generate_series_test] 31 | pgbench_command: pgbench -c 32 -j 16 -f -T 120 -P 5 -n fabfile/pgbench_scripts/generate_series.sql 32 | -------------------------------------------------------------------------------- /hammerdb/upload-results.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u 5 | # exit immediately if a command fails 6 | set -e 7 | # echo commands 8 | set -x 9 | 10 | hammerdb_version=$(cat ~/HAMMERDB_VERSION) 11 | hammerdb_dir="${HOME}"/HammerDB-"${hammerdb_version}" 12 | rg_name=$1 13 | 14 | cd "${hammerdb_dir}" 15 | 16 | cp build.tcl ./results 17 | cp run.tcl ./results 18 | 19 | cp -r "${HOME}"/test-automation/hammerdb ./results/hammerdb 20 | 21 | cd "$HOME" 22 | 23 | # add github to known hosts 24 | echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" >> ~/.ssh/known_hosts 25 | 26 | git clone https://github.com/citusdata/release-test-results.git 27 | 28 | git config --global user.email "citus-bot@microsoft.com" 29 | git config --global user.name "citus bot" 30 | 31 | now=$(date +"%m_%d_%Y_%s") 32 | 33 | mv "${hammerdb_dir}"/results "${HOME}"/release-test-results/hammerdb/"${now}" 34 | 35 | cd "${HOME}"/release-test-results 36 | 37 | git checkout -b "${rg_name}"/"${now}" 38 | git add -A 39 | git commit -m "add test results for hammerdb tests ${rg_name}" 40 | git push origin "${rg_name}"/"${now}" 41 | -------------------------------------------------------------------------------- /azure/push-results.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # this scripts pushes the results under results/ directory to release-test-results repository 4 | 5 | # args # 6 | # $1 -> branch name to push results 7 | 8 | # fail if trying to reference a variable that is not set. 9 | set -u 10 | # exit immediately if a command fails 11 | set -e 12 | # fail in a pipeline if any of the commands fails 13 | set -o pipefail 14 | 15 | branch_name=$1 16 | 17 | # add github to known hosts 18 | 19 | echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" >> ~/.ssh/known_hosts 20 | 21 | git clone https://oauth2:$(cat /tmp/result-repo-token)@github.com/citusdata/release-test-results.git "${HOME}"/release-test-results 22 | 23 | git config --global user.email "citus-bot@microsoft.com" 24 | git config --global user.name "citus bot" 25 | 26 | now=$(date +"%m_%d_%Y_%s") 27 | 28 | mv "${HOME}"/results "${HOME}"/release-test-results/periodic_job_results/"${now}" 29 | 30 | cd "${HOME}"/release-test-results 31 | 32 | commit_message="add test results" 33 | 34 | git checkout -b "${branch_name}/${now}" 35 | git add -A 36 | git commit -m "$commit_message" 37 | git push origin "${branch_name}/${now}" 38 | -------------------------------------------------------------------------------- /fabfile/pgbench_scripts/scale_multi_row_insert.sql: -------------------------------------------------------------------------------- 1 | \set aid random(1, 100000) 2 | \set bid random(1, 100000) 3 | \set cid random(1, 100000) 4 | \set did random(1, 100000) 5 | 6 | INSERT INTO test_table (key, value_1, value_2, value_3, value_4) VALUES 7 | (:aid, 8 | row_to_json(row(:aid,:bid, row(:cid+10, :did+500,:aid *7), row(:cid-10, :did-500,:aid *71))), 9 | ARRAY[:aid::text, :bid::text, :cid::text, :did::text, 'Onder Kalaci', 'CitusData Rocks']::text[], 10 | int4range(:aid, :aid + :bid), 11 | (:aid, :did)::complex), 12 | (:bid, 13 | row_to_json(row(:aid,:bid, row(:cid+10, :did+500,:aid *7), row(:cid-10, :did-500,:aid *71))), 14 | ARRAY[:aid::text, :bid::text, :cid::text, :did::text, 'Onder Kalaci', 'CitusData Rocks']::text[], 15 | int4range(:aid, :aid + :bid), 16 | (:aid, :did)::complex), 17 | (:cid, 18 | row_to_json(row(:aid,:bid, row(:cid+10, :did+500,:aid *7), row(:cid-10, :did-500,:aid *71))), 19 | ARRAY[:aid::text, :bid::text, :cid::text, :did::text, 'Onder Kalaci', 'CitusData Rocks']::text[], 20 | int4range(:aid, :aid + :bid), 21 | (:aid, :did)::complex), 22 | (:did, 23 | row_to_json(row(:aid,:bid, row(:cid+10, :did+500,:aid *7), row(:cid-10, :did-500,:aid *71))), 24 | ARRAY[:aid::text, :bid::text, :cid::text, :did::text, 'Onder Kalaci', 'CitusData Rocks']::text[], 25 | int4range(:aid, :aid + :bid), 26 | (:aid, :did)::complex); 27 | -------------------------------------------------------------------------------- /tpch_2_13_0/distributed_queries/19.sql: -------------------------------------------------------------------------------- 1 | -- Query #19 2 | ------------ 3 | 4 | SELECT 5 | sum(l_extendedprice* (1 - l_discount)) as revenue 6 | FROM 7 | lineitem, 8 | part 9 | WHERE 10 | ( 11 | p_partkey = l_partkey 12 | AND p_brand = 'Brand#12' 13 | AND p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') 14 | AND l_quantity >= 1 AND l_quantity <= 1 + 10 15 | AND p_size between 1 AND 5 16 | AND l_shipmode in ('AIR', 'AIR REG') 17 | AND l_shipinstruct = 'DELIVER IN PERSON' 18 | ) 19 | OR 20 | ( 21 | p_partkey = l_partkey 22 | AND p_brand = 'Brand#23' 23 | AND p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') 24 | AND l_quantity >= 10 AND l_quantity <= 10 + 10 25 | AND p_size between 1 AND 10 26 | AND l_shipmode in ('AIR', 'AIR REG') 27 | AND l_shipinstruct = 'DELIVER IN PERSON' 28 | ) 29 | OR 30 | ( 31 | p_partkey = l_partkey 32 | AND p_brand = 'Brand#34' 33 | AND p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') 34 | AND l_quantity >= 20 AND l_quantity <= 20 + 10 35 | AND p_size between 1 AND 15 36 | AND l_shipmode in ('AIR', 'AIR REG') 37 | AND l_shipinstruct = 'DELIVER IN PERSON' 38 | ) 39 | -- the following LIMIT is not part of the benchmark suite 40 | -- but required to run the tests automatically 41 | -- see https://github.com/citusdata/test-automation/issues/120 42 | LIMIT 4; 43 | 44 | -------------------------------------------------------------------------------- /tpch_2_13_0/load_balance.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DOP=4 3 | # 4 | lock() 5 | { 6 | while [ true ] 7 | do 8 | sleep 1 9 | [ -f $LOCK ] && continue 10 | echo $$ > $LOCK 11 | [ `cat $LOCK` -ne $$ ] && continue 12 | return 13 | done 14 | } 15 | unlock() 16 | { 17 | rm -f $LOCK 18 | } 19 | usage() 20 | { 21 | echo "USAGE: `basename $0` []" 22 | echo " work through commands in , keeping tasks active" 23 | exit 24 | } 25 | # 26 | ### 27 | # MAIN 28 | ### 29 | case $# in 30 | 0) # no args, summarize usage and exit 31 | usage 32 | ;; 33 | 1) # default DOP 34 | [ $1 = "-h" ] && usage 35 | [ $1 = "--help" ] && usage 36 | TASKS=$1 37 | ;; 38 | 2) # set DOP and tasks 39 | TASKS=$1 40 | DOP=$2 41 | ;; 42 | 3) # worker invocation 43 | TASKS=$1 44 | DOP=$2 45 | JOBID=$3 46 | ;; 47 | esac 48 | # validate commands 49 | if [ ! -f "$TASKS" ] 50 | then 51 | echo "ERROR: no list of tasks found" 52 | exit 53 | fi 54 | # launch children or execute next command 55 | if [ -z "$JOBID" ] 56 | then 57 | JOBID=$$ 58 | LOCK=$JOBID.lck 59 | TID=$JOBID.task 60 | echo 1 > $TID 61 | unlock 62 | while [ $DOP -gt 0 ] 63 | do 64 | ./`basename $0` $TASKS $DOP $JOBID & 65 | DOP=`expr $DOP - 1` 66 | done 67 | wait 68 | rm -f $TID $LOCK 69 | else 70 | LOCK=$JOBID.lck 71 | TID=$JOBID.task 72 | while [ true ] 73 | do 74 | lock 75 | CMD_NUM=`cat $TID` 76 | echo `expr $CMD_NUM + 1` > $TID 77 | unlock 78 | CMD=`sed -n ${CMD_NUM}p $TASKS` 79 | [ -z "$CMD" ] && break 80 | echo -n "." 81 | $CMD >> lb_${JOBID}_$DOP.out 2>&1 82 | done 83 | fi 84 | 85 | -------------------------------------------------------------------------------- /.github/workflows/cleanup.yml: -------------------------------------------------------------------------------- 1 | name: Cleanup Resource Group 2 | on: 3 | push: 4 | branches: 5 | - delete_resource_group/* 6 | jobs: 7 | delete_resource_group: 8 | runs-on: ubuntu-latest 9 | container: 10 | image: buildpack-deps:bullseye 11 | options: --user root 12 | steps: 13 | - uses: actions/checkout@v3.5.0 14 | - name: Fix ownership issues 15 | run: git config --global --add safe.directory ${GITHUB_WORKSPACE} 16 | - name: Install Dependencies 17 | run: |- 18 | apt-get update && apt-get install -y cmake zip jq 19 | apt-get install -y ca-certificates curl apt-transport-https lsb-release gnupg 20 | curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null 21 | AZ_REPO=$(lsb_release -cs) 22 | echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | tee /etc/apt/sources.list.d/azure-cli.list 23 | apt-get update 24 | apt-get install -y azure-cli 25 | - name: Login to Azure 26 | run: |- 27 | az login --service-principal \ 28 | -u ${{ fromJson(secrets.AZURE_CREDENTIALS)['clientId']}} \ 29 | -p ${{ fromJson(secrets.AZURE_CREDENTIALS)['clientSecret']}} \ 30 | --tenant ${{ fromJson(secrets.AZURE_CREDENTIALS)['tenantId']}} 31 | - name: Delete resource group 32 | run: |- 33 | mkdir ~/.ssh 34 | cd ./azure 35 | ./delete-resource-group-job.sh 36 | shell: bash 37 | -------------------------------------------------------------------------------- /tpch_2_13_0/new55.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | # verify data set at different DOP levels 4 | ## 5 | usage() 6 | { 7 | [ $# -gt 1 ] && echo "ERROR: $2" 8 | echo "USAGE: `basename $1` []" 9 | echo " collect last rows for each table at a set of SF and DOP values" 10 | echo " and try to keep tasks running at all times" 11 | echo " NOTE: multiple values for SF/DOP must be enclosed in quotation marks and separated by spaces" 12 | exit 13 | } 14 | ## 15 | # MAIN 16 | ## 17 | PROCS=4 18 | [ -z "$DSS_PATH" ] && DSS_PATH="." 19 | export DSS_PATH 20 | 21 | # parse command line 22 | case $# in 23 | 3) 24 | PROCS=$3 25 | DOP=$2 26 | SF=$1 27 | ;; 28 | 2) 29 | DOP=$2 30 | SF=$1 31 | ;; 32 | *) usage $0;; 33 | esac 34 | 35 | # generate a task list 36 | CMDS=$$.cmds 37 | rm -f $CMDS 38 | for s in $SF 39 | do 40 | ./gen_tasks.sh $s $DOP >> $CMDS 41 | done 42 | 43 | # run parallel tasks to get last rows 44 | echo "Begining `wc -l $CMDS|cut -f1 -d\ ` tasks with PROCS=$PROCS" 45 | 46 | if [ ! -x ./dbgen ] 47 | then 48 | echo "ERROR: dbgen not found in `pwd`" 49 | exit 50 | fi 51 | ./load_balance.sh $CMDS $PROCS 52 | echo "done" 53 | 54 | # gather the results by table 55 | for f in *.last_row.* 56 | do 57 | t=`echo $f | cut -f1 -d\.` 58 | s=`echo $f | cut -f3 -d\.` 59 | d=`echo $f | cut -f4 -d\.` 60 | echo -n "$d " >> ${DSS_PATH}/$s/results.$t 61 | cat $f >> ${DSS_PATH}/$s/results.$t 62 | rm $f 63 | done 64 | 65 | # clean up 66 | rm $CMDS 67 | for f in lb_*.out 68 | do 69 | if [ -s $f ] 70 | then 71 | echo "There is output in $f. Not removed" 72 | else 73 | rm $f 74 | fi 75 | done 76 | -------------------------------------------------------------------------------- /.github/workflows/scale-tests.yml: -------------------------------------------------------------------------------- 1 | name: Performance Tests - Scale 2 | on: 3 | push: 4 | branches: 5 | - scale/* 6 | - all_performance_test/* 7 | env: 8 | RESULT_REPO_ACCESS_TOKEN: ${{ secrets.TEST_RESULTS_REPO_TOKEN }} 9 | jobs: 10 | scale: 11 | runs-on: ubuntu-latest 12 | container: 13 | image: buildpack-deps:bullseye 14 | options: --user root 15 | steps: 16 | - uses: actions/checkout@v3.5.0 17 | - name: Fix ownership issues 18 | run: git config --global --add safe.directory ${GITHUB_WORKSPACE} 19 | - name: Install Dependencies 20 | run: |- 21 | apt-get update && apt-get install -y cmake zip jq 22 | apt-get install -y ca-certificates curl apt-transport-https lsb-release gnupg 23 | curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null 24 | AZ_REPO=$(lsb_release -cs) 25 | echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | tee /etc/apt/sources.list.d/azure-cli.list 26 | apt-get update 27 | apt-get install -y azure-cli 28 | - name: Login to Azure 29 | run: |- 30 | az login --service-principal \ 31 | -u ${{ fromJson(secrets.AZURE_CREDENTIALS)['clientId']}} \ 32 | -p ${{ fromJson(secrets.AZURE_CREDENTIALS)['clientSecret']}} \ 33 | --tenant ${{ fromJson(secrets.AZURE_CREDENTIALS)['tenantId']}} 34 | - name: Run scale tests 35 | run: |- 36 | mkdir ~/.ssh 37 | cd ./azure 38 | ./citus-bot.sh citusbot_scale_test_resource_group 39 | shell: bash 40 | -------------------------------------------------------------------------------- /.github/workflows/tpch-tests.yml: -------------------------------------------------------------------------------- 1 | name: Performance Tests - TPCH 2 | on: 3 | push: 4 | branches: 5 | - tpch/* 6 | - all_performance_test/* 7 | env: 8 | RESULT_REPO_ACCESS_TOKEN: ${{ secrets.TEST_RESULTS_REPO_TOKEN }} 9 | jobs: 10 | tpch: 11 | runs-on: ubuntu-latest 12 | container: 13 | image: buildpack-deps:bullseye 14 | options: --user root 15 | steps: 16 | - uses: actions/checkout@v3.5.0 17 | - name: Fix ownership issues 18 | run: git config --global --add safe.directory ${GITHUB_WORKSPACE} 19 | - name: Install Dependencies 20 | run: |- 21 | apt-get update && apt-get install -y cmake zip jq 22 | apt-get install -y ca-certificates curl apt-transport-https lsb-release gnupg 23 | curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null 24 | AZ_REPO=$(lsb_release -cs) 25 | echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | tee /etc/apt/sources.list.d/azure-cli.list 26 | apt-get update 27 | apt-get install -y azure-cli 28 | - name: Login to Azure 29 | run: |- 30 | az login --service-principal \ 31 | -u ${{ fromJson(secrets.AZURE_CREDENTIALS)['clientId']}} \ 32 | -p ${{ fromJson(secrets.AZURE_CREDENTIALS)['clientSecret']}} \ 33 | --tenant ${{ fromJson(secrets.AZURE_CREDENTIALS)['tenantId']}} 34 | - name: Run tpch tests 35 | run: |- 36 | mkdir ~/.ssh 37 | cd ./azure 38 | ./citus-bot.sh citusbot_tpch_test_resource_group 39 | shell: bash 40 | -------------------------------------------------------------------------------- /.github/workflows/pgbench-tests.yml: -------------------------------------------------------------------------------- 1 | name: Performance Tests - PgBench 2 | on: 3 | push: 4 | branches: 5 | - pgbench/* 6 | - all_performance_test/* 7 | env: 8 | RESULT_REPO_ACCESS_TOKEN: ${{ secrets.TEST_RESULTS_REPO_TOKEN }} 9 | jobs: 10 | pgbench: 11 | runs-on: ubuntu-latest 12 | container: 13 | image: buildpack-deps:bullseye 14 | options: --user root 15 | steps: 16 | - uses: actions/checkout@v3.5.0 17 | - name: Fix ownership issues 18 | run: git config --global --add safe.directory ${GITHUB_WORKSPACE} 19 | - name: Install Dependencies 20 | run: |- 21 | apt-get update && apt-get install -y cmake zip jq 22 | apt-get install -y ca-certificates curl apt-transport-https lsb-release gnupg 23 | curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null 24 | AZ_REPO=$(lsb_release -cs) 25 | echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | tee /etc/apt/sources.list.d/azure-cli.list 26 | apt-get update 27 | apt-get install -y azure-cli 28 | - name: Login to Azure 29 | run: |- 30 | az login --service-principal \ 31 | -u ${{ fromJson(secrets.AZURE_CREDENTIALS)['clientId']}} \ 32 | -p ${{ fromJson(secrets.AZURE_CREDENTIALS)['clientSecret']}} \ 33 | --tenant ${{ fromJson(secrets.AZURE_CREDENTIALS)['tenantId']}} 34 | - name: Run pgbench tests 35 | run: |- 36 | mkdir ~/.ssh 37 | cd ./azure 38 | ./citus-bot.sh citusbot_pgbench_test_resource_group 39 | shell: bash 40 | -------------------------------------------------------------------------------- /tpch_2_13_0/last_row.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | # use named pipes to get the last row of a given DBGEN data chunk 4 | ## 5 | usage() 6 | { 7 | [ $# -gt 1 ] && echo "ERROR: $2" 8 | echo "USAGE: `basename $1` " 9 | echo " gather final row of table (e.g., -T ) at named and " 10 | exit 11 | } 12 | ## 13 | # MAIN 14 | ## 15 | SF=1 16 | DOP=1 17 | # parse command line 18 | case $# in 19 | 1) 20 | [ $1 = "-h" ] && usage $0 21 | [ $1 = "--help" ] && usage $0 22 | t=$1 23 | ;; 24 | 2) 25 | t=$1 26 | SF=$2 27 | ;; 28 | 3) 29 | t=$1 30 | SF=$2 31 | DOP=$3 32 | ;; 33 | *) 34 | usage $0 "invalid argument count" 35 | ;; 36 | esac 37 | 38 | # assure a setting for DSS_PATH 39 | [ -z "$DSS_PATH" ] && DSS_PATH="." 40 | [ ! -d "$DSS_PATH/$SF" ] && mkdir $DSS_PATH/$SF 41 | chmod 777 $DSS_PATH 42 | DSS_PATH=$DSS_PATH/$SF 43 | export DSS_PATH 44 | 45 | # set the other args 46 | case $t in 47 | n) f="nation"; DOP=1;; # special case for tiny table 48 | r) f="region"; DOP=1;; # special case for tiny table 49 | c) f="customer";; 50 | s) f="supplier";; 51 | P) f="part";; 52 | S) f="partsupp";; 53 | O) f="orders";; 54 | L) f="lineitem";; 55 | *) usage "bad table abreviation: $t" 56 | esac 57 | if [ $DOP -eq 1 ] 58 | then PIPE="$DSS_PATH/${f}.tbl" 59 | else PIPE="$DSS_PATH/${f}.tbl.${DOP}" 60 | fi 61 | 62 | # create a named pipe for each table 63 | rm -rf $PIPE 64 | mknod $PIPE p 65 | # generate data into it 66 | ./dbgen -q -f -s $SF -T $t -S $DOP -C $DOP & 67 | # assure that data is being written to the pipe 68 | sleep 30 69 | # read the last row from the pipe 70 | tail -1 $PIPE > $f.last_row.$SF.$DOP & 71 | wait 72 | rm $PIPE 73 | -------------------------------------------------------------------------------- /fabfile/multi_connections.py: -------------------------------------------------------------------------------- 1 | from fabric import Connection 2 | from invoke import Context 3 | from invoke.exceptions import Exit 4 | 5 | from connection import all_connections 6 | 7 | def is_coordinator_connection(c): 8 | # returns true if the connection is to local node or it is a context object(no connection to localhost sometimes). 9 | # Info: specifying -H flag(hostlist) will pass the @task a Connection object. if -H is not specified a Context object will be passed. 10 | 11 | if isinstance(c, Connection): 12 | return c.host == 'localhost' 13 | elif isinstance(c, Context): 14 | return True 15 | raise Exit('Unexpected connection!') 16 | 17 | def name_of_connection(c): 18 | if isinstance(c, Connection): 19 | return "{}:{}".format(c.host, c.port) 20 | elif isinstance(c, Context): 21 | return 'localhost' 22 | raise Exit('Unexpected connection!') 23 | 24 | def execute_on_all_nodes_if_no_hosts(c, task, *args, **kwargs): 25 | # fab will execute on only on local node but we modify it to execute on all nodes. 26 | # that method should be called in the first line of each @task if we want to have explained behaviour. 27 | 28 | # no -H is given if the connection is of Context (fab ) 29 | if isinstance(c, Context) and not isinstance(c, Connection) and len(all_connections) > 1: 30 | execute(all_connections, task, *args, **kwargs) 31 | return True 32 | return False 33 | 34 | def execute(connections, task, *args, **kwargs): 35 | # runs a task on given connections 36 | 37 | for connection in connections: 38 | task(connection, *args, **kwargs) 39 | -------------------------------------------------------------------------------- /.github/workflows/extension.yml: -------------------------------------------------------------------------------- 1 | name: Compatibility Tests - Extension 2 | on: 3 | push: 4 | branches: 5 | - extension/* 6 | env: 7 | RESULT_REPO_ACCESS_TOKEN: ${{ secrets.TEST_RESULTS_REPO_TOKEN }} 8 | jobs: 9 | extension: 10 | runs-on: ubuntu-latest 11 | container: 12 | image: buildpack-deps:bullseye 13 | options: --user root 14 | steps: 15 | - uses: actions/checkout@v3.5.0 16 | - name: Fix ownership issues 17 | run: git config --global --add safe.directory ${GITHUB_WORKSPACE} 18 | - name: Install Dependencies 19 | run: |- 20 | apt-get update && apt-get install -y cmake zip jq 21 | apt-get install -y ca-certificates curl apt-transport-https lsb-release gnupg 22 | curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null 23 | AZ_REPO=$(lsb_release -cs) 24 | echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | tee /etc/apt/sources.list.d/azure-cli.list 25 | apt-get update 26 | apt-get install -y azure-cli 27 | - name: Login to Azure 28 | run: |- 29 | az login --service-principal \ 30 | -u ${{ fromJson(secrets.AZURE_CREDENTIALS)['clientId']}} \ 31 | -p ${{ fromJson(secrets.AZURE_CREDENTIALS)['clientSecret']}} \ 32 | --tenant ${{ fromJson(secrets.AZURE_CREDENTIALS)['tenantId']}} 33 | - name: Run extension tests 34 | run: |- 35 | mkdir ~/.ssh 36 | cd ./azure 37 | export EXTENSION_TEST=1 38 | ./citus-bot.sh citusbot_extension_test_resource_group 39 | shell: bash 40 | -------------------------------------------------------------------------------- /fabfile/pgbench_confs/pgbench_default.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | postgres_citus_versions: [('15.1', 'main')] 3 | shard_counts_replication_factors: [(32, 1)] 4 | 5 | ; Citus flags can be added below as well. E.g. citus.shard_replication_factor = 2 6 | ; Beware of citus flags which are available in one version but not the other 7 | ; In that case, consider creating two different configs for each version 8 | postgresql_conf: [ 9 | "max_wal_size = '50GB'", 10 | "checkpoint_completion_target = 0.9", 11 | "checkpoint_timeout = '1h'", 12 | "max_connections = 1000", 13 | "max_prepared_transactions = 1000", 14 | ] 15 | 16 | [initialization] 17 | pgbench_command: pgbench -i -s 100 18 | 19 | [citus1] 20 | distribute_table_command: SELECT create_distributed_table('pgbench_accounts', 'aid'); 21 | sql_command: VACUUM ANALYZE pgbench_accounts; 22 | 23 | [citus2] 24 | distribute_table_command: SELECT create_distributed_table('pgbench_branches', 'bid'); 25 | sql_command: VACUUM ANALYZE pgbench_branches; 26 | 27 | [citus3] 28 | distribute_table_command: SELECT create_distributed_table('pgbench_history', 'tid'); 29 | sql_command: VACUUM ANALYZE pgbench_history; 30 | 31 | [citus4] 32 | distribute_table_command: SELECT create_distributed_table('pgbench_tellers', 'tid'); 33 | sql_command: VACUUM ANALYZE pgbench_tellers; 34 | 35 | [default-pgbench] 36 | pgbench_command: pgbench -c 32 -j 16 -T 600 -P 10 -r 37 | 38 | [simple-update] 39 | pgbench_command: pgbench -c 32 -j 16 -T 600 -P 10 -r -b simple-update 40 | 41 | [select-only] 42 | pgbench_command: pgbench -c 32 -j 16 -T 600 -P 10 -r -b select-only 43 | -------------------------------------------------------------------------------- /hammerdb/run_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u 5 | # exit immediately if a command fails 6 | set -e 7 | # echo commands 8 | set -x 9 | 10 | coordinator_private_ip=$1 11 | driver_private_ip=$2 12 | branch_name=$3 13 | is_tpcc=$4 14 | is_ch=$5 15 | username=$6 16 | hammerdb_version=$7 17 | cluster_rg=$8 18 | 19 | # store hammerdb version in a file so that we can get it in other scripts 20 | echo "${hammerdb_version}" > ~/HAMMERDB_VERSION 21 | 22 | # turn pager off, because queries might return results that are bigger than the page size 23 | # in that case the more process will be created, and the script will hang. 24 | echo "\pset pager off" >> ~/.psqlrc 25 | 26 | # do setup of cluster 27 | "${HOME}"/test-automation/hammerdb/setup.sh "${coordinator_private_ip}" "${username}" 28 | 29 | # for each hammerdb config, run the tests and store the results 30 | for config_file in "${HOME}/test-automation/fabfile/hammerdb_confs"/* 31 | do 32 | # get the file name from absolute path 33 | config_file=$(basename "$config_file") 34 | 35 | ssh -o "StrictHostKeyChecking no" -A "${coordinator_private_ip}" "source ~/.bash_profile;fab setup.hammerdb --config-file=${config_file} --driver-ip=${driver_private_ip}" 36 | "${HOME}"/test-automation/hammerdb/build-and-run.sh "${coordinator_private_ip}" "${config_file}" "${is_tpcc}" "${is_ch}" "${username}" 37 | done 38 | 39 | cp -r "${HOME}"/test-automation/fabfile/hammerdb_confs "${HOME}"/HammerDB-"${hammerdb_version}"/results 40 | "${HOME}"/test-automation/hammerdb/upload-results.sh "${branch_name}" 41 | "${HOME}"/test-automation/hammerdb/delete-resource-group.sh "${cluster_rg}" 42 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/cmd_base_sf1: -------------------------------------------------------------------------------- 1 | ../dbgen -s 1 -T L -C 60000 -S 1 -f -b ../dists.dss 2 | ../dbgen -s 1 -T L -C 60000 -S 19999 -f -b ../dists.dss 3 | ../dbgen -s 1 -T L -C 60000 -S 39998 -f -b ../dists.dss 4 | ../dbgen -s 1 -T L -C 60000 -S 59997 -f -b ../dists.dss 5 | ../dbgen -s 1 -T L -C 60000 -S 60000 -f -b ../dists.dss 6 | ../dbgen -s 1 -T O -C 15000 -S 1 -f -b ../dists.dss 7 | ../dbgen -s 1 -T O -C 15000 -S 4999 -f -b ../dists.dss 8 | ../dbgen -s 1 -T O -C 15000 -S 9998 -f -b ../dists.dss 9 | ../dbgen -s 1 -T O -C 15000 -S 14997 -f -b ../dists.dss 10 | ../dbgen -s 1 -T O -C 15000 -S 15000 -f -b ../dists.dss 11 | ../dbgen -s 1 -T P -C 2000 -S 1 -f -b ../dists.dss 12 | ../dbgen -s 1 -T P -C 2000 -S 666 -f -b ../dists.dss 13 | ../dbgen -s 1 -T P -C 2000 -S 1332 -f -b ../dists.dss 14 | ../dbgen -s 1 -T P -C 2000 -S 1998 -f -b ../dists.dss 15 | ../dbgen -s 1 -T P -C 2000 -S 2000 -f -b ../dists.dss 16 | ../dbgen -s 1 -T S -C 8000 -S 1 -f -b ../dists.dss 17 | ../dbgen -s 1 -T S -C 8000 -S 2666 -f -b ../dists.dss 18 | ../dbgen -s 1 -T S -C 8000 -S 5332 -f -b ../dists.dss 19 | ../dbgen -s 1 -T S -C 8000 -S 7998 -f -b ../dists.dss 20 | ../dbgen -s 1 -T S -C 8000 -S 8000 -f -b ../dists.dss 21 | ../dbgen -s 1 -T s -C 100 -S 1 -f -b ../dists.dss 22 | ../dbgen -s 1 -T s -C 100 -S 32 -f -b ../dists.dss 23 | ../dbgen -s 1 -T s -C 100 -S 64 -f -b ../dists.dss 24 | ../dbgen -s 1 -T s -C 100 -S 96 -f -b ../dists.dss 25 | ../dbgen -s 1 -T s -C 100 -S 100 -f -b ../dists.dss 26 | ../dbgen -s 1 -T c -C 1500 -S 1 -f -b ../dists.dss 27 | ../dbgen -s 1 -T c -C 1500 -S 499 -f -b ../dists.dss 28 | ../dbgen -s 1 -T c -C 1500 -S 998 -f -b ../dists.dss 29 | ../dbgen -s 1 -T c -C 1500 -S 1497 -f -b ../dists.dss 30 | ../dbgen -s 1 -T c -C 1500 -S 1500 -f -b ../dists.dss 31 | ../dbgen -T n -f -b ../dists.dss 32 | ../dbgen -T r -f -b ../dists.dss 33 | -------------------------------------------------------------------------------- /hammerdb/download-hammerdb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | cd "$(dirname "$0")" 5 | 6 | 7 | # no-op if the directory already exists 8 | if test -d "HammerDB-$1"; then 9 | echo "HammerDB-$1 exists, skipping download" 1>&2 10 | exit 11 | fi 12 | 13 | if [ "$1" == 4.4 ] 14 | then 15 | SHA1=5347e06baad690336afa6d354330cd59d98343cc 16 | OUTPUT=HammerDB-4.4-Linux.tar.gz 17 | URL=https://github.com/TPC-Council/HammerDB/releases/download/v4.4/$OUTPUT 18 | elif [ "$1" == 4.3 ] 19 | then 20 | SHA1=5bb232c45c2d3a063df09bb3dedd44d3fe5a3edf 21 | OUTPUT=HammerDB-4.3-Linux.tar.gz 22 | URL=https://github.com/TPC-Council/HammerDB/releases/download/v4.3/$OUTPUT 23 | elif [ "$1" == 4.0 ] 24 | then 25 | SHA1=a6a35c234d324077c7819ca4ae4aa8eabaff4c15 26 | OUTPUT=HammerDB-4.0-Linux.tar.gz 27 | URL=https://github.com/TPC-Council/HammerDB/releases/download/v4.0/$OUTPUT 28 | elif [ "$1" == 3.3 ] 29 | then 30 | SHA1=3a0a7552e74c0f8cb07376301b23c6a5ca29d0f1 31 | OUTPUT=HammerDB-3.3-Linux.tar.gz 32 | URL=https://github.com/TPC-Council/HammerDB/releases/download/v3.3/$OUTPUT 33 | else 34 | echo 'Expects version parameter. Supported versions: 3.3 4.0 4.3 4.4' 1>&2 35 | exit 1 36 | fi 37 | 38 | if test -f "$OUTPUT" 39 | then 40 | CHECK_OUTPUT=${OUTPUT} 41 | echo "$OUTPUT exists, skipping download" 1>&2 42 | else 43 | CHECK_OUTPUT=${OUTPUT}.tmp 44 | curl --location --output "$CHECK_OUTPUT" "$URL" 1>&2 45 | fi 46 | 47 | GZSHA1_ARRAY=($(sha1sum "$CHECK_OUTPUT")) 48 | GZSHA1=${GZSHA1_ARRAY[0]} 49 | 50 | if [ "$GZSHA1" != "$SHA1" ] 51 | then 52 | echo "Unexpected checksum $GZSHA1" 1>&2 53 | echo "Expected $SHA1" 1>&2 54 | exit 1 55 | fi 56 | 57 | if ! test -f "$OUTPUT" 58 | then 59 | mv "$CHECK_OUTPUT" "$OUTPUT" 60 | fi 61 | 62 | tar -xf "$OUTPUT" 63 | -------------------------------------------------------------------------------- /fabfile/__init__.py: -------------------------------------------------------------------------------- 1 | from invoke import task 2 | from invoke import Collection 3 | 4 | import multi_connections # multi_connections module implements some features on top of fabric 5 | import prefix # utilities for keeping track of where we're installing everything 6 | from connection import all_connections # connections for nodes 7 | 8 | import pg # tasks which control postgres 9 | import add # tasks which add things to existing clusters 10 | import setup # tasks which create clusters with preset settings 11 | import use # tasks which change some configuration future tasks read 12 | import run # tasks to run misc commands 13 | 14 | @task(default=True) 15 | def main(c): 16 | 'The default task (what happens when you type "fab"), currently setup.basic-testing' 17 | if not multi_connections.is_coordinator_connection(c): 18 | return 19 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, main): 20 | return 21 | 22 | setup.basic_testing(c) 23 | 24 | @task(positional=['prefixpath']) 25 | def set_pg_latest(c, prefixpath): 26 | 'Use this if you want multiple simultaneous installs, the README has more information' 27 | if not multi_connections.is_coordinator_connection(c): 28 | return 29 | 30 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, set_pg_latest, prefixpath): 31 | return 32 | 33 | new_prefix = prefixpath 34 | 35 | multi_connections.execute(all_connections, prefix.set_prefix, new_prefix) 36 | 37 | # control the output of "fab --list" 38 | def setup_tasks(): 39 | ns = Collection('fabfile') 40 | ns.add_task(main) 41 | ns.add_task(set_pg_latest) 42 | ns.add_collection(pg) 43 | ns.add_collection(add) 44 | ns.add_collection(setup) 45 | ns.add_collection(use) 46 | ns.add_collection(run) 47 | return ns 48 | ns = setup_tasks() 49 | -------------------------------------------------------------------------------- /fabfile/pgbench_confs/pgbench_default_without_transaction.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | postgres_citus_versions: [('17.5', 'release-13.2')] 3 | shard_counts_replication_factors: [(32, 1)] 4 | 5 | ; Citus flags can be added below as well. E.g. citus.shard_replication_factor = 2 6 | ; Beware of citus flags which are available in one version but not the other 7 | ; In that case, consider creating two different configs for each version 8 | postgresql_conf: [ 9 | "max_wal_size = '50GB'", 10 | "checkpoint_completion_target = 0.9", 11 | "checkpoint_timeout = '1h'", 12 | "max_connections = 1000", 13 | "max_prepared_transactions = 1000" 14 | ] 15 | 16 | [initialization] 17 | pgbench_command: pgbench -i -s 100 18 | 19 | [citus1] 20 | distribute_table_command: SELECT create_distributed_table('pgbench_accounts', 'aid'); 21 | sql_command: VACUUM ANALYZE pgbench_accounts; 22 | 23 | [citus2] 24 | distribute_table_command: SELECT create_distributed_table('pgbench_branches', 'bid'); 25 | sql_command: VACUUM ANALYZE pgbench_branches; 26 | 27 | [citus3] 28 | distribute_table_command: SELECT create_distributed_table('pgbench_history', 'tid'); 29 | sql_command: VACUUM ANALYZE pgbench_history; 30 | 31 | [citus4] 32 | distribute_table_command: SELECT create_distributed_table('pgbench_tellers', 'tid'); 33 | sql_command: VACUUM ANALYZE pgbench_tellers; 34 | 35 | [default-pgbench] 36 | pgbench_command: pgbench -c 32 -j 16 -T 600 -P 10 -r -f fabfile/pgbench_scripts/pgbench_default_no_transactions.sql 37 | 38 | [simple-update] 39 | pgbench_command: pgbench -c 32 -j 16 -T 600 -P 10 -r -f fabfile/pgbench_scripts/pgbench_simple_update_no_transactions.sql 40 | 41 | [select-only] 42 | pgbench_command: pgbench -c 32 -j 16 -T 600 -P 10 -r -f fabfile/pgbench_scripts/pgbench_select_only_no_transactions.sql 43 | -------------------------------------------------------------------------------- /jdbc/utils.sh: -------------------------------------------------------------------------------- 1 | function remove_string_quotations() 2 | { 3 | # https://tecadmin.net/bash-remove-double-quote-string/#:~:text=A%20single%20line%20sed%20command%20can%20remove%20quotes,will%20remove%20the%20ending%20quote%20from%20the%20string. 4 | # https://stackoverflow.com/a/35512655 5 | sed -e 's/^"//' -e 's/"$//' <<<"${1:-$( public ip of vm 12 | # $2 -> ssh port of vm 13 | ssh_execute() { 14 | ip=$1 15 | ssh_port=$2 16 | shift 2; 17 | command=$@ 18 | n=0 19 | until [ $n -ge 10 ] 20 | do 21 | ssh -o "StrictHostKeyChecking no" -A -p "${ssh_port}" pguser@${ip} "source ~/.bash_profile;${command}" && break 22 | rc=$? # get return code 23 | if [ $rc -ne 255 ] ; then 24 | # if the error code is not 255 we didn't get a connection error. 25 | exit 1 26 | fi 27 | n=$[$n+1] 28 | done 29 | 30 | if [ $n == 10 ]; then 31 | exit 1 32 | fi 33 | } 34 | 35 | # rg_get_ssh_port returns customSshPort value for the vm created in given resource group 36 | # $1 -> resource group name 37 | rg_get_ssh_port() { 38 | rg=$1 39 | 40 | ssh_port=$(az deployment group show -g "${rg}" -n azuredeploy --query properties.outputs.customSshPort.value) 41 | # remove the quotes 42 | ssh_port=$(echo "${ssh_port}" | cut -d "\"" -f 2) 43 | 44 | echo $ssh_port 45 | } 46 | 47 | # rg_get_ssh_port returns public ip for the vm created in given resource group 48 | # $1 -> resource group name 49 | rg_get_public_ip() { 50 | rg=$1 51 | 52 | public_ip=$(az deployment group show -g ${rg} -n azuredeploy --query properties.outputs.publicIP.value) 53 | # remove the quotes 54 | public_ip=$(echo ${public_ip} | cut -d "\"" -f 2) 55 | 56 | echo $public_ip 57 | } 58 | 59 | # vm_add_public_ip_to_known_hosts adds public ip of the vm to its known hosts file 60 | # $1 -> public ip of the vm 61 | # $1 -> ssh port of the vm 62 | vm_add_public_ip_to_known_hosts() { 63 | public_ip=$1 64 | ssh_port=$2 65 | 66 | echo "adding public ip to known hosts in remote" 67 | ssh_execute ${public_ip} ${ssh_port} "ssh-keyscan -H ${public_ip} >> /home/pguser/.ssh/known_hosts" 68 | } 69 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/cmd_base_sf10000: -------------------------------------------------------------------------------- 1 | ../dbgen -s 10000 -T L -C 600000000 -S 1 -f -b ../dists.dss 2 | ../dbgen -s 10000 -T L -C 600000000 -S 199999999 -f -b ../dists.dss 3 | ../dbgen -s 10000 -T L -C 600000000 -S 399999998 -f -b ../dists.dss 4 | ../dbgen -s 10000 -T L -C 600000000 -S 599999997 -f -b ../dists.dss 5 | ../dbgen -s 10000 -T L -C 600000000 -S 600000000 -f -b ../dists.dss 6 | ../dbgen -s 10000 -T O -C 150000000 -S 1 -f -b ../dists.dss 7 | ../dbgen -s 10000 -T O -C 150000000 -S 49999999 -f -b ../dists.dss 8 | ../dbgen -s 10000 -T O -C 150000000 -S 99999998 -f -b ../dists.dss 9 | ../dbgen -s 10000 -T O -C 150000000 -S 149999997 -f -b ../dists.dss 10 | ../dbgen -s 10000 -T O -C 150000000 -S 150000000 -f -b ../dists.dss 11 | ../dbgen -s 10000 -T P -C 20000000 -S 1 -f -b ../dists.dss 12 | ../dbgen -s 10000 -T P -C 20000000 -S 6666666 -f -b ../dists.dss 13 | ../dbgen -s 10000 -T P -C 20000000 -S 13333332 -f -b ../dists.dss 14 | ../dbgen -s 10000 -T P -C 20000000 -S 19999998 -f -b ../dists.dss 15 | ../dbgen -s 10000 -T P -C 20000000 -S 20000000 -f -b ../dists.dss 16 | ../dbgen -s 10000 -T S -C 80000000 -S 1 -f -b ../dists.dss 17 | ../dbgen -s 10000 -T S -C 80000000 -S 26666666 -f -b ../dists.dss 18 | ../dbgen -s 10000 -T S -C 80000000 -S 53333332 -f -b ../dists.dss 19 | ../dbgen -s 10000 -T S -C 80000000 -S 79999998 -f -b ../dists.dss 20 | ../dbgen -s 10000 -T S -C 80000000 -S 80000000 -f -b ../dists.dss 21 | ../dbgen -s 10000 -T s -C 1000000 -S 1 -f -b ../dists.dss 22 | ../dbgen -s 10000 -T s -C 1000000 -S 333332 -f -b ../dists.dss 23 | ../dbgen -s 10000 -T s -C 1000000 -S 666664 -f -b ../dists.dss 24 | ../dbgen -s 10000 -T s -C 1000000 -S 999996 -f -b ../dists.dss 25 | ../dbgen -s 10000 -T s -C 1000000 -S 1000000 -f -b ../dists.dss 26 | ../dbgen -s 10000 -T c -C 15000000 -S 1 -f -b ../dists.dss 27 | ../dbgen -s 10000 -T c -C 15000000 -S 4999999 -f -b ../dists.dss 28 | ../dbgen -s 10000 -T c -C 15000000 -S 9999998 -f -b ../dists.dss 29 | ../dbgen -s 10000 -T c -C 15000000 -S 14999997 -f -b ../dists.dss 30 | ../dbgen -s 10000 -T c -C 15000000 -S 15000000 -f -b ../dists.dss 31 | ../dbgen -T n -f -b ../dists.dss 32 | ../dbgen -T r -f -b ../dists.dss 33 | -------------------------------------------------------------------------------- /hammerdb/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u / set -o nounset 5 | # exit immediately if a command fails 6 | set -e 7 | 8 | ## Set mydir to the directory containing the script 9 | ## The ${var%pattern} format will remove the shortest match of 10 | ## pattern from the end of the string. Here, it will remove the 11 | ## script's name,. leaving only the directory. 12 | driverdir="${0%/*}" 13 | cd "${driverdir}" 14 | 15 | regions=(eastus southcentralus westus2) 16 | 17 | size=${#regions[@]} 18 | index=$((RANDOM % size)) 19 | random_region=${regions[$index]} 20 | 21 | rg="${RESOURCE_GROUP_NAME}" 22 | # if region is provided, use that. If not use the random region. 23 | region=${AZURE_REGION:=$random_region} 24 | echo "${region}" 25 | az group create -l "${region}" -n "${rg}" 26 | 27 | # get our public key programmatically so that users don't have to enter it manually. 28 | public_key=$(cat ~/.ssh/id_rsa.pub) 29 | 30 | start_time=$(date +%s) 31 | echo "waiting a long time to create cluster, this might take up to 30 mins depending on your cluster size" 32 | 33 | # https://stackoverflow.com/questions/1593051/how-to-programmatically-determine-the-current-checked-out-git-branch 34 | branch_name=$(git symbolic-ref -q HEAD) 35 | branch_name=${branch_name##refs/heads/} 36 | branch_name=${branch_name:-HEAD} 37 | 38 | # store the branch name in a file so that target user can read it. Target user cannot see the envionment variables because 39 | # we use login option in su and -p(preserving environment variables) cannot be used with login. We need to use login option 40 | # so that $HOME, $PATH are set to the target users $HOME and $PATH. 41 | export BRANCH=${branch_name} 42 | 43 | # get local public ip 44 | local_public_ip=$(curl ifconfig.me) 45 | 46 | az deployment group create -g "${rg}" --template-file azuredeploy.json --parameters @azuredeploy.parameters.json --parameters sshPublicKey="${public_key}" branchName="$BRANCH" git_username="${GIT_USERNAME}" git_token="${GIT_TOKEN}" localPublicIp="$local_public_ip" 47 | 48 | end_time=$(date +%s) 49 | echo execution time was $((end_time - start_time)) s. 50 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/cmd_base_sf30000: -------------------------------------------------------------------------------- 1 | ../dbgen -s 30000 -T L -C 1800000000 -S 1 -f -b ../dists.dss 2 | ../dbgen -s 30000 -T L -C 1800000000 -S 599999999 -f -b ../dists.dss 3 | ../dbgen -s 30000 -T L -C 1800000000 -S 1199999998 -f -b ../dists.dss 4 | ../dbgen -s 30000 -T L -C 1800000000 -S 1799999997 -f -b ../dists.dss 5 | ../dbgen -s 30000 -T L -C 1800000000 -S 1800000000 -f -b ../dists.dss 6 | ../dbgen -s 30000 -T O -C 450000000 -S 1 -f -b ../dists.dss 7 | ../dbgen -s 30000 -T O -C 450000000 -S 149999999 -f -b ../dists.dss 8 | ../dbgen -s 30000 -T O -C 450000000 -S 299999998 -f -b ../dists.dss 9 | ../dbgen -s 30000 -T O -C 450000000 -S 449999997 -f -b ../dists.dss 10 | ../dbgen -s 30000 -T O -C 450000000 -S 450000000 -f -b ../dists.dss 11 | ../dbgen -s 30000 -T P -C 60000000 -S 1 -f -b ../dists.dss 12 | ../dbgen -s 30000 -T P -C 60000000 -S 19999999 -f -b ../dists.dss 13 | ../dbgen -s 30000 -T P -C 60000000 -S 39999998 -f -b ../dists.dss 14 | ../dbgen -s 30000 -T P -C 60000000 -S 59999997 -f -b ../dists.dss 15 | ../dbgen -s 30000 -T P -C 60000000 -S 60000000 -f -b ../dists.dss 16 | ../dbgen -s 30000 -T S -C 240000000 -S 1 -f -b ../dists.dss 17 | ../dbgen -s 30000 -T S -C 240000000 -S 79999999 -f -b ../dists.dss 18 | ../dbgen -s 30000 -T S -C 240000000 -S 159999998 -f -b ../dists.dss 19 | ../dbgen -s 30000 -T S -C 240000000 -S 239999997 -f -b ../dists.dss 20 | ../dbgen -s 30000 -T S -C 240000000 -S 240000000 -f -b ../dists.dss 21 | ../dbgen -s 30000 -T s -C 3000000 -S 1 -f -b ../dists.dss 22 | ../dbgen -s 30000 -T s -C 3000000 -S 999999 -f -b ../dists.dss 23 | ../dbgen -s 30000 -T s -C 3000000 -S 1999998 -f -b ../dists.dss 24 | ../dbgen -s 30000 -T s -C 3000000 -S 2999997 -f -b ../dists.dss 25 | ../dbgen -s 30000 -T s -C 3000000 -S 3000000 -f -b ../dists.dss 26 | ../dbgen -s 30000 -T c -C 45000000 -S 1 -f -b ../dists.dss 27 | ../dbgen -s 30000 -T c -C 45000000 -S 14999999 -f -b ../dists.dss 28 | ../dbgen -s 30000 -T c -C 45000000 -S 29999998 -f -b ../dists.dss 29 | ../dbgen -s 30000 -T c -C 45000000 -S 44999997 -f -b ../dists.dss 30 | ../dbgen -s 30000 -T c -C 45000000 -S 45000000 -f -b ../dists.dss 31 | ../dbgen -T n -f -b ../dists.dss 32 | ../dbgen -T r -f -b ../dists.dss 33 | -------------------------------------------------------------------------------- /fabfile/prefix.py: -------------------------------------------------------------------------------- 1 | ''' 2 | $HOME/pg-latest (config.PG_LATEST) should always point to the 3 | installation of citus we're currently working with. This way tasks which interact with an 4 | installation can just use pg-latest and not care where it points to. (We use this instead 5 | of something like a "use" task because this is long-term state which should be kept 6 | between invocations of fab) 7 | ''' 8 | import os.path 9 | 10 | from invoke.exceptions import Exit 11 | from patchwork.files import exists 12 | 13 | import config 14 | 15 | def set_prefix(c, prefix): 16 | 'Change where pg-latest points to' 17 | 18 | if not os.path.isabs(prefix): 19 | raise Exit('{} is not an absolute path'.format(prefix)) 20 | 21 | latest = config.PG_LATEST 22 | 23 | # -f to overwrite any existing links 24 | # -n to not follow the {latest} link, if it exists, and instead replace it 25 | c.run('ln -snf {} {}'.format(prefix, latest)) 26 | c.run('mkdir -p {}'.format(prefix)) 27 | 28 | def ensure_pg_latest_exists(c, default): 29 | 'If there is no valid working directory make one and point it at prefix' 30 | latest = config.PG_LATEST 31 | 32 | # make sure pg-latest exists 33 | if not exists(c, latest): 34 | set_prefix(c, default) 35 | 36 | # make sure pg-latest is a link 37 | if c.run('stat -c %F {}'.format(latest)).stdout.strip() != 'symbolic link': 38 | raise Exit('pg-latest exists but is not a symbolic link!') 39 | 40 | # make sure the link points to something 41 | destination = c.run('readlink {}'.format(latest)).stdout.strip() 42 | c.run('mkdir -p {}'.format(destination)) 43 | 44 | def check_for_pg_latest(c): 45 | "Fail-fast if there isn't a valid working directory" 46 | latest = config.PG_LATEST 47 | 48 | if not exists(c, latest): 49 | raise Exit('There is no pg-latest symlink, run a setup.XXX task to create a cluster or the set-pg-latest task to point pg-latest to a citus installation') 50 | 51 | destination = c.run('readlink {}'.format(latest)).stdout.strip() 52 | 53 | if not exists(c, destination): 54 | raise Exit('pg-latest does not point to a valid working directory, run a setup.XXX task to create a cluster') 55 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/cmd_base_sf100000: -------------------------------------------------------------------------------- 1 | ../dbgen -s 100000 -T L -C 2000000000 -S 1 -f -b ../dists.dss 2 | ../dbgen -s 100000 -T L -C 2000000000 -S 666666666 -f -b ../dists.dss 3 | ../dbgen -s 100000 -T L -C 2000000000 -S 1333333332 -f -b ../dists.dss 4 | ../dbgen -s 100000 -T L -C 2000000000 -S 1999999998 -f -b ../dists.dss 5 | ../dbgen -s 100000 -T L -C 2000000000 -S 2000000000 -f -b ../dists.dss 6 | ../dbgen -s 100000 -T O -C 1500000000 -S 1 -f -b ../dists.dss 7 | ../dbgen -s 100000 -T O -C 1500000000 -S 499999999 -f -b ../dists.dss 8 | ../dbgen -s 100000 -T O -C 1500000000 -S 999999998 -f -b ../dists.dss 9 | ../dbgen -s 100000 -T O -C 1500000000 -S 1499999997 -f -b ../dists.dss 10 | ../dbgen -s 100000 -T O -C 1500000000 -S 1500000000 -f -b ../dists.dss 11 | ../dbgen -s 100000 -T P -C 200000000 -S 1 -f -b ../dists.dss 12 | ../dbgen -s 100000 -T P -C 200000000 -S 66666666 -f -b ../dists.dss 13 | ../dbgen -s 100000 -T P -C 200000000 -S 133333332 -f -b ../dists.dss 14 | ../dbgen -s 100000 -T P -C 200000000 -S 199999998 -f -b ../dists.dss 15 | ../dbgen -s 100000 -T P -C 200000000 -S 200000000 -f -b ../dists.dss 16 | ../dbgen -s 100000 -T S -C 800000000 -S 1 -f -b ../dists.dss 17 | ../dbgen -s 100000 -T S -C 800000000 -S 266666666 -f -b ../dists.dss 18 | ../dbgen -s 100000 -T S -C 800000000 -S 533333332 -f -b ../dists.dss 19 | ../dbgen -s 100000 -T S -C 800000000 -S 799999998 -f -b ../dists.dss 20 | ../dbgen -s 100000 -T S -C 800000000 -S 800000000 -f -b ../dists.dss 21 | ../dbgen -s 100000 -T s -C 10000000 -S 1 -f -b ../dists.dss 22 | ../dbgen -s 100000 -T s -C 10000000 -S 3333332 -f -b ../dists.dss 23 | ../dbgen -s 100000 -T s -C 10000000 -S 6666664 -f -b ../dists.dss 24 | ../dbgen -s 100000 -T s -C 10000000 -S 9999996 -f -b ../dists.dss 25 | ../dbgen -s 100000 -T s -C 10000000 -S 10000000 -f -b ../dists.dss 26 | ../dbgen -s 100000 -T c -C 150000000 -S 1 -f -b ../dists.dss 27 | ../dbgen -s 100000 -T c -C 150000000 -S 49999999 -f -b ../dists.dss 28 | ../dbgen -s 100000 -T c -C 150000000 -S 99999998 -f -b ../dists.dss 29 | ../dbgen -s 100000 -T c -C 150000000 -S 149999997 -f -b ../dists.dss 30 | ../dbgen -s 100000 -T c -C 150000000 -S 150000000 -f -b ../dists.dss 31 | ../dbgen -T n -f -b ../dists.dss 32 | ../dbgen -T r -f -b ../dists.dss 33 | -------------------------------------------------------------------------------- /routes.ps1: -------------------------------------------------------------------------------- 1 | # Self-elevate the script if required 2 | if (-Not ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] 'Administrator')) { 3 | if ([int](Get-CimInstance -Class Win32_OperatingSystem | Select-Object -ExpandProperty BuildNumber) -ge 6000) { 4 | $CommandLine = "-File `"" + $MyInvocation.MyCommand.Path + "`" " + $MyInvocation.UnboundArguments 5 | Start-Process -FilePath PowerShell.exe -Verb Runas -ArgumentList $CommandLine 6 | Exit 7 | } 8 | } 9 | 10 | $python_script = @" 11 | from urllib.request import Request, urlopen 12 | import re 13 | import json 14 | import ipaddress 15 | import subprocess 16 | 17 | 18 | req = Request( 19 | "https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519", 20 | headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0"}, 21 | ) 22 | 23 | with urlopen(req) as response: 24 | response_content = response.read().decode("utf-8") 25 | ips_url = re.search( 26 | '"(https://[^"]*ServiceTags_Public[^"]*)"', response_content 27 | ).group(1) 28 | 29 | req = Request( 30 | ips_url, 31 | headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0"}, 32 | ) 33 | with urlopen(req) as response: 34 | data = json.load(response) 35 | 36 | for ip_info in data["values"]: 37 | if ip_info["name"] == "AzureCloud": 38 | all_cidrs = ip_info["properties"]["addressPrefixes"] 39 | break 40 | else: 41 | raise Exception("Could not find ip info for AzureCloud") 42 | 43 | for cidr in all_cidrs: 44 | network = ipaddress.ip_network(cidr) 45 | if network.version == 4: 46 | subprocess.run( 47 | [ 48 | "route", 49 | "add", 50 | str(network.network_address), 51 | "mask", 52 | str(network.netmask), 53 | "100.64.108.75", 54 | "metric", 55 | "311", 56 | ], 57 | check=True, 58 | ) 59 | else: 60 | # TODO: do something similar for IPv6 61 | ... 62 | "@ 63 | echo $python_script | python 64 | 65 | Read-Host "Press Enter to continue..." 66 | -------------------------------------------------------------------------------- /valgrind/docker/run_valgrind_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [ "$#" -ne 2 ]; then 6 | echo "Usage: $0 " 7 | exit 1 8 | fi 9 | 10 | TEST_SCHEDULE=$1 11 | MAKE_CHECK_TARGET=$2 12 | 13 | export PATH=/pgenv/pgsql/bin/:$PATH 14 | 15 | # allow valgrind to generate a coredump if a test crashes 16 | ulimit -c unlimited 17 | 18 | # At this point, we don't want to fail rest of the script if valgrind exits with 19 | # an error, so we disable exit-on-error. 20 | set +e 21 | 22 | SCHEDULE=$TEST_SCHEDULE make -C /citus/src/test/regress/ $MAKE_CHECK_TARGET 23 | 24 | shopt -s extglob nullglob 25 | 26 | # Collect all Valgrind log files into a single file (valgrind_logs.txt). 27 | # Handles both formats: 28 | # - Older Citus versions: a single file named citus_valgrind_test_log.txt 29 | # - Newer Citus versions: one file per PID, e.g. citus_valgrind_test_log.txt. 30 | valgrind_log_files=(/citus/src/test/regress/citus_valgrind_test_log.txt?(.+([0-9]))) 31 | if (( ${#valgrind_log_files[@]} )); then 32 | output=/citus/src/test/regress/valgrind_logs.txt 33 | touch "$output" # truncate/create output file 34 | 35 | # For each log file, add a header with its name and then append its contents 36 | for valgrind_log_file in "${valgrind_log_files[@]}"; do 37 | echo "+++++++++++++++++++++++++ $(basename "$valgrind_log_file") +++++++++++++++++++++++++" >> "$output" 38 | cat "$valgrind_log_file" >> "$output" 39 | echo "" >> "$output" 40 | done 41 | fi 42 | 43 | # For each core file that valgrind generated in case of a process crash (if any), 44 | # we run gdb and save the backtrace to a file. 45 | core_files=(/citus/src/test/regress/citus_valgrind_test_log.txt.core.+([0-9])) 46 | if (( ${#core_files[@]} )); then 47 | pushd /citus/src/test/regress/ 48 | 49 | mkdir gdb_core_backtraces 50 | 51 | for core_file_name in "${core_files[@]}"; do 52 | base_name=$(basename "$core_file_name") 53 | gdb -ex bt -ex quit postgres "$core_file_name" &> "gdb_core_backtraces/$base_name" 54 | done 55 | 56 | echo "Found core files. Stacktraces are saved under /citus/src/test/regress/gdb_core_backtraces." 57 | echo "Stacktraces will be copied back to the host machine as artifacts but you might want to further investigate the core files." 58 | 59 | popd 60 | fi 61 | -------------------------------------------------------------------------------- /hammerdb/sql/tpcc-distribute-funcs.sql: -------------------------------------------------------------------------------- 1 | -- THESE SHOULD BE UNCOMMENTED IF STORED PRODECURES ARE FALSE IN CONFIGS 2 | 3 | -- SELECT create_distributed_function('public.dbms_random (integer, integer)'); 4 | -- SELECT create_distributed_function('public.delivery (integer, integer)', '$1', colocate_with := 'warehouse'); 5 | -- SELECT create_distributed_function('public.neword ( integer, integer, integer, integer, integer, integer)', '$1', colocate_with := 'warehouse'); 6 | -- SELECT create_distributed_function('public.payment ( integer, integer, integer, integer, numeric, integer, numeric, character varying, character varying, numeric)', '$1', colocate_with := 'warehouse'); 7 | -- SELECT create_distributed_function('public.slev ( integer, integer, integer)', '$1', colocate_with := 'warehouse'); 8 | -- SELECT create_distributed_function('public.ostat ( integer, integer, integer, integer, character varying)', '$1', colocate_with := 'warehouse'); 9 | 10 | 11 | -- THESE SHOULD BE COMMENTED IF STORED PRODECURES IS FALSE IN CONFIGS 12 | 13 | SELECT create_distributed_function('public.payment(integer, integer, integer, integer, integer, numeric, character, character varying, numeric, character varying, character varying, character varying, character, character, character varying, character varying, character varying, character, character, character varying, character, character varying, character varying, character varying, character, character, character, timestamp without time zone, numeric, numeric, numeric, character varying, timestamp without time zone)', '$1', colocate_with:='warehouse'); 14 | SELECT create_distributed_function('public.neword( integer, integer, integer, integer, integer, numeric, character varying, character varying, numeric, numeric, integer, timestamp without time zone)', '$1', colocate_with:='warehouse'); 15 | SELECT create_distributed_function('dbms_random(int,int)'); 16 | SELECT create_distributed_function('public.ostat( integer, integer, integer, integer, character varying, character varying, character varying, numeric, integer, timestamp without time zone, integer, text)', '$1', colocate_with:='warehouse'); 17 | SELECT create_distributed_function('public.slev(integer, integer, integer, integer)', '$1', colocate_with:='warehouse'); 18 | SELECT create_distributed_function('DELIVERY (INTEGER, INTEGER, Timestamp without time zone)', '$1', colocate_with:='warehouse'); 19 | -------------------------------------------------------------------------------- /fabfile/use.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This namespace doesn't perform any installation, it's only used to change configuration 3 | which later tasks read. The "citus" task, for instance, specifies which git ref to use 4 | when building Citus. 5 | ''' 6 | from invoke import task 7 | 8 | import setup 9 | import config 10 | import multi_connections 11 | import utils 12 | 13 | @task(positional=['version']) 14 | def citus(c, version): 15 | 'Choose a citus version. For example: fab use.citus v6.0.1 setup.basic-testing (defaults to master)' 16 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, citus, version): 17 | return 18 | 19 | git_ref = version 20 | 21 | # set community repo specific variables 22 | config.settings[config.REPO_PATH] = config.CITUS_REPO 23 | config.settings[config.BUILD_CITUS_FUNC] = setup.build_citus 24 | 25 | # check if we can clone citus successfully, then remove it 26 | path = "/tmp/tmp_citus" 27 | c.run('rm -rf {} || true'.format(path)) 28 | c.run('git clone -q https://github.com/citusdata/citus.git {}'.format(path)) 29 | c.run('cd {} && git checkout {}'.format(path, git_ref)) 30 | c.run('rm -rf {} || true'.format(path)) 31 | 32 | config.settings['citus-git-ref'] = git_ref 33 | 34 | @task(positional=['version']) 35 | def postgres(c, version): 36 | 'Choose a postgres version. For example: fab use.postgres 9.6.1 setup.basic-testing' 37 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, postgres, version): 38 | return 39 | 40 | config.PG_VERSION = version 41 | utils.download_pg(c) # Check that this doesn't 404 42 | 43 | @task 44 | def hammerdb(c): 45 | # we use git tokens for authentication in hammerdb 46 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, hammerdb): 47 | return 48 | 49 | config.settings[config.IS_SSH_KEYS_USED] = False 50 | 51 | @task 52 | def asserts(c): 53 | 'Enable asserts in pg (and therefore citus) by passing --enable-cassert' 54 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, asserts): 55 | return 56 | 57 | config.PG_CONFIGURE_FLAGS.append('--enable-cassert') 58 | 59 | @task 60 | def debug_mode(c): 61 | '''ps's configure is passed: '--enable-debug --enable-cassert CFLAGS="-ggdb -Og -g3 -fno-omit-frame-pointer"' ''' 62 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, debug_mode): 63 | return 64 | 65 | config.PG_CONFIGURE_FLAGS.append('--enable-debug --enable-cassert CFLAGS="-ggdb -Og -g3 -fno-omit-frame-pointer"') 66 | -------------------------------------------------------------------------------- /fabfile/pgbench_confs/scale_test_100_columns.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | postgres_citus_versions: [('12.1', 'release-9.2'), ('12.1', 'release-9.2')] 3 | shard_counts_replication_factors: [(32, 1)] 4 | postgresql_conf: [ 5 | "max_wal_size = '50GB'", 6 | "checkpoint_completion_target = 0.9", 7 | "checkpoint_timeout = '1h'", 8 | "max_connections = 1000", 9 | "max_prepared_transactions = 1000", 10 | "shared_buffers = '256MB'", 11 | "citus.multi_shard_commit_protocol = '2pc'" 12 | ] 13 | 14 | [create_table] 15 | sql_command: CREATE TABLE test_table (key int, value_1 int, value_2 int, value_3 int, value_4 text, value_5 text, value_6 text, value_7 text, value_8 text, 16 | value_9 text, value_10 text, value_11 text, value_12 text, value_13 text, value_14 text, value_15 text, value_16 text, 17 | value_17 text, value_18 text, value_19 text, value_20 text, value_21 text, value_22 text, value_23 text, value_24 text, 18 | value_25 text, value_26 text, value_27 text, value_28 text, value_29 text, value_30 text, value_31 text, value_32 text, 19 | value_33 text, value_34 text, value_35 text, value_36 text, value_37 text, value_38 text, value_39 text, value_40 text, 20 | value_41 text, value_42 text, value_43 text, value_44 text, value_45 text, value_46 text, value_47 text, value_48 text, 21 | value_49 text, value_50 text, value_51 text, value_52 text, value_53 text, value_54 text, value_55 text, value_56 text, 22 | value_57 text,value_58 text, value_59 text, value_60 text, value_61 text, value_62 text, value_63 text, value_64 text, 23 | value_65 text, value_66 text, value_67 text, value_68 text, value_69 text, value_70 text, value_71 text, value_72 text, 24 | value_73 text, value_74 text, value_75 text, value_76 text, value_77 text, value_78 text, value_79 text, value_80 text, 25 | value_81 text, value_82 text,value_83 text, value_84 text, value_85 text, value_86 text, value_87 text, value_88 text, 26 | value_89 text, value_90 text, value_91 text, value_92 text, value_93 text, value_94 jsonb, value_95 jsonb, value_96 jsonb, 27 | value_97 jsonb, value_98 jsonb, value_99 jsonb); 28 | 29 | distribute_table_command: SELECT create_distributed_table('test_table', 'key'); 30 | 31 | [single_insert] 32 | pgbench_command: pgbench -c128 -j16 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert.sql 33 | -------------------------------------------------------------------------------- /fabfile/pg.py: -------------------------------------------------------------------------------- 1 | from invoke import task 2 | 3 | import config 4 | import multi_connections 5 | import prefix 6 | from utils import psql 7 | 8 | @task 9 | def create(c): 10 | 'Create the database in pg-latest' 11 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, create): 12 | return 13 | 14 | prefix.check_for_pg_latest(c) 15 | 16 | with c.cd(config.PG_LATEST): 17 | c.run('bin/initdb -D data') 18 | 19 | @task 20 | def start(c): 21 | 'Start the database in pg-latest' 22 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, start): 23 | return 24 | 25 | prefix.check_for_pg_latest(c) 26 | 27 | with c.cd(config.PG_LATEST): 28 | # "set -m" spawns postgres in a new process group so it runs in the background 29 | c.run('set -m; bin/pg_ctl --timeout=1000 -D data -l logfile start') 30 | 31 | @task 32 | def stop(c): 33 | 'Stop the database in pg-latest' 34 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, stop): 35 | return 36 | 37 | prefix.check_for_pg_latest(c) 38 | 39 | with c.cd(config.PG_LATEST): 40 | c.run('set -m; bin/pg_ctl -D data stop') 41 | 42 | @task 43 | def restart(c): 44 | 'Restart the database in pg-latest' 45 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, restart): 46 | return 47 | 48 | prefix.check_for_pg_latest(c) 49 | 50 | with c.cd(config.PG_LATEST): 51 | c.run('set -m; bin/pg_ctl -D data -l logfile restart') 52 | 53 | # TODO: Maybe also check that the server started properly. And if it didn't tail the log file? 54 | 55 | @task(positional=['key']) 56 | def read_config(c, key): 57 | 'Returns the present value of the requested key e.x `fab pg.read-config max_connections`' 58 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, read_config, key): 59 | return 60 | 61 | prefix.check_for_pg_latest(c) 62 | 63 | psql(c, 'SHOW {}'.format(key)) 64 | 65 | @task(positional=['key', 'value']) 66 | def set_config(c, key, value): 67 | 'Changes the postgres configuration: e.g. `fab pg.set-config max_connections 200`' 68 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, set_config, key, value): 69 | return 70 | 71 | prefix.check_for_pg_latest(c) 72 | 73 | psql(c, 'ALTER SYSTEM SET {} TO {}'.format(key, value)) 74 | 75 | @task 76 | def set_config_str(c, config): 77 | if multi_connections.execute_on_all_nodes_if_no_hosts(c, set_config_str, config): 78 | return 79 | 80 | prefix.check_for_pg_latest(c) 81 | 82 | psql(c, 'ALTER SYSTEM SET {}'.format(config)) 83 | -------------------------------------------------------------------------------- /tpch_2_13_0/dss.ri: -------------------------------------------------------------------------------- 1 | -- Sccsid: @(#)dss.ri 2.1.8.1 2 | -- TPCD Benchmark Version 8.0 3 | 4 | CONNECT TO TPCD; 5 | 6 | --ALTER TABLE TPCD.REGION DROP PRIMARY KEY; 7 | --ALTER TABLE TPCD.NATION DROP PRIMARY KEY; 8 | --ALTER TABLE TPCD.PART DROP PRIMARY KEY; 9 | --ALTER TABLE TPCD.SUPPLIER DROP PRIMARY KEY; 10 | --ALTER TABLE TPCD.PARTSUPP DROP PRIMARY KEY; 11 | --ALTER TABLE TPCD.ORDERS DROP PRIMARY KEY; 12 | --ALTER TABLE TPCD.LINEITEM DROP PRIMARY KEY; 13 | --ALTER TABLE TPCD.CUSTOMER DROP PRIMARY KEY; 14 | 15 | 16 | -- For table REGION 17 | ALTER TABLE TPCD.REGION 18 | ADD PRIMARY KEY (R_REGIONKEY); 19 | 20 | -- For table NATION 21 | ALTER TABLE TPCD.NATION 22 | ADD PRIMARY KEY (N_NATIONKEY); 23 | 24 | ALTER TABLE TPCD.NATION 25 | ADD FOREIGN KEY NATION_FK1 (N_REGIONKEY) references TPCD.REGION; 26 | 27 | COMMIT WORK; 28 | 29 | -- For table PART 30 | ALTER TABLE TPCD.PART 31 | ADD PRIMARY KEY (P_PARTKEY); 32 | 33 | COMMIT WORK; 34 | 35 | -- For table SUPPLIER 36 | ALTER TABLE TPCD.SUPPLIER 37 | ADD PRIMARY KEY (S_SUPPKEY); 38 | 39 | ALTER TABLE TPCD.SUPPLIER 40 | ADD FOREIGN KEY SUPPLIER_FK1 (S_NATIONKEY) references TPCD.NATION; 41 | 42 | COMMIT WORK; 43 | 44 | -- For table PARTSUPP 45 | ALTER TABLE TPCD.PARTSUPP 46 | ADD PRIMARY KEY (PS_PARTKEY,PS_SUPPKEY); 47 | 48 | COMMIT WORK; 49 | 50 | -- For table CUSTOMER 51 | ALTER TABLE TPCD.CUSTOMER 52 | ADD PRIMARY KEY (C_CUSTKEY); 53 | 54 | ALTER TABLE TPCD.CUSTOMER 55 | ADD FOREIGN KEY CUSTOMER_FK1 (C_NATIONKEY) references TPCD.NATION; 56 | 57 | COMMIT WORK; 58 | 59 | -- For table LINEITEM 60 | ALTER TABLE TPCD.LINEITEM 61 | ADD PRIMARY KEY (L_ORDERKEY,L_LINENUMBER); 62 | 63 | COMMIT WORK; 64 | 65 | -- For table ORDERS 66 | ALTER TABLE TPCD.ORDERS 67 | ADD PRIMARY KEY (O_ORDERKEY); 68 | 69 | COMMIT WORK; 70 | 71 | -- For table PARTSUPP 72 | ALTER TABLE TPCD.PARTSUPP 73 | ADD FOREIGN KEY PARTSUPP_FK1 (PS_SUPPKEY) references TPCD.SUPPLIER; 74 | 75 | COMMIT WORK; 76 | 77 | ALTER TABLE TPCD.PARTSUPP 78 | ADD FOREIGN KEY PARTSUPP_FK2 (PS_PARTKEY) references TPCD.PART; 79 | 80 | COMMIT WORK; 81 | 82 | -- For table ORDERS 83 | ALTER TABLE TPCD.ORDERS 84 | ADD FOREIGN KEY ORDERS_FK1 (O_CUSTKEY) references TPCD.CUSTOMER; 85 | 86 | COMMIT WORK; 87 | 88 | -- For table LINEITEM 89 | ALTER TABLE TPCD.LINEITEM 90 | ADD FOREIGN KEY LINEITEM_FK1 (L_ORDERKEY) references TPCD.ORDERS; 91 | 92 | COMMIT WORK; 93 | 94 | ALTER TABLE TPCD.LINEITEM 95 | ADD FOREIGN KEY LINEITEM_FK2 (L_PARTKEY,L_SUPPKEY) references 96 | TPCD.PARTSUPP; 97 | 98 | COMMIT WORK; 99 | 100 | 101 | -------------------------------------------------------------------------------- /valgrind/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | # check n args 6 | if [ "$#" -ne 4 ]; then 7 | echo "Usage: $0 " 8 | exit 1 9 | fi 10 | 11 | PG_VERSION=$1 12 | CITUS_VERSION=$2 13 | TEST_SCHEDULE=$3 14 | ARTIFACTS_DIR=$4 15 | 16 | image_name="citus-vg-$PG_VERSION-$CITUS_VERSION" 17 | 18 | docker build -t "$image_name" --build-arg PG_VERSION="$PG_VERSION" --build-arg CITUS_VERSION="$CITUS_VERSION" docker/ 19 | 20 | container_name="$image_name-$TEST_SCHEDULE-$(date +"%Y-%m-%d-%H-%M-%S")" 21 | 22 | test_artifacts_dir="$ARTIFACTS_DIR/$container_name" 23 | mkdir -p "$test_artifacts_dir" 24 | 25 | # determine make target based on the test schedule 26 | if [[ "$TEST_SCHEDULE" == *"failure"* ]]; then 27 | # Actually, Citus repo supports failure schedules under valgrind via 28 | # "check-failure-custom-schedule-vg" but the Dockerfile we use for 29 | # valgrind doesn't yet have the necessary dependencies to run the 30 | # failure schedules. 31 | echo "failure schedules are not supported yet" 32 | exit 1 33 | elif [[ "$TEST_SCHEDULE" == *"columnar_isolation"* ]]; then 34 | make_check_target="check-columnar-isolation-custom-schedule-vg" 35 | elif [[ "$TEST_SCHEDULE" == *"isolation"* ]]; then 36 | make_check_target="check-isolation-custom-schedule-vg" 37 | elif [[ "$TEST_SCHEDULE" == *"columnar"* ]]; then 38 | make_check_target="check-columnar-custom-schedule-vg" 39 | else 40 | make_check_target="check-custom-schedule-vg" 41 | fi 42 | 43 | # At this point, we don't want to fail copying the artifacts if the container run 44 | # or a prior copy fails, so we disable exit-on-error. 45 | set +e 46 | 47 | start_time=$(date +%s) 48 | 49 | docker run --name "$container_name" "$image_name" /run_valgrind_test.sh "$TEST_SCHEDULE" "$make_check_target" 50 | 51 | end_time=$(date +%s) 52 | 53 | echo "Copying artifacts from container to host" 54 | 55 | docker cp "$container_name":/citus/src/test/regress/regression.diffs "$test_artifacts_dir/" 56 | docker cp "$container_name":/citus/src/test/regress/regression.out "$test_artifacts_dir/" 57 | docker cp "$container_name":/citus/src/test/regress/valgrind_logs.txt "$test_artifacts_dir/" 58 | docker cp "$container_name":/citus/src/test/regress/gdb_core_backtraces "$test_artifacts_dir/" 59 | 60 | # finally, also dump some information about the test run 61 | echo "PG_VERSION=$PG_VERSION" > "$test_artifacts_dir/meta" 62 | echo "CITUS_VERSION=$CITUS_VERSION" >> "$test_artifacts_dir/meta" 63 | echo "TEST_SCHEDULE=$TEST_SCHEDULE" >> "$test_artifacts_dir/meta" 64 | echo "MAKE_CHECK_TARGET=$make_check_target" >> "$test_artifacts_dir/meta" 65 | echo "IMAGE_NAME=$image_name" >> "$test_artifacts_dir/meta" 66 | echo "CONTAINER_NAME=$container_name" >> "$test_artifacts_dir/meta" 67 | echo "TIME_TAKEN_SEC=$((end_time - start_time))" >> "$test_artifacts_dir/meta" 68 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q21.out: -------------------------------------------------------------------------------- 1 | s_name | numwait 2 | Supplier#000002829 | 20 3 | Supplier#000005808 | 18 4 | Supplier#000000262 | 17 5 | Supplier#000000496 | 17 6 | Supplier#000002160 | 17 7 | Supplier#000002301 | 17 8 | Supplier#000002540 | 17 9 | Supplier#000003063 | 17 10 | Supplier#000005178 | 17 11 | Supplier#000008331 | 17 12 | Supplier#000002005 | 16 13 | Supplier#000002095 | 16 14 | Supplier#000005799 | 16 15 | Supplier#000005842 | 16 16 | Supplier#000006450 | 16 17 | Supplier#000006939 | 16 18 | Supplier#000009200 | 16 19 | Supplier#000009727 | 16 20 | Supplier#000000486 | 15 21 | Supplier#000000565 | 15 22 | Supplier#000001046 | 15 23 | Supplier#000001047 | 15 24 | Supplier#000001161 | 15 25 | Supplier#000001336 | 15 26 | Supplier#000001435 | 15 27 | Supplier#000003075 | 15 28 | Supplier#000003335 | 15 29 | Supplier#000005649 | 15 30 | Supplier#000006027 | 15 31 | Supplier#000006795 | 15 32 | Supplier#000006800 | 15 33 | Supplier#000006824 | 15 34 | Supplier#000007131 | 15 35 | Supplier#000007382 | 15 36 | Supplier#000008913 | 15 37 | Supplier#000009787 | 15 38 | Supplier#000000633 | 14 39 | Supplier#000001960 | 14 40 | Supplier#000002323 | 14 41 | Supplier#000002490 | 14 42 | Supplier#000002993 | 14 43 | Supplier#000003101 | 14 44 | Supplier#000004489 | 14 45 | Supplier#000005435 | 14 46 | Supplier#000005583 | 14 47 | Supplier#000005774 | 14 48 | Supplier#000007579 | 14 49 | Supplier#000008180 | 14 50 | Supplier#000008695 | 14 51 | Supplier#000009224 | 14 52 | Supplier#000000357 | 13 53 | Supplier#000000436 | 13 54 | Supplier#000000610 | 13 55 | Supplier#000000788 | 13 56 | Supplier#000000889 | 13 57 | Supplier#000001062 | 13 58 | Supplier#000001498 | 13 59 | Supplier#000002056 | 13 60 | Supplier#000002312 | 13 61 | Supplier#000002344 | 13 62 | Supplier#000002596 | 13 63 | Supplier#000002615 | 13 64 | Supplier#000002978 | 13 65 | Supplier#000003048 | 13 66 | Supplier#000003234 | 13 67 | Supplier#000003727 | 13 68 | Supplier#000003806 | 13 69 | Supplier#000004472 | 13 70 | Supplier#000005236 | 13 71 | Supplier#000005906 | 13 72 | Supplier#000006241 | 13 73 | Supplier#000006326 | 13 74 | Supplier#000006384 | 13 75 | Supplier#000006394 | 13 76 | Supplier#000006624 | 13 77 | Supplier#000006629 | 13 78 | Supplier#000006682 | 13 79 | Supplier#000006737 | 13 80 | Supplier#000006825 | 13 81 | Supplier#000007021 | 13 82 | Supplier#000007417 | 13 83 | Supplier#000007497 | 13 84 | Supplier#000007602 | 13 85 | Supplier#000008134 | 13 86 | Supplier#000008234 | 13 87 | Supplier#000009435 | 13 88 | Supplier#000009436 | 13 89 | Supplier#000009564 | 13 90 | Supplier#000009896 | 13 91 | Supplier#000000379 | 12 92 | Supplier#000000673 | 12 93 | Supplier#000000762 | 12 94 | Supplier#000000811 | 12 95 | Supplier#000000821 | 12 96 | Supplier#000001337 | 12 97 | Supplier#000001916 | 12 98 | Supplier#000001925 | 12 99 | Supplier#000002039 | 12 100 | Supplier#000002357 | 12 101 | Supplier#000002483 | 12 102 | -------------------------------------------------------------------------------- /hammerdb/create-run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u 5 | # exit immediately if a command fails 6 | set -e 7 | # echo commands 8 | set -x 9 | 10 | is_tpcc=${IS_TPCC:=true} # set to true if you want tpcc to be run, otherwise set to false 11 | is_ch=${IS_CH:=false} # set to true if you want ch benchmark to be run, otherwise set to false 12 | username=pguser # username of the database 13 | hammerdb_version=4.4 # (4.4+ recommended) find available versions in download-hammerdb.sh 14 | 15 | # ssh_execute is used to run a command multiple times on ssh, this is because we sometimes get timeouts 16 | # while trying to ssh, and it shouldn't make the script fail. If a command actually fails, it will always 17 | # fail no matter how many times we try. 18 | ssh_execute() { 19 | ip=$1 20 | shift; 21 | command=$* 22 | n=0 23 | until [ $n -ge 10 ] 24 | do 25 | ssh -o "StrictHostKeyChecking no" -A -p "$ssh_port" pguser@"${ip}" "source ~/.bash_profile;${command}" && break 26 | n=$((n+1)) 27 | done 28 | 29 | if [ $n == 10 ]; then 30 | exit 1 31 | fi 32 | } 33 | 34 | get_cluster_output() { 35 | output_name=$1 36 | value=$(az deployment group show -g "${cluster_rg}" -n azuredeploy --query properties.outputs."${output_name}".value) 37 | # remove the quotes 38 | value=$(echo "${value}" | cut -d "\"" -f 2) 39 | echo "$value" 40 | } 41 | 42 | hammerdb_dir="${0%/*}" 43 | cd "${hammerdb_dir}" 44 | export topdir=${hammerdb_dir}/.. 45 | 46 | cluster_rg="${RESOURCE_GROUP_NAME}" 47 | 48 | # https://stackoverflow.com/questions/1593051/how-to-programmatically-determine-the-current-checked-out-git-branch 49 | branch_name=$(git symbolic-ref -q HEAD) 50 | branch_name=${branch_name##refs/heads/} 51 | branch_name=${branch_name:-HEAD} 52 | 53 | cd "${topdir}"/hammerdb 54 | # create the cluster with driver node 55 | ./create.sh 56 | 57 | ssh_port=$(get_cluster_output customSshPort) 58 | 59 | cluster_ip=$(get_cluster_output publicIP) 60 | echo "${cluster_ip}" 61 | 62 | coordinator_private_ip=$(get_cluster_output coordinatorPrivateIP) 63 | echo "${coordinator_private_ip}" 64 | 65 | driver_ip=$(get_cluster_output driverPublicIP) 66 | echo "${driver_ip}" 67 | 68 | driver_private_ip=$(get_cluster_output driverPrivateIP) 69 | echo "${driver_private_ip}" 70 | 71 | # add the public key of coordinator to the driver node so that driver can connect to the coordinator 72 | # without getting permission error. 73 | ssh_execute "${driver_ip}" "/home/pguser/test-automation/hammerdb/send_pubkey.sh ${coordinator_private_ip}" 74 | 75 | set +e 76 | # run hammerdb test, this will be run in a detached session. 77 | ssh_execute "${driver_ip}" "screen -d -m -L /home/pguser/test-automation/hammerdb/run_all.sh ${coordinator_private_ip} ${driver_private_ip} ${branch_name} ${is_tpcc} ${is_ch} ${username} ${hammerdb_version} ${cluster_rg}" 78 | set -e 79 | 80 | echo "Successfully started the benchmark(There can still be runtime errors)!" 81 | -------------------------------------------------------------------------------- /fabfile/utils.py: -------------------------------------------------------------------------------- 1 | ''' 2 | A grab bag of functions used by other modules 3 | ''' 4 | 5 | import os.path 6 | import os 7 | import socket 8 | 9 | from patchwork.files import append, exists 10 | from invoke.exceptions import Exit 11 | 12 | import config 13 | 14 | # TODO:: this can be replaced with makedirs(..., exists_ok=True) when we upgrade to python3 15 | def mkdir_if_not_exists(path): 16 | if not os.path.exists(path): 17 | os.mkdir(path) 18 | 19 | 20 | def rmdir(c, path, force=False): 21 | 'Better than rm because it is silent when the file does not exist' 22 | flag = '-f' if force else '' 23 | if exists(c, path): 24 | c.run('rm {} -r {}'.format(flag, path)) 25 | 26 | 27 | def psql(c, command='', filepath='', connectionURI=''): 28 | if command == '' and filepath == '': 29 | raise Exit('psql needs at least one of the -c or -f options!') 30 | 31 | with c.cd(config.PG_LATEST): 32 | psql_command = 'bin/psql {} '.format(connectionURI) 33 | 34 | if command != '': 35 | psql_command += '-c "{}" '.format(command) 36 | if filepath != '': 37 | psql_command += '-f {} '.format(filepath) 38 | return c.run(psql_command) 39 | 40 | 41 | def add_github_to_known_hosts(c): 42 | 'Removes prompts from github checkouts asking whether you want to trust the remote' 43 | key = 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' 44 | append(c, os.path.join(config.HOME_DIR, '.ssh/known_hosts'), key) 45 | 46 | 47 | def download_pg(c): 48 | "Idempotent, does not download if file already exists. Returns the file's location" 49 | version = config.PG_VERSION 50 | url = pg_url_for_version(version) 51 | 52 | target_dir = config.PG_SOURCE_BALLS 53 | c.run('mkdir -p {}'.format(target_dir)) 54 | 55 | target_file = '{}/{}'.format(target_dir, os.path.basename(url)) 56 | 57 | if exists(c, target_file): 58 | return target_file 59 | 60 | c.run('wget -O {} --no-verbose {}'.format(target_file, url)) 61 | return target_file 62 | 63 | def pg_contrib_dir(): 64 | version = config.PG_VERSION 65 | src_dir = os.path.join(config.PG_SOURCE_BALLS, 'postgresql-{}'.format(version)) 66 | contrib_dir = os.path.join(src_dir, 'contrib') 67 | return contrib_dir 68 | 69 | def pg_url_for_version(version): 70 | return 'https://ftp.postgresql.org/pub/source/v{0}/postgresql-{0}.tar.bz2'.format(version) 71 | 72 | def get_local_ip(): 73 | return socket.gethostbyname(socket.gethostname()) 74 | 75 | def get_preload_libs_string(preloaded_libs): 76 | return "shared_preload_libraries = \'{}\'".format(",".join(preloaded_libs)) 77 | 78 | def get_core_count(): 79 | core_count = os.cpu_count() 80 | return core_count 81 | -------------------------------------------------------------------------------- /fabfile/pgbench_confs/scale_test_no_index.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | postgres_citus_versions: [('12.1', 'release-9.2'), ('12.1', 'release-9.2')] 3 | shard_counts_replication_factors: [(32, 1)] 4 | postgresql_conf: [ 5 | "max_wal_size = '50GB'", 6 | "checkpoint_completion_target = 0.9", 7 | "checkpoint_timeout = '1h'", 8 | "max_connections = 1000", 9 | "max_prepared_transactions = 1000", 10 | "shared_buffers = '256MB'", 11 | "citus.multi_shard_commit_protocol = '2pc'" 12 | ] 13 | 14 | [create_type] 15 | sql_command: CREATE TYPE complex AS (r double precision, i double precision); 16 | 17 | [distribute_type] 18 | sql_command: SELECT run_command_on_workers('CREATE TYPE complex AS (r double precision, i double precision)'); 19 | 20 | [create_table] 21 | sql_command: CREATE TABLE test_table (key int, occurred_at timestamp DEFAULT now(), value_1 jsonb, value_2 text[], value_3 int4range, value_4 complex NOT NULL); 22 | distribute_table_command: SELECT create_distributed_table('test_table', 'key'); 23 | 24 | [single_insert] 25 | pgbench_command: pgbench -c128 -j16 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert_complex.sql 26 | 27 | [muti_row_insert] 28 | pgbench_command: pgbench -c32 -j16 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/scale_multi_row_insert.sql 29 | 30 | [router_select] 31 | pgbench_command: pgbench -c128 -j16 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/router_select.sql 32 | 33 | [realtime_select] 34 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/realtime_select.sql 35 | 36 | [insert_select_pushdown] 37 | sql_command: CREATE TABLE test_table_target (key int, occurred_at timestamp DEFAULT now(), value_1 jsonb, value_2 text[], value_3 int4range, value_4 complex NOT NULL); 38 | distribute_table_command: SELECT create_distributed_table('test_table_target', 'key'); 39 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert_select_pushdown.sql 40 | 41 | [insert_select_coordinator] 42 | pgbench_command: pgbench -c16 -j8 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert_select_coordinator.sql 43 | 44 | [copy] 45 | sql_command: COPY (SELECT * FROM test_table LIMIT 100) TO '${HOME}/scale_test_data.csv'; 46 | pgbench_command: pgbench -c8 -j8 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/scale_copy.sql -D HOME=${HOME} 47 | 48 | [copy_and_multi_row] 49 | pgbench_command: pgbench -c16 -j8 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/scale_copy.sql@1 -f fabfile/pgbench_scripts/scale_multi_row_insert.sql@25 -D HOME=${HOME} 50 | 51 | [mix_them_all] 52 | pgbench_command: pgbench -c8 -j4 -T 600 -P 10 -n -r \ 53 | -f fabfile/pgbench_scripts/insert_complex.sql@5 \ 54 | -f fabfile/pgbench_scripts/scale_multi_row_insert.sql@5 \ 55 | -f fabfile/pgbench_scripts/router_select.sql@5 \ 56 | -f fabfile/pgbench_scripts/realtime_select.sql@1 \ 57 | -f fabfile/pgbench_scripts/insert_select_pushdown.sql@1 \ 58 | -f fabfile/pgbench_scripts/insert_select_coordinator.sql@1 \ 59 | -f fabfile/pgbench_scripts/scale_copy.sql@1 \ 60 | -D HOME=${HOME} 61 | -------------------------------------------------------------------------------- /azure/create-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # fail if trying to reference a variable that is not set. 4 | set -u / set -o nounset 5 | # exit immediately if a command fails 6 | set -e 7 | 8 | ## Set mydir to the directory containing the script 9 | ## The ${var%pattern} format will remove the shortest match of 10 | ## pattern from the end of the string. Here, it will remove the 11 | ## script's name,. leaving only the directory. 12 | azuredir="${0%/*}" 13 | cd ${azuredir} 14 | 15 | regions=(eastus southcentralus westus2) 16 | 17 | size=${#regions[@]} 18 | index=$(($RANDOM % $size)) 19 | random_region=${regions[$index]} 20 | 21 | rg=${RESOURCE_GROUP_NAME} 22 | region=${AZURE_REGION:=$random_region} 23 | echo ${region} 24 | az group create -l ${region} -n ${rg} 25 | 26 | # given we might have more then one ssh key loaded we simply take the 27 | # first one according to ssh-add as the public key to use for the vm 28 | # we create in azure. 29 | # jobs that run in multiple stages should have the same set of keys 30 | # added to their invocations. 31 | # make sure the key is in rsa format since other formats are not supported 32 | # from Azure 33 | public_key=$(ssh-add -L | grep ssh-rsa | head -n1) 34 | 35 | start_time=`date +%s` 36 | echo "waiting a long time to create cluster, this might take up to 30 mins depending on your cluster size" 37 | 38 | # if this is run on a job, use the branch for the job, otherwise use the local (running in local or remote without a job) 39 | # store the branch name in a file so that target user can read it. Target user cannot see the envionment variables because 40 | # we use login option in su and -p(preserving environment variables) cannot be used with login. We need to use login option 41 | # so that $HOME, $PATH are set to the target users $HOME and $PATH. 42 | 43 | # https://stackoverflow.com/questions/6245570/how-to-get-the-current-branch-name-in-git 44 | current_branch_name=$(git symbolic-ref --short HEAD 2>/dev/null) 45 | export BRANCH=${CIRCLE_BRANCH:=$current_branch_name} 46 | 47 | # get local public ip 48 | local_public_ip=$(curl ifconfig.me) 49 | 50 | # below is the default create cluster command 51 | CREATE_CLUSTER_COMMAND=(az deployment group create -g ${rg} --template-file azuredeploy.json --parameters @azuredeploy.parameters.json 52 | --parameters sshPublicKey="${public_key}" branchName="$BRANCH" localPublicIp="$local_public_ip") 53 | 54 | # if EXTENSION_TEST variable is not exported, set it to 0 55 | is_extension_test=${EXTENSION_TEST:=0} 56 | 57 | # override numberOfWorkers param if it is extension testing 58 | if [ "$is_extension_test" != "0" ]; then 59 | CREATE_CLUSTER_COMMAND+=(--parameters numberOfWorkers=0) 60 | fi 61 | 62 | # run CREATE_CLUSTER_COMMAND 63 | "${CREATE_CLUSTER_COMMAND[@]}" 64 | 65 | end_time=`date +%s` 66 | echo execution time was `expr $end_time - $start_time` s. 67 | 68 | 69 | connection_string=$(az deployment group show -g ${rg} -n azuredeploy --query properties.outputs.ssh.value) 70 | 71 | # remove the quotes 72 | connection_string=$(echo ${connection_string} | cut -d "\"" -f 2) 73 | 74 | echo "run './connect.sh' to connect to the coordinator, or ALTERNATIVELY RUN THE FOLLOWING:" 75 | 76 | echo ${connection_string} 77 | -------------------------------------------------------------------------------- /tpch_2_13_0/reference/README.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * $Id: README.txt,v 1.4 2006/08/07 17:36:46 jms Exp $ 3 | */ 4 | 5 | 1. Goal 6 | This document summarizes the steps required to validate the data set generated 7 | by DBGEN, relative to the reference data published by the TPC. 8 | 9 | 2. Pre-Requesites 10 | Toolset executable: This readme file is included as part of the toolset 11 | distributed by the TPC. The DBGEN and QGEN executables must be built from 12 | the same distribution. 13 | 14 | Reference Data Set(s): For each permissible scale factor, the TPC has 15 | produced a separate zip file that defines the reference point for the base 16 | data set, the update data set, and the output of QGEN. These must be 17 | downloaded separately from the TPC web site. 18 | 19 | 3. Validation Process 20 | 21 | a. Setup 22 | The commands that are included in this distribution assume that they are 23 | being executed from /reference, where is directory into which 24 | the DBGEN distribution was unpacked. If that is not the case for your 25 | environemnt, the paths to executables and associated files will need to be 26 | edited to match your directory structure. 27 | 28 | b. Base Data Validation 29 | The base data set is produced using cmd_base_sf where is the scale 30 | factor to be generated. The resulting files will be produced in the current 31 | working directory. The generated files will be of the form .tbl., 32 | where will is the name of one of the tables in the TPCH schema, and 33 | identifies a particular data generation step. 34 | 35 | The file set produced by genbaserefdata.sh should match the .tbl. 36 | files found in the reference data set for the same scale factor. 37 | 38 | c. Update Data Validation 39 | The update data set is produced using cmd_update_sf where is the scale 40 | factor to be generated. The resulting files will be produced in the current 41 | working directory. The generated files will be of the form 42 | .tbl.u. and delete.u., where will is the 43 | name of one of the tables in the TPCH schema, and and identify 44 | a particular data generation step. 45 | 46 | For each file produced by DBGEN, the first 100 lines of the file should 47 | match the delete.tbl.u. and .tbl.u. files found in the 48 | reference data set. Where the files from the reference data set contain 49 | fewer than 100 lines, the file should match in their entriety. 50 | 51 | d. QGEN Parameter Generation 52 | The qgen paramter sets are produced using the cmd_qgen_sf files, where 53 | matches the scale factor being validated. The resulting files will 54 | be produced in the current working directory. The generated files will be of 55 | the form subparam_, where identifies a particular qgen subsituttion 56 | set. 57 | 58 | The file set produced by cmd_qgen_ script should match the subparam. 59 | files found in the reference data set. 60 | 61 | /********************* 62 | * Revision History 63 | * =================== 64 | * $Log: README.txt,v $ 65 | * Revision 1.4 2006/08/07 17:36:46 jms 66 | * typo corrections 67 | * 68 | * Revision 1.3 2006/03/30 22:12:36 jms 69 | * reduce to simple scripts 70 | * 71 | -------------------------------------------------------------------------------- /tpch_2_13_0/permute.h: -------------------------------------------------------------------------------- 1 | /* 2 | * $Id: permute.h,v 1.2 2005/01/03 20:08:59 jms Exp $ 3 | * 4 | * Revision History 5 | * =================== 6 | * $Log: permute.h,v $ 7 | * Revision 1.2 2005/01/03 20:08:59 jms 8 | * change line terminations 9 | * 10 | * Revision 1.1.1.1 2004/11/24 23:31:47 jms 11 | * re-establish external server 12 | * 13 | * Revision 1.1.1.1 2003/04/03 18:54:21 jms 14 | * recreation after CVS crash 15 | * 16 | * Revision 1.1.1.1 2003/04/03 18:54:21 jms 17 | * initial checkin 18 | * 19 | * 20 | */ 21 | long permutation[41][22] = 22 | { 23 | {14, 2, 9,20, 6,17,18, 8,21,13, 3,22,16, 4,11,15, 1,10,19, 5, 7,12}, 24 | {21, 3,18, 5,11, 7, 6,20,17,12,16,15,13,10, 2, 8,14,19, 9,22, 1, 4}, 25 | { 6,17,14,16,19,10, 9, 2,15, 8, 5,22,12, 7,13,18, 1, 4,20, 3,11,21}, 26 | { 8, 5, 4, 6,17, 7, 1,18,22,14, 9,10,15,11,20, 2,21,19,13,16,12, 3}, 27 | { 5,21,14,19,15,17,12, 6, 4, 9, 8,16,11, 2,10,18, 1,13, 7,22, 3,20}, 28 | {21,15, 4, 6, 7,16,19,18,14,22,11,13, 3, 1, 2, 5, 8,20,12,17,10, 9}, 29 | {10, 3,15,13, 6, 8, 9, 7, 4,11,22,18,12, 1, 5,16, 2,14,19,20,17,21}, 30 | {18, 8,20,21, 2, 4,22,17, 1,11, 9,19, 3,13, 5, 7,10,16, 6,14,15,12}, 31 | {19, 1,15,17, 5, 8, 9,12,14, 7, 4, 3,20,16, 6,22,10,13, 2,21,18,11}, 32 | { 8,13, 2,20,17, 3, 6,21,18,11,19,10,15, 4,22, 1, 7,12, 9,14, 5,16}, 33 | { 6,15,18,17,12, 1, 7, 2,22,13,21,10,14, 9, 3,16,20,19,11, 4, 8, 5}, 34 | {15,14,18,17,10,20,16,11, 1, 8, 4,22, 5,12, 3, 9,21, 2,13, 6,19, 7}, 35 | { 1, 7,16,17,18,22,12, 6, 8, 9,11, 4, 2, 5,20,21,13,10,19, 3,14,15}, 36 | {21,17, 7, 3, 1,10,12,22, 9,16, 6,11, 2, 4, 5,14, 8,20,13,18,15,19}, 37 | { 2, 9, 5, 4,18, 1,20,15,16,17, 7,21,13,14,19, 8,22,11,10, 3,12, 6}, 38 | {16, 9,17, 8,14,11,10,12, 6,21, 7, 3,15, 5,22,20, 1,13,19, 2, 4,18}, 39 | { 1, 3, 6, 5, 2,16,14,22,17,20, 4, 9,10,11,15, 8,12,19,18,13, 7,21}, 40 | { 3,16, 5,11,21, 9, 2,15,10,18,17, 7, 8,19,14,13, 1, 4,22,20, 6,12}, 41 | {14, 4,13, 5,21,11, 8, 6, 3,17, 2,20, 1,19,10, 9,12,18,15, 7,22,16}, 42 | { 4,12,22,14, 5,15,16, 2, 8,10,17, 9,21, 7, 3, 6,13,18,11,20,19, 1}, 43 | {16,15,14,13, 4,22,18,19, 7, 1,12,17, 5,10,20, 3, 9,21,11, 2, 6, 8}, 44 | {20,14,21,12,15,17, 4,19,13,10,11, 1,16, 5,18, 7, 8,22, 9, 6, 3, 2}, 45 | {16,14,13, 2,21,10,11, 4, 1,22,18,12,19, 5, 7, 8, 6, 3,15,20, 9,17}, 46 | {18,15, 9,14,12, 2, 8,11,22,21,16, 1, 6,17, 5,10,19, 4,20,13, 3, 7}, 47 | { 7, 3,10,14,13,21,18, 6,20, 4, 9, 8,22,15, 2, 1, 5,12,19,17,11,16}, 48 | {18, 1,13, 7,16,10,14, 2,19, 5,21,11,22,15, 8,17,20, 3, 4,12, 6, 9}, 49 | {13, 2,22, 5,11,21,20,14, 7,10, 4, 9,19,18, 6, 3, 1, 8,15,12,17,16}, 50 | {14,17,21, 8, 2, 9, 6, 4, 5,13,22, 7,15, 3, 1,18,16,11,10,12,20,19}, 51 | {10,22, 1,12,13,18,21,20, 2,14,16, 7,15, 3, 4,17, 5,19, 6, 8, 9,11}, 52 | {10, 8, 9,18,12, 6, 1, 5,20,11,17,22,16, 3,13, 2,15,21,14,19, 7, 4}, 53 | { 7,17,22, 5, 3,10,13,18, 9, 1,14,15,21,19,16,12, 8, 6,11,20, 4, 2}, 54 | { 2, 9,21, 3, 4, 7, 1,11,16, 5,20,19,18, 8,17,13,10,12,15, 6,14,22}, 55 | {15,12, 8, 4,22,13,16,17,18, 3, 7, 5, 6, 1, 9,11,21,10,14,20,19, 2}, 56 | {15,16, 2,11,17, 7, 5,14,20, 4,21, 3,10, 9,12, 8,13, 6,18,19,22, 1}, 57 | { 1,13,11, 3, 4,21, 6,14,15,22,18, 9, 7, 5,10,20,12,16,17, 8,19, 2}, 58 | {14,17,22,20, 8,16, 5,10, 1,13, 2,21,12, 9, 4,18, 3, 7, 6,19,15,11}, 59 | { 9,17, 7, 4, 5,13,21,18,11, 3,22, 1, 6,16,20,14,15,10, 8, 2,12,19}, 60 | {13,14, 5,22,19,11, 9, 6,18,15, 8,10, 7, 4,17,16, 3, 1,12, 2,21,20}, 61 | {20, 5, 4,14,11, 1, 6,16, 8,22, 7, 3, 2,12,21,19,17,13,10,15,18, 9}, 62 | { 3, 7,14,15, 6, 5,21,20,18,10, 4,16,19, 1,13, 9, 8,17,11,12,22, 2}, 63 | {13,15,17, 1,22,11, 3, 4, 7,20,14,21, 9, 8, 2,18,16, 6,10,12, 5,19} 64 | }; 65 | -------------------------------------------------------------------------------- /fabfile/pgbench_confs/scale_test_reference.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | postgres_citus_versions: [('12.1', 'release-9.2'), ('12.1', 'release-9.2')] 3 | shard_counts_replication_factors: [(32, 1)] 4 | postgresql_conf: [ 5 | "max_wal_size = '50GB'", 6 | "checkpoint_completion_target = 0.9", 7 | "checkpoint_timeout = '1h'", 8 | "max_connections = 1000", 9 | "max_prepared_transactions = 1000", 10 | "shared_buffers = '256MB'", 11 | "citus.multi_shard_commit_protocol = '2pc'" 12 | ] 13 | 14 | [create_type] 15 | sql_command: CREATE TYPE complex AS (r double precision, i double precision); 16 | 17 | [distribute_type] 18 | sql_command: SELECT run_command_on_workers('CREATE TYPE complex AS (r double precision, i double precision)'); 19 | 20 | [create_table] 21 | sql_command: CREATE TABLE test_table (key int, occurred_at timestamp DEFAULT now(), value_1 jsonb, value_2 text[], value_3 int4range, value_4 complex NOT NULL); 22 | distribute_table_command: SELECT create_reference_table('test_table'); 23 | 24 | [create_indexes_key] 25 | sql_command: CREATE INDEX i1 ON test_table(key); 26 | 27 | [create_indexes_value_1] 28 | sql_command: CREATE INDEX i2 ON test_table USING HASH(value_1); CREATE INDEX i3 ON test_table USING GIN(value_1); 29 | 30 | [create_indexes_value_2] 31 | sql_command: CREATE INDEX i4 ON test_table(value_2); CREATE INDEX i5 ON test_table USING GIN(value_2); 32 | 33 | [create_indexes_value_3] 34 | sql_command: CREATE INDEX i6 ON test_table USING GIST(value_3); 35 | 36 | [single_insert] 37 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert_complex.sql 38 | 39 | [muti_row_insert] 40 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/scale_multi_row_insert.sql 41 | 42 | [router_select] 43 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/router_select.sql 44 | 45 | [realtime_select] 46 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/realtime_select.sql 47 | 48 | [insert_select_pushdown] 49 | sql_command: CREATE TABLE test_table_target (key int, occurred_at timestamp DEFAULT now(), value_1 jsonb, value_2 text[], value_3 int4range, value_4 complex NOT NULL); 50 | distribute_table_command: SELECT create_distributed_table('test_table_target', 'key'); 51 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert_select_pushdown.sql 52 | 53 | [insert_select_coordinator] 54 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert_select_coordinator.sql 55 | 56 | [copy] 57 | sql_command: COPY (SELECT * FROM test_table LIMIT 100) TO '${HOME}/scale_test_data.csv'; 58 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/scale_copy.sql -D HOME=${HOME} 59 | 60 | [copy_and_multi_row] 61 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/scale_copy.sql@1 -f fabfile/pgbench_scripts/scale_multi_row_insert.sql@25 -D HOME=${HOME} 62 | 63 | [mix_them_all] 64 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r \ 65 | -f fabfile/pgbench_scripts/insert_complex.sql@5 \ 66 | -f fabfile/pgbench_scripts/scale_multi_row_insert.sql@5 \ 67 | -f fabfile/pgbench_scripts/router_select.sql@5 \ 68 | -f fabfile/pgbench_scripts/realtime_select.sql@1 \ 69 | -f fabfile/pgbench_scripts/insert_select_pushdown.sql@1 \ 70 | -f fabfile/pgbench_scripts/insert_select_coordinator.sql@1 \ 71 | -f fabfile/pgbench_scripts/scale_copy.sql@1 \ 72 | -D HOME=${HOME} 73 | -------------------------------------------------------------------------------- /fabfile/pgbench_confs/scale_test.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | postgres_citus_versions: [('12.1', 'release-9.2'), ('12.1', 'release-9.2')] 3 | shard_counts_replication_factors: [(32, 1)] 4 | postgresql_conf: [ 5 | "max_wal_size = '50GB'", 6 | "checkpoint_completion_target = 0.9", 7 | "checkpoint_timeout = '1h'", 8 | "max_connections = 1000", 9 | "max_prepared_transactions = 1000", 10 | "shared_buffers = '256MB'", 11 | "citus.multi_shard_commit_protocol = '2pc'" 12 | ] 13 | 14 | [create_type] 15 | sql_command: CREATE TYPE complex AS (r double precision, i double precision); 16 | 17 | [distribute_type] 18 | sql_command: SELECT run_command_on_workers('CREATE TYPE complex AS (r double precision, i double precision)'); 19 | 20 | [create_table] 21 | sql_command: CREATE TABLE test_table (key int, occurred_at timestamp DEFAULT now(), value_1 jsonb, value_2 text[], value_3 int4range, value_4 complex NOT NULL); 22 | distribute_table_command: SELECT create_distributed_table('test_table', 'key'); 23 | 24 | [create_indexes_key] 25 | sql_command: CREATE INDEX i1 ON test_table(key); 26 | 27 | [create_indexes_value_1] 28 | sql_command: CREATE INDEX i2 ON test_table USING HASH(value_1); CREATE INDEX i3 ON test_table USING GIN(value_1); 29 | 30 | [create_indexes_value_2] 31 | sql_command: CREATE INDEX i4 ON test_table(value_2); CREATE INDEX i5 ON test_table USING GIN(value_2); 32 | 33 | [create_indexes_value_3] 34 | sql_command: CREATE INDEX i6 ON test_table USING GIST(value_3); 35 | 36 | [single_insert] 37 | pgbench_command: pgbench -c128 -j16 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert_complex.sql 38 | 39 | [muti_row_insert] 40 | pgbench_command: pgbench -c32 -j16 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/scale_multi_row_insert.sql 41 | 42 | [router_select] 43 | pgbench_command: pgbench -c128 -j16 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/router_select.sql 44 | 45 | [realtime_select] 46 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/realtime_select.sql 47 | 48 | [insert_select_pushdown] 49 | sql_command: CREATE TABLE test_table_target (key int, occurred_at timestamp DEFAULT now(), value_1 jsonb, value_2 text[], value_3 int4range, value_4 complex NOT NULL); 50 | distribute_table_command: SELECT create_distributed_table('test_table_target', 'key'); 51 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert_select_pushdown.sql 52 | 53 | [insert_select_coordinator] 54 | pgbench_command: pgbench -c16 -j8 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert_select_coordinator.sql 55 | 56 | [copy] 57 | sql_command: COPY (SELECT * FROM test_table LIMIT 100) TO '${HOME}/scale_test_data.csv'; 58 | pgbench_command: pgbench -c8 -j8 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/scale_copy.sql -D HOME=${HOME} 59 | 60 | [copy_and_multi_row] 61 | pgbench_command: pgbench -c16 -j8 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/scale_copy.sql@1 -f fabfile/pgbench_scripts/scale_multi_row_insert.sql@25 -D HOME=${HOME} 62 | 63 | [mix_them_all] 64 | pgbench_command: pgbench -c8 -j4 -T 600 -P 10 -n -r \ 65 | -f fabfile/pgbench_scripts/insert_complex.sql@5 \ 66 | -f fabfile/pgbench_scripts/scale_multi_row_insert.sql@5 \ 67 | -f fabfile/pgbench_scripts/router_select.sql@5 \ 68 | -f fabfile/pgbench_scripts/realtime_select.sql@1 \ 69 | -f fabfile/pgbench_scripts/insert_select_pushdown.sql@1 \ 70 | -f fabfile/pgbench_scripts/insert_select_coordinator.sql@1 \ 71 | -f fabfile/pgbench_scripts/scale_copy.sql@1 \ 72 | -D HOME=${HOME} 73 | -------------------------------------------------------------------------------- /tpch_2_13_0/answers/q10.out: -------------------------------------------------------------------------------- 1 | c_custkey | c_name | revenue | c_acctbal | n_name | c_address | c_phone | c_comment 2 | 57040 | Customer#000057040 | 734235.25 | 632.87 | JAPAN | Eioyzjf4pp | 22-895-641-3466 | sits. slyly regular requests sleep alongside of the regular inst 3 | 143347 | Customer#000143347 | 721002.69 | 2557.47 | EGYPT | 1aReFYv,Kw4 | 14-742-935-3718 | ggle carefully enticing requests. final deposits use bold, bold pinto beans. ironic, idle re 4 | 60838 | Customer#000060838 | 679127.31 | 2454.77 | BRAZIL | 64EaJ5vMAHWJlBOxJklpNc2RJiWE | 12-913-494-9813 | need to boost against the slyly regular account 5 | 101998 | Customer#000101998 | 637029.57 | 3790.89 | UNITED KINGDOM | 01c9CILnNtfOQYmZj | 33-593-865-6378 | ress foxes wake slyly after the bold excuses. ironic platelets are furiously carefully bold theodolites 6 | 125341 | Customer#000125341 | 633508.09 | 4983.51 | GERMANY | S29ODD6bceU8QSuuEJznkNaK | 17-582-695-5962 | arefully even depths. blithely even excuses sleep furiously. foxes use except the dependencies. ca 7 | 25501 | Customer#000025501 | 620269.78 | 7725.04 | ETHIOPIA | W556MXuoiaYCCZamJI,Rn0B4ACUGdkQ8DZ | 15-874-808-6793 | he pending instructions wake carefully at the pinto beans. regular, final instructions along the slyly fina 8 | 115831 | Customer#000115831 | 596423.87 | 5098.10 | FRANCE | rFeBbEEyk dl ne7zV5fDrmiq1oK09wV7pxqCgIc | 16-715-386-3788 | l somas sleep. furiously final deposits wake blithely regular pinto b 9 | 84223 | Customer#000084223 | 594998.02 | 528.65 | UNITED KINGDOM | nAVZCs6BaWap rrM27N 2qBnzc5WBauxbA | 33-442-824-8191 | slyly final deposits haggle regular, pending dependencies. pending escapades wake 10 | 54289 | Customer#000054289 | 585603.39 | 5583.02 | IRAN | vXCxoCsU0Bad5JQI ,oobkZ | 20-834-292-4707 | ely special foxes are quickly finally ironic p 11 | 39922 | Customer#000039922 | 584878.11 | 7321.11 | GERMANY | Zgy4s50l2GKN4pLDPBU8m342gIw6R | 17-147-757-8036 | y final requests. furiously final foxes cajole blithely special platelets. f 12 | 6226 | Customer#000006226 | 576783.76 | 2230.09 | UNITED KINGDOM | 8gPu8,NPGkfyQQ0hcIYUGPIBWc,ybP5g, | 33-657-701-3391 | ending platelets along the express deposits cajole carefully final 13 | 922 | Customer#000000922 | 576767.53 | 3869.25 | GERMANY | Az9RFaut7NkPnc5zSD2PwHgVwr4jRzq | 17-945-916-9648 | luffily fluffy deposits. packages c 14 | 147946 | Customer#000147946 | 576455.13 | 2030.13 | ALGERIA | iANyZHjqhyy7Ajah0pTrYyhJ | 10-886-956-3143 | ithely ironic deposits haggle blithely ironic requests. quickly regu 15 | 115640 | Customer#000115640 | 569341.19 | 6436.10 | ARGENTINA | Vtgfia9qI 7EpHgecU1X | 11-411-543-4901 | ost slyly along the patterns; pinto be 16 | 73606 | Customer#000073606 | 568656.86 | 1785.67 | JAPAN | xuR0Tro5yChDfOCrjkd2ol | 22-437-653-6966 | he furiously regular ideas. slowly 17 | 110246 | Customer#000110246 | 566842.98 | 7763.35 | VIETNAM | 7KzflgX MDOq7sOkI | 31-943-426-9837 | egular deposits serve blithely above the fl 18 | 142549 | Customer#000142549 | 563537.24 | 5085.99 | INDONESIA | ChqEoK43OysjdHbtKCp6dKqjNyvvi9 | 19-955-562-2398 | sleep pending courts. ironic deposits against the carefully unusual platelets cajole carefully express accounts. 19 | 146149 | Customer#000146149 | 557254.99 | 1791.55 | ROMANIA | s87fvzFQpU | 29-744-164-6487 | of the slyly silent accounts. quickly final accounts across the 20 | 52528 | Customer#000052528 | 556397.35 | 551.79 | ARGENTINA | NFztyTOR10UOJ | 11-208-192-3205 | deposits hinder. blithely pending asymptotes breach slyly regular re 21 | 23431 | Customer#000023431 | 554269.54 | 3381.86 | ROMANIA | HgiV0phqhaIa9aydNoIlb | 29-915-458-2654 | nusual, even instructions: furiously stealthy n 22 | -------------------------------------------------------------------------------- /tpch_2_13_0/tpch_create_tables.ddl: -------------------------------------------------------------------------------- 1 | -- Sccsid: @(#)dss.ddl 2.1.8.1 2 | 3 | CREATE TABLE nation 4 | ( 5 | n_nationkey INTEGER not null, 6 | n_name CHAR(25) not null, 7 | n_regionkey INTEGER not null, 8 | n_comment VARCHAR(152) 9 | ); 10 | 11 | select create_reference_table('nation'); 12 | 13 | 14 | CREATE TABLE region 15 | ( 16 | r_regionkey INTEGER not null, 17 | r_name CHAR(25) not null, 18 | r_comment VARCHAR(152) 19 | ); 20 | 21 | select create_reference_table('region'); 22 | 23 | CREATE TABLE part 24 | ( 25 | p_partkey INTEGER not null, 26 | p_name VARCHAR(55) not null, 27 | p_mfgr CHAR(25) not null, 28 | p_brand CHAR(10) not null, 29 | p_type VARCHAR(25) not null, 30 | p_size INTEGER not null, 31 | p_container CHAR(10) not null, 32 | p_retailprice DOUBLE PRECISION not null, 33 | p_comment VARCHAR(23) not null 34 | ); 35 | 36 | select create_distributed_table('part', 'p_partkey'); 37 | 38 | 39 | CREATE TABLE supplier 40 | ( 41 | s_suppkey INTEGER not null, 42 | s_name CHAR(25) not null, 43 | s_address VARCHAR(40) not null, 44 | s_nationkey INTEGER not null, 45 | s_phone CHAR(15) not null, 46 | s_acctbal DOUBLE PRECISION not null, 47 | s_comment VARCHAR(101) not null 48 | ); 49 | 50 | select create_reference_table('supplier'); 51 | 52 | 53 | CREATE TABLE partsupp 54 | ( 55 | ps_partkey INTEGER not null, 56 | ps_suppkey INTEGER not null, 57 | ps_availqty INTEGER not null, 58 | ps_supplycost DOUBLE PRECISION not null, 59 | ps_comment VARCHAR(199) not null 60 | ); 61 | 62 | select create_distributed_table('partsupp', 'ps_partkey'); 63 | 64 | 65 | CREATE TABLE customer 66 | ( 67 | c_custkey INTEGER not null, 68 | c_name VARCHAR(25) not null, 69 | c_address VARCHAR(40) not null, 70 | c_nationkey INTEGER not null, 71 | c_phone CHAR(15) not null, 72 | c_acctbal DOUBLE PRECISION not null, 73 | c_mktsegment CHAR(10) not null, 74 | c_comment VARCHAR(117) not null 75 | ); 76 | 77 | select create_distributed_table('customer', 'c_custkey'); 78 | 79 | 80 | CREATE TABLE orders 81 | ( 82 | o_orderkey BIGINT not null, 83 | o_custkey INTEGER not null, 84 | o_orderstatus CHAR(1) not null, 85 | o_totalprice DOUBLE PRECISION not null, 86 | o_orderdate DATE not null, 87 | o_orderpriority CHAR(15) not null, 88 | o_clerk CHAR(15) not null, 89 | o_shippriority INTEGER not null, 90 | o_comment VARCHAR(79) not null 91 | ); 92 | 93 | select create_distributed_table('orders', 'o_orderkey'); 94 | 95 | 96 | CREATE TABLE lineitem 97 | ( 98 | l_orderkey BIGINT not null, 99 | l_partkey INTEGER not null, 100 | l_suppkey INTEGER not null, 101 | l_linenumber INTEGER not null, 102 | l_quantity DOUBLE PRECISION not null, 103 | l_extendedprice DOUBLE PRECISION not null, 104 | l_discount DOUBLE PRECISION not null, 105 | l_tax DOUBLE PRECISION not null, 106 | l_returnflag CHAR(1) not null, 107 | l_linestatus CHAR(1) not null, 108 | l_shipdate DATE not null, 109 | l_commitdate DATE not null, 110 | l_receiptdate DATE not null, 111 | l_shipinstruct CHAR(25) not null, 112 | l_shipmode CHAR(10) not null, 113 | l_comment VARCHAR(44) not null 114 | ); 115 | 116 | select create_distributed_table('lineitem', 'l_orderkey'); 117 | -------------------------------------------------------------------------------- /tpch_2_13_0/tpch_create_hash_partitioned_tables.ddl: -------------------------------------------------------------------------------- 1 | -- Sccsid: @(#)dss.ddl 2.1.8.1 2 | 3 | CREATE TABLE nation 4 | ( 5 | n_nationkey INTEGER not null, 6 | n_name CHAR(25) not null, 7 | n_regionkey INTEGER not null, 8 | n_comment VARCHAR(152) 9 | ); 10 | 11 | select create_distributed_table('nation', 'n_nationkey'); 12 | 13 | 14 | CREATE TABLE region 15 | ( 16 | r_regionkey INTEGER not null, 17 | r_name CHAR(25) not null, 18 | r_comment VARCHAR(152) 19 | ); 20 | 21 | select create_distributed_table('region', 'r_regionkey'); 22 | 23 | CREATE TABLE part 24 | ( 25 | p_partkey INTEGER not null, 26 | p_name VARCHAR(55) not null, 27 | p_mfgr CHAR(25) not null, 28 | p_brand CHAR(10) not null, 29 | p_type VARCHAR(25) not null, 30 | p_size INTEGER not null, 31 | p_container CHAR(10) not null, 32 | p_retailprice DOUBLE PRECISION not null, 33 | p_comment VARCHAR(23) not null 34 | ); 35 | 36 | select create_distributed_table('part', 'p_partkey'); 37 | 38 | 39 | CREATE TABLE supplier 40 | ( 41 | s_suppkey INTEGER not null, 42 | s_name CHAR(25) not null, 43 | s_address VARCHAR(40) not null, 44 | s_nationkey INTEGER not null, 45 | s_phone CHAR(15) not null, 46 | s_acctbal DOUBLE PRECISION not null, 47 | s_comment VARCHAR(101) not null 48 | ); 49 | 50 | select create_distributed_table('supplier', 's_suppkey'); 51 | 52 | 53 | CREATE TABLE partsupp 54 | ( 55 | ps_partkey INTEGER not null, 56 | ps_suppkey INTEGER not null, 57 | ps_availqty INTEGER not null, 58 | ps_supplycost DOUBLE PRECISION not null, 59 | ps_comment VARCHAR(199) not null 60 | ); 61 | 62 | select create_distributed_table('partsupp', 'ps_partkey'); 63 | 64 | 65 | CREATE TABLE customer 66 | ( 67 | c_custkey INTEGER not null, 68 | c_name VARCHAR(25) not null, 69 | c_address VARCHAR(40) not null, 70 | c_nationkey INTEGER not null, 71 | c_phone CHAR(15) not null, 72 | c_acctbal DOUBLE PRECISION not null, 73 | c_mktsegment CHAR(10) not null, 74 | c_comment VARCHAR(117) not null 75 | ); 76 | 77 | select create_distributed_table('customer', 'c_custkey'); 78 | 79 | 80 | CREATE TABLE ORDERS 81 | ( 82 | o_orderkey BIGINT not null, 83 | o_custkey INTEGER not null, 84 | o_orderstatus CHAR(1) not null, 85 | o_totalprice DOUBLE PRECISION not null, 86 | o_orderdate DATE not null, 87 | o_orderpriority CHAR(15) not null, 88 | o_clerk CHAR(15) not null, 89 | o_shippriority INTEGER not null, 90 | o_comment VARCHAR(79) not null 91 | ); 92 | 93 | select create_distributed_table('ORDERS', 'o_orderkey'); 94 | 95 | 96 | CREATE TABLE LINEITEM 97 | ( 98 | l_orderkey BIGINT not null, 99 | l_partkey INTEGER not null, 100 | l_suppkey INTEGER not null, 101 | l_linenumber INTEGER not null, 102 | l_quantity DOUBLE PRECISION not null, 103 | l_extendedprice DOUBLE PRECISION not null, 104 | l_discount DOUBLE PRECISION not null, 105 | l_tax DOUBLE PRECISION not null, 106 | l_returnflag CHAR(1) not null, 107 | l_linestatus CHAR(1) not null, 108 | l_shipdate DATE not null, 109 | l_commitdate DATE not null, 110 | l_receiptdate DATE not null, 111 | l_shipinstruct CHAR(25) not null, 112 | l_shipmode CHAR(10) not null, 113 | l_comment VARCHAR(44) not null 114 | ); 115 | 116 | select create_distributed_table('LINEITEM', 'l_orderkey'); 117 | -------------------------------------------------------------------------------- /fabfile/pgbench_confs/scale_test_prepared.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | postgres_citus_versions: [('12.1', 'release-9.2'), ('12.1', 'release-9.2')] 3 | shard_counts_replication_factors: [(32, 1)] 4 | postgresql_conf: [ 5 | "max_wal_size = '50GB'", 6 | "checkpoint_completion_target = 0.9", 7 | "checkpoint_timeout = '1h'", 8 | "max_connections = 1000", 9 | "max_prepared_transactions = 1000", 10 | "shared_buffers = '256MB'", 11 | "citus.multi_shard_commit_protocol = '2pc'" 12 | ] 13 | 14 | [create_type] 15 | sql_command: CREATE TYPE complex AS (r double precision, i double precision); 16 | 17 | [distribute_type] 18 | sql_command: SELECT run_command_on_workers('CREATE TYPE complex AS (r double precision, i double precision)'); 19 | 20 | [create_table] 21 | sql_command: CREATE TABLE test_table (key int, occurred_at timestamp DEFAULT now(), value_1 jsonb, value_2 text[], value_3 int4range, value_4 complex); 22 | distribute_table_command: SELECT create_distributed_table('test_table', 'key'); 23 | 24 | [create_indexes_key] 25 | sql_command: CREATE INDEX i1 ON test_table(key); 26 | 27 | [create_indexes_value_1] 28 | sql_command: CREATE INDEX i2 ON test_table USING HASH(value_1); CREATE INDEX i3 ON test_table USING GIN(value_1); 29 | 30 | [create_indexes_value_2] 31 | sql_command: CREATE INDEX i4 ON test_table(value_2); CREATE INDEX i5 ON test_table USING GIN(value_2); 32 | 33 | [create_indexes_value_3] 34 | sql_command: CREATE INDEX i6 ON test_table USING GIST(value_3); 35 | 36 | [single_insert] 37 | pgbench_command: pgbench -c128 -j16 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert_complex_prepared.sql -M prepared 38 | 39 | [muti_row_insert] 40 | pgbench_command: pgbench -c32 -j16 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/multi_row_insert_prepared.sql -M prepared 41 | 42 | [router_select] 43 | pgbench_command: pgbench -c128 -j16 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/router_select.sql -M prepared 44 | 45 | [realtime_select] 46 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/realtime_select.sql -M prepared 47 | 48 | [insert_select_pushdown] 49 | sql_command: CREATE TABLE test_table_target (key int, occurred_at timestamp DEFAULT now(), value_1 jsonb, value_2 text[], value_3 int4range, value_4 complex); 50 | distribute_table_command: SELECT create_distributed_table('test_table_target', 'key'); 51 | pgbench_command: pgbench -c4 -j4 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert_select_pushdown.sql -M prepared 52 | 53 | [insert_select_coordinator] 54 | pgbench_command: pgbench -c16 -j8 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/insert_select_coordinator.sql -M prepared 55 | 56 | [copy] 57 | sql_command: COPY (SELECT * FROM test_table LIMIT 100) TO '${HOME}/scale_test_data.csv'; 58 | pgbench_command: pgbench -c8 -j8 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/scale_copy.sql -M prepared -D HOME=${HOME} 59 | 60 | [copy_and_multi_row] 61 | pgbench_command: pgbench -c16 -j8 -T 600 -P 10 -n -r -f fabfile/pgbench_scripts/scale_copy.sql@1 -f fabfile/pgbench_scripts/multi_row_insert_prepared.sql@25 -M prepared -D HOME=${HOME} 62 | 63 | [mix_them_all] 64 | pgbench_command: pgbench -c8 -j4 -T 600 -P 10 -n -r -M prepared \ 65 | -f fabfile/pgbench_scripts/insert_complex_prepared.sql@5 \ 66 | -f fabfile/pgbench_scripts/multi_row_insert_prepared.sql@5 \ 67 | -f fabfile/pgbench_scripts/router_select.sql@5 \ 68 | -f fabfile/pgbench_scripts/realtime_select.sql@1 \ 69 | -f fabfile/pgbench_scripts/insert_select_pushdown.sql@1 \ 70 | -f fabfile/pgbench_scripts/insert_select_coordinator.sql@1 \ 71 | -f fabfile/pgbench_scripts/scale_copy.sql@1 \ 72 | -D HOME=${HOME} 73 | --------------------------------------------------------------------------------