├── executors ├── grpc │ ├── protoc.sh │ ├── nautilus_rpc.proto │ └── nautilus_rpc_pb2_grpc.py └── executor.py ├── spaces ├── postgres-none.py ├── ycsbB │ └── postgres-1K-backend.py ├── ycsbA │ └── postgres-all-8K-posthoc.py ├── tpcc │ └── postgres-all-8K-ycsbA.py ├── common.py └── definitions │ └── postgres-9.6.json ├── requirements.txt ├── configs ├── ycsb │ ├── ycsbA │ │ ├── ycsbA.all.ini │ │ ├── ycsbA.all.mkbo.ini │ │ ├── ycsbA.all.top8.ph.ini │ │ ├── low-dim-sensitivity │ │ │ ├── ycsbA.all.hesbo_8.ini │ │ │ ├── ycsbA.all.rembo_8.ini │ │ │ ├── ycsbA.all.hesbo_16.ini │ │ │ ├── ycsbA.all.hesbo_24.ini │ │ │ ├── ycsbA.all.rembo_16.ini │ │ │ └── ycsbA.all.rembo_24.ini │ │ ├── ablation-study │ │ │ ├── ycsbA.all.hesbo_16.ini │ │ │ └── ycsbA.all.hesbo_16_bs.ini │ │ ├── biased-sampling-sensitivity │ │ │ ├── ycsbA.all.bs.10.ini │ │ │ ├── ycsbA.all.bs.30.ini │ │ │ ├── ycsbA.all.bs.5.ini │ │ │ └── ycsbA.all.bs.20.ini │ │ ├── bucketization-sensitivity │ │ │ ├── ycsbA.all.q10K.ini │ │ │ ├── ycsbA.all.q1K.ini │ │ │ ├── ycsbA.all.q20K.ini │ │ │ └── ycsbA.all.q5K.ini │ │ ├── ycsbA.all.mkbo.llama.ini │ │ └── ycsbA.all.llama.ini │ └── ycsbB │ │ ├── ycsbB.all.ini │ │ ├── ycsbB.1K.backend_bs.ini │ │ ├── ycsbB.all.mkbo.ini │ │ ├── low-dim-sensitivity │ │ ├── ycsbB.all.hesbo_8.ini │ │ ├── ycsbB.all.rembo_8.ini │ │ ├── ycsbB.all.hesbo_16.ini │ │ ├── ycsbB.all.hesbo_24.ini │ │ ├── ycsbB.all.rembo_16.ini │ │ └── ycsbB.all.rembo_24.ini │ │ ├── ablation-study │ │ ├── ycsbB.all.hesbo_16.ini │ │ └── ycsbB.all.hesbo_16_bs.ini │ │ ├── biased-sampling-sensitivity │ │ ├── ycsbB.all.bs.10.ini │ │ ├── ycsbB.all.bs.30.ini │ │ ├── ycsbB.all.bs.5.ini │ │ └── ycsbB.all.bs.20.ini │ │ ├── bucketization-sensitivity │ │ ├── ycsbB.all.q5K.ini │ │ ├── ycsbB.all.q10K.ini │ │ ├── ycsbB.all.q1K.ini │ │ └── ycsbB.all.q20K.ini │ │ ├── ycsbB.all.mkbo.llama.ini │ │ └── ycsbB.all.llama.ini └── benchbase │ ├── seats │ ├── seats.all.ini │ ├── seats.all.lat.ini │ ├── seats.all.mkbo.ini │ ├── seats.all.llama.ini │ ├── seats.all.lat.llama.ini │ └── seats.all.mkbo.llama.ini │ ├── tpcc │ ├── tpcc.all.ini │ ├── tpcc.all.lat.ini │ ├── tpcc.all.top8-ycsbA.ini │ ├── tpcc.all.mkbo.ini │ ├── ablation-study │ │ ├── tpcc.all.hesbo_16.ini │ │ └── tpcc.all.hesbo_16_bs.ini │ ├── tpcc.all.llama.ini │ ├── tpcc.all.lat.llama.ini │ └── tpcc.all.mkbo.llama.ini │ ├── twitter │ ├── twitter.all.ini │ ├── twitter.all.lat.ini │ ├── twitter.all.mkbo.ini │ ├── twitter.all.llama.ini │ ├── twitter.all.lat.llama.ini │ └── twitter.all.mkbo.llama.ini │ └── resourcestresser │ ├── resourcestresser.all.ini │ ├── resourcestresser.all.lat.ini │ ├── resourcestresser.all.mkbo.ini │ ├── resourcestresser.all.llama.ini │ ├── resourcestresser.all.lat.llama.ini │ └── resourcestresser.all.mkbo.llama.ini ├── adapters ├── __init__.py ├── configspace │ ├── quantization.py │ └── low_embeddings.py └── bias_sampling.py ├── ddpg ├── prm.py └── ddpg.py ├── config.py ├── storage.py ├── run-smac.py ├── run-ddpg.py ├── space.py └── optimizer.py /executors/grpc/protoc.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | protoc --python_out=./ nautilus_rpc.proto 3 | -------------------------------------------------------------------------------- /spaces/postgres-none.py: -------------------------------------------------------------------------------- 1 | # NOTE: Default values are gathered from PostgreSQL-9.6 on CloudLab c220g5 2 | 3 | from spaces.common import finalize_conf, unfinalize_conf 4 | 5 | KNOBS = [ ] 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio 2 | grpcio_tools 3 | pandas 4 | prettytable>=2 5 | scikit-learn 6 | 7 | # required 8 | ConfigSpace==0.4.20 9 | smac==1.1.0 10 | numpy 11 | 12 | # cpu-only torch 13 | torch==1.10.1+cpu -------------------------------------------------------------------------------- /spaces/ycsbB/postgres-1K-backend.py: -------------------------------------------------------------------------------- 1 | # NOTE: Default values are gathered from PostgreSQL-9.6 on CloudLab c220g5 2 | 3 | from spaces.common import finalize_conf, unfinalize_conf 4 | 5 | KNOBS = [ 6 | "backend_flush_after", 7 | ] 8 | -------------------------------------------------------------------------------- /spaces/ycsbA/postgres-all-8K-posthoc.py: -------------------------------------------------------------------------------- 1 | from spaces.common import finalize_conf, unfinalize_conf 2 | 3 | KNOBS = [ 4 | "autovacuum_analyze_scale_factor", 5 | "autovacuum_vacuum_scale_factor", 6 | "commit_delay", 7 | "full_page_writes", 8 | "geqo_selection_bias", 9 | "max_wal_size", 10 | "shared_buffers", 11 | "wal_writer_flush_after", 12 | ] 13 | -------------------------------------------------------------------------------- /spaces/tpcc/postgres-all-8K-ycsbA.py: -------------------------------------------------------------------------------- 1 | # NOTE: Default values are gathered from PostgreSQL-9.6 on CloudLab c220g5 2 | 3 | from spaces.common import finalize_conf, unfinalize_conf 4 | 5 | KNOBS = [ 6 | "autovacuum_vacuum_threshold", 7 | "autovacuum_vacuum_scale_factor", 8 | "commit_delay", 9 | "enable_seqscan", 10 | "full_page_writes", 11 | "geqo_selection_bias", 12 | "shared_buffers", 13 | "wal_writer_flush_after", 14 | ] 15 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/ycsbA.all.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | target_metric=throughput 28 | 29 | [storage] 30 | classname=FileTablesStorage 31 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/ycsbB.all.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | target_metric=throughput 28 | 29 | [storage] 30 | classname=FileTablesStorage 31 | outdir=results -------------------------------------------------------------------------------- /configs/benchbase/seats/seats.all.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=seats 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | target_metric=throughput 28 | 29 | [storage] 30 | classname=FileTablesStorage 31 | outdir=results 32 | -------------------------------------------------------------------------------- /configs/benchbase/tpcc/tpcc.all.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=tpcc 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | target_metric=throughput 28 | 29 | [storage] 30 | classname=FileTablesStorage 31 | outdir=results 32 | -------------------------------------------------------------------------------- /configs/benchbase/seats/seats.all.lat.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=seats-lat 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | target_metric=latency 28 | 29 | [storage] 30 | classname=FileTablesStorage 31 | outdir=results 32 | -------------------------------------------------------------------------------- /configs/benchbase/tpcc/tpcc.all.lat.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=tpcc-lat 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | target_metric=latency 28 | 29 | [storage] 30 | classname=FileTablesStorage 31 | outdir=results 32 | -------------------------------------------------------------------------------- /configs/benchbase/twitter/twitter.all.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=twitter 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | target_metric=throughput 28 | 29 | [storage] 30 | classname=FileTablesStorage 31 | outdir=results 32 | -------------------------------------------------------------------------------- /configs/benchbase/twitter/twitter.all.lat.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=twitter-lat 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | target_metric=latency 28 | 29 | [storage] 30 | classname=FileTablesStorage 31 | outdir=results 32 | -------------------------------------------------------------------------------- /configs/benchbase/tpcc/tpcc.all.top8-ycsbA.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=tpcc 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | include=tpcc.postgres-all-8K-ycsbA 26 | adapter_alias=none 27 | target_metric=throughput 28 | 29 | [storage] 30 | classname=FileTablesStorage 31 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/ycsbA.all.mkbo.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | model_type=mkbo 18 | rand_percentage=0.1 19 | n_estimators=100 20 | 21 | [policy] 22 | classname=NoPrunning 23 | 24 | [spaces] 25 | definition=postgres-9.6 26 | ignore=postgres-none 27 | adapter_alias=none 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/ycsbA.all.top8.ph.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | include=ycsbA.postgres-all-8K-posthoc 26 | adapter_alias=none 27 | target_metric=throughput 28 | 29 | [storage] 30 | classname=FileTablesStorage 31 | outdir=results 32 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/ycsbB.1K.backend_bs.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | include=ycsbB.postgres-1K-backend 26 | adapter_alias=none 27 | target_metric=throughput 28 | 29 | [storage] 30 | classname=FileTablesStorage 31 | outdir=results 32 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/ycsbB.all.mkbo.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | model_type=mkbo 18 | rand_percentage=0.1 19 | n_estimators=100 20 | 21 | [policy] 22 | classname=NoPrunning 23 | 24 | [spaces] 25 | definition=postgres-9.6 26 | ignore=postgres-none 27 | adapter_alias=none 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/benchbase/seats/seats.all.mkbo.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=seats 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | model_type=mkbo 18 | rand_percentage=0.1 19 | n_estimators=100 20 | 21 | [policy] 22 | classname=NoPrunning 23 | 24 | [spaces] 25 | definition=postgres-9.6 26 | ignore=postgres-none 27 | adapter_alias=none 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/benchbase/tpcc/tpcc.all.mkbo.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=tpcc 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | model_type=mkbo 18 | rand_percentage=0.1 19 | n_estimators=100 20 | 21 | [policy] 22 | classname=NoPrunning 23 | 24 | [spaces] 25 | definition=postgres-9.6 26 | ignore=postgres-none 27 | adapter_alias=none 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/benchbase/resourcestresser/resourcestresser.all.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=resourcestresser 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | target_metric=throughput 28 | 29 | [storage] 30 | classname=FileTablesStorage 31 | outdir=results 32 | -------------------------------------------------------------------------------- /configs/benchbase/twitter/twitter.all.mkbo.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=twitter 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | model_type=mkbo 18 | rand_percentage=0.1 19 | n_estimators=100 20 | 21 | [policy] 22 | classname=NoPrunning 23 | 24 | [spaces] 25 | definition=postgres-9.6 26 | ignore=postgres-none 27 | adapter_alias=none 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/low-dim-sensitivity/ycsbA.all.hesbo_8.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=8 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/low-dim-sensitivity/ycsbA.all.rembo_8.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=rembo 27 | le_low_dim=8 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/low-dim-sensitivity/ycsbB.all.hesbo_8.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=8 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/low-dim-sensitivity/ycsbB.all.rembo_8.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=rembo 27 | le_low_dim=8 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /adapters/__init__.py: -------------------------------------------------------------------------------- 1 | # smac 2 | from adapters.configspace.low_embeddings import LinearEmbeddingConfigSpace 3 | from adapters.bias_sampling import \ 4 | PostgresBiasSampling, LHDesignWithBiasedSampling, special_value_scaler, \ 5 | UniformIntegerHyperparameterWithSpecialValue 6 | from adapters.configspace.quantization import Quantization 7 | 8 | __all__ = [ 9 | # smac 10 | 'LinearEmbeddingConfigSpace', 11 | 'PostgresBiasSampling', 12 | 'Quantization', 13 | 'LHDesignWithBiasedSampling', 14 | 'special_value_scaler', 15 | 'UniformIntegerHyperparameterWithSpecialValue', 16 | ] -------------------------------------------------------------------------------- /configs/benchbase/resourcestresser/resourcestresser.all.lat.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=resourcestresser-lat 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | target_metric=latency 28 | 29 | [storage] 30 | classname=FileTablesStorage 31 | outdir=results 32 | -------------------------------------------------------------------------------- /configs/benchbase/tpcc/ablation-study/tpcc.all.hesbo_16.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=tpcc 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/ablation-study/ycsbA.all.hesbo_16.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/low-dim-sensitivity/ycsbA.all.hesbo_16.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/low-dim-sensitivity/ycsbA.all.hesbo_24.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=24 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/low-dim-sensitivity/ycsbA.all.rembo_16.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=rembo 27 | le_low_dim=16 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/low-dim-sensitivity/ycsbA.all.rembo_24.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=rembo 27 | le_low_dim=24 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/ablation-study/ycsbB.all.hesbo_16.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/low-dim-sensitivity/ycsbB.all.hesbo_16.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/low-dim-sensitivity/ycsbB.all.hesbo_24.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=24 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/low-dim-sensitivity/ycsbB.all.rembo_16.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=rembo 27 | le_low_dim=16 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/low-dim-sensitivity/ycsbB.all.rembo_24.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=rembo 27 | le_low_dim=24 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/biased-sampling-sensitivity/ycsbA.all.bs.10.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | bias_prob_sv=0.1 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/biased-sampling-sensitivity/ycsbA.all.bs.30.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | bias_prob_sv=0.3 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/biased-sampling-sensitivity/ycsbA.all.bs.5.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | bias_prob_sv=0.05 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/biased-sampling-sensitivity/ycsbB.all.bs.10.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | bias_prob_sv=0.1 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/biased-sampling-sensitivity/ycsbB.all.bs.30.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | bias_prob_sv=0.3 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/biased-sampling-sensitivity/ycsbB.all.bs.5.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | bias_prob_sv=0.05 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/biased-sampling-sensitivity/ycsbA.all.bs.20.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | bias_prob_sv=0.2 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/biased-sampling-sensitivity/ycsbB.all.bs.20.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | bias_prob_sv=0.20 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/bucketization-sensitivity/ycsbB.all.q5K.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | quantization_factor=5000 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/bucketization-sensitivity/ycsbA.all.q10K.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | quantization_factor=10000 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/bucketization-sensitivity/ycsbA.all.q1K.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | quantization_factor=1000 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/bucketization-sensitivity/ycsbA.all.q20K.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | quantization_factor=20000 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/bucketization-sensitivity/ycsbA.all.q5K.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | quantization_factor=5000 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/bucketization-sensitivity/ycsbB.all.q10K.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | quantization_factor=10000 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/bucketization-sensitivity/ycsbB.all.q1K.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | quantization_factor=1000 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/bucketization-sensitivity/ycsbB.all.q20K.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=none 27 | quantization_factor=20000 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/benchbase/resourcestresser/resourcestresser.all.mkbo.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=resourcestresser 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | model_type=mkbo 18 | rand_percentage=0.1 19 | n_estimators=100 20 | 21 | [policy] 22 | classname=NoPrunning 23 | 24 | [spaces] 25 | definition=postgres-9.6 26 | ignore=postgres-none 27 | adapter_alias=none 28 | target_metric=throughput 29 | 30 | [storage] 31 | classname=FileTablesStorage 32 | outdir=results 33 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/ablation-study/ycsbA.all.hesbo_16_bs.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | bias_prob_sv=0.2 29 | target_metric=throughput 30 | 31 | [storage] 32 | classname=FileTablesStorage 33 | outdir=results -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/ablation-study/ycsbB.all.hesbo_16_bs.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | bias_prob_sv=0.2 29 | target_metric=throughput 30 | 31 | [storage] 32 | classname=FileTablesStorage 33 | outdir=results -------------------------------------------------------------------------------- /configs/benchbase/tpcc/ablation-study/tpcc.all.hesbo_16_bs.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=tpcc 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | bias_prob_sv=0.2 29 | target_metric=throughput 30 | 31 | [storage] 32 | classname=FileTablesStorage 33 | outdir=results 34 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/ycsbA.all.mkbo.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | model_type=mkbo 18 | rand_percentage=0.1 19 | n_estimators=100 20 | 21 | [policy] 22 | classname=NoPrunning 23 | 24 | [spaces] 25 | definition=postgres-9.6 26 | ignore=postgres-none 27 | adapter_alias=hesbo 28 | le_low_dim=16 29 | bias_prob_sv=0.2 30 | target_metric=throughput 31 | 32 | [storage] 33 | classname=FileTablesStorage 34 | outdir=results 35 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/ycsbB.all.mkbo.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | model_type=mkbo 18 | rand_percentage=0.1 19 | n_estimators=100 20 | 21 | [policy] 22 | classname=NoPrunning 23 | 24 | [spaces] 25 | definition=postgres-9.6 26 | ignore=postgres-none 27 | adapter_alias=hesbo 28 | le_low_dim=16 29 | bias_prob_sv=0.2 30 | target_metric=throughput 31 | 32 | [storage] 33 | classname=FileTablesStorage 34 | outdir=results 35 | -------------------------------------------------------------------------------- /configs/benchbase/tpcc/tpcc.all.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=tpcc 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | bias_prob_sv=0.2 29 | quantization_factor=10000 30 | target_metric=throughput 31 | 32 | [storage] 33 | classname=FileTablesStorage 34 | outdir=results 35 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbA/ycsbA.all.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloada 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | bias_prob_sv=0.2 29 | quantization_factor=10000 30 | target_metric=throughput 31 | 32 | [storage] 33 | classname=FileTablesStorage 34 | outdir=results 35 | -------------------------------------------------------------------------------- /configs/ycsb/ycsbB/ycsbB.all.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=ycsb 3 | workload=workloadb 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | bias_prob_sv=0.2 29 | quantization_factor=10000 30 | target_metric=throughput 31 | 32 | [storage] 33 | classname=FileTablesStorage 34 | outdir=results 35 | -------------------------------------------------------------------------------- /configs/benchbase/seats/seats.all.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=seats 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | bias_prob_sv=0.2 29 | quantization_factor=10000 30 | target_metric=throughput 31 | 32 | [storage] 33 | classname=FileTablesStorage 34 | outdir=results 35 | -------------------------------------------------------------------------------- /configs/benchbase/tpcc/tpcc.all.lat.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=tpcc-lat 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | bias_prob_sv=0.2 29 | quantization_factor=10000 30 | target_metric=latency 31 | 32 | [storage] 33 | classname=FileTablesStorage 34 | outdir=results 35 | -------------------------------------------------------------------------------- /configs/benchbase/seats/seats.all.lat.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=seats-lat 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | bias_prob_sv=0.2 29 | quantization_factor=10000 30 | target_metric=latency 31 | 32 | [storage] 33 | classname=FileTablesStorage 34 | outdir=results 35 | -------------------------------------------------------------------------------- /configs/benchbase/twitter/twitter.all.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=twitter 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | bias_prob_sv=0.2 29 | quantization_factor=10000 30 | target_metric=throughput 31 | 32 | [storage] 33 | classname=FileTablesStorage 34 | outdir=results 35 | -------------------------------------------------------------------------------- /configs/benchbase/twitter/twitter.all.lat.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=twitter-lat 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | bias_prob_sv=0.2 29 | quantization_factor=10000 30 | target_metric=latency 31 | 32 | [storage] 33 | classname=FileTablesStorage 34 | outdir=results 35 | -------------------------------------------------------------------------------- /configs/benchbase/seats/seats.all.mkbo.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=seats 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | model_type=mkbo 18 | rand_percentage=0.1 19 | n_estimators=100 20 | 21 | [policy] 22 | classname=NoPrunning 23 | 24 | [spaces] 25 | definition=postgres-9.6 26 | ignore=postgres-none 27 | adapter_alias=hesbo 28 | le_low_dim=16 29 | bias_prob_sv=0.2 30 | quantization_factor=10000 31 | target_metric=throughput 32 | 33 | 34 | [storage] 35 | classname=FileTablesStorage 36 | outdir=results 37 | -------------------------------------------------------------------------------- /configs/benchbase/resourcestresser/resourcestresser.all.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=resourcestresser 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | bias_prob_sv=0.2 29 | quantization_factor=10000 30 | target_metric=throughput 31 | 32 | [storage] 33 | classname=FileTablesStorage 34 | outdir=results 35 | -------------------------------------------------------------------------------- /configs/benchbase/resourcestresser/resourcestresser.all.lat.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=resourcestresser-lat 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | rand_percentage=0.1 18 | n_estimators=100 19 | 20 | [policy] 21 | classname=NoPrunning 22 | 23 | [spaces] 24 | definition=postgres-9.6 25 | ignore=postgres-none 26 | adapter_alias=hesbo 27 | le_low_dim=16 28 | bias_prob_sv=0.2 29 | quantization_factor=10000 30 | target_metric=latency 31 | 32 | [storage] 33 | classname=FileTablesStorage 34 | outdir=results 35 | -------------------------------------------------------------------------------- /configs/benchbase/twitter/twitter.all.mkbo.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=twitter 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | model_type=mkbo 18 | rand_percentage=0.1 19 | n_estimators=100 20 | 21 | [policy] 22 | classname=NoPrunning 23 | 24 | [spaces] 25 | definition=postgres-9.6 26 | ignore=postgres-none 27 | adapter_alias=hesbo 28 | le_low_dim=16 29 | bias_prob_sv=0.2 30 | quantization_factor=10000 31 | 32 | target_metric=throughput 33 | 34 | [storage] 35 | classname=FileTablesStorage 36 | outdir=results 37 | -------------------------------------------------------------------------------- /configs/benchbase/tpcc/tpcc.all.mkbo.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=tpcc 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | model_type=mkbo 18 | rand_percentage=0.1 19 | n_estimators=100 20 | 21 | [policy] 22 | classname=NoPrunning 23 | 24 | [spaces] 25 | definition=postgres-9.6 26 | ignore=postgres-none 27 | adapter_alias=hesbo 28 | le_low_dim=16 29 | bias_prob_sv=0.2 30 | quantization_factor=10000 31 | target_metric=throughput 32 | 33 | 34 | 35 | 36 | 37 | [storage] 38 | classname=FileTablesStorage 39 | outdir=results 40 | -------------------------------------------------------------------------------- /configs/benchbase/resourcestresser/resourcestresser.all.mkbo.llama.ini: -------------------------------------------------------------------------------- 1 | [benchmark_info] 2 | name=benchbase 3 | workload=resourcestresser 4 | 5 | [dbms_info] 6 | name=postgres 7 | 8 | [executor] 9 | classname=DummyExecutor 10 | host=localhost 11 | port=1337 12 | 13 | [global] 14 | iters=100 15 | 16 | [optimizer] 17 | model_type=mkbo 18 | rand_percentage=0.1 19 | n_estimators=100 20 | 21 | [policy] 22 | classname=NoPrunning 23 | 24 | [spaces] 25 | definition=postgres-9.6 26 | ignore=postgres-none 27 | adapter_alias=hesbo 28 | le_low_dim=16 29 | bias_prob_sv=0.2 30 | quantization_factor=10000 31 | target_metric=throughput 32 | 33 | 34 | [storage] 35 | classname=FileTablesStorage 36 | outdir=results 37 | -------------------------------------------------------------------------------- /spaces/common.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | def is_valid_conf_10knobs(conf, knobs_dict, check_cost_order=True): 4 | # Validate knob values 5 | for k, v in conf.items(): 6 | d = knobs_dict[k] 7 | 8 | if 'choices' in d: 9 | # Validate that value is among choices 10 | if v not in d['choices']: 11 | print(f'{k}: {v} not in {d["choices"]}') 12 | return False 13 | continue 14 | 15 | #v = v_type(v[:-len(suffix)]) 16 | min_v, max_v = d['min'], d['max'] 17 | if not (min_v <= v <= max_v): 18 | print(f'{k}: {v} is not between {min_v} and {max_v}') 19 | return False 20 | 21 | return True 22 | 23 | def finalize_conf(conf, knobs_dict, n_decimals=2): 24 | new_conf = { } 25 | for k, v in conf.items(): 26 | if k not in knobs_dict: 27 | raise ValueError(f'Invalid knob value: "{k}"') 28 | d = knobs_dict[k] 29 | 30 | # Truncate decimals 31 | if d['type'] == 'real': 32 | v = round(v, n_decimals) 33 | 34 | new_conf[k] = v 35 | 36 | return new_conf 37 | 38 | def unfinalize_conf(conf, knobs_dict): 39 | new_conf = { } 40 | for k, v in conf.items(): 41 | if k not in knobs_dict: 42 | raise ValueError(f'Invalid knob value: "{k}"') 43 | d = knobs_dict[k] 44 | 45 | if d['type'] == 'integer': 46 | v = int(v) 47 | elif d['type'] == 'real': 48 | v = float(v) 49 | 50 | new_conf[k] = v 51 | 52 | return new_conf 53 | -------------------------------------------------------------------------------- /executors/grpc/nautilus_rpc.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "google/protobuf/struct.proto"; 4 | import "google/protobuf/timestamp.proto"; 5 | 6 | service ExecutionService { 7 | rpc Execute (ExecuteRequest) returns (ExecuteReply) {} 8 | rpc Heartbeat (EmptyMessage) returns (StatsReply) {} 9 | } 10 | 11 | message ExecuteRequest{ 12 | // DBMS Info structure 13 | message DBMSInfo { 14 | string name = 1; 15 | google.protobuf.Struct config = 2; 16 | } 17 | 18 | //// WorkloadInfo 19 | 20 | // YCSB 21 | message YCSBWorkloadProperties { 22 | uint64 recordcount = 1; 23 | uint64 operationcount = 2; 24 | uint32 threadcount = 3; 25 | } 26 | message YCSBInfo { 27 | string name = 1; 28 | string workload = 2; 29 | int32 warmup_duration = 3; 30 | int32 benchmark_duration = 4; 31 | optional YCSBWorkloadProperties workload_properties = 5; 32 | } 33 | 34 | // OLTP-Bench 35 | message OLTPBenchWorkloadProperties { 36 | uint32 scalefactor = 1; 37 | uint32 terminals = 2; 38 | string work_rate = 3; 39 | } 40 | message OLTPBenchInfo { 41 | string name = 1; 42 | string workload = 2; 43 | int32 warmup_duration = 3; 44 | int32 benchmark_duration = 4; 45 | optional OLTPBenchWorkloadProperties workload_properties = 5; 46 | bool capture_raw_perf_stats = 6; 47 | } 48 | 49 | // BenchBase 50 | message BenchBaseWorkloadProperties { 51 | uint32 scalefactor = 1; 52 | uint32 terminals = 2; 53 | string work_rate = 3; 54 | } 55 | message BenchBaseInfo { 56 | string name = 1; 57 | string workload = 2; 58 | int32 warmup_duration = 3; 59 | int32 benchmark_duration = 4; 60 | optional BenchBaseWorkloadProperties workload_properties = 5; 61 | bool capture_raw_perf_stats = 6; 62 | } 63 | 64 | oneof _dbms_info { 65 | DBMSInfo dbms_info = 1; 66 | } 67 | 68 | oneof _benchmark_info { 69 | YCSBInfo ycsb_info = 2; 70 | OLTPBenchInfo oltpbench_info = 3; 71 | BenchBaseInfo benchbase_info = 4; 72 | } 73 | } 74 | 75 | message ExecuteReply { 76 | google.protobuf.Struct results = 1; 77 | } 78 | 79 | message StatsReply { 80 | google.protobuf.Timestamp alive_since = 1; 81 | google.protobuf.Timestamp time_now = 2; 82 | uint32 jobs_finished = 3; 83 | } 84 | 85 | message EmptyMessage { 86 | // Empty message 87 | } 88 | -------------------------------------------------------------------------------- /ddpg/prm.py: -------------------------------------------------------------------------------- 1 | # from: https://raw.githubusercontent.com/cmu-db/ottertune/master/server/analysis/ddpg/prioritized_replay_memory.py 2 | 3 | import random 4 | import pickle 5 | import numpy as np 6 | 7 | 8 | class SumTree(object): 9 | write = 0 10 | 11 | def __init__(self, capacity): 12 | self.capacity = capacity 13 | self.tree = np.zeros(2 * capacity - 1) 14 | self.data = np.zeros(capacity, dtype=object) 15 | self.num_entries = 0 16 | 17 | def _propagate(self, idx, change): 18 | parent = (idx - 1) // 2 19 | self.tree[parent] += change 20 | if parent != 0: 21 | self._propagate(parent, change) 22 | 23 | def _retrieve(self, idx, s): 24 | left = 2 * idx + 1 25 | right = left + 1 26 | 27 | if left >= len(self.tree): 28 | return idx 29 | 30 | if s <= self.tree[left]: 31 | return self._retrieve(left, s) 32 | else: 33 | return self._retrieve(right, s - self.tree[left]) 34 | 35 | def total(self): 36 | return self.tree[0] 37 | 38 | def add(self, p, data): 39 | idx = self.write + self.capacity - 1 40 | 41 | self.data[self.write] = data 42 | self.update(idx, p) 43 | 44 | self.write += 1 45 | if self.write >= self.capacity: 46 | self.write = 0 47 | if self.num_entries < self.capacity: 48 | self.num_entries += 1 49 | 50 | def update(self, idx, p): 51 | change = p - self.tree[idx] 52 | 53 | self.tree[idx] = p 54 | self._propagate(idx, change) 55 | 56 | def get(self, s): 57 | idx = self._retrieve(0, s) 58 | data_idx = idx - self.capacity + 1 59 | return [idx, self.tree[idx], self.data[data_idx]] 60 | 61 | 62 | class PrioritizedReplayMemory(object): 63 | 64 | def __init__(self, capacity): 65 | self.tree = SumTree(capacity) 66 | self.capacity = capacity 67 | self.e = 0.01 # pylint: disable=invalid-name 68 | self.a = 0.6 # pylint: disable=invalid-name 69 | self.beta = 0.4 70 | self.beta_increment_per_sampling = 0.001 71 | 72 | def _get_priority(self, error): 73 | return (error + self.e) ** self.a 74 | 75 | def add(self, error, sample): 76 | # (s, a, r, s, t) 77 | p = self._get_priority(error) 78 | self.tree.add(p, sample) 79 | 80 | def __len__(self): 81 | return self.tree.num_entries 82 | 83 | def sample(self, n): 84 | batch = [] 85 | idxs = [] 86 | segment = self.tree.total() / n 87 | priorities = [] 88 | 89 | self.beta = np.min([1., self.beta + self.beta_increment_per_sampling]) 90 | 91 | for i in range(n): 92 | a = segment * i 93 | b = segment * (i + 1) 94 | 95 | s = random.uniform(a, b) 96 | (idx, p, data) = self.tree.get(s) 97 | priorities.append(p) 98 | batch.append(data) 99 | idxs.append(idx) 100 | return batch, idxs 101 | 102 | # sampling_probabilities = priorities / self.tree.total() 103 | # is_weight = np.power(self.tree.num_entries * sampling_probabilities, -self.beta) 104 | # is_weight /= is_weight.max() 105 | 106 | def update(self, idx, error): 107 | p = self._get_priority(error) 108 | self.tree.update(idx, p) 109 | 110 | def save(self, path): 111 | f = open(path, 'wb') 112 | pickle.dump({"tree": self.tree}, f) 113 | f.close() 114 | 115 | def load_memory(self, path): 116 | with open(path, 'rb') as f: 117 | _memory = pickle.load(f) 118 | self.tree = _memory['tree'] 119 | 120 | def get(self): 121 | return pickle.dumps({"tree": self.tree}) 122 | 123 | def set(self, binary): 124 | self.tree = pickle.loads(binary)['tree'] -------------------------------------------------------------------------------- /adapters/configspace/quantization.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import ConfigSpace as CS 4 | import ConfigSpace.hyperparameters as CSH 5 | import numpy as np 6 | from sklearn.preprocessing import MinMaxScaler 7 | 8 | import logging 9 | logger = logging.getLogger(__name__) 10 | 11 | class Quantization: 12 | def __init__(self, 13 | adaptee: CS.ConfigurationSpace, seed: int, max_num_values: int, 14 | ): 15 | 16 | self._adaptee: CS.ConfigurationSpace = adaptee 17 | self._target: CS.ConfigurationSpace = None 18 | self._seed: int = seed 19 | self._max_num_values: int = max_num_values 20 | assert max_num_values > 1 21 | 22 | self._rs = np.random.RandomState(seed=self._seed) 23 | 24 | self._build_space() 25 | 26 | def _build_space(self): 27 | self._knobs_scalers = { } 28 | 29 | root_hyperparams = [ ] 30 | for adaptee_hp in self.adaptee.get_hyperparameters(): 31 | if not isinstance(adaptee_hp, CSH.UniformIntegerHyperparameter): 32 | # Leave float & categorical hps as is 33 | root_hyperparams.append(adaptee_hp) 34 | continue 35 | 36 | if not self._needs_quantization(adaptee_hp): 37 | # If values range <= max values, then leave as is 38 | root_hyperparams.append(adaptee_hp) 39 | continue 40 | 41 | # quantize knob 42 | lower, upper = adaptee_hp.lower, adaptee_hp.upper 43 | scaler = MinMaxScaler(feature_range=(lower, upper)) 44 | scaler.fit([[1], [self._max_num_values]]) 45 | self._knobs_scalers[adaptee_hp.name] = scaler 46 | 47 | default_value = round( 48 | scaler.inverse_transform([[ adaptee_hp.default_value ]])[0][0]) 49 | quantized_hp = CSH.UniformIntegerHyperparameter( 50 | f'{adaptee_hp.name}|q', 1, self._max_num_values, 51 | default_value=default_value, 52 | ) 53 | root_hyperparams.append(quantized_hp) 54 | 55 | root = CS.ConfigurationSpace( 56 | name=self._adaptee.name, 57 | seed=self._seed, 58 | ) 59 | root.add_hyperparameters(root_hyperparams) 60 | self._target = root 61 | 62 | def _needs_quantization(self, hp): 63 | return (hp.upper - hp.lower + 1) > self._max_num_values 64 | 65 | @property 66 | def adaptee(self) -> CS.ConfigurationSpace: 67 | return self._adaptee 68 | 69 | @property 70 | def target(self) -> CS.ConfigurationSpace: 71 | return self._target 72 | 73 | def project_point(self, point): 74 | raise NotImplementedError() 75 | 76 | def unproject_point(self, point: CS.Configuration) -> dict: 77 | coords = point.get_dictionary() 78 | valid_dim_names = [ dim.name for dim in self.adaptee.get_hyperparameters() ] 79 | unproject_coords = { } 80 | for name, value in coords.items(): 81 | dequantize = name.endswith('|q') 82 | if not dequantize: 83 | unproject_coords[name] = value 84 | continue 85 | 86 | # de-quantize 87 | dim_name = name[:-2] 88 | assert dim_name in valid_dim_names and dim_name in self._knobs_scalers 89 | 90 | scaler, value = self._knobs_scalers[dim_name], coords[name] 91 | lower, upper = scaler.feature_range 92 | # transform value 93 | value = int(scaler.transform([[ value ]])[0][0]) 94 | value = max(lower, min(upper, value)) 95 | unproject_coords[dim_name] = value 96 | 97 | return unproject_coords 98 | 99 | def project_dataframe(self, df, in_place: bool): 100 | raise NotImplementedError() 101 | 102 | def unproject_dataframe(self, df, in_place: bool): 103 | raise NotImplementedError() 104 | 105 | 106 | -------------------------------------------------------------------------------- /executors/grpc/nautilus_rpc_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | """Client and server classes corresponding to protobuf-defined services.""" 3 | import grpc 4 | 5 | import executors.grpc.nautilus_rpc_pb2 as nautilus__rpc__pb2 # pylint: disable=import-error 6 | 7 | 8 | class ExecutionServiceStub(object): 9 | """Missing associated documentation comment in .proto file.""" 10 | 11 | def __init__(self, channel): 12 | """Constructor. 13 | 14 | Args: 15 | channel: A grpc.Channel. 16 | """ 17 | self.Execute = channel.unary_unary( 18 | '/ExecutionService/Execute', 19 | request_serializer=nautilus__rpc__pb2.ExecuteRequest.SerializeToString, 20 | response_deserializer=nautilus__rpc__pb2.ExecuteReply.FromString, 21 | ) 22 | self.Heartbeat = channel.unary_unary( 23 | '/ExecutionService/Heartbeat', 24 | request_serializer=nautilus__rpc__pb2.EmptyMessage.SerializeToString, 25 | response_deserializer=nautilus__rpc__pb2.StatsReply.FromString, 26 | ) 27 | 28 | 29 | class ExecutionServiceServicer(object): 30 | """Missing associated documentation comment in .proto file.""" 31 | 32 | def Execute(self, request, context): 33 | """Missing associated documentation comment in .proto file.""" 34 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 35 | context.set_details('Method not implemented!') 36 | raise NotImplementedError('Method not implemented!') 37 | 38 | def Heartbeat(self, request, context): 39 | """Missing associated documentation comment in .proto file.""" 40 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 41 | context.set_details('Method not implemented!') 42 | raise NotImplementedError('Method not implemented!') 43 | 44 | 45 | def add_ExecutionServiceServicer_to_server(servicer, server): 46 | rpc_method_handlers = { 47 | 'Execute': grpc.unary_unary_rpc_method_handler( 48 | servicer.Execute, 49 | request_deserializer=nautilus__rpc__pb2.ExecuteRequest.FromString, 50 | response_serializer=nautilus__rpc__pb2.ExecuteReply.SerializeToString, 51 | ), 52 | 'Heartbeat': grpc.unary_unary_rpc_method_handler( 53 | servicer.Heartbeat, 54 | request_deserializer=nautilus__rpc__pb2.EmptyMessage.FromString, 55 | response_serializer=nautilus__rpc__pb2.StatsReply.SerializeToString, 56 | ), 57 | } 58 | generic_handler = grpc.method_handlers_generic_handler( 59 | 'ExecutionService', rpc_method_handlers) 60 | server.add_generic_rpc_handlers((generic_handler,)) 61 | 62 | 63 | # This class is part of an EXPERIMENTAL API. 64 | class ExecutionService(object): 65 | """Missing associated documentation comment in .proto file.""" 66 | 67 | @staticmethod 68 | def Execute(request, 69 | target, 70 | options=(), 71 | channel_credentials=None, 72 | call_credentials=None, 73 | insecure=False, 74 | compression=None, 75 | wait_for_ready=None, 76 | timeout=None, 77 | metadata=None): 78 | return grpc.experimental.unary_unary(request, target, '/ExecutionService/Execute', 79 | nautilus__rpc__pb2.ExecuteRequest.SerializeToString, 80 | nautilus__rpc__pb2.ExecuteReply.FromString, 81 | options, channel_credentials, 82 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 83 | 84 | @staticmethod 85 | def Heartbeat(request, 86 | target, 87 | options=(), 88 | channel_credentials=None, 89 | call_credentials=None, 90 | insecure=False, 91 | compression=None, 92 | wait_for_ready=None, 93 | timeout=None, 94 | metadata=None): 95 | return grpc.experimental.unary_unary(request, target, '/ExecutionService/Heartbeat', 96 | nautilus__rpc__pb2.EmptyMessage.SerializeToString, 97 | nautilus__rpc__pb2.StatsReply.FromString, 98 | options, channel_credentials, 99 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 100 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | from configparser import SafeConfigParser 2 | 3 | import logging 4 | logging.basicConfig(level=logging.INFO) 5 | logger = logging.getLogger() 6 | 7 | # YCSB 8 | ycsb_defaut_workload_properties = dict( 9 | recordcount=18000000, 10 | operationcount=2000000000, 11 | threadcount=40 12 | ) 13 | 14 | # Benchbase 15 | benchbase_default_workload_properties = { 16 | 'resourcestresser': dict( 17 | scalefactor=20000, # ~20 GB 18 | terminals=40, 19 | work_rate='unlimited', 20 | ), 21 | 'seats': dict( 22 | scalefactor=50, # ~20 GB 23 | terminals=40, 24 | work_rate='unlimited' 25 | ), 26 | 'tpcc': dict( 27 | scalefactor=100, # ~20 GB 28 | terminals=40, 29 | work_rate='unlimited' 30 | ), 31 | 'twitter': dict( 32 | scalefactor=1500, # ~20 GB 33 | terminals=40, 34 | work_rate='unlimited' 35 | ), 36 | 37 | # Rate-limited 38 | 'tpcc-lat': dict( 39 | scalefactor=100, # ~20 GB 40 | terminals=40, 41 | work_rate="2000", # ~50% of max 42 | ), 43 | 'seats-lat': dict( 44 | scalefactor=50, # ~20 GB 45 | terminals=40, 46 | work_rate="8000", # ~50% of max 47 | ), 48 | 'twitter-lat': dict( 49 | scalefactor=1500, # ~20 GB 50 | terminals=40, 51 | work_rate="60000", # ~50% of max 52 | ), 53 | } 54 | 55 | workload_defaults = { 56 | 'WARMUP_DURATION_SECS': 30, 57 | 'BENCHMARK_DURATION_SECS': 300, 58 | } 59 | 60 | class Configuration: 61 | def __init__(self): 62 | self.parser = SafeConfigParser() 63 | self.parser.optionxform = str 64 | 65 | self.dict = { } 66 | 67 | def update_from_file(self, filename): 68 | if not self.parser.read(filename): 69 | raise FileNotFoundError(f'Config file [@{filename}] not found! :(') 70 | 71 | for name in self.parser.sections(): 72 | self.dict[name] = dict(self.parser.items(name)) 73 | 74 | self._postprocess() 75 | 76 | def _postprocess(self): 77 | # Save dbms_info 78 | assert 'dbms_info' in self.dict, 'Section `dbms_info` not specified' 79 | self._dbms_info = self.dict['dbms_info'] 80 | 81 | if 'version' not in self._dbms_info: 82 | self._dbms_info['version'] = '9.6' 83 | 84 | # Save benchmark_info 85 | assert 'benchmark_info' in self.dict, 'Section `benchmark_info` not specified' 86 | self._benchmark_info = self.dict['benchmark_info'] 87 | 88 | benchmark_name, workload = ( 89 | self._benchmark_info['name'], self._benchmark_info['workload']) 90 | assert benchmark_name in ['ycsb', 'oltpbench', 'benchbase'] 91 | 92 | if benchmark_name == 'ycsb': 93 | workload_properties = ycsb_defaut_workload_properties 94 | elif benchmark_name == 'oltpbench': 95 | if workload.startswith('ycsb'): 96 | workload_properties = oltpbench_default_workload_properties['ycsb'] 97 | else: 98 | workload_properties = oltpbench_default_workload_properties[workload] 99 | else: # benchbase 100 | if workload.startswith('ycsb'): 101 | workload_properties = benchbase_default_workload_properties['ycsb'] 102 | else: 103 | workload_properties = benchbase_default_workload_properties[workload] 104 | 105 | if workload.endswith('-lat'): 106 | # Provision for latency-focused configs 107 | workload = workload[:-len('-lat')] 108 | 109 | self._benchmark_info.update( 110 | workload=workload, 111 | warmup_duration=workload_defaults['WARMUP_DURATION_SECS'], 112 | benchmark_duration=workload_defaults['BENCHMARK_DURATION_SECS'], 113 | workload_properties=workload_properties, 114 | ) 115 | if benchmark_name != 'ycsb': 116 | self._benchmark_info.update( 117 | capture_raw_perf_stats=False, 118 | ) 119 | 120 | # global section 121 | self._iters = int(self.dict['global']['iters']) 122 | 123 | def __getitem__(self, key): 124 | return self.dict[key] 125 | 126 | @property 127 | def seed(self): 128 | return self._seed 129 | 130 | @seed.setter 131 | def seed(self, value): 132 | self._seed = value 133 | 134 | @property 135 | def dbms_info(self): 136 | return self._dbms_info 137 | 138 | @property 139 | def benchmark_info(self): 140 | return self._benchmark_info 141 | 142 | @property 143 | def iters(self): 144 | return self._iters 145 | 146 | config = Configuration() 147 | -------------------------------------------------------------------------------- /storage.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import uuid 4 | from abc import ABC, abstractmethod 5 | from copy import copy, deepcopy 6 | from pathlib import Path 7 | from string import Template 8 | 9 | from prettytable import PrettyTable 10 | 11 | # pylint: disable=import-error 12 | logging.basicConfig(level=logging.INFO) 13 | logger = logging.getLogger() 14 | 15 | LOG_FILENAME = 'log.txt' 16 | 17 | class StorageInterface: 18 | def __init__(self, *args, **kwargs): 19 | self._outdir = "N/A" 20 | 21 | @property 22 | def outdir(self): 23 | return self._outdir 24 | 25 | @abstractmethod 26 | def store_knobs_importances(self, iter_, knob_importances): 27 | raise NotImplementedError 28 | 29 | @abstractmethod 30 | def store_executor_result(self, iter_, result): 31 | raise NotImplementedError 32 | 33 | @abstractmethod 34 | def store_result_summary(self, summary): 35 | raise NotImplementedError 36 | 37 | class NoStorage(StorageInterface): 38 | def store_knobs_importances(self, iter_, knob_importances): 39 | pass 40 | 41 | def store_executor_result(self, iter_, result): 42 | pass 43 | 44 | def store_result_summary(self, summary): 45 | pass 46 | 47 | class FileTablesStorage(StorageInterface): 48 | 49 | def __init__(self, outdir=None, columns=None, inner_path=None): 50 | assert outdir != None, 'Need to provide outdir' 51 | assert logger != None, 'Need to provide instance of logger' 52 | 53 | # Create base results dir if not exists 54 | outdir = Path(outdir) if not isinstance(outdir, Path) else outdir 55 | outdir.mkdir(exist_ok=True) 56 | 57 | if inner_path is None: 58 | # Create unique dir inside 59 | experiment_path = outdir / Path(uuid.uuid4().hex[:8]) 60 | while experiment_path.exists(): 61 | experiment_path = outdir / Path(uuid.uuid4().hex[:8]) 62 | else: 63 | experiment_path = outdir / inner_path 64 | if experiment_path.exists(): 65 | raise FileExistsError('Experiment output path already exists') 66 | experiment_path.mkdir(parents=True) 67 | self._outdir = experiment_path 68 | 69 | logger.addHandler( 70 | logging.FileHandler(self.outdir / LOG_FILENAME)) 71 | logger.info(f'Results will be saved @ "{self.outdir}"\n\n') 72 | 73 | # Summary table initialization 74 | self.table = PrettyTable() 75 | self.table.field_names = copy(columns) 76 | self.table_txt_filepath = self.outdir / 'optimizer.txt' 77 | self.table_csv_filepath = self.outdir / 'optimizer.csv' 78 | self.executor_result_filename_template = Template('iter-${iter}.json') 79 | 80 | def store_knobs_importances(self, iter_, knob_importances): 81 | # Save to table 82 | ki_table = PrettyTable() 83 | ki_table.field_names = ['Knob', 'Importance Score'] 84 | for knob, importance in knob_importances: 85 | logger.info(f'\t{knob}:\t{importance:.3f}') 86 | ki_table.add_row([knob, importance]) 87 | 88 | ki_table_filepath = self.outdir / f'ki-{iter_}.txt' 89 | ki_table_csv_filepath = self.outdir / f'ki-{iter_}.csv' 90 | with open(ki_table_filepath, 'w') as f: 91 | f.write(ki_table.get_string()) 92 | with open(ki_table_csv_filepath, 'w') as f: 93 | f.write(ki_table.get_csv_string()) 94 | 95 | logger.info(f'Knobs importances saved @ {ki_table_filepath}') 96 | 97 | def store_executor_result(self, iter_, result): 98 | filename = self.executor_result_filename_template.substitute(iter=iter_) 99 | filepath = self.outdir / filename 100 | with open(filepath, 'w') as f: 101 | json.dump(result, f) 102 | 103 | def store_result_summary(self, summary): 104 | assert set(summary.keys()) == set(self.table.field_names) 105 | 106 | row = [ summary[k] for k in self.table.field_names ] 107 | self.table.add_row(row) 108 | 109 | with open(self.table_txt_filepath, 'w') as f: 110 | f.write(self.table.get_string()) 111 | with open(self.table_csv_filepath, 'w') as f: 112 | f.write(self.table.get_csv_string()) 113 | 114 | 115 | class StorageFactory: 116 | concrete_classes = { 117 | 'FileTablesStorage': FileTablesStorage, 118 | 'NoStorage': NoStorage, 119 | } 120 | 121 | @staticmethod 122 | def from_config(config, **extra_kwargs): 123 | storage_config = deepcopy(config['storage']) 124 | 125 | classname = storage_config.pop('classname', None) 126 | assert classname != None, 'Please specify the storage class name' 127 | 128 | try: 129 | class_ = StorageFactory.concrete_classes[classname] 130 | except KeyError: 131 | raise ValueError(f'Storage class "{classname}" not found. ' 132 | f'Options are [{", ".join(StorageFactory.concrete_classes.keys())}]') 133 | 134 | # Override with local 135 | storage_config.update(**extra_kwargs) 136 | 137 | return class_(**storage_config) 138 | -------------------------------------------------------------------------------- /run-smac.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import random 3 | from copy import copy, deepcopy 4 | from pathlib import Path 5 | 6 | import pandas as pd 7 | pd.set_option('display.max_columns', None) 8 | import numpy as np 9 | 10 | from config import config 11 | from executors.executor import ExecutorFactory 12 | from optimizer import get_new_optimizer, get_smac_optimizer 13 | from space import ConfigSpaceGenerator 14 | from storage import StorageFactory 15 | 16 | from smac.facade.smac_hpo_facade import SMAC4AC 17 | 18 | # pylint: disable=logging-fstring-interpolation 19 | import logging 20 | logging.basicConfig(level=logging.INFO) 21 | logger = logging.getLogger() 22 | 23 | def fix_global_random_state(seed=None): 24 | random.seed(seed) 25 | np.random.seed(seed) 26 | 27 | class ExperimentState: 28 | def __init__(self, dbms_info, benchmark_info, results_path: Path, target_metric: str): 29 | self.iter = 0 30 | self.best_conf = None 31 | self.best_perf = None 32 | self.worse_perf = None 33 | self.default_perf_stats = None 34 | 35 | assert target_metric in ['throughput', 'latency'], \ 36 | f'Unsupported target metric: {target_metric}' 37 | self.minimize = target_metric != 'throughput' 38 | self._target_metric = target_metric 39 | 40 | self._dbms_info = dbms_info 41 | self._benchmark_info = benchmark_info 42 | 43 | assert results_path.exists() 44 | self._results_path = str(results_path) 45 | 46 | @property 47 | def benchmark_info(self): 48 | return self._benchmark_info 49 | 50 | @property 51 | def dbms_info(self): 52 | return self._dbms_info 53 | 54 | @property 55 | def results_path(self) -> str: 56 | return self._results_path 57 | 58 | @property 59 | def target_metric(self) -> str: 60 | return self._target_metric 61 | 62 | def is_better_perf(self, perf, other): 63 | return (perf > other) if not self.minimize else (perf < other) 64 | 65 | def __str__(self): 66 | fields = ['iter', 'best_conf', 'best_perf', 'worse_perf', 67 | 'default_perf_stats', 'target_metric'] 68 | return ':\n' + \ 69 | '\n'.join([ f'{f}: \t{getattr(self, f)}' for f in fields ]) 70 | 71 | 72 | def evaluate_dbms_conf(sample, state=None): 73 | global spaces, executor 74 | 75 | logger.info(f'\n\n{25*"="} Iteration {state.iter:2d} {25*"="}\n\n') 76 | logger.info('Sample from optimizer: ') 77 | logger.info(sample) 78 | 79 | if state.iter > 0: # if not default conf 80 | sample = spaces.unproject_input_point(sample) 81 | conf = spaces.finalize_conf(sample) 82 | logger.info(f'Evaluating Configuration:\n{conf}') 83 | 84 | ## Send configuration task to Nautilus 85 | dbms_info = dict( 86 | name=state.dbms_info['name'], 87 | config=conf, 88 | version=state.dbms_info['version'] 89 | ) 90 | perf_stats = executor.evaluate_configuration(dbms_info, state.benchmark_info) 91 | logger.info(f'Performance Statistics:\n{perf_stats}') 92 | 93 | if state.default_perf_stats is None: 94 | state.default_perf_stats = perf_stats 95 | 96 | target_metric = state.target_metric 97 | if perf_stats is None: 98 | # Error while evaluating conf -- set some reasonable value for target metric 99 | runtime, perf = 0, state.worse_perf 100 | else: 101 | if target_metric == 'latency' and \ 102 | perf_stats['throughput'] < state.default_perf_stats['throughput']: 103 | # Throughput less than default -- invalid round 104 | runtime, perf = perf_stats['runtime'], state.worse_perf 105 | else: 106 | runtime, perf = perf_stats['runtime'], perf_stats[target_metric] 107 | 108 | logger.info(f'Evaluation took {runtime} seconds') 109 | if target_metric == 'throughput': 110 | logger.info(f'Throughput: {perf} ops/sec') 111 | elif target_metric == 'latency': 112 | logger.info(f'95-th Latency: {perf} milliseconds') 113 | else: 114 | raise NotADirectoryError() 115 | 116 | if (perf_stats is not None) and ('sample' in perf_stats): 117 | sample = perf_stats['sample'] 118 | logger.info(f'Point evaluated was: {sample}') 119 | 120 | # Keep best-conf updated 121 | if (state.best_perf is None) or state.is_better_perf(perf, state.best_perf): 122 | state.best_conf, state.best_perf = copy(sample), perf 123 | if (state.worse_perf is None) or not state.is_better_perf(perf, state.worse_perf): 124 | state.worse_perf = perf 125 | 126 | # Update optimizer results 127 | storage.store_result_summary( 128 | dict(zip(columns, [state.iter, perf, state.best_perf, runtime]))) 129 | state.iter += 1 130 | 131 | # Register sample to the optimizer -- optimizer always minimizes 132 | return perf if state.minimize else -perf 133 | 134 | 135 | # Parse args 136 | parser = argparse.ArgumentParser() 137 | parser.add_argument('conf_filepath') 138 | parser.add_argument('seed', type=int) 139 | args = parser.parse_args() 140 | 141 | config.update_from_file(args.conf_filepath) 142 | config.seed = args.seed 143 | 144 | # Set global random state 145 | fix_global_random_state(seed=config.seed) 146 | 147 | # init input & output space 148 | spaces = ConfigSpaceGenerator.from_config(config) 149 | target_metric = spaces.target_metric 150 | 151 | # init storage class 152 | perf_label = 'Throughput' if target_metric == 'throughput' else 'Latency' 153 | columns = ['Iteration', perf_label, 'Optimum', 'Runtime'] 154 | 155 | benchmark, workload = ( 156 | config['benchmark_info']['name'], config['benchmark_info']['workload']) 157 | 158 | inner_path = Path(f'{benchmark}.{workload}') / f'seed{args.seed}' 159 | storage = StorageFactory.from_config(config, columns=columns, inner_path=inner_path) 160 | 161 | 162 | # store dbms & benchmark info in experiment state object 163 | benchmark_info_config = config.benchmark_info 164 | dbms_info_config = config.dbms_info 165 | results_path = Path(config['storage']['outdir']) / inner_path 166 | 167 | exp_state = ExperimentState( 168 | dbms_info_config, benchmark_info_config, results_path, target_metric) 169 | 170 | # Create a new optimizer 171 | optimizer = get_smac_optimizer(config, spaces, evaluate_dbms_conf, exp_state) 172 | 173 | # init executor 174 | executor = ExecutorFactory.from_config(config, spaces, storage) 175 | 176 | # evaluate on default config 177 | default_config = spaces.get_default_configuration() 178 | logger.info('Evaluating Default Configuration') 179 | logger.debug(default_config) 180 | perf = evaluate_dbms_conf(default_config, state=exp_state) 181 | perf = perf if exp_state.minimize else -perf 182 | assert perf >= 0, \ 183 | f'Performance should not be negative: perf={perf}, metric={target_metric}' 184 | 185 | # set starting point for worse performance 186 | exp_state.worse_perf = perf * 4 if exp_state.minimize else perf / 4 187 | 188 | # Start optimization loop 189 | if hasattr(optimizer, 'optimize'): 190 | optimizer.optimize() # SMAC 191 | else: 192 | optimizer.run() # OpenBox 193 | 194 | 195 | # Print final stats 196 | logger.info(f'\nBest Configuration:\n{exp_state.best_conf}') 197 | if target_metric == 'throughput': 198 | logger.info(f'Throughput: {exp_state.best_perf} ops/sec') 199 | else: 200 | logger.info(f'95-th Latency: {exp_state.best_perf} milliseconds') 201 | logger.info(f'Saved @ {storage.outdir}') 202 | -------------------------------------------------------------------------------- /run-ddpg.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import random 3 | from copy import copy, deepcopy 4 | from pathlib import Path 5 | 6 | import pandas as pd 7 | pd.set_option('display.max_columns', None) 8 | import numpy as np 9 | 10 | from config import config 11 | from executors.executor import ExecutorFactory 12 | from optimizer import get_new_optimizer, get_ddpg_optimizer 13 | from space import ConfigSpaceGenerator 14 | from storage import StorageFactory 15 | 16 | from smac.facade.smac_hpo_facade import SMAC4AC 17 | 18 | # pylint: disable=logging-fstring-interpolation 19 | import logging 20 | logging.basicConfig(level=logging.INFO) 21 | logger = logging.getLogger() 22 | 23 | def fix_global_random_state(seed=None): 24 | random.seed(seed) 25 | np.random.seed(seed) 26 | 27 | class ExperimentState: 28 | def __init__(self, dbms_info, benchmark_info, results_path: Path, target_metric: str): 29 | self.iter = 0 30 | self.best_conf = None 31 | self.best_perf = None 32 | self.worse_perf = None 33 | self.default_perf_stats = None 34 | 35 | assert target_metric in ['throughput', 'latency'], \ 36 | f'Unsupported target metric: {target_metric}' 37 | self.minimize = target_metric != 'throughput' 38 | self._target_metric = target_metric 39 | 40 | self._dbms_info = dbms_info 41 | self._benchmark_info = benchmark_info 42 | 43 | assert results_path.exists() 44 | self._results_path = str(results_path) 45 | 46 | @property 47 | def benchmark_info(self): 48 | return self._benchmark_info 49 | 50 | @property 51 | def dbms_info(self): 52 | return self._dbms_info 53 | 54 | @property 55 | def results_path(self) -> str: 56 | return self._results_path 57 | 58 | @property 59 | def target_metric(self) -> str: 60 | return self._target_metric 61 | 62 | @property 63 | def default_perf(self) -> float: 64 | return self.default_perf_stats[self.target_metric] 65 | 66 | def is_better_perf(self, perf, other): 67 | return (perf > other) if not self.minimize else (perf < other) 68 | 69 | def __str__(self): 70 | fields = ['iter', 'best_conf', 'best_perf', 'worse_perf', 71 | 'default_perf_stats', 'target_metric'] 72 | return ':\n' + \ 73 | '\n'.join([ f'{f}: \t{getattr(self, f)}' for f in fields ]) 74 | 75 | 76 | def evaluate_dbms_conf(sample, state=None): 77 | global spaces, executor 78 | 79 | logger.info(f'\n\n{25*"="} Iteration {state.iter:2d} {25*"="}\n\n') 80 | logger.info('Sample from optimizer: ') 81 | logger.info(sample) 82 | 83 | if state.iter > 0: # if not default conf 84 | sample = spaces.unproject_input_point(sample) 85 | conf = spaces.finalize_conf(sample) 86 | logger.info(f'Evaluating Configuration:\n{conf}') 87 | 88 | ## Send configuration task to Nautilus 89 | dbms_info = dict( 90 | name=state.dbms_info['name'], 91 | config=conf, 92 | version=state.dbms_info['version'] 93 | ) 94 | perf_stats, metrics = executor.evaluate_configuration(dbms_info, state.benchmark_info) 95 | logger.info(f'Performance Statistics:\n{perf_stats}') 96 | 97 | if state.default_perf_stats is None: 98 | state.default_perf_stats = perf_stats 99 | 100 | target_metric = state.target_metric 101 | if perf_stats is None: 102 | # Error while evaluating conf -- set some reasonable value for target metric 103 | runtime, perf = 0, state.worse_perf 104 | else: 105 | if target_metric == 'latency' and \ 106 | perf_stats['throughput'] < state.default_perf_stats['throughput']: 107 | # Throughput less than default -- invalid round 108 | runtime, perf = perf_stats['runtime'], state.worse_perf 109 | else: 110 | runtime, perf = perf_stats['runtime'], perf_stats[target_metric] 111 | 112 | logger.info(f'Evaluation took {runtime} seconds') 113 | if target_metric == 'throughput': 114 | logger.info(f'Throughput: {perf} ops/sec') 115 | elif target_metric == 'latency': 116 | logger.info(f'95-th Latency: {perf} milliseconds') 117 | else: 118 | raise NotADirectoryError() 119 | 120 | if (perf_stats is not None) and ('sample' in perf_stats): 121 | sample = perf_stats['sample'] 122 | logger.info(f'Point evaluated was: {sample}') 123 | 124 | # Keep best-conf updated 125 | if (state.best_perf is None) or state.is_better_perf(perf, state.best_perf): 126 | state.best_conf, state.best_perf = copy(sample), perf 127 | if (state.worse_perf is None) or not state.is_better_perf(perf, state.worse_perf): 128 | state.worse_perf = perf 129 | 130 | # Update optimizer results 131 | storage.store_result_summary( 132 | dict(zip(columns, [state.iter, perf, state.best_perf, runtime]))) 133 | state.iter += 1 134 | 135 | # Register sample to the optimizer -- optimizer always minimizes 136 | perf = perf if state.minimize else -perf 137 | return perf, metrics 138 | 139 | 140 | # Parse args 141 | parser = argparse.ArgumentParser() 142 | parser.add_argument('conf_filepath') 143 | parser.add_argument('seed', type=int) 144 | args = parser.parse_args() 145 | 146 | config.update_from_file(args.conf_filepath) 147 | config.seed = args.seed 148 | ### number of DBMS internal metrics being sampled 149 | config.num_dbms_metrics = 27 150 | 151 | # Set global random state 152 | fix_global_random_state(seed=config.seed) 153 | 154 | # init input & output space 155 | spaces = ConfigSpaceGenerator.from_config(config) 156 | target_metric = spaces.target_metric 157 | 158 | # init storage class 159 | perf_label = 'Throughput' if target_metric == 'throughput' else 'Latency' 160 | columns = ['Iteration', perf_label, 'Optimum', 'Runtime'] 161 | 162 | benchmark, workload = ( 163 | config['benchmark_info']['name'], config['benchmark_info']['workload']) 164 | 165 | inner_path = Path(f'{benchmark}.{workload}') / f'seed{args.seed}' 166 | storage = StorageFactory.from_config(config, columns=columns, inner_path=inner_path) 167 | 168 | 169 | # store dbms & benchmark info in experiment state object 170 | benchmark_info_config = config.benchmark_info 171 | dbms_info_config = config.dbms_info 172 | results_path = Path(config['storage']['outdir']) / inner_path 173 | 174 | exp_state = ExperimentState( 175 | dbms_info_config, benchmark_info_config, results_path, target_metric) 176 | 177 | 178 | # Create a new optimizer 179 | optimizer = get_ddpg_optimizer(config, spaces, evaluate_dbms_conf, exp_state) 180 | 181 | # init executor 182 | executor = ExecutorFactory.from_config(config, spaces, storage, parse_metrics=True, 183 | num_dbms_metrics=config.num_dbms_metrics) 184 | 185 | # evaluate on default config 186 | default_config = spaces.get_default_configuration() 187 | logger.info('Evaluating Default Configuration') 188 | logger.debug(default_config) 189 | perf, default_metrics = evaluate_dbms_conf(default_config, state=exp_state) 190 | perf = perf if exp_state.minimize else -perf 191 | assert perf >= 0, \ 192 | f'Performance should not be negative: perf={perf}, metric={target_metric}' 193 | assert len(default_metrics) == config.num_dbms_metrics, \ 194 | ('DBMS metrics number does not match with expected: ' 195 | f'[ret={len(default_metrics)}] [exp={config.num_dbms_metrics}]') 196 | 197 | # set starting point for worse performance 198 | exp_state.worse_perf = perf * 4 if exp_state.minimize else perf / 4 199 | 200 | # run DDPG 201 | optimizer.run() 202 | 203 | # Print final stats 204 | logger.info(f'\nBest Configuration:\n{exp_state.best_conf}') 205 | if target_metric == 'throughput': 206 | logger.info(f'Throughput: {exp_state.best_perf} ops/sec') 207 | else: 208 | logger.info(f'95-th Latency: {exp_state.best_perf} milliseconds') 209 | logger.info(f'Saved @ {storage.outdir}') 210 | -------------------------------------------------------------------------------- /space.py: -------------------------------------------------------------------------------- 1 | import json 2 | from copy import deepcopy 3 | from importlib import import_module 4 | from pathlib import Path 5 | 6 | import ConfigSpace as CS 7 | import ConfigSpace.hyperparameters as CSH 8 | 9 | from adapters import * 10 | 11 | class ConfigSpaceGenerator: 12 | knob_types = ['enum', 'integer', 'real'] 13 | valid_adapter_aliases = ['none', 'tree', 'rembo', 'hesbo'] 14 | 15 | def __init__(self, definition=None, include=None, ignore=None, target_metric=None, 16 | adapter_alias='none', le_low_dim=None, bias_prob_sv=None, 17 | quantization_factor=None, finalize_conf_func=None, unfinalize_conf_func=None): 18 | if (ignore is not None) and (include is not None): 19 | raise ValueError("Define either `ignore_knobs' or `include_knobs'") 20 | assert isinstance(target_metric, str), 'Target metric should be of type string' 21 | assert adapter_alias in self.valid_adapter_aliases, \ 22 | f"Valid values for `adapter_alias' are: {self.valid_adapter_aliases}" 23 | 24 | assert finalize_conf_func != None, 'Space should define "finalize_conf" function' 25 | self.finalize_conf_func = staticmethod(finalize_conf_func) 26 | assert unfinalize_conf_func != None, 'Space should define "unfinalize_conf" function' 27 | self.unfinalize_conf_func = staticmethod(unfinalize_conf_func) 28 | 29 | if bias_prob_sv is not None: 30 | bias_prob_sv = float(bias_prob_sv) 31 | assert 0 < bias_prob_sv < 1, 'Bias sampling prob should be in (0, 1) range' 32 | self._bias_prob_sv = bias_prob_sv 33 | 34 | if quantization_factor is not None: 35 | quantization_factor = int(quantization_factor) 36 | assert quantization_factor > 1, \ 37 | 'Quantization should be an integer value, larger than 1' 38 | assert (adapter_alias == 'none' and bias_prob_sv is None) or \ 39 | adapter_alias in ['rembo', 'hesbo'] 40 | self._quantization_factor = quantization_factor 41 | 42 | all_knobs = set([ d['name'] for d in definition ]) 43 | include_knobs = include if include is not None else all_knobs - set(ignore) 44 | 45 | self.knobs = [ info for info in definition if info['name'] in include_knobs ] 46 | self.knobs_dict = { d['name']: d for d in self.knobs } 47 | 48 | self._target_metric = target_metric # to be used by the property 49 | self._adapter_alias = adapter_alias 50 | 51 | if adapter_alias in ['rembo', 'hesbo']: 52 | assert le_low_dim is not None, 'Linear embedding target dimensions not defined' 53 | self._le_low_dim = int(le_low_dim) 54 | assert self._le_low_dim < len(self.knobs), \ 55 | 'Linear embedding target dim should be less than original space' 56 | 57 | @property 58 | def target_metric(self): 59 | return self._target_metric 60 | 61 | def generate_input_space(self, seed: int, ignore_extra_knobs=None): 62 | ignore_extra_knobs = ignore_extra_knobs or [ ] 63 | 64 | input_dimensions = [ ] 65 | for info in self.knobs: 66 | name, knob_type = info['name'], info['type'] 67 | if name in ignore_extra_knobs: 68 | continue 69 | 70 | if knob_type not in self.knob_types: 71 | raise NotImplementedError(f'Knob type of "{knob_type}" is not supported :(') 72 | 73 | ## Categorical 74 | if knob_type == 'enum': 75 | dim = CSH.CategoricalHyperparameter( 76 | name=name, 77 | choices=info['choices'], 78 | default_value=info['default']) 79 | ## Numerical 80 | elif knob_type == 'integer': 81 | dim = CSH.UniformIntegerHyperparameter( 82 | name=name, 83 | lower=info['min'], 84 | upper=info['max'], 85 | default_value=info['default']) 86 | elif knob_type == 'real': 87 | dim = CSH.UniformFloatHyperparameter( 88 | name=name, 89 | lower=info['min'], 90 | upper=info['max'], 91 | default_value=info['default']) 92 | 93 | input_dimensions.append(dim) 94 | 95 | input_space = CS.ConfigurationSpace(name="input", seed=seed) 96 | input_space.add_hyperparameters(input_dimensions) 97 | 98 | if hasattr(self, 'input_space'): 99 | print('WARNING: Overriding input_space class variable') 100 | self.input_space = input_space 101 | 102 | self._input_space_adapter = None 103 | if self._adapter_alias == 'none': 104 | if self._bias_prob_sv is not None: 105 | # biased sampling 106 | self._input_space_adapter = PostgresBiasSampling( 107 | input_space, seed, self._bias_prob_sv) 108 | return self._input_space_adapter.target 109 | 110 | if self._quantization_factor is not None: 111 | self._input_space_adapter = Quantization( 112 | input_space, seed, self._quantization_factor) 113 | return self._input_space_adapter.target 114 | 115 | return input_space 116 | 117 | elif self._adapter_alias == 'tree': 118 | self._input_space_adapter = PostgresTreeConfigSpace( 119 | input_space, seed, bias_prob_sv=self._bias_prob_sv) 120 | return self._input_space_adapter.target 121 | 122 | else: 123 | assert self._adapter_alias in ['rembo', 'hesbo'] 124 | 125 | if self._bias_prob_sv is not None: 126 | # biased sampling 127 | input_space = PostgresBiasSampling( 128 | input_space, seed, self._bias_prob_sv).target 129 | 130 | self._input_space_adapter = LinearEmbeddingConfigSpace.create( 131 | input_space, seed, 132 | method=self._adapter_alias, 133 | target_dim=self._le_low_dim, 134 | bias_prob_sv=self._bias_prob_sv, 135 | max_num_values=self._quantization_factor) 136 | return self._input_space_adapter.target 137 | 138 | def get_default_configuration(self): 139 | return self.input_space.get_default_configuration() 140 | 141 | def finalize_conf(self, conf, n_decimals=2): 142 | return self.finalize_conf_func.__func__( 143 | conf, self.knobs_dict, n_decimals=n_decimals) 144 | 145 | def unfinalize_conf(self, conf): 146 | return self.unfinalize_conf_func.__func__(conf, self.knobs_dict) 147 | 148 | def unproject_input_point(self, point): 149 | if self._input_space_adapter: 150 | return self._input_space_adapter.unproject_point(point) 151 | return point 152 | 153 | 154 | @classmethod 155 | def from_config(cls, config): 156 | valid_config_fields = ['definition', 'include', 'ignore', 'target_metric', 157 | 'adapter_alias', 'le_low_dim', 'bias_prob_sv', 'quantization_factor'] 158 | 159 | spaces_config = deepcopy(config['spaces']) 160 | assert all(field in valid_config_fields for field in spaces_config), \ 161 | 'Configuration contains invalid fields: ' \ 162 | f'{set(spaces_config.keys()) - set(valid_config_fields)}' 163 | assert 'definition' in spaces_config, 'Spaces section should contain "definition" key' 164 | assert 'include' in spaces_config or 'ignore' in spaces_config, \ 165 | 'Spaces section should contain "include" or "ignore" key' 166 | assert not ('include' in spaces_config and 'ignore' in spaces_config), \ 167 | 'Spaces section should not contain both "include" and "ignore" keys' 168 | assert 'target_metric' in spaces_config, \ 169 | 'Spaces section should contain "target_metric" key' 170 | if 'le_low_dim' in spaces_config: 171 | assert spaces_config['adapter_alias'] in ['rembo', 'hesbo'], \ 172 | 'Linear embedding low dimension is only valid in REMBO & HesBO' 173 | 174 | # Read space definition from json file 175 | definition_fp = Path('./spaces/definitions') / f"{spaces_config['definition']}.json" 176 | with open(definition_fp, 'r') as f: 177 | definition = json.load(f) 178 | all_knobs_name = [ d['name'] for d in definition ] 179 | 180 | # Import designated module and utilize knobs 181 | include = 'include' in spaces_config 182 | module_name = spaces_config['include'] if include else spaces_config['ignore'] 183 | import_path = module_name.replace('/', '.') 184 | module = import_module(f"spaces.{import_path}") 185 | knobs = getattr(module, 'KNOBS') 186 | assert isinstance(knobs, list) and all([ k in all_knobs_name for k in knobs]) 187 | finalize_conf_func = getattr(module, 'finalize_conf') 188 | unfinalize_conf_func = getattr(module, 'unfinalize_conf') 189 | 190 | return cls(definition=definition, 191 | include=knobs if include else None, 192 | ignore=knobs if not include else None, 193 | target_metric=spaces_config['target_metric'], 194 | adapter_alias=spaces_config.get('adapter_alias', 'none'), 195 | le_low_dim=spaces_config.get('le_low_dim', None), 196 | bias_prob_sv=spaces_config.get('bias_prob_sv', None), 197 | quantization_factor=spaces_config.get('quantization_factor', None), 198 | finalize_conf_func=finalize_conf_func, 199 | unfinalize_conf_func=unfinalize_conf_func) 200 | -------------------------------------------------------------------------------- /adapters/configspace/low_embeddings.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from typing import Optional 3 | 4 | import ConfigSpace as CS 5 | import ConfigSpace.hyperparameters as CSH 6 | import numpy as np 7 | from sklearn.preprocessing import MinMaxScaler 8 | 9 | from adapters.bias_sampling import UniformIntegerHyperparameterWithSpecialValue, special_value_scaler 10 | 11 | import logging 12 | logger = logging.getLogger(__name__) 13 | 14 | class LinearEmbeddingConfigSpace(ABC): 15 | def __init__(self, 16 | adaptee: CS.ConfigurationSpace, seed: int, target_dim: int, 17 | bias_prob_sv: Optional[float] = None, # biased-sampling 18 | max_num_values: Optional[int] = None, # quantization 19 | ): 20 | 21 | self._adaptee: CS.ConfigurationSpace = adaptee 22 | self._target: CS.ConfigurationSpace = None 23 | self._seed: int = seed 24 | self._target_dim: int = target_dim 25 | self._bias_prob_sv: Optional[float] = bias_prob_sv 26 | self._max_num_values: Optional[int] = max_num_values 27 | 28 | self._rs = np.random.RandomState(seed=self._seed) 29 | 30 | self._build_space() 31 | 32 | @abstractmethod 33 | def _build_space(self): 34 | raise NotImplementedError() 35 | 36 | def _get_active_hps(self): 37 | # NOTE: add multi-choice categorical vars 38 | return self.adaptee.get_hyperparameters() 39 | 40 | @property 41 | def adaptee(self) -> CS.ConfigurationSpace: 42 | return self._adaptee 43 | 44 | @property 45 | def target(self) -> CS.ConfigurationSpace: 46 | return self._target 47 | 48 | def project_point(self, point): 49 | raise NotImplementedError() 50 | 51 | @abstractmethod 52 | def unproject_point(self, point:CS.Configuration) -> dict: 53 | raise NotImplementedError() 54 | 55 | def project_dataframe(self, df, in_place: bool): 56 | raise NotImplementedError() 57 | 58 | def unproject_dataframe(self, df, in_place: bool): 59 | raise NotImplementedError() 60 | 61 | @staticmethod 62 | def create(*args, method: str = 'hesbo', **kwargs): 63 | if method not in ['hesbo', 'rembo']: 64 | raise ValueError("Supported methods are 'rembo', 'hesbo'") 65 | 66 | if method == 'rembo': 67 | return REMBOConfigSpace(*args, **kwargs) 68 | elif method == 'hesbo': 69 | return HesBOConfigSpace(*args, **kwargs) 70 | else: 71 | raise NotImplementedError() 72 | 73 | 74 | class REMBOConfigSpace(LinearEmbeddingConfigSpace): 75 | 76 | def _build_space(self): 77 | self.active_hps = self._get_active_hps() 78 | 79 | # Create lower dimensionality configuration space 80 | # NOTE: space bounds are [-sqrt(low_dim), sqrt(low_dim)] rather than [-1, 1] 81 | box_bound = np.sqrt(self._target_dim) 82 | target = CS.ConfigurationSpace( 83 | name=self._adaptee.name, seed=self._seed) 84 | 85 | if self._max_num_values is None: 86 | hps = [ 87 | CS.UniformFloatHyperparameter( 88 | name=f'rembo_{idx}', lower=-box_bound, upper=box_bound) 89 | for idx in range(self._target_dim) 90 | ] 91 | q_scaler = None 92 | else: 93 | logger.info(f'Using quantization: q={self._max_num_values}') 94 | hps = [ 95 | CS.UniformIntegerHyperparameter( 96 | name=f'rembo_{idx}', lower=1, upper=self._max_num_values) 97 | for idx in range(self._target_dim) 98 | ] 99 | # (1, q) -> (-sqrt(low_dim), sqrt(low_dim)) scaling 100 | q_scaler = MinMaxScaler(feature_range=(-box_bound, box_bound)) 101 | ones = np.ones(shape=self._target_dim) 102 | q_scaler.fit([ones, ones * self._max_num_values]) 103 | 104 | target.add_hyperparameters(hps) 105 | logger.info(target) 106 | 107 | self._target = target 108 | self._q_scaler = q_scaler 109 | 110 | # (-sqrt, sqrt) -> (0, 1) scaling 111 | self._scaler = MinMaxScaler(feature_range=(0, 1)) 112 | bbound_vector = np.ones(len(self.active_hps)) * box_bound 113 | # use two points (minimum & maximum) 114 | self._scaler.fit( 115 | np.array([-bbound_vector, bbound_vector])) 116 | 117 | # Create random project matrix: A ~ N(0,1) 118 | self._A = self._rs.normal( 119 | 0, 1, (len(self.active_hps), self._target_dim)) 120 | 121 | def unproject_point(self, point: CS.Configuration) -> dict: 122 | low_dim_point = np.array([ 123 | point.get(f'rembo_{idx}') for idx in range(len(point)) ]) 124 | 125 | if self._max_num_values is not None: 126 | assert self._q_scaler is not None # self-validate 127 | low_dim_point = self._q_scaler.transform([low_dim_point])[0] 128 | 129 | high_dim_point = [ 130 | np.dot(self._A[idx, :], low_dim_point) 131 | for idx in range(len(self.active_hps)) 132 | ] 133 | high_dim_point = self._scaler.transform([high_dim_point])[0] 134 | 135 | high_dim_conf = { } 136 | dims_clipped = 0 137 | for hp, value in zip(self.active_hps, high_dim_point): 138 | if value <= 0 or value >= 1: 139 | logger.warning('Point clipped in dim: %s' % hp.name) 140 | dims_clipped += 1 141 | # clip value to [0, 1] 142 | value = np.clip(value, 0., 1.) 143 | 144 | if isinstance(hp, CS.CategoricalHyperparameter): 145 | index = int(value * len(hp.choices)) # truncate integer part 146 | index = max(0, min(len(hp.choices) - 1, index)) 147 | # NOTE: rounding here would be unfair to first & last values 148 | value = hp.choices[index] 149 | elif isinstance(hp, CS.hyperparameters.NumericalHyperparameter): 150 | if isinstance(hp, UniformIntegerHyperparameterWithSpecialValue): 151 | value = special_value_scaler(hp, value) # bias special value 152 | 153 | value = hp._transform(value) 154 | value = max(hp.lower, min(hp.upper, value)) 155 | assert value >= hp.lower and value <= hp.upper 156 | else: 157 | raise NotImplementedError() 158 | 159 | high_dim_conf[hp.name] = value 160 | 161 | if dims_clipped > 0: 162 | logger.info('# dimensions clipped: %d' % dims_clipped) 163 | 164 | return high_dim_conf 165 | 166 | 167 | class HesBOConfigSpace(LinearEmbeddingConfigSpace): 168 | 169 | def _build_space(self): 170 | self.active_hps = self._get_active_hps() 171 | 172 | # Create lower dimensionality configuration space 173 | target = CS.ConfigurationSpace( 174 | name=self._adaptee.name, seed=self._seed) 175 | 176 | if self._max_num_values is None: 177 | hps = [ 178 | CS.UniformFloatHyperparameter( 179 | name=f'hesbo_{idx}', lower=-1, upper=1) 180 | for idx in range(self._target_dim) 181 | ] 182 | else: 183 | logger.info(f'Using quantization: q={self._max_num_values}') 184 | q = 2. / self._max_num_values 185 | hps = [ 186 | CS.UniformFloatHyperparameter( 187 | name=f'hesbo_{idx}', lower=-1, upper=1, q=q) 188 | for idx in range(self._target_dim) 189 | ] 190 | 191 | target.add_hyperparameters(hps) 192 | logger.info(target) 193 | 194 | self._target = target 195 | 196 | # (-1, 1) -> (0, 1) scaling 197 | self._scaler = MinMaxScaler(feature_range=(0, 1)) 198 | ones = np.ones(len(self.active_hps)) 199 | # use two points (minimum & maximum) 200 | self._scaler.fit( 201 | np.array([-ones, ones])) 202 | 203 | # Implicitely define matrix S' 204 | self._h = self._rs.choice( 205 | range(self._target_dim), len(self.active_hps)) 206 | self._sigma = self._rs.choice([-1, 1], len(self.active_hps)) 207 | 208 | def unproject_point(self, point: CS.Configuration) -> dict: 209 | low_dim_point = [ 210 | point.get(f'hesbo_{idx}') for idx in range(len(point)) ] 211 | 212 | # OLD METHOD OF QUANTIZATION 213 | #if self._max_num_values is not None and self.q_use_integer: 214 | # assert self._q_scaler is not None # self-validate 215 | # low_dim_point = self._q_scaler.transform([low_dim_point])[0] 216 | 217 | high_dim_point = [ 218 | self._sigma[idx] * low_dim_point[self._h[idx]] 219 | for idx in range(len(self.active_hps)) 220 | ] 221 | high_dim_point = self._scaler.transform([high_dim_point])[0] 222 | 223 | high_dim_conf = { } 224 | for hp, value in zip(self.active_hps, high_dim_point): 225 | # hesbo does not project values outside of range 226 | # NOTE: need this cause of weird floating point errors 227 | value = max(0, min(1, value)) 228 | 229 | if isinstance(hp, CS.CategoricalHyperparameter): 230 | index = int(value * len(hp.choices)) # truncate integer part 231 | index = max(0, min(len(hp.choices) - 1, index)) 232 | # NOTE: rounding here would be unfair to first & last values 233 | value = hp.choices[index] 234 | elif isinstance(hp, CS.hyperparameters.NumericalHyperparameter): 235 | if isinstance(hp, UniformIntegerHyperparameterWithSpecialValue): 236 | value = special_value_scaler(hp, value) # bias special value 237 | 238 | value = hp._transform(value) 239 | value = max(hp.lower, min(hp.upper, value)) 240 | assert value >= hp.lower and value <= hp.upper 241 | else: 242 | raise NotImplementedError() 243 | 244 | high_dim_conf[hp.name] = value 245 | 246 | return high_dim_conf 247 | -------------------------------------------------------------------------------- /optimizer.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from functools import partial 3 | 4 | import numpy as np 5 | import pandas as pd 6 | 7 | from config import config 8 | from adapters import LHDesignWithBiasedSampling 9 | 10 | # SMAC 11 | from smac.facade.smac_hpo_facade import SMAC4HPO 12 | from smac.facade.smac_bb_facade import SMAC4BB 13 | from smac.scenario.scenario import Scenario 14 | 15 | logging.basicConfig(level=logging.INFO) 16 | logger = logging.getLogger() 17 | 18 | def initialize_optimizer(optimization_problem, rand_percentage=0.1, n_estimators=100): 19 | optimizer_config = bayesian_optimizer_config_store.default 20 | # set the fraction of randomly sampled configuration to 10% of suggestions 21 | optimizer_config.experiment_designer_config.fraction_random_suggestions = rand_percentage 22 | # Set multiplier for the confidence bound 23 | optimizer_config.experiment_designer_config.confidence_bound_utility_function_config.alpha = 0.1 24 | 25 | # configure the random forest surrogate mode 26 | random_forest_config = optimizer_config.homogeneous_random_forest_regression_model_config 27 | random_forest_config.decision_tree_regression_model_config.n_new_samples_before_refit = 1 28 | # Use the best split in trees (not random as in extremely randomized trees) 29 | random_forest_config.decision_tree_regression_model_config.splitter = 'best' 30 | # right now we're sampling without replacement so we need to subsample 31 | # to make the trees different when using the 'best' splitter 32 | #random_forest_config.samples_fraction_per_estimator = .9 33 | random_forest_config.samples_fraction_per_estimator = 1 34 | random_forest_config.n_estimators = n_estimators 35 | 36 | optimizer_factory = BayesianOptimizerFactory() 37 | return optimizer_factory.create_local_optimizer( 38 | optimization_problem=optimization_problem, 39 | optimizer_config=optimizer_config 40 | ) 41 | 42 | def get_new_optimizer(spaces, ignore_knobs=None, bootstrap_points=None): 43 | bootstrap_points = bootstrap_points or [ ] 44 | 45 | logger.info('Constructing new optimizer...') 46 | logger.info(f'Len(Evaluated points): {len(bootstrap_points)}') 47 | logger.info(f'Ignored knobs: {ignore_knobs}') 48 | 49 | # Generate input (i.e. knobs) and output (i.e. perf metric) space 50 | input_space = spaces.generate_input_space(ignore_extra_knobs=ignore_knobs) 51 | output_space = spaces.generate_output_space() 52 | 53 | logger.debug(f'\ninput space:\n{input_space}') 54 | logger.debug(f'\noutput space:\n{output_space}') 55 | 56 | # Initialize optimizer 57 | optimization_problem = OptimizationProblem( 58 | parameter_space=input_space, 59 | objective_space=output_space, 60 | objectives=[ 61 | Objective(name=config['spaces']['target_metric'], minimize=False)] 62 | ) 63 | rand_percentage = float(config['optimizer']['rand_percentage']) 64 | assert 0 <= rand_percentage <= 1, 'Optimizer rand optimizer must be between 0 and 1' 65 | 66 | n_estimators = int(config['optimizer']['n_estimators']) 67 | optimizer = initialize_optimizer(optimization_problem, 68 | rand_percentage=rand_percentage, n_estimators=n_estimators) 69 | 70 | # Fix optimizer random state 71 | fix_optimizer_random_state(optimizer, seed=config.seed) 72 | 73 | # Train optimizer with previously-evaluated points 74 | if len(bootstrap_points) > 0: 75 | samples, outputs = map(pd.concat, zip(*bootstrap_points)) 76 | samples = samples.drop(columns=ignore_knobs) # remove pruned columns 77 | 78 | optimizer.register(samples, outputs) # fit optimizer with points 79 | 80 | return optimizer 81 | 82 | def get_smac_optimizer(config, spaces, tae_runner, state, 83 | ignore_knobs=None, run_history=None): 84 | 85 | logger.info('Constructing new optimizer...') 86 | logger.info(f'Run History: {run_history}') 87 | logger.info(f'Ignored knobs: {ignore_knobs}') 88 | 89 | # Generate input (i.e. knobs) and output (i.e. perf metric) space 90 | input_space = spaces.generate_input_space( 91 | config.seed, ignore_extra_knobs=ignore_knobs) 92 | 93 | logger.info(f'\ninput space:\n{input_space}') 94 | 95 | scenario = Scenario({ 96 | "run_obj": "quality", 97 | "runcount-limit": config.iters, 98 | "cs": input_space, 99 | "deterministic": "true", 100 | "always_race_default": "false", 101 | # disables pynisher, which allows for shared state 102 | "limit_resources": "false", 103 | "output_dir": state.results_path, 104 | }) 105 | # Latin Hypercube design, with 10 iters 106 | init_rand_samples = int(config['optimizer'].get('init_rand_samples', 10)) 107 | initial_design = LHDesignWithBiasedSampling 108 | initial_design_kwargs = { 109 | "init_budget": init_rand_samples, 110 | "max_config_fracs": 1, 111 | } 112 | 113 | # Get RF params from config 114 | rand_percentage = float(config['optimizer']['rand_percentage']) 115 | assert 0 <= rand_percentage <= 1, 'Optimizer rand optimizer must be between 0 and 1' 116 | n_estimators = int(config['optimizer']['n_estimators']) 117 | 118 | # how often to evaluate a random sample 119 | random_configuration_chooser_kwargs = { 120 | 'prob': rand_percentage, 121 | } 122 | tae_runner = partial(tae_runner, state=state) 123 | 124 | model_type = config['optimizer'].get('model_type', 'rf') # default is RF-SMACHPO 125 | assert model_type in ['rf', 'gp', 'mkbo'], 'Model type %s not supported' % model_type 126 | 127 | if model_type == 'rf': 128 | # RF model params -- similar to MLOS ones 129 | model_kwargs = { 130 | 'num_trees': n_estimators, 131 | 'log_y': False, # no log scale 132 | 'ratio_features': 1, # 133 | 'min_samples_split': 2, # min number of samples to perform a split 134 | 'min_samples_leaf': 3, # min number of smaples on a leaf node 135 | 'max_depth': 2**20, # max depth of tree 136 | } 137 | optimizer = SMAC4HPO( 138 | scenario=scenario, 139 | tae_runner=tae_runner, 140 | rng=config.seed, 141 | model_kwargs=model_kwargs, 142 | initial_design=initial_design, 143 | initial_design_kwargs=initial_design_kwargs, 144 | random_configuration_chooser_kwargs=random_configuration_chooser_kwargs, 145 | ) 146 | 147 | elif model_type == 'gp': 148 | optimizer = SMAC4BB( 149 | model_type='gp', 150 | scenario=scenario, 151 | tae_runner=tae_runner, 152 | rng=config.seed, 153 | initial_design=initial_design, 154 | initial_design_kwargs=initial_design_kwargs, 155 | random_configuration_chooser_kwargs=random_configuration_chooser_kwargs, 156 | ) 157 | 158 | elif model_type == 'mkbo': 159 | # OpenBox 160 | import openbox 161 | openbox.utils.limit._platform = 'Windows' # Patch to avoid objective function wrapping 162 | optimizer = openbox.Optimizer( 163 | tae_runner, 164 | input_space, 165 | num_objs=1, 166 | num_constraints=0, 167 | max_runs=config.iters, 168 | surrogate_type='gp', 169 | acq_optimizer_type='local_random', 170 | initial_runs=10, 171 | init_strategy='random_explore_first', 172 | time_limit_per_trial=10**6, 173 | logging_dir=state.results_path, 174 | random_state=config.seed, 175 | ) 176 | 177 | logger.info(optimizer) 178 | return optimizer 179 | 180 | 181 | def get_ddpg_optimizer(config, spaces, tae_runner, state): 182 | logger.info('Constructing new optimizer...') 183 | 184 | # Generate input (i.e. knobs) and output (i.e. perf metric) space 185 | input_space = spaces.generate_input_space( 186 | config.seed, ignore_extra_knobs=None) 187 | logger.info(f'\ninput space:\n{input_space}') 188 | 189 | # random number generator 190 | rng = np.random.RandomState(seed=config.seed) 191 | 192 | from smac.stats.stats import Stats 193 | from smac.utils.io.traj_logging import TrajLogger 194 | from smac.utils.io.output_directory import create_output_directory 195 | 196 | scenario = Scenario({ 197 | "run_obj": "quality", 198 | "runcount-limit": config.iters, 199 | "cs": input_space, 200 | "deterministic": "true", 201 | "always_race_default": "false", 202 | # disables pynisher, which allows for shared state 203 | "limit_resources": "false", 204 | "output_dir": state.results_path, 205 | }) 206 | 207 | output_dir = create_output_directory(scenario, 0) 208 | stats = Stats(scenario) 209 | traj_logger = TrajLogger(output_dir=output_dir, stats=stats) 210 | 211 | # Latin Hypercube design, with 10 iters 212 | init_rand_samples = int(config['optimizer'].get('init_rand_samples', 10)) 213 | init_design_def_kwargs = { 214 | "cs": input_space, 215 | "rng": rng, 216 | "traj_logger": traj_logger, # required 217 | "ta_run_limit": 99999999999, 218 | "max_config_fracs": 1, 219 | "init_budget": init_rand_samples, 220 | } 221 | initial_design = LHDesignWithBiasedSampling(**init_design_def_kwargs) 222 | 223 | # Random conf chooser 224 | from smac.optimizer.random_configuration_chooser import ChooserProb 225 | 226 | rand_percentage = float(config['optimizer']['rand_percentage']) 227 | assert 0 <= rand_percentage <= 1, 'Optimizer rand optimizer must be between 0 and 1' 228 | rcc_rng = np.random.RandomState(seed=config.seed) 229 | rand_conf_chooser = ChooserProb(rcc_rng, rand_percentage) 230 | 231 | # DDPG Model 232 | from ddpg.ddpg import DDPG, DDPGOptimizer 233 | n_states = config.num_dbms_metrics 234 | n_actions = len(input_space) 235 | 236 | model = DDPG(n_states, n_actions, model_name='ddpg_model') 237 | 238 | tae_runner = partial(tae_runner, state=state) 239 | optimizer = DDPGOptimizer(state, model, tae_runner, initial_design, rand_conf_chooser, 240 | config.iters, logging_dir=state.results_path) 241 | 242 | return optimizer 243 | -------------------------------------------------------------------------------- /adapters/bias_sampling.py: -------------------------------------------------------------------------------- 1 | import io 2 | import logging 3 | from functools import partial 4 | from typing import Optional, Union, List 5 | 6 | import ConfigSpace as CS 7 | import ConfigSpace.hyperparameters as CSH 8 | import numpy as np 9 | 10 | from smac.initial_design.latin_hypercube_design import LHDesign 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | KNOBS_WITH_SPECIAL_VALUES = { 15 | 'autovacuum_vacuum_cost_delay': { 16 | # Value of -1 infers the value from `vacuum_cost_delay' 17 | 'subgrid': 'autovacuum', 18 | 'special_value': -1, 19 | }, 20 | 'autovacuum_vacuum_cost_limit': { 21 | # Value of -1 infers the value from `vacuum_cost_limit' (default) 22 | 'subgrid': 'autovacuum', 23 | 'special_value': -1, 24 | }, 25 | 'autovacuum_work_mem': { 26 | # Value of -1 infers the value from `maintenance_work_mem' param (default) 27 | 'subgrid': 'autovacuum', 28 | 'special_value': -1, 29 | }, 30 | 'backend_flush_after': { 31 | # Value of 0 disables forced writeback (default) 32 | 'special_value': 0, 33 | }, 34 | 'bgwriter_flush_after': { 35 | # Value of 0 disables forced writeback 36 | 'subgrid': 'bgwriter', 37 | 'special_value': 0, 38 | }, 39 | 'bgwriter_lru_maxpages': { 40 | # Value of 0 disables background writing 41 | 'subgrid': True, 42 | 'special_value': 0, 43 | }, 44 | 'checkpoint_flush_after': { 45 | # Value of 0, disables forced writeback (default) 46 | 'special_value': 0, 47 | }, 48 | 'effective_io_concurrency': { 49 | # Value of 0 disables issuance of asynchronous I/O requests 50 | 'special_value': 0, 51 | }, 52 | 'geqo_generations': { 53 | # Value will be infered from `geqo_effort' knob (default) 54 | 'subgrid': 'geqo', 55 | 'special_value': 0, 56 | }, 57 | 'geqo_pool_size': { 58 | # Value will be infered from `geqo_effort' knob (default) 59 | 'subgrid': 'geqo', 60 | 'special_value': 0, 61 | }, 62 | 'max_parallel_workers_per_gather': { 63 | # Value of 0 disables parallel query execution (default) 64 | 'special_value': 0, 65 | }, 66 | 'max_prepared_transactions': { 67 | # Value of 0 disables the prepared-transaction feature (default) 68 | 'special_value': 0, 69 | }, 70 | 'old_snapshot_threshold': { 71 | # Value of -1 disables the feature (default) 72 | 'special_value': -1, 73 | }, 74 | 'temp_file_limit': { 75 | # Value of -1 means no limit 76 | 'special_value': -1, 77 | }, 78 | 'vacuum_cost_delay': { 79 | # Value of 0, disables the cost-based vacuum delay feature (default) 80 | 'subgrid': True, 81 | 'special_value': 0, 82 | }, 83 | 'wal_buffers': { 84 | # Value of 1 infers value from `shared_buffers' (default) 85 | # More specifically: wal_buffers = 1/32 * shared_buffers 86 | 'special_value': -1, 87 | }, 88 | 'wal_writer_flush_after': { 89 | # Value of 0 forces WAL data to be flushed immediately 90 | 'special_value': 0, 91 | }, 92 | } 93 | 94 | def special_value_scaler(hp, value): 95 | if value < hp._special_value_prob: 96 | # Fix value to special value 97 | return hp._inverse_transform(hp._special_value) 98 | # uniformly scale remaining percentage to all other values 99 | return hp._inverse_transform(hp._transform_scalar( 100 | (value - hp._special_value_prob) / (1. - hp._special_value_prob))) 101 | 102 | 103 | class UniformIntegerHyperparameterWithSpecialValue(CS.UniformIntegerHyperparameter): 104 | def __init__(self, *args, special_value: Optional[int] = None, 105 | special_value_prob: Optional[float], **kwargs): 106 | super().__init__(*args, **kwargs) 107 | 108 | assert special_value >= self.lower and special_value <= self.upper, \ 109 | ('Special value [=%d] should be inside the range' % special_value) 110 | assert special_value_prob > 0 and special_value_prob < 1, \ 111 | ('Probability for special value should be in (0, 1)') 112 | assert special_value == self.lower, \ 113 | ('For now implementation supports only special value to be on the lower end') 114 | 115 | self._special_value = special_value 116 | self._special_value_prob = special_value_prob 117 | 118 | def _sample(self, rs: np.random.RandomState, size: Optional[int] = None 119 | ) -> Union[np.ndarray, float]: 120 | 121 | # Bias sample on the special value 122 | samples = rs.uniform(size=size) 123 | 124 | if size is None: 125 | return special_value_scaler(self, value) 126 | 127 | special_value_scaler_vector = partial(special_value_scaler, self) 128 | return np.array(list(map(special_value_scaler_vector, samples))) 129 | 130 | def __repr__(self) -> str: 131 | repr_str = io.StringIO() 132 | repr_str.write("%s, Type: UniformIntegerWithSpecialValues, Range: [%s, %s], Default: %s" 133 | % (self.name, repr(self.lower), 134 | repr(self.upper), repr(self.default_value))) 135 | if self.log: 136 | repr_str.write(", on log-scale") 137 | if self.q is not None: 138 | repr_str.write(", Q: %s" % repr(self.q)) 139 | repr_str.seek(0) 140 | return repr_str.getvalue() 141 | 142 | 143 | class LHDesignWithBiasedSampling(LHDesign): 144 | def _transform_continuous_designs(self, 145 | design: np.ndarray, 146 | origin: str, 147 | cs: CS.ConfigurationSpace) -> List[CS.Configuration]: 148 | 149 | params = cs.get_hyperparameters() 150 | for idx, param in enumerate(params): 151 | if isinstance(param, CSH.NumericalHyperparameter): 152 | if not isinstance(param, UniformIntegerHyperparameterWithSpecialValue): 153 | continue 154 | 155 | # handle bias to the special value 156 | special_value, special_value_prob = \ 157 | param._special_value, param._special_value_prob 158 | 159 | for row, v in enumerate(design[:, idx]): 160 | if v < special_value_prob: 161 | v = param._inverse_transform(special_value) 162 | else: 163 | v = param._inverse_transform(param._transform_scalar( 164 | (v - special_value_prob) / (1. - special_value_prob))) 165 | design[row, idx] = v 166 | 167 | elif isinstance(param, CSH.Constant): 168 | # add a vector with zeros 169 | design_ = np.zeros(np.array(design.shape) + np.array((0, 1))) 170 | design_[:, :idx] = design[:, :idx] 171 | design_[:, idx + 1:] = design[:, idx:] 172 | design = design_ 173 | elif isinstance(param, CSH.CategoricalHyperparameter): 174 | v_design = design[:, idx] 175 | v_design[v_design == 1] = 1 - 10**-10 176 | design[:, idx] = np.array(v_design * len(param.choices), dtype=np.int) 177 | elif isinstance(param, CSH.OrdinalHyperparameter): 178 | v_design = design[:, idx] 179 | v_design[v_design == 1] = 1 - 10**-10 180 | design[:, idx] = np.array(v_design * len(param.sequence), dtype=np.int) 181 | else: 182 | raise ValueError("Hyperparameter not supported in LHD") 183 | 184 | self.logger.debug("Initial Design") 185 | configs = [] 186 | for vector in design: 187 | conf = CS.util.deactivate_inactive_hyperparameters(configuration=None, 188 | configuration_space=cs, 189 | vector=vector) 190 | conf.origin = origin 191 | configs.append(conf) 192 | self.logger.debug(conf) 193 | 194 | self.logger.debug("Size of initial design: %d" % (len(configs))) 195 | 196 | return configs 197 | 198 | 199 | class PostgresBiasSampling: 200 | def __init__(self, adaptee: CS.ConfigurationSpace, seed: int, bias_prob_sv: float): 201 | self._adaptee: CS.ConfigurationSpace = adaptee 202 | self._target: CS.ConfigurationSpace = None 203 | self._seed: int = seed 204 | 205 | assert bias_prob_sv > 0 and bias_prob_sv < 1, \ 206 | 'Bias on the special values should be in (0, 1) range' 207 | self._bias_prob_sv: float = bias_prob_sv 208 | 209 | self._build_biased_config_space() 210 | 211 | def _build_biased_config_space(self): 212 | root_hyperparams = [ ] 213 | for adaptee_hp in self.adaptee.get_hyperparameters(): 214 | 215 | if adaptee_hp.name not in KNOBS_WITH_SPECIAL_VALUES.keys(): 216 | # no need to bias sampling 217 | root_hyperparams.append(adaptee_hp) 218 | continue 219 | 220 | # NOTE: assume splitting occurs in the lower point 221 | info = KNOBS_WITH_SPECIAL_VALUES[adaptee_hp.name] 222 | assert isinstance(adaptee_hp, CS.UniformIntegerHyperparameter) 223 | assert adaptee_hp.lower == info['special_value'] 224 | 225 | biased_hyperparam = UniformIntegerHyperparameterWithSpecialValue( 226 | adaptee_hp.name, adaptee_hp.lower, adaptee_hp.upper, 227 | default_value=adaptee_hp.default_value, 228 | special_value=info['special_value'], 229 | special_value_prob=self._bias_prob_sv, 230 | ) 231 | root_hyperparams.append(biased_hyperparam) 232 | 233 | root = CS.ConfigurationSpace( 234 | name=self._adaptee.name, 235 | seed=self._seed, 236 | ) 237 | root.add_hyperparameters(root_hyperparams) 238 | 239 | self._target = root 240 | 241 | @property 242 | def adaptee(self) -> CS.ConfigurationSpace: 243 | return self._adaptee 244 | 245 | @property 246 | def target(self) -> CS.ConfigurationSpace: 247 | return self._target 248 | 249 | def project_point(self, point): 250 | raise NotImplementedError() 251 | 252 | def unproject_point(self, point): 253 | coords = point.get_dictionary() 254 | valid_dim_names = [ dim.name for dim in self.adaptee.get_hyperparameters() ] 255 | assert all(dim in valid_dim_names for dim in coords.keys()) 256 | return coords 257 | 258 | def project_dataframe(self, df, in_place: bool): 259 | raise NotImplementedError 260 | 261 | def unproject_dataframe(self, df, in_place: bool): 262 | raise NotImplementedError 263 | -------------------------------------------------------------------------------- /ddpg/ddpg.py: -------------------------------------------------------------------------------- 1 | # from: https://raw.githubusercontent.com/cmu-db/ottertune/master/server/analysis/ddpg/ddpg.py 2 | 3 | import pickle 4 | import torch 5 | torch.set_num_threads(1) 6 | import numpy as np 7 | import torch.nn as nn 8 | import torch.optim as optimizer 9 | from torch.autograd import Variable 10 | 11 | from ddpg.prm import PrioritizedReplayMemory 12 | 13 | class Actor(nn.Module): 14 | 15 | def __init__(self, n_states, n_actions, hidden_sizes, use_default): 16 | super(Actor, self).__init__() 17 | if use_default: 18 | self.layers = nn.Sequential( 19 | nn.Linear(n_states, 128), 20 | nn.LeakyReLU(negative_slope=0.2), 21 | nn.BatchNorm1d(hidden_sizes[0]), 22 | nn.Linear(128, 128), 23 | nn.Tanh(), 24 | nn.Dropout(0.3), 25 | nn.Linear(128, 128), 26 | nn.Tanh(), 27 | nn.Linear(128, 64), 28 | nn.Linear(64, n_actions) 29 | ) 30 | else: 31 | self.layers = nn.Sequential( 32 | nn.Linear(n_states, hidden_sizes[0]), 33 | nn.LeakyReLU(negative_slope=0.2), 34 | nn.BatchNorm1d(hidden_sizes[0]), 35 | nn.Linear(hidden_sizes[0], hidden_sizes[1]), 36 | nn.Tanh(), 37 | nn.Dropout(0.3), 38 | nn.BatchNorm1d(hidden_sizes[1]), 39 | nn.Linear(hidden_sizes[1], hidden_sizes[2]), 40 | nn.Tanh(), 41 | nn.Dropout(0.3), 42 | nn.BatchNorm1d(hidden_sizes[2]), 43 | nn.Linear(hidden_sizes[2], n_actions) 44 | ) 45 | # This act layer maps the output to (0, 1) 46 | self.act = nn.Sigmoid() 47 | self._init_weights() 48 | 49 | def _init_weights(self): 50 | 51 | for m in self.layers: 52 | if isinstance(m, nn.Linear): 53 | m.weight.data.normal_(0.0, 1e-2) 54 | m.bias.data.uniform_(-0.1, 0.1) 55 | 56 | def forward(self, states): # pylint: disable=arguments-differ 57 | 58 | actions = self.act(self.layers(states)) 59 | return actions 60 | 61 | 62 | class Critic(nn.Module): 63 | 64 | def __init__(self, n_states, n_actions, hidden_sizes, use_default): 65 | super(Critic, self).__init__() 66 | self.act = nn.Tanh() 67 | if use_default: 68 | self.state_input = nn.Linear(n_states, 128) 69 | self.action_input = nn.Linear(n_actions, 128) 70 | self.layers = nn.Sequential( 71 | nn.Linear(256, 256), 72 | nn.LeakyReLU(negative_slope=0.2), 73 | nn.BatchNorm1d(256), 74 | nn.Linear(256, 256), 75 | nn.Linear(256, 64), 76 | nn.Tanh(), 77 | nn.Dropout(0.3), 78 | nn.BatchNorm1d(64), 79 | nn.Linear(64, 1) 80 | ) 81 | else: 82 | self.state_input = nn.Linear(n_states, hidden_sizes[0]) 83 | self.action_input = nn.Linear(n_actions, hidden_sizes[0]) 84 | self.layers = nn.Sequential( 85 | nn.Linear(hidden_sizes[0] * 2, hidden_sizes[1]), 86 | nn.LeakyReLU(negative_slope=0.2), 87 | nn.Dropout(0.3), 88 | nn.BatchNorm1d(hidden_sizes[1]), 89 | nn.Linear(hidden_sizes[1], hidden_sizes[2]), 90 | nn.Tanh(), 91 | nn.Dropout(0.3), 92 | nn.BatchNorm1d(hidden_sizes[2]), 93 | nn.Linear(hidden_sizes[2], 1) 94 | ) 95 | self._init_weights() 96 | 97 | def _init_weights(self): 98 | self.state_input.weight.data.normal_(0.0, 1e-2) 99 | self.state_input.bias.data.uniform_(-0.1, 0.1) 100 | 101 | self.action_input.weight.data.normal_(0.0, 1e-2) 102 | self.action_input.bias.data.uniform_(-0.1, 0.1) 103 | 104 | for m in self.layers: 105 | if isinstance(m, nn.Linear): 106 | m.weight.data.normal_(0.0, 1e-2) 107 | m.bias.data.uniform_(-0.1, 0.1) 108 | 109 | def forward(self, states, actions): # pylint: disable=arguments-differ 110 | states = self.act(self.state_input(states)) 111 | actions = self.act(self.action_input(actions)) 112 | 113 | _input = torch.cat([states, actions], dim=1) 114 | value = self.layers(_input) 115 | return value 116 | 117 | class OUProcess(object): 118 | def __init__(self, n_actions, theta=0.15, mu=0, sigma=0.1, ): 119 | 120 | self.n_actions = n_actions 121 | self.theta = theta 122 | self.mu = mu 123 | self.sigma = sigma 124 | self.current_value = np.ones(self.n_actions) * self.mu 125 | 126 | def reset(self, sigma=0, theta=0): 127 | self.current_value = np.ones(self.n_actions) * self.mu 128 | if sigma != 0: 129 | self.sigma = sigma 130 | if theta != 0: 131 | self.theta = theta 132 | 133 | def noise(self): 134 | x = self.current_value 135 | dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(len(x)) 136 | self.current_value = x + dx 137 | return self.current_value 138 | 139 | 140 | class DDPG(object): 141 | 142 | def __init__(self, n_states, n_actions, model_name='', alr=0.001, clr=0.001, 143 | gamma=0.9, batch_size=32, tau=0.002, shift=0, memory_size=100000, 144 | a_hidden_sizes=[128, 128, 64], c_hidden_sizes=[128, 256, 64], 145 | use_default=False): 146 | self.n_states = n_states 147 | self.n_actions = n_actions 148 | self.alr = alr 149 | self.clr = clr 150 | self.model_name = model_name 151 | self.batch_size = batch_size 152 | self.gamma = gamma 153 | self.tau = tau 154 | self.a_hidden_sizes = a_hidden_sizes 155 | self.c_hidden_sizes = c_hidden_sizes 156 | self.shift = shift 157 | self.use_default = use_default 158 | 159 | self._build_network() 160 | 161 | self.replay_memory = PrioritizedReplayMemory(capacity=memory_size) 162 | self.noise = OUProcess(n_actions) 163 | 164 | @staticmethod 165 | def totensor(x): 166 | return Variable(torch.FloatTensor(x)) 167 | 168 | def _build_network(self): 169 | self.actor = Actor(self.n_states, self.n_actions, self.a_hidden_sizes, self.use_default) 170 | self.target_actor = Actor(self.n_states, self.n_actions, self.a_hidden_sizes, 171 | self.use_default) 172 | self.critic = Critic(self.n_states, self.n_actions, self.c_hidden_sizes, self.use_default) 173 | self.target_critic = Critic(self.n_states, self.n_actions, self.c_hidden_sizes, 174 | self.use_default) 175 | 176 | # Copy actor's parameters 177 | self._update_target(self.target_actor, self.actor, tau=1.0) 178 | 179 | # Copy critic's parameters 180 | self._update_target(self.target_critic, self.critic, tau=1.0) 181 | 182 | self.loss_criterion = nn.MSELoss() 183 | self.actor_optimizer = optimizer.Adam(lr=self.alr, params=self.actor.parameters(), 184 | weight_decay=1e-5) 185 | self.critic_optimizer = optimizer.Adam(lr=self.clr, params=self.critic.parameters(), 186 | weight_decay=1e-5) 187 | 188 | @staticmethod 189 | def _update_target(target, source, tau): 190 | for (target_param, param) in zip(target.parameters(), source.parameters()): 191 | target_param.data.copy_( 192 | target_param.data * (1 - tau) + param.data * tau 193 | ) 194 | 195 | def reset(self, sigma, theta): 196 | self.noise.reset(sigma, theta) 197 | 198 | def _sample_batch(self): 199 | batch, idx = self.replay_memory.sample(self.batch_size) 200 | states = list(map(lambda x: x[0].tolist(), batch)) # pylint: disable=W0141 201 | actions = list(map(lambda x: x[1].tolist(), batch)) # pylint: disable=W0141 202 | rewards = list(map(lambda x: x[2], batch)) # pylint: disable=W0141 203 | next_states = list(map(lambda x: x[3].tolist(), batch)) # pylint: disable=W0141 204 | 205 | return idx, states, next_states, actions, rewards 206 | 207 | def add_sample(self, state, action, reward, next_state): 208 | self.critic.eval() 209 | self.actor.eval() 210 | self.target_critic.eval() 211 | self.target_actor.eval() 212 | batch_state = self.totensor([state.tolist()]) 213 | batch_next_state = self.totensor([next_state.tolist()]) 214 | current_value = self.critic(batch_state, self.totensor([action.tolist()])) 215 | target_action = self.target_actor(batch_next_state) 216 | target_value = self.totensor([reward]) \ 217 | + self.target_critic(batch_next_state, target_action) * self.gamma 218 | error = float(torch.abs(current_value - target_value).data.numpy()[0]) 219 | 220 | self.target_actor.train() 221 | self.actor.train() 222 | self.critic.train() 223 | self.target_critic.train() 224 | self.replay_memory.add(error, (state, action, reward, next_state)) 225 | 226 | def update(self): 227 | idxs, states, next_states, actions, rewards = self._sample_batch() 228 | batch_states = self.totensor(states) 229 | batch_next_states = self.totensor(next_states) 230 | batch_actions = self.totensor(actions) 231 | batch_rewards = self.totensor(rewards) 232 | 233 | target_next_actions = self.target_actor(batch_next_states).detach() 234 | target_next_value = self.target_critic(batch_next_states, target_next_actions).detach() 235 | current_value = self.critic(batch_states, batch_actions) 236 | batch_rewards = batch_rewards[:, None] 237 | next_value = batch_rewards + target_next_value * self.gamma + self.shift 238 | 239 | # update prioritized memory 240 | if isinstance(self.replay_memory, PrioritizedReplayMemory): 241 | error = torch.abs(current_value - next_value).data.numpy() 242 | for i in range(self.batch_size): 243 | idx = idxs[i] 244 | self.replay_memory.update(idx, error[i][0]) 245 | 246 | # Update Critic 247 | loss = self.loss_criterion(current_value, next_value) 248 | self.critic_optimizer.zero_grad() 249 | loss.backward() 250 | self.critic_optimizer.step() 251 | 252 | # Update Actor 253 | self.critic.eval() 254 | policy_loss = -self.critic(batch_states, self.actor(batch_states)) 255 | policy_loss = policy_loss.mean() 256 | self.actor_optimizer.zero_grad() 257 | policy_loss.backward() 258 | 259 | self.actor_optimizer.step() 260 | self.critic.train() 261 | 262 | self._update_target(self.target_critic, self.critic, tau=self.tau) 263 | self._update_target(self.target_actor, self.actor, tau=self.tau) 264 | 265 | return loss.data, policy_loss.data 266 | 267 | def choose_action(self, states): 268 | self.actor.eval() 269 | act = self.actor(self.totensor([states.tolist()])).squeeze(0) 270 | self.actor.train() 271 | action = act.data.numpy() 272 | action += self.noise.noise() 273 | return action.clip(0, 1) 274 | 275 | def set_model(self, actor_dict, critic_dict): 276 | self.actor.load_state_dict(pickle.loads(actor_dict)) 277 | self.critic.load_state_dict(pickle.loads(critic_dict)) 278 | 279 | def get_model(self): 280 | return pickle.dumps(self.actor.state_dict()), pickle.dumps(self.critic.state_dict()) 281 | 282 | 283 | import ConfigSpace as CS 284 | import ConfigSpace.hyperparameters as CSH 285 | 286 | class DDPGOptimizer: 287 | def __init__(self, state, model, func, initial_design, rand_conf_chooser, 288 | n_iters, logging_dir=None): 289 | assert state.target_metric == 'throughput' 290 | 291 | self.state = state 292 | self.model = model 293 | self.func = func 294 | self.initial_design = initial_design 295 | self.input_space = initial_design.cs 296 | self.rand_conf_chooser = rand_conf_chooser 297 | self.n_iters = n_iters 298 | self.n_epochs = 2 # same value as CDBTune 299 | 300 | self.logging_dir = logging_dir 301 | 302 | def run(self): 303 | prev_perf = self.state.default_perf 304 | assert prev_perf >= 0 305 | 306 | results = [ ] 307 | 308 | # Bootstrap with random samples 309 | init_configurations = self.initial_design.select_configurations() 310 | for i, knob_data in enumerate(init_configurations): 311 | print(f'Iter {i} -- RANDOM') 312 | ### reward, metric_data = env.simulate(knob_data) 313 | # metrics & perf 314 | perf, metric_data = self.func(knob_data) 315 | perf = -perf # maximize 316 | assert perf >= 0 317 | # compute reward 318 | reward = self.get_reward(perf, prev_perf) 319 | # LOG 320 | print(f'Iter {i} -- PERF = {perf}') 321 | print(f'Iter {i} -- METRICS = {metric_data}') 322 | print(f'Iter {i} -- REWARD = {reward}') 323 | 324 | if i > 0: 325 | self.model.add_sample(prev_metric_data, prev_knob_data, prev_reward, metric_data) 326 | 327 | prev_metric_data = metric_data 328 | prev_knob_data = knob_data.get_array() # scale to [0, 1] 329 | prev_reward = reward 330 | prev_perf = perf 331 | 332 | # add last random sample 333 | self.model.add_sample(prev_metric_data, prev_knob_data, prev_reward, metric_data) 334 | 335 | # Start guided search 336 | for i in range(len(init_configurations), self.n_iters): 337 | 338 | if self.rand_conf_chooser.check(i): 339 | print(f'Iter {i} -- RANDOM SAMPLE') 340 | # get random sample from config space 341 | knob_data = self.input_space.sample_configuration() 342 | knob_data_vector = knob_data.get_array() 343 | else: 344 | print(f'Iter {i} -- GUIDED') 345 | # get next recommendation from DDPG 346 | knob_data_vector = self.model.choose_action(prev_metric_data) 347 | knob_data = self.convert_from_vector(knob_data_vector) 348 | 349 | # metrics & perf 350 | perf, metric_data = self.func(knob_data) 351 | perf = -perf # maximize 352 | assert perf >= 0 353 | # compute reward 354 | reward = self.get_reward(perf, prev_perf) 355 | # LOG 356 | print(f'Iter {i} -- PERF = {perf}') 357 | print(f'Iter {i} -- METRICS = {metric_data}') 358 | print(f'Iter {i} -- REWARD = {reward}') 359 | 360 | # register point to the optimizer 361 | self.model.add_sample(prev_metric_data, prev_knob_data, prev_reward, metric_data) 362 | 363 | prev_metric_data = metric_data 364 | prev_knob_data = knob_data_vector 365 | prev_reward = reward 366 | prev_perf = perf 367 | 368 | # update DDPG model 369 | if len(self.model.replay_memory) >= self.model.batch_size: 370 | for _ in range(self.n_epochs): 371 | self.model.update() 372 | print('AFTER UPDATE') 373 | 374 | def get_reward(self, perf, prev_perf): 375 | """ Reward calculation same as CDBTune paper -- Section 4.2 """ 376 | def calculate_reward(delta_default, delta_prev): 377 | if delta_default > 0: 378 | reward = ((1 + delta_default) ** 2 - 1) * np.abs(1 + delta_prev) 379 | else: 380 | reward = - ((1 - delta_default) ** 2 - 1) * np.abs(1 - delta_prev) 381 | 382 | # no improvement over last evaluation -- 0 reward 383 | if reward > 0 and delta_prev < 0: 384 | reward = 0 385 | 386 | return reward 387 | 388 | if perf == self.state.worse_perf: 389 | return 0 390 | 391 | # perf diff from default / prev evaluation 392 | delta_default = (perf - self.state.default_perf) / self.state.default_perf 393 | delta_prev = (perf - prev_perf) / prev_perf 394 | 395 | return calculate_reward(delta_default, delta_prev) 396 | 397 | def convert_from_vector(self, vector): 398 | from adapters.bias_sampling import special_value_scaler, UniformIntegerHyperparameterWithSpecialValue 399 | 400 | values = { } 401 | for hp, value in zip(self.input_space.get_hyperparameters(), vector): 402 | 403 | if isinstance(hp, CSH.NumericalHyperparameter): 404 | if isinstance(hp, UniformIntegerHyperparameterWithSpecialValue): 405 | value = special_value_scaler(hp, value) 406 | else: 407 | value = hp._transform(value) 408 | elif isinstance(hp, CSH.CategoricalHyperparameter): 409 | assert hp.num_choices == 2 410 | value = hp.choices[0] if value <= 0.5 else hp.choices[1] 411 | else: 412 | raise NotImplementedError() 413 | 414 | values[hp.name] = value 415 | 416 | conf = CS.Configuration(self.input_space, values=values) 417 | self.input_space.check_configuration(conf) 418 | return conf 419 | 420 | -------------------------------------------------------------------------------- /executors/executor.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import random 5 | import subprocess 6 | import time 7 | from abc import ABC, abstractmethod 8 | from copy import deepcopy 9 | from datetime import datetime 10 | from pathlib import Path 11 | 12 | import pandas as pd 13 | import numpy as np 14 | from scipy.spatial.distance import euclidean, cityblock 15 | 16 | import grpc 17 | from google.protobuf.struct_pb2 import Struct 18 | from google.protobuf.json_format import MessageToDict 19 | 20 | from sklearn.preprocessing import StandardScaler 21 | 22 | # pylint: disable=import-error 23 | from executors.grpc.nautilus_rpc_pb2 import ExecuteRequest, EmptyMessage 24 | from executors.grpc.nautilus_rpc_pb2_grpc import ExecutionServiceStub 25 | 26 | logging.basicConfig(level=logging.INFO) 27 | logger = logging.getLogger() 28 | 29 | def run_command(cmd, **kwargs): 30 | logger.debug(f'Running command: `{cmd}`...') 31 | logger.debug(50 * '=') 32 | 33 | cp = None 34 | try: 35 | cp = subprocess.run(cmd, shell=True, **kwargs) 36 | if cp.returncode != 0: 37 | logger.warn(f'Non-zero code [{cp.returncode}] for command `{cmd}`') 38 | 39 | except Exception as err: 40 | logger.error(err) 41 | logger.error(f'Error while running command `{cmd}`') 42 | 43 | return cp 44 | 45 | def trim_disks(): 46 | logger.info('Executing TRIM on all mount points') 47 | try: 48 | run_command('sudo fstrim -av', check=True) 49 | except Exception as err: 50 | logger.warn(f'Error while TRIMing: {repr(err)}') 51 | 52 | def get_measured_performance(perf_stats, benchmark): 53 | """ Return throughput & 95-th latency percentile """ 54 | if benchmark == 'ycsb': 55 | overall_stats = perf_stats['ycsb']['groups']['overall']['statistics'] 56 | throughput, runtime = ( 57 | overall_stats['Throughput(ops/sec)'], 58 | overall_stats['RunTime(ms)'] / 1000.0) 59 | 60 | # Check if Return=ERROR in read/update results 61 | error = False 62 | try: 63 | read_stats = perf_stats['ycsb']['groups']['read']['statistics'] 64 | assert 'Return=ERROR' not in read_stats.keys() 65 | update_stats = perf_stats['ycsb']['groups']['update']['statistics'] 66 | assert 'Return=ERROR' not in update_stats.keys() 67 | except AssertionError: 68 | logger.warning('Return=ERROR found in YCSB; Treating it as failed run!') 69 | throughput, latency = 0.1, 2 ** 30 70 | error = True 71 | 72 | if not error: 73 | # Manually compute latency (weighted by ops) 74 | groups = [ 75 | g for name, g in perf_stats['ycsb']['groups'].items() 76 | if name != 'overall' 77 | ] 78 | latency_info = [ # latencies are in micro-seconds 79 | (float(g['statistics']['p95']), int(g['statistics']['Return=OK'])) 80 | for g in groups 81 | ] 82 | latencies, weights = tuple(zip(*latency_info)) 83 | latency = np.average(latencies, weights=weights) / 1000.0 84 | 85 | elif benchmark == 'oltpbench': 86 | summary_stats = perf_stats['oltpbench_summary'] 87 | throughput, latency, runtime = ( 88 | summary_stats['throughput(req/sec)'], 89 | summary_stats['95th_lat(ms)'], 90 | summary_stats['time(sec)']) 91 | elif benchmark == 'benchbase': 92 | summary_stats = perf_stats['benchbase_summary'] 93 | throughput, latency, runtime = ( 94 | summary_stats['throughput(req/sec)'], 95 | summary_stats['95th_lat(ms)'], 96 | summary_stats['time(sec)']) 97 | else: 98 | raise NotImplementedError(f'Benchmark `{benchmark}` is not supported') 99 | 100 | return { 101 | 'throughput': throughput, 102 | 'latency': latency, 103 | 'runtime': runtime, 104 | } 105 | 106 | def get_dbms_metrics(results, num_expected): 107 | """ Parses DBMS metrics and returns their mean as a numpy array 108 | 109 | NOTE: Currently only DB-wide metrics are parsed; not table-wide ones 110 | """ 111 | GLOBAL_STAT_VIEWS = ['pg_stat_bgwriter', 'pg_stat_database'] 112 | PER_TABLE_STAT_VIEWS = [ # Not used currently 113 | 'pg_stat_user_tables', 114 | 'pg_stat_user_indexes', 115 | 'pg_statio_user_tables', 116 | 'pg_statio_user_indexes' 117 | ] 118 | try: 119 | metrics = json.loads(results['samplers']['db_metrics'])['postgres'] 120 | samples = metrics['samples'] 121 | except Exception as err: 122 | logger.error(f'Error while *retrieving* DBMS metrics: {repr(err)}') 123 | logger.info('Returning dummy (i.e., all zeros) metrics') 124 | return np.zeros(num_expected) 125 | 126 | try: 127 | global_dfs = [ ] 128 | for k in GLOBAL_STAT_VIEWS: 129 | s = samples[k] 130 | v = [ l for l in s if l != None ] 131 | cols = [ f'{k}_{idx}' for idx in range(len(v[0])) ] 132 | 133 | df = pd.DataFrame(data=v, columns=cols) 134 | df.dropna(axis=1, inplace=True) 135 | df = df.select_dtypes(['number']) 136 | global_dfs.append(df) 137 | 138 | df = pd.concat(global_dfs, axis=1) 139 | metrics = df.mean(axis=0).to_numpy() 140 | except Exception as err: 141 | logger.error(f'Error while *parsing* DBMS metrics: {repr(err)}') 142 | logger.info('Returning dummy (i.e., all zeros) metrics') 143 | return np.zeros(num_expected) 144 | 145 | if len(metrics) != num_expected: 146 | logger.error(f'Num of metrics [{len(metrics)}] is different than expected [{num_expected}] :(') 147 | logger.info('Returning dummy (i.e., all zeros) metrics') 148 | return np.zeros(num_expected) 149 | 150 | return metrics 151 | 152 | def is_result_valid(results, benchmark): 153 | # Check results 154 | run_info, perf_stats = results['run_info'], results['performance_stats'] 155 | 156 | if benchmark == 'ycsb': 157 | check_fields = [ 158 | run_info['warm_up']['result'], 159 | run_info['benchmark']['result'], 160 | perf_stats['ycsb_result'], 161 | perf_stats['ycsb_raw_result'], 162 | ] 163 | elif benchmark == 'oltpbench': 164 | check_fields = [ 165 | run_info['benchmark']['result'], 166 | perf_stats['oltpbench_summary_result'], 167 | ] 168 | elif benchmark == 'benchbase': 169 | check_fields = [ 170 | run_info['benchmark']['result'], 171 | perf_stats['benchbase_summary_result'], 172 | ] 173 | else: 174 | raise NotImplementedError(f'Benchmark `{benchmark}` is not supported') 175 | 176 | return all(v == 'ok' for v in check_fields) 177 | 178 | class ExecutorInterface(ABC): 179 | def __init__(self, spaces, storage, **kwargs): 180 | self.spaces = spaces 181 | self.storage = storage 182 | 183 | @abstractmethod 184 | def evaluate_configuration(self, dbms_info, benchmark_info): 185 | raise NotImplementedError 186 | 187 | class DummyExecutor(ExecutorInterface): 188 | def __init__(self, spaces, storage, parse_metrics=False, num_dbms_metrics=None, **kwargs): 189 | self.parse_metrics = parse_metrics 190 | self.num_dbms_metrics = num_dbms_metrics 191 | 192 | def evaluate_configuration(self, dbms_info, benchmark_info): 193 | perf = { 194 | 'throughput': float(random.randint(1000, 10000)), 195 | 'latency': float(random.randint(1000, 10000)), 196 | 'runtime': 0, 197 | } 198 | 199 | if not self.parse_metrics: 200 | return perf 201 | 202 | metrics = np.random.rand(self.num_dbms_metrics) 203 | return perf, metrics 204 | 205 | class NautilusExecutor(ExecutorInterface): 206 | GRPC_MAX_MESSAGE_LENGTH = 32 * (2 ** 20) # 32MB 207 | # NOTE: Nautilus already has a soft time limit (default is *1.5 hour*) 208 | EXECUTE_TIMEOUT_SECS = 2 * 60 * 60 # 4 hours 209 | 210 | def __init__(self, spaces, storage, host=None, port=None, n_retries=10, 211 | parse_metrics=False, num_dbms_metrics=None): 212 | super().__init__(spaces, storage) 213 | 214 | self.host, self.port = host, port 215 | self.iter = 0 216 | self.parse_metrics = parse_metrics 217 | if parse_metrics: 218 | assert(num_dbms_metrics >= 0) 219 | self.num_dbms_metrics = num_dbms_metrics 220 | 221 | delay = 2 222 | for idx in range(1, n_retries + 1): 223 | logger.debug(f'Trying connecting to Nautilus [#={idx}]...') 224 | try: 225 | self._try_connect(timeout=5) 226 | except Exception as err: 227 | logger.debug(f'Failed to connect: {repr(err)}') 228 | logger.debug(f'Trying again in {delay} seconds') 229 | time.sleep(delay) 230 | delay *= 2 231 | else: 232 | logger.info('Connected to Nautilus!') 233 | return 234 | 235 | raise RuntimeError(f'Cannot connect to Nautilus @ {host}:{port}') 236 | 237 | def evaluate_configuration(self, dbms_info, benchmark_info): 238 | """ Call Nautilus executor RPC """ 239 | # trim disks before sending request 240 | trim_disks() 241 | 242 | # NOTE: protobuf explicitely converts ints to floats; this is a workaround 243 | # https://stackoverflow.com/questions/51818125/how-to-use-ints-in-a-protobuf-struct 244 | config = { } 245 | for k, v in dbms_info['config'].items(): 246 | if isinstance(v, int): 247 | config[k] = str(v) 248 | else: config[k] = v 249 | dbms_info['config'] = config 250 | 251 | # Construct request 252 | config = Struct() 253 | config.update(dbms_info['config']) # pylint: disable=no-member 254 | dbms_info = ExecuteRequest.DBMSInfo( 255 | name=dbms_info['name'], config=config, version=dbms_info['version']) 256 | 257 | if benchmark_info['name'] == 'ycsb': 258 | request = ExecuteRequest( 259 | dbms_info=dbms_info, ycsb_info=benchmark_info) 260 | elif benchmark_info['name'] == 'oltpbench': 261 | request = ExecuteRequest( 262 | dbms_info=dbms_info, oltpbench_info=benchmark_info) 263 | elif benchmark_info['name'] == 'benchbase': 264 | request = ExecuteRequest( 265 | dbms_info=dbms_info, benchbase_info=benchmark_info) 266 | else: 267 | raise ValueError(f"Benchmark `{benchmark_info['name']}' not found") 268 | 269 | # Do RPC 270 | logger.debug(f'Calling Nautilus RPC with request:\n{request}') 271 | try: 272 | response = self.stub.Execute(request, timeout=self.EXECUTE_TIMEOUT_SECS) 273 | except Exception as err: 274 | logger.error(f'Error while submitting task: {repr(err)}') 275 | with open('error.txt', 'a') as f: 276 | f.write(repr(err)) 277 | 278 | logger.info(f'Received response JSON [len={len(response.results)}]') 279 | results = MessageToDict(response)['results'] 280 | 281 | # Save results 282 | self.storage.store_executor_result(self.iter, results) 283 | self.iter += 1 284 | 285 | # Check results 286 | try: 287 | is_valid = is_result_valid(results, benchmark_info['name']) 288 | except Exception as err: 289 | logger.error(f'Exception while trying to check result: {str(err)}') 290 | is_valid = False 291 | finally: 292 | if not is_valid: 293 | logger.error('Nautilus experienced an error.. check logs :(') 294 | return None if not self.parse_metrics else (None, np.zeros(self.num_dbms_metrics)) 295 | 296 | # Retrieve throughput & latency stats 297 | perf = get_measured_performance( 298 | results['performance_stats'], benchmark_info['name']) 299 | if not self.parse_metrics: 300 | return perf 301 | 302 | # Parse DBMS metrics and return along with perf 303 | return perf, get_dbms_metrics(results, self.num_dbms_metrics) 304 | 305 | 306 | def close(self): 307 | """ Close connection to Nautilus """ 308 | self.channel.close() 309 | 310 | def _try_connect(self, **kwargs): 311 | """ Attempt to connect to host:port address """ 312 | self.channel = grpc.insecure_channel( 313 | f'{self.host}:{self.port}', 314 | options=[ # send/recv up to 32 MB of messages (4MB default) 315 | ('grpc.max_send_message_length', self.GRPC_MAX_MESSAGE_LENGTH), 316 | ('grpc.max_receive_message_length', self.GRPC_MAX_MESSAGE_LENGTH), 317 | ]) 318 | self.stub = ExecutionServiceStub(self.channel) 319 | 320 | response = self.stub.Heartbeat(EmptyMessage(), **kwargs) 321 | logger.info(f'{10*"="} Nautilus Info {10*"="}') 322 | logger.info(f'Alive since : {response.alive_since.ToDatetime()}') 323 | logger.info(f'Current time : {response.time_now.ToDatetime()}') 324 | logger.info(f'Jobs finished: {response.jobs_finished}') 325 | logger.info(f'{35 * "="}') 326 | 327 | class QueryFromDatasetExecutor(ExecutorInterface): 328 | def __init__(self, spaces, storage, dataset=None): 329 | super().__init__(spaces, storage) 330 | assert dataset != None, 'Please provide dataset filepath' 331 | 332 | filepath = Path('./datasets') / dataset 333 | assert filepath.exists(), f'Dataset filepath [@{filepath}] does not exist' 334 | 335 | configs, perfs = self._parse(filepath) 336 | 337 | # Find which knobs are numeric, and which categorical 338 | self.numerical_knobs = np.array([ col for col in configs.columns 339 | if pd.api.types.is_numeric_dtype(configs[col]) ]) 340 | self.categorical_knobs = np.array([ col for col in configs.columns 341 | if pd.api.types.is_string_dtype(configs[col]) ]) 342 | assert (len(self.numerical_knobs) + len(self.categorical_knobs) == len(configs.columns)) 343 | 344 | logger.info(f'Numerical knobs: {self.numerical_knobs}') 345 | logger.info(f'Categorical knobs: {self.categorical_knobs}') 346 | 347 | # save samples 348 | self._samples = pd.concat([configs, perfs], axis=1) 349 | self.samples_used = np.zeros(len(self.samples), dtype=np.bool) 350 | 351 | # pre-compute normalization of numerical samples for easier closest-point search 352 | self._store_normalize_numerical_knobs_of_samples(configs, self.numerical_knobs) 353 | 354 | # default values to be used for pruned knobs 355 | self.default_config = spaces.get_default_config_point() 356 | 357 | def _parse(self, filepath): 358 | with open(filepath, 'r') as f: 359 | lines = f.readlines() 360 | logger.info(f'Read {len(lines)} samples from dataset [@ {filepath}]') 361 | 362 | self.benchmark_info = None 363 | configs, perfs = [ ], [ ] 364 | 365 | confs_seen = set() 366 | 367 | # Retrieve config, performance tuples 368 | for result in map(json.loads, lines): 369 | dbms_config = result['task_args']['dbms']['config'] 370 | if dbms_config == None: # default configuration 371 | continue 372 | 373 | benchmark_info = result['task_args']['benchmark'] 374 | self.benchmark_info = self.benchmark_info or benchmark_info 375 | if self.benchmark_info != benchmark_info: # TODO: deepdiff 376 | logger.error('Benchmark info differs across samples -- skip sample') 377 | continue 378 | 379 | # Check result 380 | if not is_result_valid(result, benchmark_info['name']): 381 | logger.warning('Skipping non-valid result... :(') 382 | continue 383 | 384 | # Check if we have already seen this conf 385 | conf_fs = frozenset(dbms_config.items()) 386 | if conf_fs in confs_seen: 387 | logger.warning('Skipping already seen configuration...') 388 | continue 389 | confs_seen |= { conf_fs } 390 | 391 | dbms_config = self.spaces.unfinalize_conf(dbms_config) 392 | configs.append(dbms_config) 393 | 394 | # Retrieve throughput 395 | throughput, _ = get_measured_throughput( 396 | result['performance_stats'], benchmark_info['name']) 397 | perfs.append(throughput) 398 | 399 | configs, perfs = pd.DataFrame(configs), pd.DataFrame(perfs, columns=['throughput']) 400 | logger.info(f'Found {len(configs)} valid samples!') 401 | 402 | return configs, perfs 403 | 404 | def _store_normalize_numerical_knobs_of_samples(self, samples, knobs): 405 | logger.info(f'(Re)normalizing {len(samples)} samples...') 406 | logger.info(f'Keeping following numerical knobs: {knobs}') 407 | 408 | # normalize samples (used for distance computation) 409 | scaler = StandardScaler() 410 | scaled_configs = scaler.fit_transform(samples[knobs]) 411 | self.scaled_samples = pd.DataFrame(data=scaled_configs, columns=knobs) 412 | 413 | # store scaler & number of num knobs 414 | self.scaler = scaler 415 | self.n_numerical_knobs = len(knobs) 416 | 417 | @property 418 | def samples(self): 419 | return self._samples 420 | 421 | def evaluate_configuration(self, dbms_info, benchmark_info): 422 | #if self.benchmark_info != benchmark_info: 423 | # logger.error('Benchmark info from dataset differs from the provided\n' 424 | # f'Dataset:\t{self.benchmark_info}\nExperiment:\t{benchmark_info}') 425 | start = datetime.now() 426 | 427 | # remove suffixes, string to numeric, etc 428 | config = self.spaces.unfinalize_conf(dbms_info['config']) 429 | 430 | # find which knobs are tuned (may be less than all due to pruning) 431 | categorical_knobs = [ col for col in self.categorical_knobs if col in config ] 432 | numerical_knobs = [ col for col in self.numerical_knobs if col in config ] 433 | 434 | # NOTE: old way of handling pruned knobs -- i.e. re-train scaler 435 | #if len(numerical_knobs) < self.n_numerical_knobs: 436 | # # pruning occured -- need to update scaler & re-normalize samples 437 | # self._store_normalize_numerical_knobs_of_samples( 438 | # self.samples[numerical_knobs], numerical_knobs) 439 | 440 | # fill pruned/missing knob values with default ones 441 | missing_numeric_knobs = set(self.numerical_knobs) - set(numerical_knobs) 442 | for knob in missing_numeric_knobs: 443 | config[knob] = self.default_config[knob] 444 | config = pd.Series(config) 445 | 446 | if len(missing_numeric_knobs) > 0: 447 | logger.info(f'Filled config with default values:\n{config}') 448 | numerical_knobs = self.numerical_knobs 449 | 450 | # normize given config values 451 | scaled_values = self.scaler.transform([ config[numerical_knobs] ])[0] 452 | scaled_config = pd.Series(data=scaled_values, index=numerical_knobs) 453 | 454 | # consider samples where all categorical knobs match to given config 455 | samples = self.samples 456 | for col in categorical_knobs: 457 | samples = samples.loc[samples[col] == config[col]] 458 | logger.info(f'Found {len(samples)} candidate points after fixing categorical knobs\' values') 459 | 460 | # find "closest" previously evaluated sample 461 | closest_dist, closest_idx = None, None 462 | for idx, _ in samples.iterrows(): 463 | if self.samples_used[idx]: 464 | # skip already used point 465 | continue 466 | 467 | # compute distance 468 | total_dist = euclidean( 469 | self.scaled_samples.iloc[idx][numerical_knobs].values, 470 | scaled_config[numerical_knobs].values 471 | ) 472 | # update closer point 473 | if (closest_dist == None) or (total_dist < closest_dist): 474 | closest_dist, closest_idx, = total_dist, idx 475 | 476 | # Find and mark out used point 477 | closest_sample = self.samples.iloc[closest_idx] 478 | self.samples_used[closest_idx] = True 479 | 480 | logger.info(f'Closest sample is:\n{closest_sample}') 481 | logger.info(f'L2-norm Distance is: {closest_dist: .3f}') 482 | 483 | return { 484 | 'sample': self.spaces.point_from_dict({ k: v 485 | for k, v in zip(closest_sample.index.values, closest_sample.values) 486 | if k != 'throughput'}), 487 | 'throughput': closest_sample['throughput'], 488 | 'runtime': (datetime.now() - start).total_seconds(), 489 | } 490 | 491 | 492 | class ExecutorFactory: 493 | concrete_classes = { 494 | 'NautilusExecutor': NautilusExecutor, 495 | 'QueryFromDatasetExecutor': QueryFromDatasetExecutor, 496 | 'DummyExecutor': DummyExecutor, 497 | } 498 | 499 | @staticmethod 500 | def from_config(config, spaces, storage, **extra_kwargs): 501 | executor_config = deepcopy(config['executor']) 502 | 503 | classname = executor_config.pop('classname', None) 504 | assert classname != None, 'Please specify the *executor* class name' 505 | 506 | try: 507 | class_ = ExecutorFactory.concrete_classes[classname] 508 | except KeyError: 509 | raise ValueError(f'Executor class "{classname}" not found. ' 510 | f'Options are [{", ".join(ExecutorFactory.concrete_classes.keys())}]') 511 | 512 | # Override with local 513 | executor_config.update(**extra_kwargs) 514 | 515 | return class_(spaces, storage, **executor_config) 516 | -------------------------------------------------------------------------------- /spaces/definitions/postgres-9.6.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 7, 4 | "name": "autovacuum", 5 | "type": "enum", 6 | "default": "on", 7 | "description": "Starts the autovacuum subprocess", 8 | "choices": [ 9 | "on", 10 | "off" 11 | ] 12 | }, 13 | { 14 | "id": 8, 15 | "name": "autovacuum_analyze_scale_factor", 16 | "type": "real", 17 | "default": 0.1, 18 | "description": "Number of tuple inserts, updates, or deletes prior to analyze as a fraction of reltuples", 19 | "min": 0.0, 20 | "max": 100.0 21 | }, 22 | { 23 | "id": 9, 24 | "name": "autovacuum_analyze_threshold", 25 | "type": "integer", 26 | "default": 50, 27 | "description": "Minimum number of tuple inserts, updates, or deletes prior to analyze", 28 | "min": 0, 29 | "max": 1000 30 | }, 31 | { 32 | "id": 10, 33 | "name": "autovacuum_freeze_max_age", 34 | "type": "integer", 35 | "default": 200000000, 36 | "description": "Age at which to autovacuum a table to prevent transaction ID wraparound", 37 | "min": 100000, 38 | "max": 2000000000 39 | }, 40 | { 41 | "id": 11, 42 | "name": "autovacuum_max_workers", 43 | "type": "integer", 44 | "default": 3, 45 | "description": "Sets the maximum number of simultaneously running autovacuum worker processes", 46 | "min": 1, 47 | "max": 10000 48 | }, 49 | { 50 | "id": 12, 51 | "name": "autovacuum_multixact_freeze_max_age", 52 | "type": "integer", 53 | "default": 400000000, 54 | "description": "Multixact age at which to autovacuum a table to prevent multixact wraparound", 55 | "min": 10000, 56 | "max": 2000000000 57 | }, 58 | { 59 | "id": 13, 60 | "name": "autovacuum_naptime", 61 | "type": "integer", 62 | "default": 60, 63 | "description": "Time to sleep between autovacuum runs", 64 | "min": 1, 65 | "max": 2147483 66 | }, 67 | { 68 | "id": 14, 69 | "name": "autovacuum_vacuum_cost_delay", 70 | "type": "integer", 71 | "default": 20, 72 | "description": "Vacuum cost delay in milliseconds, for autovacuum", 73 | "min": -1, 74 | "max": 100 75 | }, 76 | { 77 | "id": 15, 78 | "name": "autovacuum_vacuum_cost_limit", 79 | "type": "integer", 80 | "default": -1, 81 | "description": "Vacuum cost amount available before napping, for autovacuum", 82 | "min": -1, 83 | "max": 10000 84 | }, 85 | { 86 | "id": 16, 87 | "name": "autovacuum_vacuum_scale_factor", 88 | "type": "real", 89 | "default": 0.2, 90 | "description": "Number of tuple updates or deletes prior to vacuum as a fraction of reltuples", 91 | "min": 0.0, 92 | "max": 100.0 93 | }, 94 | { 95 | "id": 17, 96 | "name": "autovacuum_vacuum_threshold", 97 | "type": "integer", 98 | "default": 50, 99 | "description": "Minimum number of tuple updates or deletes prior to vacuum", 100 | "min": 0, 101 | "max": 2147483647 102 | }, 103 | { 104 | "id": 18, 105 | "name": "autovacuum_work_mem", 106 | "type": "integer", 107 | "default": -1, 108 | "description": "Sets the maximum memory to be used by each autovacuum worker process", 109 | "min": -1, 110 | "max": 1048576 111 | }, 112 | { 113 | "id": 19, 114 | "name": "backend_flush_after", 115 | "type": "integer", 116 | "default": 0, 117 | "description": "Number of pages after which previously performed writes are flushed to disk", 118 | "min": 0, 119 | "max": 256 120 | }, 121 | { 122 | "id": 21, 123 | "name": "bgwriter_delay", 124 | "type": "integer", 125 | "default": 200, 126 | "description": "Background writer sleep time between rounds", 127 | "min": 10, 128 | "max": 4000 129 | }, 130 | { 131 | "id": 22, 132 | "name": "bgwriter_flush_after", 133 | "type": "integer", 134 | "default": 64, 135 | "description": "Number of pages after which previously performed writes are flushed to disk", 136 | "min": 0, 137 | "max": 256 138 | }, 139 | { 140 | "id": 23, 141 | "name": "bgwriter_lru_maxpages", 142 | "type": "integer", 143 | "default": 100, 144 | "description": "Background writer maximum number of LRU pages to flush per round", 145 | "min": 0, 146 | "max": 1000 147 | }, 148 | { 149 | "id": 24, 150 | "name": "bgwriter_lru_multiplier", 151 | "type": "real", 152 | "default": 2.0, 153 | "description": "Multiple of the average buffer usage to free per round", 154 | "min": 0.0, 155 | "max": 8.0 156 | }, 157 | { 158 | "id": 29, 159 | "name": "checkpoint_completion_target", 160 | "type": "real", 161 | "default": 0.5, 162 | "description": "Time spent flushing dirty buffers during checkpoint, as fraction of checkpoint interval", 163 | "min": 0.0, 164 | "max": 1.0 165 | }, 166 | { 167 | "id": 30, 168 | "name": "checkpoint_flush_after", 169 | "type": "integer", 170 | "default": 32, 171 | "description": "Number of pages after which previously performed writes are flushed to disk", 172 | "min": 0, 173 | "max": 256 174 | }, 175 | { 176 | "id": 31, 177 | "name": "checkpoint_timeout", 178 | "type": "integer", 179 | "default": 300, 180 | "description": "Sets the maximum time between automatic WAL checkpoints", 181 | "min": 30, 182 | "max": 86400 183 | }, 184 | { 185 | "id": 36, 186 | "name": "commit_delay", 187 | "type": "integer", 188 | "default": 0, 189 | "description": "Sets the delay in microseconds between transaction commit and flushing WAL to disk", 190 | "min": 0, 191 | "max": 4096 192 | }, 193 | { 194 | "id": 37, 195 | "name": "commit_siblings", 196 | "type": "integer", 197 | "default": 5, 198 | "description": "Sets the minimum concurrent open transactions before performing commit_delay", 199 | "min": 0, 200 | "max": 16 201 | }, 202 | { 203 | "id": 40, 204 | "name": "cpu_index_tuple_cost", 205 | "type": "real", 206 | "default": 0.005, 207 | "description": "Sets the planner's estimate of the cost of processing each index entry during an index scan", 208 | "min": 0.0, 209 | "max": 4 210 | }, 211 | { 212 | "id": 41, 213 | "name": "cpu_operator_cost", 214 | "type": "real", 215 | "default": 0.0025, 216 | "description": "Sets the planner's estimate of the cost of processing each operator or function call", 217 | "min": 0.0, 218 | "max": 4 219 | }, 220 | { 221 | "id": 42, 222 | "name": "cpu_tuple_cost", 223 | "type": "real", 224 | "default": 0.01, 225 | "description": "Sets the planner's estimate of the cost of processing each tuple (row)", 226 | "min": 0.0, 227 | "max": 4 228 | }, 229 | { 230 | "id": 43, 231 | "name": "cursor_tuple_fraction", 232 | "type": "real", 233 | "default": 0.1, 234 | "description": "Sets the planner's estimate of the fraction of a cursor's rows that will be retrieved", 235 | "min": 0.0, 236 | "max": 1.0 237 | }, 238 | { 239 | "id": 45, 240 | "name": "data_sync_retry", 241 | "type": "enum", 242 | "default": "off", 243 | "description": "Whether to continue running after a failure to sync data files", 244 | "choices": [ 245 | "on", 246 | "off" 247 | ] 248 | }, 249 | { 250 | "id": 48, 251 | "name": "deadlock_timeout", 252 | "type": "integer", 253 | "default": 1000, 254 | "description": "Sets the time to wait on a lock before checking for deadlock", 255 | "min": 1, 256 | "max": 5000 257 | }, 258 | { 259 | "id": 53, 260 | "name": "default_statistics_target", 261 | "type": "integer", 262 | "default": 100, 263 | "description": "Sets the default statistics target", 264 | "min": 1, 265 | "max": 5120 266 | }, 267 | { 268 | "id": 56, 269 | "name": "default_transaction_deferrable", 270 | "type": "enum", 271 | "default": "off", 272 | "description": "Sets the default deferrable status of new transactions", 273 | "choices": [ 274 | "on", 275 | "off" 276 | ] 277 | }, 278 | { 279 | "id": 62, 280 | "name": "effective_cache_size", 281 | "type": "integer", 282 | "default": 524288, 283 | "description": "Sets the planner's assumption about the total size of the data caches", 284 | "min": 1, 285 | "max": 2097152 286 | }, 287 | { 288 | "id": 63, 289 | "name": "effective_io_concurrency", 290 | "type": "integer", 291 | "default": 1, 292 | "description": "Number of simultaneous requests that can be handled efficiently by the disk subsystem", 293 | "min": 0, 294 | "max": 512 295 | }, 296 | { 297 | "id": 64, 298 | "name": "enable_bitmapscan", 299 | "type": "enum", 300 | "default": "on", 301 | "description": "Enables the planner's use of bitmap-scan plans", 302 | "choices": [ 303 | "on", 304 | "off" 305 | ] 306 | }, 307 | { 308 | "id": 65, 309 | "name": "enable_hashagg", 310 | "type": "enum", 311 | "default": "on", 312 | "description": "Enables the planner's use of hashed aggregation plans", 313 | "choices": [ 314 | "on", 315 | "off" 316 | ] 317 | }, 318 | { 319 | "id": 66, 320 | "name": "enable_hashjoin", 321 | "type": "enum", 322 | "default": "on", 323 | "description": "Enables the planner's use of hash join plans", 324 | "choices": [ 325 | "on", 326 | "off" 327 | ] 328 | }, 329 | { 330 | "id": 67, 331 | "name": "enable_indexonlyscan", 332 | "type": "enum", 333 | "default": "on", 334 | "description": "Enables the planner's use of index-only-scan plans", 335 | "choices": [ 336 | "on", 337 | "off" 338 | ] 339 | }, 340 | { 341 | "id": 68, 342 | "name": "enable_indexscan", 343 | "type": "enum", 344 | "default": "on", 345 | "description": "Enables the planner's use of index-scan plans", 346 | "choices": [ 347 | "on", 348 | "off" 349 | ] 350 | }, 351 | { 352 | "id": 69, 353 | "name": "enable_material", 354 | "type": "enum", 355 | "default": "on", 356 | "description": "Enables the planner's use of materialization", 357 | "choices": [ 358 | "on", 359 | "off" 360 | ] 361 | }, 362 | { 363 | "id": 70, 364 | "name": "enable_mergejoin", 365 | "type": "enum", 366 | "default": "on", 367 | "description": "Enables the planner's use of merge join plans", 368 | "choices": [ 369 | "on", 370 | "off" 371 | ] 372 | }, 373 | { 374 | "id": 71, 375 | "name": "enable_nestloop", 376 | "type": "enum", 377 | "default": "on", 378 | "description": "Enables the planner's use of nested-loop join plans", 379 | "choices": [ 380 | "on", 381 | "off" 382 | ] 383 | }, 384 | { 385 | "id": 72, 386 | "name": "enable_seqscan", 387 | "type": "enum", 388 | "default": "on", 389 | "description": "Enables the planner's use of sequential-scan plans", 390 | "choices": [ 391 | "on", 392 | "off" 393 | ] 394 | }, 395 | { 396 | "id": 73, 397 | "name": "enable_sort", 398 | "type": "enum", 399 | "default": "on", 400 | "description": "Enables the planner's use of explicit sort steps", 401 | "choices": [ 402 | "on", 403 | "off" 404 | ] 405 | }, 406 | { 407 | "id": 74, 408 | "name": "enable_tidscan", 409 | "type": "enum", 410 | "default": "on", 411 | "description": "Enables the planner's use of TID scan plans", 412 | "choices": [ 413 | "on", 414 | "off" 415 | ] 416 | }, 417 | { 418 | "id": 81, 419 | "name": "from_collapse_limit", 420 | "type": "integer", 421 | "default": 8, 422 | "description": "Sets the FROM-list size beyond which subqueries are not collapsed", 423 | "min": 1, 424 | "max": 50 425 | }, 426 | { 427 | "id": 83, 428 | "name": "full_page_writes", 429 | "type": "enum", 430 | "default": "on", 431 | "description": "Writes full pages to WAL when first modified after a checkpoint", 432 | "choices": [ 433 | "on", 434 | "off" 435 | ] 436 | }, 437 | { 438 | "id": 84, 439 | "name": "geqo", 440 | "type": "enum", 441 | "default": "on", 442 | "description": "Enables genetic query optimization", 443 | "choices": [ 444 | "on", 445 | "off" 446 | ] 447 | }, 448 | { 449 | "id": 85, 450 | "name": "geqo_effort", 451 | "type": "integer", 452 | "default": 5, 453 | "description": "GEQO: effort is used to set the default for other GEQO parameters", 454 | "min": 1, 455 | "max": 10 456 | }, 457 | { 458 | "id": 86, 459 | "name": "geqo_generations", 460 | "type": "integer", 461 | "default": 0, 462 | "description": "GEQO: number of iterations of the algorithm", 463 | "min": 0, 464 | "max": 1000 465 | }, 466 | { 467 | "id": 87, 468 | "name": "geqo_pool_size", 469 | "type": "integer", 470 | "default": 0, 471 | "description": "GEQO: number of individuals in the population", 472 | "min": 0, 473 | "max": 1000 474 | }, 475 | { 476 | "id": 88, 477 | "name": "geqo_seed", 478 | "type": "real", 479 | "default": 0.0, 480 | "description": "GEQO: seed for random path selection", 481 | "min": 0.0, 482 | "max": 1.0 483 | }, 484 | { 485 | "id": 89, 486 | "name": "geqo_selection_bias", 487 | "type": "real", 488 | "default": 2.0, 489 | "description": "GEQO: selective pressure within the population", 490 | "min": 1.5, 491 | "max": 2.0 492 | }, 493 | { 494 | "id": 90, 495 | "name": "geqo_threshold", 496 | "type": "integer", 497 | "default": 12, 498 | "description": "Sets the threshold of FROM items beyond which GEQO is used", 499 | "min": 2, 500 | "max": 50 501 | }, 502 | { 503 | "id": 102, 504 | "name": "join_collapse_limit", 505 | "type": "integer", 506 | "default": 8, 507 | "description": "Sets the FROM-list size beyond which JOIN constructs are not flattened", 508 | "min": 1, 509 | "max": 50 510 | }, 511 | { 512 | "id": 141, 513 | "name": "maintenance_work_mem", 514 | "type": "integer", 515 | "default": 65536, 516 | "description": "Sets the maximum memory to be used for maintenance operations", 517 | "min": 1024, 518 | "max": 1048576 519 | }, 520 | { 521 | "id": 142, 522 | "name": "max_connections", 523 | "type": "integer", 524 | "default": 100, 525 | "description": "Sets the maximum number of concurrent connections", 526 | "min": 5, 527 | "max": 10000 528 | }, 529 | { 530 | "id": 143, 531 | "name": "max_files_per_process", 532 | "type": "integer", 533 | "default": 1000, 534 | "description": "Sets the maximum number of simultaneously open files for each server process", 535 | "min": 25, 536 | "max": 50000 537 | }, 538 | { 539 | "id": 144, 540 | "name": "max_locks_per_transaction", 541 | "type": "integer", 542 | "default": 64, 543 | "description": "Sets the maximum number of locks per transaction", 544 | "min": 10, 545 | "max": 1024 546 | }, 547 | { 548 | "id": 145, 549 | "name": "max_parallel_workers_per_gather", 550 | "type": "integer", 551 | "default": 0, 552 | "description": "Sets the maximum number of parallel processes per executor node", 553 | "min": 0, 554 | "max": 256 555 | }, 556 | { 557 | "id": 146, 558 | "name": "max_pred_locks_per_transaction", 559 | "type": "integer", 560 | "default": 64, 561 | "description": "Sets the maximum number of predicate locks per transaction", 562 | "min": 10, 563 | "max": 1024 564 | }, 565 | { 566 | "id": 147, 567 | "name": "max_prepared_transactions", 568 | "type": "integer", 569 | "default": 0, 570 | "description": "Sets the maximum number of simultaneously prepared transactions", 571 | "min": 0, 572 | "max": 1024 573 | }, 574 | { 575 | "id": 149, 576 | "name": "max_stack_depth", 577 | "type": "integer", 578 | "default": 100, 579 | "description": "Sets the maximum stack depth, in kilobytes", 580 | "min": 100, 581 | "max": 7680 582 | }, 583 | { 584 | "id": 153, 585 | "name": "max_wal_size", 586 | "type": "integer", 587 | "default": 64, 588 | "description": "Sets the WAL size that triggers a checkpoint", 589 | "min": 2, 590 | "max": 256 591 | }, 592 | { 593 | "id": 154, 594 | "name": "max_worker_processes", 595 | "type": "integer", 596 | "default": 8, 597 | "description": "Maximum number of concurrent worker processes", 598 | "min": 0, 599 | "max": 256 600 | }, 601 | { 602 | "id": 155, 603 | "name": "min_parallel_relation_size", 604 | "type": "integer", 605 | "default": 1024, 606 | "description": "Sets the minimum size of relations to be considered for parallel scan", 607 | "min": 0, 608 | "max": 131072 609 | }, 610 | { 611 | "id": 156, 612 | "name": "min_wal_size", 613 | "type": "integer", 614 | "default": 5, 615 | "description": "Sets the minimum size to shrink the WAL to", 616 | "min": 2, 617 | "max": 16 618 | }, 619 | { 620 | "id": 157, 621 | "name": "old_snapshot_threshold", 622 | "type": "integer", 623 | "default": -1, 624 | "description": "Time before a snapshot is too old to read pages changed after the snapshot was taken", 625 | "min": -1, 626 | "max": 86400 627 | }, 628 | { 629 | "id": 159, 630 | "name": "parallel_setup_cost", 631 | "type": "real", 632 | "default": 1000.0, 633 | "description": "Sets the planner's estimate of the cost of starting up worker processes for parallel query", 634 | "min": 0.0, 635 | "max": 2500 636 | }, 637 | { 638 | "id": 160, 639 | "name": "parallel_tuple_cost", 640 | "type": "real", 641 | "default": 0.1, 642 | "description": "Sets the planner's estimate of the cost of passing each tuple (row) from worker to master backend", 643 | "min": 0.0, 644 | "max": 4 645 | }, 646 | { 647 | "id": 165, 648 | "name": "quote_all_identifiers", 649 | "type": "enum", 650 | "default": "off", 651 | "description": "When generating SQL fragments, quote all identifiers", 652 | "choices": [ 653 | "on", 654 | "off" 655 | ] 656 | }, 657 | { 658 | "id": 166, 659 | "name": "random_page_cost", 660 | "type": "real", 661 | "default": 4.0, 662 | "description": "Sets the planner's estimate of the cost of a nonsequentially fetched disk page", 663 | "min": 0.0, 664 | "max": 8 665 | }, 666 | { 667 | "id": 167, 668 | "name": "replacement_sort_tuples", 669 | "type": "integer", 670 | "default": 150000, 671 | "description": "Sets the maximum number of tuples to be sorted using replacement selection", 672 | "min": 0, 673 | "max": 2147483647 674 | }, 675 | { 676 | "id": 171, 677 | "name": "seq_page_cost", 678 | "type": "real", 679 | "default": 1.0, 680 | "description": "Sets the planner's estimate of the cost of a sequentially fetched disk page", 681 | "min": 0.0, 682 | "max": 4 683 | }, 684 | { 685 | "id": 174, 686 | "name": "shared_buffers", 687 | "type": "integer", 688 | "default": 1024, 689 | "description": "Sets the number of shared memory buffers used by the server", 690 | "min": 16, 691 | "max": 1048576 692 | }, 693 | { 694 | "id": 199, 695 | "name": "temp_buffers", 696 | "type": "integer", 697 | "default": 1024, 698 | "description": "Sets the maximum number of temporary buffers used by each session", 699 | "min": 100, 700 | "max": 65536 701 | }, 702 | { 703 | "id": 200, 704 | "name": "temp_file_limit", 705 | "type": "integer", 706 | "default": -1, 707 | "description": "Limits the total size of all temporary files used by each process", 708 | "min": -1, 709 | "max": 20971520 710 | }, 711 | { 712 | "id": 221, 713 | "name": "vacuum_cost_delay", 714 | "type": "integer", 715 | "default": 0, 716 | "description": "Vacuum cost delay in milliseconds", 717 | "min": 0, 718 | "max": 100 719 | }, 720 | { 721 | "id": 222, 722 | "name": "vacuum_cost_limit", 723 | "type": "integer", 724 | "default": 200, 725 | "description": "Vacuum cost amount available before napping", 726 | "min": 1, 727 | "max": 10000 728 | }, 729 | { 730 | "id": 223, 731 | "name": "vacuum_cost_page_dirty", 732 | "type": "integer", 733 | "default": 20, 734 | "description": "Vacuum cost for a page dirtied by vacuum", 735 | "min": 0, 736 | "max": 10000 737 | }, 738 | { 739 | "id": 224, 740 | "name": "vacuum_cost_page_hit", 741 | "type": "integer", 742 | "default": 1, 743 | "description": "Vacuum cost for a page found in the buffer cache", 744 | "min": 0, 745 | "max": 10000 746 | }, 747 | { 748 | "id": 225, 749 | "name": "vacuum_cost_page_miss", 750 | "type": "integer", 751 | "default": 10, 752 | "description": "Vacuum cost for a page not found in the buffer cache", 753 | "min": 0, 754 | "max": 10000 755 | }, 756 | { 757 | "id": 227, 758 | "name": "vacuum_freeze_min_age", 759 | "type": "integer", 760 | "default": 50000000, 761 | "description": "Minimum age at which VACUUM should freeze a table row", 762 | "min": 0, 763 | "max": 1000000000 764 | }, 765 | { 766 | "id": 228, 767 | "name": "vacuum_freeze_table_age", 768 | "type": "integer", 769 | "default": 150000000, 770 | "description": "Age at which VACUUM should scan whole table to freeze tuples", 771 | "min": 0, 772 | "max": 2000000000 773 | }, 774 | { 775 | "id": 229, 776 | "name": "vacuum_multixact_freeze_min_age", 777 | "type": "integer", 778 | "default": 5000000, 779 | "description": "Minimum age at which VACUUM should freeze a MultiXactId in a table row", 780 | "min": 0, 781 | "max": 1000000000 782 | }, 783 | { 784 | "id": 230, 785 | "name": "vacuum_multixact_freeze_table_age", 786 | "type": "integer", 787 | "default": 150000000, 788 | "description": "Multixact age at which VACUUM should scan whole table to freeze tuples", 789 | "min": 0, 790 | "max": 2000000000 791 | }, 792 | { 793 | "id": 231, 794 | "name": "wal_buffers", 795 | "type": "integer", 796 | "default": -1, 797 | "description": "Sets the number of disk-page buffers in shared memory for WAL", 798 | "min": -1, 799 | "max": 65536 800 | }, 801 | { 802 | "id": 232, 803 | "name": "wal_compression", 804 | "type": "enum", 805 | "default": "off", 806 | "description": "Compresses full-page writes written in WAL file", 807 | "choices": [ 808 | "on", 809 | "off" 810 | ] 811 | }, 812 | { 813 | "id": 235, 814 | "name": "wal_log_hints", 815 | "type": "enum", 816 | "default": "off", 817 | "description": "Writes full pages to WAL when first modified after a checkpoint, even for a non-critical modification", 818 | "choices": [ 819 | "on", 820 | "off" 821 | ] 822 | }, 823 | { 824 | "id": 241, 825 | "name": "wal_writer_delay", 826 | "type": "integer", 827 | "default": 200, 828 | "description": "Time between WAL flushes performed in the WAL writer", 829 | "min": 1, 830 | "max": 4096 831 | }, 832 | { 833 | "id": 242, 834 | "name": "wal_writer_flush_after", 835 | "type": "integer", 836 | "default": 128, 837 | "description": "Amount of WAL written out by WAL writer that triggers a flush", 838 | "min": 0, 839 | "max": 32768 840 | }, 841 | { 842 | "id": 243, 843 | "name": "work_mem", 844 | "type": "integer", 845 | "default": 4096, 846 | "description": "Sets the maximum memory to be used for query workspaces", 847 | "min": 64, 848 | "max": 262144 849 | } 850 | ] --------------------------------------------------------------------------------