├── data ├── sim_results │ ├── within_session │ │ └── .gitkeep │ ├── multi_batch │ │ ├── one_shot │ │ │ └── .gitkeep │ │ ├── interactive │ │ │ └── .gitkeep │ │ └── probit_noise │ │ │ └── .gitkeep │ └── .DS_Store └── processed │ ├── multi_batch.pickle │ ├── within_session.pickle │ └── final_batch_diff_interactive_vs_oneshot.pickle ├── fixed_init_X_dict.pickle ├── plots ├── example_eubo.pdf ├── probit_noise.pdf ├── all_within_session.pdf ├── mt_within_session.pdf ├── all_multi_batch_oneshot.pdf ├── mt_multi_batch_oneshot.pdf ├── mt_multi_batch_interactive.pdf ├── all_multi_batch_interactive.pdf ├── interactive_oneshot_diff_in_util.pdf └── mt_interactive_oneshot_diff_in_util.pdf ├── CONTRIBUTING.md ├── constants.py ├── LICENSE ├── run_multi_sim.sh ├── run_within_sim.sh ├── helper_classes.py ├── run_sim.sh ├── requirements.txt ├── README.md ├── CODE_OF_CONDUCT.md ├── pbo.py ├── notebooks ├── determining_probit_noise.ipynb └── create_fixed_init_X.ipynb ├── within_session_sim.py ├── acquisition_functions.py ├── multi_batch_sim.py ├── test_functions.py └── sim_helpers.py /data/sim_results/within_session/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/sim_results/multi_batch/one_shot/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/sim_results/multi_batch/interactive/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/sim_results/multi_batch/probit_noise/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fixed_init_X_dict.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/fixed_init_X_dict.pickle -------------------------------------------------------------------------------- /plots/example_eubo.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/plots/example_eubo.pdf -------------------------------------------------------------------------------- /plots/probit_noise.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/plots/probit_noise.pdf -------------------------------------------------------------------------------- /data/sim_results/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/data/sim_results/.DS_Store -------------------------------------------------------------------------------- /plots/all_within_session.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/plots/all_within_session.pdf -------------------------------------------------------------------------------- /plots/mt_within_session.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/plots/mt_within_session.pdf -------------------------------------------------------------------------------- /data/processed/multi_batch.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/data/processed/multi_batch.pickle -------------------------------------------------------------------------------- /plots/all_multi_batch_oneshot.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/plots/all_multi_batch_oneshot.pdf -------------------------------------------------------------------------------- /plots/mt_multi_batch_oneshot.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/plots/mt_multi_batch_oneshot.pdf -------------------------------------------------------------------------------- /data/processed/within_session.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/data/processed/within_session.pickle -------------------------------------------------------------------------------- /plots/mt_multi_batch_interactive.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/plots/mt_multi_batch_interactive.pdf -------------------------------------------------------------------------------- /plots/all_multi_batch_interactive.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/plots/all_multi_batch_interactive.pdf -------------------------------------------------------------------------------- /plots/interactive_oneshot_diff_in_util.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/plots/interactive_oneshot_diff_in_util.pdf -------------------------------------------------------------------------------- /plots/mt_interactive_oneshot_diff_in_util.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/plots/mt_interactive_oneshot_diff_in_util.pdf -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | This repo is for experiment replication of the paper. 2 | 3 | You may use the code to reproduce the results in the paper and use any under the MIT license. 4 | -------------------------------------------------------------------------------- /data/processed/final_batch_diff_interactive_vs_oneshot.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/preference-exploration/HEAD/data/processed/final_batch_diff_interactive_vs_oneshot.pickle -------------------------------------------------------------------------------- /constants.py: -------------------------------------------------------------------------------- 1 | # Constants 2 | RAW_SAMPLES = 256 3 | NUM_RESTARTS = 16 4 | BATCH_LIMIT = 4 5 | QNEHVI_RAW_SAMPLES = 512 6 | QNEHVI_BATCH_LIMIT = 2 7 | NUM_TRUE_UTIL_SAMPLES = 512 8 | NUM_PAREGO_SAMPLES = 256 9 | NUM_LEARN_PREF_SAMPLES_UNEIPM = 256 10 | NUM_LEARN_PREF_SAMPLE_UNEI = 8 11 | NUM_LEARN_OUTCOME_SAMPLE_UNEI = 32 12 | NUM_EVAL_PREF_SAMPLES = 16 13 | NUM_EVAL_OUTCOME_SAMPLES = 32 14 | FTOL = 1e-5 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Meta Platforms, Inc. and affiliates. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /run_multi_sim.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/zsh 2 | 3 | if [ "$#" -ne 4 ] 4 | then 5 | echo "Incorrect number of arguments" 6 | exit 1 7 | fi 8 | 9 | while getopts g:b:e:c: flag 10 | do 11 | case "${flag}" in 12 | g) gpu=${OPTARG};; 13 | b) begin_init_seed=${OPTARG};; 14 | e) end_init_seed=${OPTARG};; 15 | c) cnt=${OPTARG};; 16 | esac 17 | done 18 | 19 | echo "Using $(which python)" 20 | 21 | problem_strs=( 22 | "vehiclesafety_5d3d_kumaraswamyproduct" 23 | "dtlz2_8d4d_negl1dist" 24 | "osy_6d8d_piecewiselinear" 25 | "carcabdesign_7d9d_piecewiselinear" 26 | 27 | "vehiclesafety_5d3d_piecewiselinear" 28 | "dtlz2_8d4d_piecewiselinear" 29 | "osy_6d8d_sigmodconstraints" 30 | "carcabdesign_7d9d_linear" 31 | ) 32 | 33 | for i in {$begin_init_seed..$end_init_seed} 34 | do 35 | echo "Running the simulation round $i" 36 | 37 | shuffled=( $(shuf -e "${problem_strs[@]}") ) 38 | 39 | for problem_str in ${shuffled}; do 40 | # echo "GPU=$gpu runs=$num_sim $problem_str" 41 | c="CUDA_VISIBLE_DEVICES=$gpu nice -n 19 python multi_batch_sim.py --problem_str=$problem_str --noisy=False --init_seed=$i --kernel=default --comp_noise_type=$cnt --device=$gpu" 42 | echo $c 43 | eval $c 44 | done 45 | done 46 | 47 | -------------------------------------------------------------------------------- /run_within_sim.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/zsh 2 | 3 | if [ "$#" -ne 4 ] 4 | then 5 | echo "Incorrect number of arguments $#" 6 | exit 1 7 | fi 8 | 9 | while getopts g:b:e:c: flag 10 | do 11 | case "${flag}" in 12 | g) gpu=${OPTARG};; 13 | b) begin_init_seed=${OPTARG};; 14 | e) end_init_seed=${OPTARG};; 15 | c) cnt=${OPTARG};; 16 | esac 17 | done 18 | 19 | echo "Using $(which python)" 20 | 21 | problem_strs=( 22 | "vehiclesafety_5d3d_kumaraswamyproduct" 23 | "dtlz2_8d4d_negl1dist" 24 | "osy_6d8d_piecewiselinear" 25 | "carcabdesign_7d9d_piecewiselinear" 26 | 27 | "vehiclesafety_5d3d_piecewiselinear" 28 | "dtlz2_8d4d_piecewiselinear" 29 | "osy_6d8d_sigmodconstraints" 30 | "carcabdesign_7d9d_linear" 31 | ) 32 | 33 | for i in {$begin_init_seed..$end_init_seed} 34 | do 35 | echo "Running the simulation round $i" 36 | 37 | shuffled=( $(shuf -e "${problem_strs[@]}") ) 38 | 39 | for problem_str in ${shuffled}; do 40 | # echo "GPU=$gpu runs=$num_sim $problem_str" 41 | c="CUDA_VISIBLE_DEVICES=$gpu nice -n 19 python within_session_sim.py --problem_str=$problem_str --noisy=False --init_seed=$i --gen_method=qnei --kernel=default --comp_noise_type=$cnt --device=$gpu" 42 | echo $c 43 | eval $c 44 | done 45 | done 46 | 47 | -------------------------------------------------------------------------------- /helper_classes.py: -------------------------------------------------------------------------------- 1 | from botorch.acquisition import MCAcquisitionObjective 2 | from botorch.models.model import Model 3 | from botorch.posteriors import Posterior 4 | from botorch.sampling.samplers import MCSampler, SobolQMCNormalSampler 5 | from torch import Tensor 6 | 7 | 8 | class PosteriorMeanDummySampler(MCSampler): 9 | def __init__(self, model: Model): 10 | super().__init__() 11 | self.model = model 12 | 13 | def _construct_base_samples(self): 14 | pass 15 | 16 | def forward(self, posterior: Posterior): 17 | return posterior.mean.unsqueeze(0) 18 | 19 | 20 | class LearnedPrefereceObjective(MCAcquisitionObjective): 21 | def __init__(self, pref_model, sampler=None, use_mean=False): 22 | super().__init__() 23 | self.use_mean = use_mean 24 | self.sampler = None 25 | if not self.use_mean: 26 | if sampler is None: 27 | self.sampler = SobolQMCNormalSampler(num_samples=1) 28 | else: 29 | self.sampler = sampler 30 | 31 | self.pref_model = pref_model 32 | 33 | def forward(self, samples: Tensor, X: Tensor = None): 34 | if self.use_mean: 35 | output_sample = self.pref_model.posterior(samples).mean.squeeze(-1) 36 | else: 37 | output_sample = self.sampler(self.pref_model.posterior(samples)).squeeze(-1) 38 | output_sample = output_sample.reshape((-1,) + output_sample.shape[2:]) 39 | return output_sample 40 | -------------------------------------------------------------------------------- /run_sim.sh: -------------------------------------------------------------------------------- 1 | ./run_within_sim.sh -g4 -b0 -e99 -cconstant 2 | 3 | ./run_within_sim.sh -g3 -b0 -e19 -cconstant 4 | ./run_within_sim.sh -g4 -b20 -e39 -cconstant 5 | ./run_within_sim.sh -g4 -b40 -e59 -cconstant 6 | ./run_within_sim.sh -g4 -b60 -e79 -cconstant 7 | ./run_within_sim.sh -g4 -b80 -e99 -cconstant 8 | 9 | ./run_multi_sim.sh -gcpu -b0 -e1 -cconstant 10 | ./run_multi_sim.sh -gcpu -b2 -e3 -cconstant 11 | ./run_multi_sim.sh -gcpu -b4 -e5 -cconstant 12 | ./run_multi_sim.sh -gcpu -b6 -e7 -cconstant 13 | ./run_multi_sim.sh -gcpu -b8 -e9 -cconstant 14 | 15 | ./run_multi_sim.sh -g3 -b0 -e14 -cconstant 16 | ./run_multi_sim.sh -g4 -b15 -e29 -cconstant 17 | 18 | 19 | ./run_multi_sim.sh -g3 -b0 -e2 -cconstant 20 | ./run_multi_sim.sh -g4 -b3 -e5 -cconstant 21 | ./run_multi_sim.sh -g5 -b6 -e8 -cconstant 22 | ./run_multi_sim.sh -g6 -b9 -e11 -cconstant 23 | ./run_multi_sim.sh -g7 -b12 -e14 -cconstant 24 | 25 | ./run_multi_sim.sh -g3 -b15 -e17 -cconstant 26 | ./run_multi_sim.sh -g4 -b18 -e20 -cconstant 27 | ./run_multi_sim.sh -g5 -b21 -e23 -cconstant 28 | ./run_multi_sim.sh -g6 -b24 -e26 -cconstant 29 | ./run_multi_sim.sh -g7 -b27 -e29 -cconstant 30 | 31 | ./run_multi_sim.sh -g3 -b0 -e3 -cconstant 32 | ./run_multi_sim.sh -g4 -b4 -e7 -cconstant 33 | ./run_multi_sim.sh -g5 -b8 -e11 -cconstant 34 | ./run_multi_sim.sh -g6 -b12 -e15 -cconstant 35 | ./run_multi_sim.sh -g7 -b16 -e19 -cconstant 36 | 37 | ./run_multi_sim.sh -g3 -b20 -e23 -cconstant 38 | ./run_multi_sim.sh -g4 -b24 -e27 -cconstant 39 | ./run_multi_sim.sh -g5 -b28 -e31 -cconstant 40 | ./run_multi_sim.sh -g6 -b32 -e35 -cconstant 41 | ./run_multi_sim.sh -g7 -b36 -e39 -cconstant 42 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | appdirs==1.4.4 2 | argcomplete==1.12.3 3 | argon2-cffi==20.1.0 4 | astroid==2.7.2 5 | async-generator==1.10 6 | attrs==21.2.0 7 | autopep8==1.5.7 8 | backcall==0.2.0 9 | black==21.7b0 10 | bleach==4.1.0 11 | botorch==0.6.0 12 | certifi==2021.5.30 13 | cffi==1.14.6 14 | click==8.0.1 15 | cma==3.1.0 16 | coverage==5.5 17 | cycler==0.10.0 18 | debugpy==1.4.1 19 | decorator==5.0.9 20 | defusedxml==0.7.1 21 | entrypoints==0.3 22 | fire==0.4.0 23 | flake8==3.9.2 24 | future==0.18.2 25 | gpytorch==1.6.0 26 | importlib-metadata==4.7.1 27 | iniconfig==1.1.1 28 | ipykernel==6.2.0 29 | ipython==7.27.0 30 | ipython-genutils==0.2.0 31 | ipywidgets==7.6.3 32 | isort==5.9.3 33 | jedi==0.18.0 34 | Jinja2==3.0.1 35 | joblib==1.0.1 36 | jsonschema==3.2.0 37 | jupyter-client==7.0.1 38 | jupyter-core==4.7.1 39 | jupyterlab-pygments==0.1.2 40 | jupyterlab-widgets==1.0.0 41 | kiwisolver==1.3.2 42 | lazy-object-proxy==1.6.0 43 | MarkupSafe==2.0.1 44 | matplotlib==3.4.3 45 | matplotlib-inline==0.1.2 46 | mccabe==0.6.1 47 | memory-profiler==0.58.0 48 | mistune==0.8.4 49 | mypy-extensions==0.4.3 50 | nbclient==0.5.4 51 | nbconvert==6.1.0 52 | nbformat==5.1.3 53 | nest-asyncio==1.5.1 54 | notebook==6.4.3 55 | numpy==1.21.2 56 | packaging==21.0 57 | pandas==1.3.2 58 | pandocfilters==1.4.3 59 | parso==0.8.2 60 | pathspec==0.9.0 61 | pexpect==4.8.0 62 | pickleshare==0.7.5 63 | Pillow==8.3.1 64 | platformdirs==2.2.0 65 | pluggy==0.13.1 66 | prometheus-client==0.11.0 67 | prompt-toolkit==3.0.20 68 | psutil==5.8.0 69 | ptyprocess==0.7.0 70 | py==1.10.0 71 | pycodestyle==2.7.0 72 | pycparser==2.20 73 | pyflakes==2.3.1 74 | Pygments==2.10.0 75 | pykeops==1.5 76 | pylint==2.10.2 77 | pyparsing==2.4.7 78 | pyrsistent==0.18.0 79 | pytest==6.2.4 80 | pytest-cov==2.12.1 81 | python-dateutil==2.8.2 82 | pytz==2021.1 83 | pyzmq==22.2.1 84 | regex==2021.8.28 85 | scikit-learn==1.0 86 | scipy==1.7.1 87 | seaborn==0.11.2 88 | Send2Trash==1.8.0 89 | six==1.16.0 90 | termcolor==1.1.0 91 | terminado==0.11.1 92 | testpath==0.5.0 93 | threadpoolctl==2.2.0 94 | toml==0.10.2 95 | tomli==1.2.1 96 | torch==1.9.0 97 | tornado==6.1 98 | tqdm==4.62.2 99 | traitlets==5.0.5 100 | typed-ast==1.4.3 101 | typing-extensions==3.10.0.0 102 | wcwidth==0.2.5 103 | webencodings==0.5.1 104 | widgetsnbextension==3.5.1 105 | wrapt==1.12.1 106 | zipp==3.5.0 107 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Preference Exploration 2 | Code for replicating experiments from the paper, Preference Exploration for Efficient Bayesian Optimization with Multiple Outcomes, published in AISTATS 2022. 3 | 4 | Also see https://botorch.org/tutorials/bope for BoTorch's BOPE tutorial. 5 | 6 | ## Recreaing figures in the paper 7 | Folder `plots` contains all figures we used in the paper generated using data under `data/processed`. 8 | `notebooks/illustrative_plot.ipynb` creates Figure 1. 9 | `notebooks/plot.ipynb` creates all other figures. 10 | 11 | ## Rerunning the experiments 12 | We have provided processed data for plotting and analysis under folder `data/processed`. However, if you wish to re-run the experiments, please follow the instructions below. 13 | Results will be saved under folder `data/sim_results`. 14 | `notebooks/clean_sim_data.ipynb` turns raw data under `data/sim_results` into the processed format. 15 | 16 | 17 | #### Identifying high utility designs with PE 18 | To re-run the experiment in Section 5.1 *Identifying High Utility Designs with PE*, run the following command: 19 | ``` 20 | ./run_within_sim.sh -g[gpu index or "cpu"] -b[begin of random seeds] -e[end of random seeds] -c[comparison noise type] 21 | ``` 22 | 23 | Args: 24 | - `g`: indicate whether you want to run the simulation on a gpu or cpu. 25 | - `b`: Begin of random seeds used, inclusive. This experiment will be run using a range of random seeds between 0 and 255. `e - b + 1` will be the total number of replications we run using random seed b, b+1, ..., e. 26 | - `e`: End of random seeds used, inclusive. 27 | - `c`: comparison noise type. It should be either "constant" or "probit" 28 | 29 | Example: 30 | `./run_within_sim.sh -gcpu -b0 -e99 -cconstant` 31 | 32 | 33 | #### BOPE with a single or multiple PE stages 34 | To re-run the experiment in Section 5.2 and 5.3, run the following command: 35 | ``` 36 | ./run_multi_sim.sh -g[gpu index or "cpu"] -b[begin of random seeds] -e[end of random seeds] -c[comparison noise type] 37 | ``` 38 | The arguments follow the same pattern as above. 39 | 40 | Example: 41 | `./run_multi_sim.sh -gcpu -b0 -e29 -cconstant` 42 | 43 | ## Reference 44 | Lin, Zhiyuan Jerry, Raul Astudillo, Peter I. Frazier, and Eytan Bakshy. "Preference Exploration for Efficient Bayesian Optimization with Multiple Outcomes" International Conference on Artificial Intelligence and Statistics, 2022. 45 | 46 | #### Bibtex 47 | ``` 48 | @inproceedings{lin2022preference, 49 | author = {Lin, Zhiyuan Jerry and Astudillo, Raul and Frazier, Peter I. and Bakshy, Eytan}, 50 | booktitle = {International Conference on Artificial Intelligence and Statistics}, 51 | title = {Preference Exploration for Efficient Bayesian Optimization with Multiple Outcomes}, 52 | year = {2022} 53 | } 54 | ``` 55 | 56 | ## License 57 | This code repo is MIT licensed, as found in the LICENSE file. 58 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to make participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies within all project spaces, and it also applies when 49 | an individual is representing the project or its community in public spaces. 50 | Examples of representing a project or community include using an official 51 | project e-mail address, posting via an official social media account, or acting 52 | as an appointed representative at an online or offline event. Representation of 53 | a project may be further defined and clarified by project maintainers. 54 | 55 | This Code of Conduct also applies outside the project spaces when there is a 56 | reasonable belief that an individual's behavior may have a negative impact on 57 | the project or its community. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported by contacting the project team at . All 63 | complaints will be reviewed and investigated and will result in a response that 64 | is deemed necessary and appropriate to the circumstances. The project team is 65 | obligated to maintain confidentiality with regard to the reporter of an incident. 66 | Further details of specific enforcement policies may be posted separately. 67 | 68 | Project maintainers who do not follow or enforce the Code of Conduct in good 69 | faith may face temporary or permanent repercussions as determined by other 70 | members of the project's leadership. 71 | 72 | ## Attribution 73 | 74 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 75 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 76 | 77 | [homepage]: https://www.contributor-covenant.org 78 | 79 | For answers to common questions about this code of conduct, see 80 | https://www.contributor-covenant.org/faq 81 | -------------------------------------------------------------------------------- /pbo.py: -------------------------------------------------------------------------------- 1 | import random 2 | import time 3 | import warnings 4 | from copy import deepcopy 5 | from itertools import permutations 6 | 7 | import numpy as np 8 | import torch 9 | from botorch.acquisition.analytic import PosteriorMean 10 | from botorch.acquisition.monte_carlo import qExpectedImprovement 11 | from botorch.optim.optimize import optimize_acqf 12 | from botorch.utils.gp_sampling import get_gp_samples 13 | from scipy.optimize import minimize 14 | 15 | from acquisition_functions import ExpectedUtility 16 | from constants import * # noqa: F403, F401 17 | from sim_helpers import fit_pref_model, organize_comparisons 18 | 19 | warnings.filterwarnings("ignore", message="Could not update `train_inputs` with transformed inputs") 20 | 21 | 22 | def pref2rff(pref_model, n_samples): 23 | # assume pref_model on cpu 24 | pref_model = pref_model.eval().double() 25 | # force the model to infer utility 26 | pref_model.posterior(pref_model.datapoints) 27 | modified_pref_model = deepcopy(pref_model) 28 | 29 | class LikelihoodForRFF: 30 | noise = torch.tensor(1.0).double() 31 | 32 | modified_pref_model.likelihood = LikelihoodForRFF() 33 | modified_pref_model.train_targets = pref_model.utility 34 | modified_pref_model.input_transforms = None 35 | 36 | gp_samples = get_gp_samples( 37 | model=modified_pref_model, 38 | num_outputs=1, 39 | n_samples=n_samples, 40 | num_rff_features=500, 41 | ) 42 | gp_samples.input_transform = deepcopy(pref_model.input_transform) 43 | 44 | # gp_samples = gp_samples.to(device=device) 45 | 46 | return gp_samples 47 | 48 | 49 | def get_pbo_pe_comparisons( 50 | outcome_X, 51 | train_comps, 52 | problem, 53 | utils, 54 | init_round, 55 | total_training_round, 56 | comp_noise_type, 57 | comp_noise, 58 | pe_strategy, 59 | ): 60 | """ 61 | Generate TS-based comparisons on previously observed points 62 | 63 | Args: 64 | outcome_X ([type]): [description] 65 | train_comps ([type]): [description] 66 | problem ([type]): [description] 67 | utils ([type]): [description] 68 | init_round ([type]): [description] 69 | total_training_round ([type]): [description] 70 | comp_noise_type ([type]): [description] 71 | comp_noise ([type]): [description] 72 | pe_strategy (str): being either "random", "ts" or "eubo" 73 | 74 | Returns: 75 | [type]: [description] 76 | """ 77 | 78 | all_pairs = torch.combinations(torch.tensor(range(outcome_X.shape[-2])), r=2).to(train_comps) 79 | 80 | for i in range(total_training_round): 81 | pbo_pe_start_time = time.time() 82 | if ( 83 | (pe_strategy != "random") 84 | and (train_comps is not None) 85 | and (train_comps.shape[-2] >= init_round) 86 | ): 87 | pbo_pref_model = fit_pref_model( 88 | outcome_X, 89 | train_comps, 90 | kernel="default", 91 | transform_input=True, 92 | Y_bounds=problem.bounds, 93 | ) 94 | 95 | if ( 96 | (pe_strategy == "random") 97 | or (train_comps is None) 98 | or (train_comps.shape[-2] < init_round) 99 | ): 100 | cand_comps = all_pairs[ 101 | torch.randint(high=all_pairs.shape[-2], size=(1,)), 102 | ] 103 | elif pe_strategy == "ts": 104 | cand_comps = None 105 | # use TS to draw comparisons 106 | comp1 = pbo_pref_model.posterior(outcome_X).sample().argmax(dim=-2) 107 | # exclude the first sample 108 | sample2 = pbo_pref_model.posterior(outcome_X).sample() 109 | sample2[:, comp1.squeeze(), :] = -float("Inf") 110 | comp2 = sample2.argmax(dim=-2) 111 | # Create candidate comparisons 112 | cand_comps = torch.cat((comp1, comp2), dim=-1) 113 | elif pe_strategy == "eubo": 114 | eubo_acqf = ExpectedUtility( 115 | preference_model=pbo_pref_model, 116 | outcome_model=None, 117 | previous_winner=None, 118 | search_space_type="y", 119 | ) 120 | cand_comps = None 121 | max_eubo_val = -np.inf 122 | for j in range(all_pairs.shape[-2]): 123 | X_pair = outcome_X[all_pairs[j, :]] 124 | eubo_val = eubo_acqf(X_pair).item() 125 | 126 | if eubo_val > max_eubo_val: 127 | max_eubo_val = eubo_val 128 | cand_comps = all_pairs[[j], :] 129 | else: 130 | raise ValueError("Unsupported PE strategy for PBO") 131 | cand_comps = organize_comparisons(utils, cand_comps, comp_noise_type, comp_noise) 132 | pbo_pe_time = time.time() - pbo_pe_start_time 133 | 134 | train_comps = cand_comps if train_comps is None else torch.cat((train_comps, cand_comps)) 135 | print( 136 | f"PBO with PE strategy {pe_strategy} gen time: {pbo_pe_time:.2f}s, train_comps shape: {train_comps.shape}" 137 | ) 138 | 139 | return train_comps 140 | 141 | 142 | def gen_pbo_candidates(outcome_X, train_comps, q, problem, pbo_gen_method): 143 | """generate pbo candidates 144 | 145 | Args: 146 | outcome_X (_type_): _description_ 147 | train_comps (_type_): _description_ 148 | q (_type_): _description_ 149 | problem (_type_): _description_ 150 | pbo_gen_method (_type_): _description_ 151 | """ 152 | if pbo_gen_method == "ts": 153 | problem_cpu = deepcopy(problem).cpu() 154 | pref_model = fit_pref_model( 155 | outcome_X, train_comps, kernel="default", transform_input=True, Y_bounds=problem.bounds 156 | ) 157 | 158 | outcome_cand_X = [] 159 | for _ in range(q): 160 | gp_samples = pref2rff(pref_model.cpu(), n_samples=1) 161 | 162 | acqf = PosteriorMean(gp_samples) 163 | single_outcome_cand_X, _ = optimize_acqf( 164 | acqf, 165 | bounds=problem_cpu.bounds, 166 | q=1, 167 | num_restarts=NUM_RESTARTS, 168 | raw_samples=RAW_SAMPLES, 169 | options={"batch_limit": 1}, 170 | ) 171 | 172 | outcome_cand_X.append(single_outcome_cand_X) 173 | 174 | outcome_cand_X = torch.cat(outcome_cand_X).to(outcome_X) 175 | elif pbo_gen_method == "ei": 176 | pref_model = fit_pref_model( 177 | outcome_X, train_comps, kernel="default", transform_input=True, Y_bounds=problem.bounds 178 | ) 179 | 180 | # to fill in utility values 181 | pref_model.posterior(pref_model.datapoints) 182 | 183 | acqf = qExpectedImprovement(model=pref_model, best_f=pref_model.utility.max().item()) 184 | 185 | outcome_cand_X, _ = optimize_acqf( 186 | acqf, 187 | bounds=problem.bounds, 188 | q=q, 189 | num_restarts=NUM_RESTARTS, 190 | raw_samples=RAW_SAMPLES, 191 | sequential=True, 192 | ) 193 | else: 194 | raise ValueError("Unsupported gen_method for PBO") 195 | 196 | return outcome_cand_X 197 | -------------------------------------------------------------------------------- /notebooks/determining_probit_noise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "vehiclesafety_5d3d_kumaraswamyproduct, noisy: False, noise_std: 0\n", 13 | "dtlz2_8d4d_negl1dist, noisy: False, noise_std: 0\n", 14 | "osy_6d8d_piecewiselinear, noisy: False, noise_std: 0\n", 15 | "carcabdesign_7d9d_piecewiselinear, noisy: False, noise_std: 0\n", 16 | "vehiclesafety_5d3d_piecewiselinear, noisy: False, noise_std: 0\n", 17 | "dtlz2_8d4d_piecewiselinear, noisy: False, noise_std: 0\n", 18 | "osy_6d8d_sigmodconstraints, noisy: False, noise_std: 0\n", 19 | "carcabdesign_7d9d_linear, noisy: False, noise_std: 0\n" 20 | ] 21 | } 22 | ], 23 | "source": [ 24 | "%load_ext autoreload\n", 25 | "%autoreload 2\n", 26 | "\n", 27 | "import os\n", 28 | "import sys\n", 29 | "module_path = os.path.abspath(os.path.join('..'))\n", 30 | "if module_path not in sys.path:\n", 31 | " sys.path.append(module_path)\n", 32 | "\n", 33 | "import pandas as pd\n", 34 | "import pickle\n", 35 | "import numpy as np\n", 36 | "import torch\n", 37 | "import matplotlib.pylab as plt\n", 38 | "from test_functions import problem_setup\n", 39 | "from sim_helpers import gen_rand_X\n", 40 | "from scipy.optimize import minimize\n", 41 | "\n", 42 | "plt.rc(\"axes.spines\", top=False, right=False)\n", 43 | "colors_palette = plt.rcParams['axes.prop_cycle'].by_key()['color']\n", 44 | "\n", 45 | "problem_strs_dict = {\n", 46 | " \"vehiclesafety_5d3d_kumaraswamyproduct\": \"Vehicle safety (d=5, k=3) \\n Product of Kumaraswamy CDFs\",\n", 47 | " 'dtlz2_8d4d_negl1dist': 'DTLZ2 (d=8, k=4) \\n L1 distance',\n", 48 | " 'osy_6d8d_piecewiselinear': 'OSY (d=6, k=8) \\n Piece-wise linear',\n", 49 | " \"carcabdesign_7d9d_piecewiselinear\": \"Car cab design (d=7, k=9) \\n Piece-wise linear\", \n", 50 | " \n", 51 | " \"vehiclesafety_5d3d_piecewiselinear\": \"Vehicle safety (d=5, k=3) \\n Piece-wise linear\",\n", 52 | " \"dtlz2_8d4d_piecewiselinear\": \"DTLZ2 (d=8, k=4) \\n Piece-wise linear\",\n", 53 | " 'osy_6d8d_sigmodconstraints': 'OSY (d=6, k=8) \\n Exp. func. sum with sigmoid constraints',\n", 54 | " \"carcabdesign_7d9d_linear\": \"Car cab design (d=7, k=9) \\n Linear\",\n", 55 | "}\n", 56 | "\n", 57 | "max_Xs = {}\n", 58 | "max_utils = {}\n", 59 | "for problem_str in problem_strs_dict.keys():\n", 60 | " X_dim, Y_dim, problem, util_type, get_util, Y_bounds, probit_noise = problem_setup(problem_str, dtype=torch.double) " 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 2, 66 | "metadata": {}, 67 | "outputs": [ 68 | { 69 | "name": "stdout", 70 | "output_type": "stream", 71 | "text": [ 72 | "vehiclesafety_5d3d_kumaraswamyproduct, noisy: False, noise_std: 0\n", 73 | "vehiclesafety_5d3d_kumaraswamyproduct error rate: 0.100 with probit_noise=0.01980, util_range=0.30\n", 74 | "\n", 75 | "dtlz2_8d4d_negl1dist, noisy: False, noise_std: 0\n", 76 | "dtlz2_8d4d_negl1dist error rate: 0.100 with probit_noise=0.04560, util_range=0.73\n", 77 | "\n", 78 | "osy_6d8d_piecewiselinear, noisy: False, noise_std: 0\n", 79 | "osy_6d8d_piecewiselinear error rate: 0.100 with probit_noise=2.41970, util_range=20.06\n", 80 | "\n", 81 | "carcabdesign_7d9d_piecewiselinear, noisy: False, noise_std: 0\n", 82 | "carcabdesign_7d9d_piecewiselinear error rate: 0.100 with probit_noise=0.11010, util_range=1.28\n", 83 | "\n", 84 | "vehiclesafety_5d3d_piecewiselinear, noisy: False, noise_std: 0\n", 85 | "vehiclesafety_5d3d_piecewiselinear error rate: 0.100 with probit_noise=0.15440, util_range=2.46\n", 86 | "\n", 87 | "dtlz2_8d4d_piecewiselinear, noisy: False, noise_std: 0\n", 88 | "dtlz2_8d4d_piecewiselinear error rate: 0.100 with probit_noise=0.18960, util_range=2.86\n", 89 | "\n", 90 | "osy_6d8d_sigmodconstraints, noisy: False, noise_std: 0\n", 91 | "osy_6d8d_sigmodconstraints error rate: 0.100 with probit_noise=0.03010, util_range=3.67\n", 92 | "\n", 93 | "carcabdesign_7d9d_linear, noisy: False, noise_std: 0\n", 94 | "carcabdesign_7d9d_linear error rate: 0.100 with probit_noise=0.04460, util_range=0.67\n", 95 | "\n" 96 | ] 97 | } 98 | ], 99 | "source": [ 100 | "std_norm = torch.distributions.normal.Normal(0, 1)\n", 101 | "\n", 102 | "def estimate_error_rate(x, utils, true_comps):\n", 103 | " choose_0_prob = std_norm.cdf((utils[:, 0] - utils[:, 1]) / x)\n", 104 | " correct_prob = torch.cat((choose_0_prob[true_comps], 1 - choose_0_prob[~true_comps]))\n", 105 | " error_rate = 1 - correct_prob.mean()\n", 106 | " return error_rate.item()\n", 107 | "\n", 108 | "def error_rate_loss(x, utils, true_comps):\n", 109 | " return abs(estimate_error_rate(x, utils, true_comps) - target_error_size)\n", 110 | "\n", 111 | "probit_noise_dict = {}\n", 112 | "\n", 113 | "top1_perc_avg_util = {}\n", 114 | "\n", 115 | "for problem_str in problem_strs_dict.keys():\n", 116 | " step_size = 0.001\n", 117 | " \n", 118 | " # get top random points\n", 119 | " top_proportion = 0.1\n", 120 | " target_error_size = 0.1\n", 121 | " n_samples = int(10000 / top_proportion)\n", 122 | " X_dim, Y_dim, problem, util_type, get_util, Y_bounds, probit_noise = problem_setup(problem_str)\n", 123 | " X = gen_rand_X(n_samples, problem)\n", 124 | " Y = problem(X)\n", 125 | " utils = get_util(Y)\n", 126 | " # take top 10 percent\n", 127 | " utils = utils.sort().values[-int(n_samples * top_proportion):]\n", 128 | " # reshuffle\n", 129 | " utils = utils[torch.randperm(utils.shape[0])]\n", 130 | " utils = utils.reshape(-1, 2)\n", 131 | " top1_perc_avg_util[problem_str] = utils.mean().item()\n", 132 | " \n", 133 | " util_range = (utils.max() - utils.min()).item()\n", 134 | "\n", 135 | " # estimate probit error\n", 136 | " true_comps = (utils[:, 0] >= utils[:, 1])\n", 137 | "\n", 138 | " res = minimize(error_rate_loss, x0=0.01, args=(utils, true_comps))\n", 139 | " probit_noise = res.x[0]\n", 140 | " probit_noise = round(probit_noise, 4)\n", 141 | " \n", 142 | " error_rate = estimate_error_rate(probit_noise, utils, true_comps)\n", 143 | " \n", 144 | " probit_noise_dict[problem_str] = probit_noise\n", 145 | " print(f\"{problem_str} error rate: {error_rate:.3f} with probit_noise={probit_noise:.5f}, util_range={util_range:.2f}\")\n", 146 | " print()" 147 | ] 148 | } 149 | ], 150 | "metadata": { 151 | "kernelspec": { 152 | "display_name": "pref", 153 | "language": "python", 154 | "name": "pref" 155 | }, 156 | "language_info": { 157 | "codemirror_mode": { 158 | "name": "ipython", 159 | "version": 3 160 | }, 161 | "file_extension": ".py", 162 | "mimetype": "text/x-python", 163 | "name": "python", 164 | "nbconvert_exporter": "python", 165 | "pygments_lexer": "ipython3", 166 | "version": "3.7.4" 167 | } 168 | }, 169 | "nbformat": 4, 170 | "nbformat_minor": 4 171 | } 172 | -------------------------------------------------------------------------------- /within_session_sim.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import random 4 | import warnings 5 | from copy import deepcopy 6 | 7 | import fire 8 | import torch 9 | from gpytorch.utils.warnings import NumericalWarning 10 | 11 | from sim_helpers import fit_outcome_model, run_one_round_sim 12 | from test_functions import problem_setup 13 | 14 | warnings.filterwarnings( 15 | "ignore", 16 | message="Could not update `train_inputs` with transformed inputs", 17 | ) 18 | 19 | 20 | def run_pref_sim(problem_str, noisy, init_seed, gen_method, kernel, comp_noise_type, tkwargs): 21 | problem_prefix = "_".join(problem_str.split("_")[:2]) 22 | fixed_init_X_dict = pickle.load(open("fixed_init_X_dict.pickle", "rb")) 23 | ( 24 | X_dim, 25 | Y_dim, 26 | problem, 27 | util_type, 28 | get_util, 29 | Y_bounds, 30 | probit_noise, 31 | ) = problem_setup(problem_str, noisy=noisy, **tkwargs) 32 | 33 | # init_strategy, learn_strategy, keep_winner_prob, sample_outcome 34 | # if keep_winner_prob is None, we never enter learning phase 35 | exp_configs = [ 36 | # EUBO family 37 | ("random_ps", "eubo_one_sample", 0, False), 38 | ("random_ps", "eubo_rff", 0, False), 39 | ("uncorrelated", "eubo_y", 0, False), 40 | # BALD family 41 | ("random_ps", "bald_rff", 0, False), 42 | ("uncorrelated", "bald_yspace", 0, False), 43 | # Random baselines 44 | ("random_ps", "random_ps", 0, True), 45 | ("uncorrelated", "uncorrelated", 0, None), 46 | ] 47 | 48 | # shuffle the order in case experiments auto-restarted 49 | # so that we have roughly even number of exp run for each config 50 | random.shuffle(exp_configs) 51 | 52 | output_filepath = ( 53 | f"data/sim_results/within_session/camera_ready/sim_{problem_str}_{kernel}.pickle" 54 | ) 55 | 56 | if comp_noise_type == "constant": 57 | comp_noise = 0.1 58 | elif comp_noise_type == "probit": 59 | comp_noise = probit_noise 60 | elif comp_noise_type == "none": 61 | comp_noise_type = "constant" 62 | comp_noise = 0.0 63 | else: 64 | raise RuntimeError("Invalid comp_noise_type! Must be constant, probit, or none") 65 | 66 | large_batch_size = False 67 | total_training_round = 80 68 | init_round = Y_dim * 2 69 | 70 | # ========= start simulation from here ========== 71 | if X_dim <= 5: 72 | outcome_n = 16 73 | else: 74 | outcome_n = 32 75 | if large_batch_size: 76 | outcome_n = outcome_n * 2 77 | 78 | print(f"Running init_seed: {init_seed}, outcome_n: {outcome_n}") 79 | 80 | # initial observation 81 | outcome_X = fixed_init_X_dict[problem_prefix][init_seed].to(Y_bounds) 82 | outcome_Y = problem(outcome_X) 83 | 84 | outcome_model = fit_outcome_model(outcome_X, outcome_Y, X_bounds=problem.bounds) 85 | for init_strategy, learn_strategy, keep_winner_prob, sample_outcome in exp_configs: 86 | # read past experiments to check repeated experiments 87 | # re-read the file every time as it can be updated by other processses 88 | if os.path.isfile(output_filepath): 89 | past_sim_results = pickle.load(open(output_filepath, "rb")) 90 | else: 91 | past_sim_results = [] 92 | exp_set = set() 93 | for sr in past_sim_results: 94 | past_exp_signature = ( 95 | sr["init_seed"], 96 | sr["problem_str"], 97 | sr["init_strategy"], 98 | sr["learn_strategy"], 99 | sr["comp_noise_type"], 100 | sr["comp_noise"], 101 | ) 102 | exp_set.add(past_exp_signature) 103 | 104 | curr_exp_signature = ( 105 | init_seed, 106 | problem_str, 107 | init_strategy, 108 | learn_strategy, 109 | comp_noise_type, 110 | comp_noise, 111 | ) 112 | if curr_exp_signature not in exp_set: 113 | exp_set.add(curr_exp_signature) 114 | else: 115 | print(f"Experiment {curr_exp_signature} was previously run! Skipping...") 116 | continue 117 | 118 | print( 119 | f"Running {init_strategy} {learn_strategy} {keep_winner_prob} {sample_outcome} on {problem_str}, init_seed: {init_seed}" 120 | ) 121 | selected_pairs = [] 122 | ( 123 | train_X, 124 | train_Y, 125 | train_comps, 126 | acq_run_times, 127 | run_times, 128 | post_mean_X, 129 | post_mean_idx, 130 | selected_pairs, 131 | ) = run_one_round_sim( 132 | total_training_round=total_training_round, 133 | init_round=init_round, 134 | problem_str=problem_str, 135 | noisy=noisy, 136 | comp_noise_type=comp_noise_type, 137 | comp_noise=comp_noise, 138 | outcome_model=outcome_model, 139 | outcome_X=outcome_X, 140 | outcome_Y=outcome_Y, 141 | train_X=None, 142 | train_Y=None, 143 | train_comps=None, 144 | init_strategy=init_strategy, 145 | learn_strategy=learn_strategy, 146 | gen_method=gen_method, 147 | keep_winner_prob=keep_winner_prob, 148 | sample_outcome=sample_outcome, 149 | kernel=kernel, 150 | check_post_mean=True, 151 | check_post_mean_every_k=5, 152 | tkwargs=tkwargs, 153 | selected_pairs=selected_pairs, 154 | ) 155 | 156 | if len(post_mean_X) != 0: 157 | real_eval_util = get_util(problem.evaluate_true(post_mean_X)) 158 | post_mean_X = deepcopy(post_mean_X).detach().cpu() 159 | real_eval_util = deepcopy(real_eval_util).detach().cpu() 160 | else: 161 | post_mean_X = None 162 | real_eval_util = None 163 | 164 | single_result = { 165 | "problem_str": problem_str, 166 | "init_seed": init_seed, 167 | "kernel": kernel, 168 | "gen_method": gen_method, 169 | "init_round": init_round, 170 | "noise_std": problem.noise_std, 171 | "comp_noise_type": comp_noise_type, 172 | "comp_noise": comp_noise, 173 | "outcome_n": outcome_n, 174 | "init_strategy": init_strategy, 175 | "learn_strategy": learn_strategy, 176 | "keep_winner_prob": keep_winner_prob, 177 | "sample_outcome": sample_outcome, 178 | "outcome_X": deepcopy(outcome_X).detach().cpu(), 179 | "outcome_Y": deepcopy(outcome_Y).detach().cpu(), 180 | "train_X": deepcopy(train_X).detach().cpu(), 181 | "train_Y": deepcopy(train_Y).detach().cpu(), 182 | "train_comps": deepcopy(train_comps).detach().cpu(), 183 | "post_mean_idx": post_mean_idx, 184 | "eval_X": post_mean_X, 185 | "eval_util": real_eval_util, 186 | "acq_run_times": acq_run_times, 187 | "run_times": run_times, 188 | } 189 | 190 | if os.path.isfile(output_filepath): 191 | sim_results = pickle.load(open(output_filepath, "rb")) 192 | else: 193 | sim_results = [] 194 | sim_results.append(single_result) 195 | pickle.dump(sim_results, open(output_filepath, "wb")) 196 | torch.cuda.empty_cache() 197 | 198 | 199 | def main(problem_str, noisy, init_seed, gen_method, kernel, comp_noise_type, device): 200 | """ 201 | Args: 202 | problem_str: problem string. see definition in test_functions.py 203 | noisy: whether inject noise into the test function 204 | init_seed: initialization seed 205 | gen_methods: acquisition functions used, one of "ts", "qnei", or "qei" 206 | kernel: "default" (RBF) or "linear" (might not be numerically stable) 207 | comp_noise_type: "constant" or "probit" 208 | """ 209 | assert isinstance(noisy, bool) 210 | 211 | dtype = torch.double 212 | if device == "cpu": 213 | device = torch.device("cpu") 214 | else: 215 | # set device env variable externally 216 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 217 | 218 | tkwargs = { 219 | "dtype": dtype, 220 | "device": device, 221 | } 222 | 223 | warnings.filterwarnings("ignore", category=NumericalWarning) 224 | 225 | print(problem_str, bool(noisy), init_seed, gen_method, kernel) 226 | run_pref_sim( 227 | problem_str=problem_str, 228 | noisy=noisy, 229 | init_seed=init_seed, 230 | gen_method=gen_method, 231 | kernel=kernel, 232 | comp_noise_type=comp_noise_type, 233 | tkwargs=tkwargs, 234 | ) 235 | 236 | 237 | if __name__ == "__main__": 238 | fire.Fire(main) 239 | -------------------------------------------------------------------------------- /notebooks/create_fixed_init_X.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 25, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "The autoreload extension is already loaded. To reload it, use:\n", 13 | " %reload_ext autoreload\n" 14 | ] 15 | } 16 | ], 17 | "source": [ 18 | "%load_ext autoreload\n", 19 | "%autoreload 2\n", 20 | "\n", 21 | "import os\n", 22 | "import sys\n", 23 | "module_path = os.path.abspath(os.path.join('..'))\n", 24 | "if module_path not in sys.path:\n", 25 | " sys.path.append(module_path)\n", 26 | "\n", 27 | "from test_functions import problem_setup\n", 28 | "from sim_helpers import (\n", 29 | " gen_initial_real_data,\n", 30 | " fit_outcome_model,\n", 31 | " gen_random_candidates,\n", 32 | " fit_pref_model,\n", 33 | " gen_rand_X,\n", 34 | " gen_rand_points,\n", 35 | " PosteriorMeanDummySampler,\n", 36 | " gen_comps,\n", 37 | ")\n", 38 | "import pickle" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 26, 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "problem_strs_dict = {\n", 48 | " 'dtlz2_6d4d_restrictednegsixhump': '6d-4d DTLZ2 (4d outputs) \\n restricted negative six hump sum',\n", 49 | " 'dtlz2_9d8d_restrictednegsixhump': '9d-8d DTLZ2 (8d outputs) \\n restricted negative six hump sum',\n", 50 | " 'dtlz2_6d4d_rotatedrestrictednegsixhump': '6d-4d DTLZ2 (4d outputs) \\n rotated restricted neg. six hump sum',\n", 51 | " 'dtlz2_9d8d_rotatedrestrictednegsixhump': '9d-8d DTLZ2 (8d outputs) \\n rotated restricted neg. six hump sum',\n", 52 | "\n", 53 | " 'gpdraw_4d6d_kumaraswamy': '4d-6d GP Draw (6d outputs) \\n Kumaraswamy CDFs sum w/ interaction',\n", 54 | " 'gpdraw_4d6d_piecewiselinear': '4d-6d GP Draw (6d outputs) \\n piece-wise linear',\n", 55 | "\n", 56 | " \"dtlz2_6d4d_piecewise\": \"6d-4d DTLZ2 (4d outputs) \\n quartic-linear piece-wise\",\n", 57 | " \"dtlz2_6d4d_kumaraswamy\": \"6d-4d DTLZ2 (4d outputs) \\n Kumaraswamy CDFs sum w/ interaction\",\n", 58 | " \"dtlz2_9d8d_kumaraswamy\": \"9d-8d DTLZ2 (8d outputs) \\n Kumaraswamy CDFs sum w/ interaction\",\n", 59 | " \"dtlz2_9d8d_piecewise\": \"9d-8d DTLZ2 (8d outputs) \\n quartic-linear piece-wise\",\n", 60 | " \n", 61 | " 'dtlz2_6d4d_negsixhump': '6d-4d DTLZ2 (4d outputs) \\n negative six hump sum',\n", 62 | " 'dtlz2_9d8d_negsixhump': '9d-8d DTLZ2 (8d outputs) \\n negative six hump sum',\n", 63 | " 'dtlz2_6d4d_quadraticsum': '6d-4d DTLZ2 (4d outputs) \\n quadratic functions sum', \n", 64 | " 'dtlz2_9d8d_quadraticsum': '9d-8d DTLZ2 (8d outputs) \\n quadratic functions sum', \n", 65 | "\n", 66 | " 'augdtlz2_4d8d_negl1dist': 'Augmented DTLZ2 (d=4, k=8) \\n Negative L1 distance',\n", 67 | " 'augdtlz2_8d8d_negl1dist': 'Augmented DTLZ2 (d=8, k=8) \\n Negative L1 distance',\n", 68 | " 'augdtlz2_8d12d_negl1dist': 'Augmented DTLZ2 (d=8, k=12) \\n Negative L1 distance',\n", 69 | " 'dtlz2_8d4d_negl1dist': 'DTLZ2 (d=8, k=4) \\n Negative L1 distance',\n", 70 | " 'dtlz2_8d4d_negl2dist': 'DTLZ2 (d=8, k=4) \\n Negative L2 distance',\n", 71 | " 'augdtlz2_8d16d_negl2dist': 'Augmented DTLZ2 (d=8, k=16) \\n Negative L2 distance',\n", 72 | " 'augdtlz2_8d16d_negl1dist': 'Augmented DTLZ2 (d=8, k=16) \\n Negative L1 distance',\n", 73 | "\n", 74 | " \"dtlz2_8d4d_idealnegl1dist\": 'DTLZ2 (d=8, k=4) \\n L1 distance from desideratum',\n", 75 | " \"augdtlz2_8d16d_idealnegl1dist\": 'Augmented DTLZ2 (d=8, k=16) \\n L1 distance from desideratum',\n", 76 | " \"dtlz2_8d4d_idealnegl2dist\": 'DTLZ2 (d=8, k=4) \\n L2 distance from desideratum',\n", 77 | " \"augdtlz2_8d16d_idealnegl2dist\": 'Augmented DTLZ2 (d=8, k=16) \\n L2 distance from desideratum',\n", 78 | " \n", 79 | " \"dtlz2_8d4d_idealnegl1dist\": 'DTLZ2 (d=8, k=4) \\n L1 distance from desideratum',\n", 80 | " \"augdtlz2_8d16d_idealnegl1dist\": 'Augmented DTLZ2 (d=8, k=16) \\n L1 distance from desideratum',\n", 81 | " \"dtlz2_8d4d_idealnegl2dist\": 'DTLZ2 (d=8, k=4) \\n L2 distance from desideratum',\n", 82 | " \"augdtlz2_8d16d_idealnegl2dist\": 'Augmented DTLZ2 (d=8, k=16) \\n L2 distance from desideratum',\n", 83 | "\n", 84 | " \"carcabdesign_7d9d_piecewiselinear\": \"Car cab design (d=7, k=9) \\n piece-wise linear\", \n", 85 | " \"vehiclesafety_5d3d_kumaraswamyproduct\": \"Vehicle safety (d=5, k=3) \\n product of Kumaraswamy CDFs\",\n", 86 | " 'osy_6d8d_piecewiselinear': 'OSY (d=6, k=8) \\n piece-wise linear',\n", 87 | " 'osy_6d8d_kumaraswamy': 'OSY (d=6, k=8) \\n Kumaraswamy CDFs sum w/ interaction',\n", 88 | " \"carcabdesign_7d9d_linear\": \"Car cab design (d=7, k=9) \\n linear\",\n", 89 | " \"vehiclesafety_5d3d_linear\": \"Vehicle safety (d=5, k=3) \\n linear\",\n", 90 | "}" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 29, 96 | "metadata": {}, 97 | "outputs": [ 98 | { 99 | "name": "stdout", 100 | "output_type": "stream", 101 | "text": [ 102 | "dtlz2_6d4d_restrictednegsixhump, noisy: False, noise_std: 0\n", 103 | "dtlz2_9d8d_restrictednegsixhump, noisy: False, noise_std: 0\n", 104 | "dtlz2_6d4d_rotatedrestrictednegsixhump, noisy: False, noise_std: 0\n", 105 | "dtlz2_9d8d_rotatedrestrictednegsixhump, noisy: False, noise_std: 0\n", 106 | "gpdraw_4d6d_kumaraswamy, noisy: False, noise_std: 0\n", 107 | "gpdraw_4d6d_piecewiselinear, noisy: False, noise_std: 0\n", 108 | "dtlz2_6d4d_piecewise, noisy: False, noise_std: 0\n", 109 | "dtlz2_6d4d_kumaraswamy, noisy: False, noise_std: 0\n", 110 | "dtlz2_9d8d_kumaraswamy, noisy: False, noise_std: 0\n", 111 | "dtlz2_9d8d_piecewise, noisy: False, noise_std: 0\n", 112 | "dtlz2_6d4d_negsixhump, noisy: False, noise_std: 0\n", 113 | "dtlz2_9d8d_negsixhump, noisy: False, noise_std: 0\n", 114 | "dtlz2_6d4d_quadraticsum, noisy: False, noise_std: 0\n", 115 | "dtlz2_9d8d_quadraticsum, noisy: False, noise_std: 0\n", 116 | "augdtlz2_4d8d_negl1dist, noisy: False, noise_std: 0\n", 117 | "augdtlz2_8d8d_negl1dist, noisy: False, noise_std: 0\n", 118 | "augdtlz2_8d12d_negl1dist, noisy: False, noise_std: 0\n", 119 | "dtlz2_8d4d_negl1dist, noisy: False, noise_std: 0\n", 120 | "dtlz2_8d4d_negl2dist, noisy: False, noise_std: 0\n", 121 | "augdtlz2_8d16d_negl2dist, noisy: False, noise_std: 0\n", 122 | "augdtlz2_8d16d_negl1dist, noisy: False, noise_std: 0\n", 123 | "dtlz2_8d4d_idealnegl1dist, noisy: False, noise_std: 0\n", 124 | "augdtlz2_8d16d_idealnegl1dist, noisy: False, noise_std: 0\n", 125 | "dtlz2_8d4d_idealnegl2dist, noisy: False, noise_std: 0\n", 126 | "augdtlz2_8d16d_idealnegl2dist, noisy: False, noise_std: 0\n", 127 | "carcabdesign_7d9d_piecewiselinear, noisy: False, noise_std: 0\n", 128 | "vehiclesafety_5d3d_kumaraswamyproduct, noisy: False, noise_std: 0\n", 129 | "osy_6d8d_piecewiselinear, noisy: False, noise_std: 0\n", 130 | "osy_6d8d_kumaraswamy, noisy: False, noise_std: 0\n", 131 | "carcabdesign_7d9d_linear, noisy: False, noise_std: 0\n", 132 | "vehiclesafety_5d3d_linear, noisy: False, noise_std: 0\n" 133 | ] 134 | } 135 | ], 136 | "source": [ 137 | "n_init_X_batch = 256\n", 138 | "fixed_init_X_dict = {}\n", 139 | "\n", 140 | "for problem_str in problem_strs_dict.keys():\n", 141 | " problem_prefix = \"_\".join(problem_str.split(\"_\")[:2])\n", 142 | " (\n", 143 | " X_dim,\n", 144 | " Y_dim,\n", 145 | " problem,\n", 146 | " util_type,\n", 147 | " get_util,\n", 148 | " Y_bounds,\n", 149 | " probit_noise,\n", 150 | " ) = problem_setup(problem_str, noisy=False, dtype=torch.float64)\n", 151 | " \n", 152 | " if X_dim <= 5:\n", 153 | " init_n_outcome = 16\n", 154 | " else:\n", 155 | " init_n_outcome = 32\n", 156 | " \n", 157 | " if problem_prefix not in fixed_init_X_dict:\n", 158 | " fixed_init_X_dict[problem_prefix] = []\n", 159 | " for i in range(n_init_X_batch):\n", 160 | " fixed_init_X_dict[problem_prefix].append(gen_rand_X(init_n_outcome, problem))" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 30, 166 | "metadata": {}, 167 | "outputs": [], 168 | "source": [ 169 | "# pickle.dump(fixed_init_X_dict, open(\"../fixed_init_X_dict.pickle\", \"wb\"))" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": 33, 175 | "metadata": {}, 176 | "outputs": [ 177 | { 178 | "name": "stdout", 179 | "output_type": "stream", 180 | "text": [ 181 | "./run_multi_sim.sh -g3 -b1 -e1 -cconstant\n", 182 | "./run_multi_sim.sh -g4 -b2 -e2 -cconstant\n", 183 | "./run_multi_sim.sh -g5 -b3 -e3 -cconstant\n", 184 | "./run_multi_sim.sh -g6 -b4 -e4 -cconstant\n", 185 | "./run_multi_sim.sh -g7 -b5 -e5 -cconstant\n", 186 | "./run_multi_sim.sh -g3 -b6 -e6 -cconstant\n", 187 | "./run_multi_sim.sh -g4 -b7 -e7 -cconstant\n", 188 | "./run_multi_sim.sh -g5 -b8 -e8 -cconstant\n", 189 | "./run_multi_sim.sh -g6 -b9 -e9 -cconstant\n", 190 | "./run_multi_sim.sh -g7 -b10 -e10 -cconstant\n" 191 | ] 192 | } 193 | ], 194 | "source": [ 195 | "for i in range(10):\n", 196 | " print(f\"./run_multi_sim.sh -g{3+i%5} -b{i+1} -e{i+1} -cconstant\")" 197 | ] 198 | } 199 | ], 200 | "metadata": { 201 | "kernelspec": { 202 | "display_name": "pref", 203 | "language": "python", 204 | "name": "pref" 205 | }, 206 | "language_info": { 207 | "codemirror_mode": { 208 | "name": "ipython", 209 | "version": 3 210 | }, 211 | "file_extension": ".py", 212 | "mimetype": "text/x-python", 213 | "name": "python", 214 | "nbconvert_exporter": "python", 215 | "pygments_lexer": "ipython3", 216 | "version": "3.7.4" 217 | } 218 | }, 219 | "nbformat": 4, 220 | "nbformat_minor": 4 221 | } 222 | -------------------------------------------------------------------------------- /acquisition_functions.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import time 4 | from copy import deepcopy 5 | from typing import Any, Optional 6 | 7 | import torch 8 | from botorch.acquisition import AcquisitionFunction, MCAcquisitionFunction 9 | from botorch.acquisition.objective import MCAcquisitionObjective 10 | from botorch.exceptions.errors import UnsupportedError 11 | from botorch.models.model import Model 12 | from botorch.sampling.samplers import MCSampler, SobolQMCNormalSampler 13 | from botorch.utils.gp_sampling import get_gp_samples 14 | from botorch.utils.transforms import (concatenate_pending_points, 15 | match_batch_shape, 16 | t_batch_mode_transform) 17 | from torch import Tensor 18 | from torch.distributions import Bernoulli, Normal 19 | 20 | from constants import * # noqa: F403, F401 21 | from helper_classes import LearnedPrefereceObjective, PosteriorMeanDummySampler 22 | 23 | 24 | def get_rff_sample(outcome_model): 25 | om_without_transforms = deepcopy(outcome_model) 26 | om_without_transforms.input_transform = None 27 | om_without_transforms.outcome_transform = None 28 | 29 | gp_samples = get_gp_samples( 30 | model=om_without_transforms, 31 | num_outputs=om_without_transforms.num_outputs, 32 | n_samples=1, 33 | num_rff_features=500, 34 | ) 35 | 36 | gp_samples.input_transform = deepcopy(outcome_model.input_transform) 37 | gp_samples.outcome_transform = deepcopy(outcome_model.outcome_transform) 38 | return gp_samples 39 | 40 | 41 | class BALD(MCAcquisitionFunction): 42 | r"""Bayesian Active Learning by Disagreement""" 43 | 44 | def __init__( 45 | self, 46 | outcome_model: Model, 47 | pref_model: Model, 48 | search_space_type: str, 49 | sampler: Optional[MCSampler] = None, 50 | objective: Optional[MCAcquisitionObjective] = None, 51 | X_pending: Optional[Tensor] = None, 52 | **kwargs: Any, 53 | ) -> None: 54 | # sampler and objectives are placeholders and not used 55 | if sampler is None: 56 | sampler = PosteriorMeanDummySampler(model=outcome_model) 57 | 58 | if objective is None: 59 | preference_sampler = SobolQMCNormalSampler( 60 | num_samples=64, resample=False, collapse_batch_dims=True 61 | ) 62 | objective = LearnedPrefereceObjective( 63 | pref_model=pref_model, 64 | sampler=preference_sampler, 65 | use_mean=False, 66 | ) 67 | 68 | super().__init__( 69 | model=outcome_model, 70 | sampler=sampler, 71 | objective=objective, 72 | X_pending=X_pending, 73 | ) 74 | 75 | pref_model.eval() 76 | self.pref_model = pref_model 77 | self.search_space_type = search_space_type 78 | if search_space_type == "rff": 79 | self.gp_samples = get_rff_sample(outcome_model) 80 | 81 | @concatenate_pending_points 82 | @t_batch_mode_transform() 83 | def forward(self, X: Tensor) -> Tensor: 84 | # only work with q = 2 85 | assert X.shape[-2] == 2 86 | 87 | if self.search_space_type == "rff": 88 | batch_q_shape = X.shape[:-1] 89 | X_dim = X.shape[-1] 90 | Y = self.gp_samples.posterior(X.reshape(-1, X_dim)).mean.reshape(batch_q_shape + (-1,)) 91 | elif self.search_space_type == "f_mean": 92 | outcome_posterior = self.outcome_model.posterior(X) 93 | Y = outcome_posterior.mean 94 | elif self.search_space_type == "y": 95 | Y = X 96 | else: 97 | raise UnsupportedError("Unsupported search_space_type!") 98 | 99 | preference_posterior = self.pref_model(Y) 100 | preference_mean = preference_posterior.mean 101 | preference_cov = preference_posterior.covariance_matrix 102 | 103 | mu = preference_mean[..., 0] - preference_mean[..., 1] 104 | var = ( 105 | 2.0 106 | + preference_cov[..., 0, 0] 107 | + preference_cov[..., 1, 1] 108 | - preference_cov[..., 0, 1] 109 | - preference_cov[..., 1, 0] 110 | ) 111 | sigma = torch.sqrt(var) 112 | 113 | obj_samples = Normal(0, 1).cdf(Normal(mu, sigma).rsample(torch.Size([2048]))) 114 | 115 | posterior_entropies = ( 116 | Bernoulli(Normal(0, 1).cdf(mu / torch.sqrt(var + 1))).entropy().squeeze(-1) 117 | ) 118 | 119 | sample_entropies = Bernoulli(obj_samples).entropy() 120 | conditional_entropies = sample_entropies.mean(dim=0).squeeze(-1) 121 | return posterior_entropies - conditional_entropies 122 | 123 | 124 | class qPreferentialOptimal(MCAcquisitionFunction): 125 | r""" 126 | MC EUBO 127 | (y_1, y_2)^* = argmax_{y_1,y_2 \in Y} E[max{g(y_1), g(y_2)}] 128 | """ 129 | 130 | def __init__( 131 | self, 132 | outcome_model: Model, 133 | pref_model: Model, 134 | sampler: Optional[MCSampler] = None, 135 | objective: Optional[MCAcquisitionObjective] = None, 136 | X_pending: Optional[Tensor] = None, 137 | **kwargs: Any, 138 | ) -> None: 139 | r"""q-Preferential Noisy Expected Improvement. 140 | 141 | Args: 142 | outcome_model (Model): . 143 | pref_model (Model): . 144 | sampler (Optional[MCSampler], optional): . Defaults to None. 145 | objective (Optional[MCAcquisitionObjective], optional): . Defaults to None. 146 | X_pending (Optional[Tensor], optional): . Defaults to None. 147 | """ 148 | 149 | if sampler is None: 150 | sampler = PosteriorMeanDummySampler(model=outcome_model) 151 | 152 | if objective is None: 153 | preference_sampler = SobolQMCNormalSampler( 154 | num_samples=64, resample=False, collapse_batch_dims=True 155 | ) 156 | objective = LearnedPrefereceObjective( 157 | pref_model=pref_model, 158 | sampler=preference_sampler, 159 | use_mean=False, 160 | ) 161 | 162 | super().__init__( 163 | model=outcome_model, 164 | sampler=sampler, 165 | objective=objective, 166 | X_pending=X_pending, 167 | ) 168 | 169 | @concatenate_pending_points 170 | @t_batch_mode_transform() 171 | def forward(self, X: Tensor) -> Tensor: 172 | r"""Evaluate qNoisyExpectedImprovement on the candidate set `X`. 173 | 174 | Args: 175 | X: A `batch_shape x q x d`-dim Tensor of t-batches with `q` `d`-dim design 176 | points each. 177 | 178 | Returns: 179 | A `batch_shape'`-dim Tensor of Noisy Expected Improvement values at the 180 | given design points `X`, where `batch_shape'` is the broadcasted batch shape 181 | of model and input `X`. 182 | """ 183 | Y_posterior = self.model.posterior(X) 184 | Y_samples = self.sampler(Y_posterior) 185 | obj = self.objective(Y_samples) 186 | max_util_samples = obj.max(dim=-1).values 187 | exp_max_util = max_util_samples.mean(dim=0) 188 | return exp_max_util 189 | 190 | 191 | class ExpectedUtility(AcquisitionFunction): 192 | r"""Analytic Prefential Expected Utility, i.e., Analytical EUBO""" 193 | 194 | def __init__( 195 | self, 196 | preference_model: Model, 197 | outcome_model: Model, 198 | search_space_type: str = "f_mean", 199 | previous_winner: Optional[Tensor] = None, 200 | ) -> None: 201 | r"""Analytic Preferential Expected Utility. 202 | 203 | Args: 204 | preference_model (Model): . 205 | outcome_model (Model): . 206 | search_space_type (str, optional): "f_mean", "rff", or "one_sample". Defaults to "f_mean". 207 | previous_winner (Optional[Tensor], optional): Tensor representing the previous winner in Y space. 208 | Defaults to None. 209 | """ 210 | super().__init__(model=outcome_model) 211 | self.preference_model = preference_model 212 | self.outcome_model = outcome_model 213 | self.register_buffer("previous_winner", previous_winner) 214 | self.preference_model.eval() # make sure model is in eval mode 215 | if self.outcome_model is not None: 216 | self.outcome_model.eval() 217 | self.search_space_type = search_space_type 218 | dtype = preference_model.datapoints.dtype 219 | device = preference_model.datapoints.device 220 | self.std_norm = torch.distributions.normal.Normal( 221 | torch.zeros(1, dtype=dtype, device=device), 222 | torch.ones(1, dtype=dtype, device=device), 223 | ) 224 | if search_space_type == "rff": 225 | self.gp_samples = get_rff_sample(outcome_model) 226 | elif search_space_type == "one_sample": 227 | Y_dim = preference_model.datapoints.shape[-1] 228 | self.w = self.std_norm.rsample((Y_dim,)).squeeze(-1) 229 | elif search_space_type not in ("y", "f_mean"): 230 | raise UnsupportedError("Unsupported search space!") 231 | 232 | @t_batch_mode_transform() 233 | def forward(self, X: Tensor) -> Tensor: 234 | r"""Evaluate PreferentialOneStepLookahead on the candidate set X. 235 | Args: 236 | X: A `batch_shape x q x d`-dim Tensor, where `q = 2` if `previous_winner` is 237 | not `None`, and `q = 1` otherwise. 238 | Returns: 239 | The acquisition value for each batch as a tensor of shape `batch_shape`. 240 | """ 241 | assert (X.shape[-2] == 2) or ((X.shape[-2] == 1) and (self.previous_winner is not None)) 242 | 243 | if self.search_space_type == "rff": 244 | batch_q_shape = X.shape[:-1] 245 | X_dim = X.shape[-1] 246 | Y = self.gp_samples.posterior(X.reshape(-1, X_dim)).mean.reshape(batch_q_shape + (-1,)) 247 | elif self.search_space_type == "f_mean": 248 | outcome_posterior = self.outcome_model.posterior(X) 249 | Y = outcome_posterior.mean 250 | elif self.search_space_type == "one_sample": 251 | post = self.outcome_model.posterior(X) 252 | Y = post.mean + post.variance.sqrt() * self.w 253 | elif self.search_space_type == "y": 254 | Y = X 255 | else: 256 | raise UnsupportedError("Unsupported search_space_type!") 257 | 258 | if self.previous_winner is not None: 259 | Y = torch.cat([Y, match_batch_shape(self.previous_winner, Y)], dim=-2) 260 | preference_posterior = self.preference_model(Y) 261 | preference_mean = preference_posterior.mean 262 | preference_cov = preference_posterior.covariance_matrix 263 | delta = preference_mean[..., 0] - preference_mean[..., 1] 264 | sigma = torch.sqrt( 265 | preference_cov[..., 0, 0] 266 | + preference_cov[..., 1, 1] 267 | - preference_cov[..., 0, 1] 268 | - preference_cov[..., 1, 0] 269 | ) 270 | u = delta / sigma 271 | 272 | ucdf = self.std_norm.cdf(u) 273 | updf = torch.exp(self.std_norm.log_prob(u)) 274 | acqf_val = sigma * (updf + u * ucdf) 275 | if self.previous_winner is None: 276 | acqf_val += preference_mean[..., 1] 277 | return acqf_val 278 | -------------------------------------------------------------------------------- /multi_batch_sim.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import random 4 | import time 5 | import warnings 6 | from copy import deepcopy 7 | 8 | import fire 9 | import torch 10 | from gpytorch.utils.warnings import NumericalWarning 11 | 12 | from pbo import gen_pbo_candidates, get_pbo_pe_comparisons 13 | from sim_helpers import (fit_outcome_model, fit_pref_model, 14 | gen_parego_candidates, gen_pref_candidates_eval, 15 | gen_true_util_data, run_one_round_sim) 16 | from test_functions import gen_rand_X, problem_setup 17 | 18 | warnings.filterwarnings( 19 | "ignore", 20 | message="Could not update `train_inputs` with transformed inputs", 21 | ) 22 | 23 | 24 | def run_multi_batch_sim(problem_str, noisy, init_seed, kernel, comp_noise_type, tkwargs): 25 | problem_prefix = "_".join(problem_str.split("_")[:2]) 26 | fixed_init_X_dict = pickle.load(open("fixed_init_X_dict.pickle", "rb")) 27 | 28 | ( 29 | X_dim, 30 | Y_dim, 31 | problem, 32 | util_type, 33 | get_util, 34 | Y_bounds, 35 | probit_noise, 36 | ) = problem_setup(problem_str, noisy=noisy, **tkwargs) 37 | 38 | large_batch_size = False 39 | 40 | if comp_noise_type == "constant": 41 | comp_noise = 0.1 42 | elif comp_noise_type == "probit": 43 | comp_noise = probit_noise 44 | elif comp_noise_type == "none": 45 | comp_noise_type = "constant" 46 | comp_noise = 0.0 47 | else: 48 | raise RuntimeError("Invalid comp_noise_type! Must be constant, probit, or none") 49 | 50 | if X_dim <= 5: 51 | init_n_outcome = 16 52 | gen_batch_size = 8 53 | # keep the top batch_size in the generated batch to simulate cherry picking 54 | batch_size = 8 55 | else: 56 | init_n_outcome = 32 57 | gen_batch_size = 16 58 | batch_size = 16 59 | n_batch = 3 60 | keep_winner_prob = None 61 | 62 | if large_batch_size: 63 | init_n_outcome = init_n_outcome * 2 64 | gen_batch_size = gen_batch_size * 2 65 | batch_size = batch_size * 2 66 | 67 | print("START MULTI-BATCH SIM") 68 | print(f"init_n_outcome: {init_n_outcome}") 69 | print(f"gen_batch_size: {gen_batch_size}") 70 | print(f"batch_size: {batch_size}") 71 | print(f"n_batch: {n_batch}") 72 | print(f"comp_noise_type: {comp_noise_type}, comp_noise: {comp_noise}") 73 | 74 | # (training method, next batch generating methods, if one-shot, sample_outcome) 75 | policies = [ 76 | # Camera ready baselines 77 | # PBO 78 | ("pbo_ei_pe_eubo", None, False, False), # PBO EUBO 79 | ("pbo_ei_pe_ts", None, False, False), # PBO TS 80 | # EUBO family 81 | ("ri_eubo_y", "qnei", False, False), # EUBO-y0 82 | ("ri_eubo_one_sample", "qnei", False, False), # EUBO-zeta 83 | ("ri_eubo_rff", "qnei", False, False), # EUBO-f 84 | # BALD family 85 | ("ri_bald_rff", "qnei", False, False), # BALD-f 86 | ("ri_bald_yspace", "qnei", False, False), # BALD-Y0 87 | # Random baselines 88 | ("random_ps", "qnei", False, True), # Random-f 89 | ("uncorrelated", "qnei", False, None), # Random-Y0 90 | # Non-PE baselines 91 | ("parego_only", "qnei", False, False), # MOBO 92 | ("random_exp", None, False, False), # Random experiment 93 | ("true_util_seq", "qnei", False, None), # True utility 94 | # ======= one-shot baseilnes ========= 95 | # PBO 96 | ("pbo_ei_pe_ts", None, True, False), 97 | ("pbo_ei_pe_eubo", None, True, False), 98 | # EUBO family 99 | ("ri_eubo_y", "qnei", True, False), 100 | ("ri_eubo_one_sample", "qnei", True, False), 101 | ("ri_eubo_rff", "qnei", True, False), 102 | # BALD family 103 | ("ri_bald_rff", "qnei", True, False), 104 | ("ri_bald_yspace", "qnei", True, False), 105 | # Random baselines 106 | ("random_ps", "qnei", True, True), # Random f tilde 107 | ("uncorrelated", "qnei", True, None), # Random-Y 108 | ] 109 | 110 | # shuffle the order in case experiments auto-restarted 111 | # so that we have roughly even number of exp run for each config 112 | random.shuffle(policies) 113 | 114 | output_filepath = ( 115 | f"data/sim_results/multi_batch/interactive/sim_{problem_str}_{kernel}.pickle" 116 | # f"data/sim_results/multi_batch/one_shot/sim_{problem_str}_{kernel}.pickle" 117 | # f"data/sim_results/multi_batch/probit_noise/sim_{problem_str}_{kernel}.pickle" 118 | ) 119 | 120 | # ========= start simulation from here ========== 121 | # initial observation 122 | init_outcome_X = fixed_init_X_dict[problem_prefix][init_seed].to(Y_bounds) 123 | init_outcome_Y = problem(init_outcome_X) 124 | 125 | for policy, gen_method, one_shot, sample_outcome in policies: 126 | # read past experiments to check repeated experiments 127 | # re-read the file every time as it can be updated by other processses 128 | if os.path.isfile(output_filepath): 129 | past_sim_results = pickle.load(open(output_filepath, "rb")) 130 | else: 131 | past_sim_results = [] 132 | exp_set = set() 133 | for sr in past_sim_results: 134 | past_exp_signature = ( 135 | sr["init_seed"], 136 | sr["problem_str"], 137 | sr["policy"], 138 | sr["one_shot"], 139 | sr["comp_noise_type"], 140 | sr["comp_noise"], 141 | ) 142 | exp_set.add(past_exp_signature) 143 | 144 | curr_exp_signature = (init_seed, problem_str, policy, one_shot, comp_noise_type, comp_noise) 145 | if curr_exp_signature not in exp_set: 146 | exp_set.add(curr_exp_signature) 147 | else: 148 | print(f"Experiment {curr_exp_signature} was previously run! Skipping...") 149 | continue 150 | 151 | # total training round per preference session (or total rounds for one-shot case) 152 | n_pe_session_comp = 25 153 | total_training_round = (n_pe_session_comp * 3) if one_shot else n_pe_session_comp 154 | 155 | keep_winner_prob = None 156 | all_run_times = [] 157 | all_acq_run_times = [] 158 | print(policy, gen_method, one_shot, sample_outcome) 159 | 160 | outcome_X = init_outcome_X.clone() 161 | outcome_Y = init_outcome_Y.clone() 162 | 163 | # X/Y/comps for preference model 164 | # not used for Parego only policy 165 | train_X = None 166 | train_Y = None 167 | train_comps = None 168 | init_strategy = None 169 | learn_strategy = None 170 | 171 | init_round = Y_dim * 2 172 | 173 | if policy in ( 174 | "parego_only", 175 | "qnehvi", 176 | "true_util_seq", 177 | "pbo_ts", 178 | "pbo_ei", 179 | "pbo_ei_pe_ts", 180 | "pbo_ei_pe_eubo", 181 | "random_exp", 182 | ): 183 | keep_winner_prob = -1 184 | for batch_i in range(n_batch): 185 | print( 186 | f"init_seed {init_seed} - {policy}, one-shot: {one_shot}, gen_method: {gen_method}, batch: {batch_i}, on {problem_str}, total_training_round: {total_training_round}", 187 | ) 188 | outcome_model = fit_outcome_model(outcome_X, outcome_Y, X_bounds=problem.bounds) 189 | 190 | X_baseline = outcome_X 191 | # do not take Y here because the returned Y is a posterior sample 192 | acq_start_time = time.time() 193 | if policy == "parego_only": 194 | outcome_cand_X, _, _, _, = gen_parego_candidates( 195 | model=outcome_model, 196 | X=X_baseline, 197 | Y=outcome_Y, 198 | q=gen_batch_size, 199 | problem=problem, 200 | get_util=get_util, 201 | comp_noise_type=comp_noise_type, 202 | comp_noise=comp_noise, 203 | sample_outcome=sample_outcome, 204 | gen_method=gen_method, 205 | ) 206 | elif policy == "true_util_seq": 207 | outcome_cand_X, _, _, _, = gen_true_util_data( 208 | model=outcome_model, 209 | X=X_baseline, 210 | Y=outcome_Y, 211 | q=gen_batch_size, 212 | problem=problem, 213 | get_util=get_util, 214 | comp_noise_type=comp_noise_type, 215 | comp_noise=comp_noise, 216 | gen_method=gen_method, 217 | ) 218 | elif policy[:3] == "pbo": 219 | utils = get_util(outcome_Y) 220 | if (not one_shot) or (one_shot and batch_i == 0): 221 | if policy in ("pbo_ts", "pbo_ei"): 222 | pe_strategy = "random" 223 | elif policy == "pbo_ei_pe_ts": 224 | pe_strategy = "ts" 225 | elif policy == "pbo_ei_pe_eubo": 226 | pe_strategy = "eubo" 227 | else: 228 | raise ValueError("Unsupported PE strategy for PBO") 229 | 230 | train_comps = get_pbo_pe_comparisons( 231 | outcome_X, 232 | train_comps, 233 | problem, 234 | utils, 235 | init_round, 236 | total_training_round, 237 | comp_noise_type, 238 | comp_noise, 239 | pe_strategy=pe_strategy, 240 | ) 241 | 242 | observed_comp_error_rate = ( 243 | (utils[train_comps][..., 0] < utils[train_comps][..., 1]) 244 | .float() 245 | .mean() 246 | .item() 247 | ) 248 | print(f"observed_comp_error_rate:{observed_comp_error_rate:.3f}") 249 | else: 250 | print("not doing comparisons for PBO one-shot after init batch") 251 | print("train_comps shape:", train_comps.shape) 252 | 253 | if policy == "pbo_ts": 254 | pbo_gen_method = "ts" 255 | elif policy[:6] == "pbo_ei": 256 | pbo_gen_method = "ei" 257 | else: 258 | raise ValueError("Unknown PBO policy!") 259 | outcome_cand_X = gen_pbo_candidates( 260 | outcome_X=outcome_X, 261 | train_comps=train_comps, 262 | q=gen_batch_size, 263 | problem=problem, 264 | pbo_gen_method=pbo_gen_method, 265 | ) 266 | elif policy == "random_exp": 267 | outcome_cand_X = gen_rand_X(gen_batch_size, problem) 268 | else: 269 | raise ValueError("Unknown baseline policy") 270 | acq_runtime = time.time() - acq_start_time 271 | print(f"{policy} candidate gen time: {acq_runtime:.2f}s") 272 | 273 | # noisy observation 274 | outcome_cand_Y = problem(outcome_cand_X) 275 | outcome_cand_util = get_util(outcome_cand_Y) 276 | 277 | # select top candidates 278 | select_idx = outcome_cand_util.topk(k=batch_size).indices 279 | outcome_cand_X = outcome_cand_X[select_idx, :] 280 | outcome_cand_Y = outcome_cand_Y[select_idx, :] 281 | 282 | outcome_X = torch.cat((outcome_X, outcome_cand_X)) 283 | outcome_Y = torch.cat((outcome_Y, outcome_cand_Y)) 284 | else: 285 | print( 286 | f"init_seed {init_seed} - {policy}, {gen_method}, one-shot: {one_shot}, on {problem_str}, total_training_round: {total_training_round}", 287 | ) 288 | train_X = None 289 | train_Y = None 290 | train_comps = None 291 | selected_pairs = [] 292 | keep_winner_prob = 0 293 | if policy == "ri_bald_yspace": 294 | init_strategy = "uncorrelated" 295 | learn_strategy = "bald_yspace" 296 | elif policy == "ri_bald_correct": 297 | init_strategy = "random_ps" 298 | learn_strategy = "bald_correct" 299 | elif policy == "ri_bald_rff": 300 | init_strategy = "random_ps" 301 | learn_strategy = "bald_rff" 302 | elif policy == "ri_eubo_rff": 303 | init_strategy = "random_ps" 304 | learn_strategy = "eubo_rff" 305 | elif policy == "ri_eubo_y": 306 | init_strategy = "uncorrelated" 307 | learn_strategy = "eubo_y" 308 | elif policy == "ri_eubo_one_sample": 309 | init_strategy = "random_ps" 310 | learn_strategy = "eubo_one_sample" 311 | elif policy == "ri_bald": 312 | init_strategy = "random_ps" 313 | learn_strategy = "bald" 314 | elif policy == "random": 315 | init_strategy = "random_ps" 316 | learn_strategy = "random" 317 | elif policy == "random_ps": 318 | init_strategy = "random_ps" 319 | learn_strategy = "random_ps" 320 | elif policy == "uncorrelated": 321 | init_strategy = "uncorrelated" 322 | learn_strategy = "uncorrelated" 323 | else: 324 | raise RuntimeError("Unsupported learning policy") 325 | 326 | if one_shot: 327 | outcome_model = fit_outcome_model(outcome_X, outcome_Y, X_bounds=problem.bounds) 328 | ( 329 | train_X, 330 | train_Y, 331 | train_comps, 332 | acq_run_times, 333 | run_times, 334 | post_mean_X, 335 | post_mean_idx, 336 | selected_pairs, 337 | ) = run_one_round_sim( 338 | total_training_round=total_training_round, 339 | init_round=init_round, 340 | problem_str=problem_str, 341 | noisy=noisy, 342 | comp_noise_type=comp_noise_type, 343 | comp_noise=comp_noise, 344 | outcome_model=outcome_model, 345 | outcome_X=outcome_X, 346 | outcome_Y=outcome_Y, 347 | train_X=train_X, 348 | train_Y=train_Y, 349 | train_comps=train_comps, 350 | init_strategy=init_strategy, 351 | learn_strategy=learn_strategy, 352 | gen_method=gen_method, 353 | keep_winner_prob=keep_winner_prob, 354 | sample_outcome=sample_outcome, 355 | kernel=kernel, 356 | check_post_mean=False, 357 | check_post_mean_every_k=5, 358 | tkwargs=tkwargs, 359 | selected_pairs=selected_pairs, 360 | ) 361 | pref_model = fit_pref_model( 362 | train_Y, 363 | train_comps, 364 | kernel=kernel, 365 | transform_input=True, 366 | Y_bounds=Y_bounds, 367 | ) 368 | all_acq_run_times = all_acq_run_times + acq_run_times 369 | all_run_times = all_run_times + run_times 370 | 371 | for batch_i in range(n_batch): 372 | outcome_model = fit_outcome_model(outcome_X, outcome_Y, X_bounds=problem.bounds) 373 | if not one_shot: 374 | current_batch_init_round = max(0, init_round - batch_i * total_training_round) 375 | print(f"batch {batch_i}, current_batch_init_round: {current_batch_init_round}") 376 | ( 377 | train_X, 378 | train_Y, 379 | train_comps, 380 | acq_run_times, 381 | run_times, 382 | post_mean_X, 383 | post_mean_idx, 384 | selected_pairs, 385 | ) = run_one_round_sim( 386 | total_training_round=total_training_round, 387 | init_round=current_batch_init_round, 388 | problem_str=problem_str, 389 | noisy=noisy, 390 | comp_noise_type=comp_noise_type, 391 | comp_noise=comp_noise, 392 | outcome_model=outcome_model, 393 | outcome_X=outcome_X, 394 | outcome_Y=outcome_Y, 395 | train_X=train_X, 396 | train_Y=train_Y, 397 | train_comps=train_comps, 398 | init_strategy=init_strategy, 399 | learn_strategy=learn_strategy, 400 | gen_method=gen_method, 401 | keep_winner_prob=keep_winner_prob, 402 | sample_outcome=sample_outcome, 403 | kernel=kernel, 404 | check_post_mean=False, 405 | check_post_mean_every_k=5, 406 | tkwargs=tkwargs, 407 | selected_pairs=selected_pairs, 408 | ) 409 | 410 | pref_model = fit_pref_model( 411 | train_Y, 412 | train_comps, 413 | kernel=kernel, 414 | transform_input=True, 415 | Y_bounds=Y_bounds, 416 | ) 417 | all_acq_run_times = all_acq_run_times + acq_run_times 418 | all_run_times = all_run_times + run_times 419 | 420 | utils = get_util(train_Y) 421 | observed_comp_error_rate = ( 422 | (utils[train_comps][..., 0] < utils[train_comps][..., 1]) 423 | .float() 424 | .mean() 425 | .item() 426 | ) 427 | print(f"pref observed_comp_error_rate: {observed_comp_error_rate:.3f}") 428 | 429 | # generate next batch candidate 430 | # for baselines, only consider outcome_X as train_X are never observed for real 431 | # and could be a good point 432 | X_baseline = outcome_X 433 | acq_start_time = time.time() 434 | outcome_cand_X, _ = gen_pref_candidates_eval( 435 | outcome_model=outcome_model, 436 | pref_model=pref_model, 437 | X_baseline=X_baseline, 438 | problem=problem, 439 | gen_method=gen_method, 440 | q=gen_batch_size, 441 | tkwargs=tkwargs, 442 | ) 443 | acq_runtime = time.time() - acq_start_time 444 | print(f"pref candidate gen time: {acq_runtime:.2f}s") 445 | # noisy observation 446 | outcome_cand_Y = problem(outcome_cand_X) 447 | outcome_cand_util = get_util(outcome_cand_Y) 448 | 449 | # select top candidates 450 | select_idx = outcome_cand_util.topk(k=batch_size).indices 451 | outcome_cand_X = outcome_cand_X[select_idx, :] 452 | outcome_cand_Y = outcome_cand_Y[select_idx, :] 453 | 454 | outcome_X = torch.cat((outcome_X, outcome_cand_X)) 455 | outcome_Y = torch.cat((outcome_Y, outcome_cand_Y)) 456 | 457 | train_X = None if train_X is None else deepcopy(train_X).detach().cpu() 458 | train_Y = None if train_Y is None else deepcopy(train_Y).detach().cpu() 459 | train_comps = None if train_comps is None else deepcopy(train_comps).detach().cpu() 460 | 461 | single_result = { 462 | "init_seed": init_seed, 463 | "problem_str": problem_str, 464 | "policy": policy, 465 | "kernel": kernel, 466 | "noise_std": problem.noise_std, 467 | "init_round": init_round, 468 | "total_training_round": total_training_round, 469 | "one_shot": one_shot, 470 | "run_times": all_run_times, 471 | "acq_run_times": all_acq_run_times, 472 | # method for generating candidates, qnei or ts 473 | "gen_method": gen_method, 474 | "init_strategy": init_strategy, 475 | "learn_strategy": learn_strategy, 476 | "keep_winner_prob": keep_winner_prob, 477 | "comp_noise_type": comp_noise_type, 478 | "comp_noise": comp_noise, 479 | "sample_outcome": sample_outcome, 480 | "init_n_outcome": init_n_outcome, 481 | "gen_batch_size": gen_batch_size, 482 | "batch_size": batch_size, 483 | "n_batch": n_batch, 484 | "outcome_X": deepcopy(outcome_X).detach().cpu(), 485 | "outcome_Y": deepcopy(outcome_Y).detach().cpu(), 486 | "train_X": train_X, 487 | "train_Y": train_Y, 488 | "train_comps": train_comps, 489 | "device": str(tkwargs["device"]), 490 | "dtype": str(tkwargs["dtype"]), 491 | } 492 | 493 | if os.path.isfile(output_filepath): 494 | sim_results = pickle.load(open(output_filepath, "rb")) 495 | else: 496 | sim_results = [] 497 | sim_results.append(single_result) 498 | pickle.dump(sim_results, open(output_filepath, "wb")) 499 | torch.cuda.empty_cache() 500 | 501 | 502 | def main(problem_str, noisy, init_seed, kernel, comp_noise_type, device): 503 | """ 504 | Args: 505 | problem_str: problem string. see definition in test_functions.py 506 | noisy: whether we have noisy observation of the resopnse surface 507 | init_seed: initialization seed 508 | kernel: "default" (RBF) or "linear" (might not be numerically stable) 509 | comp_noise_type: "constant" or "probit" 510 | """ 511 | assert isinstance(noisy, bool) 512 | 513 | dtype = torch.double 514 | if device == "cpu": 515 | device = torch.device("cpu") 516 | else: 517 | # Does not really work. Need to set the env var in command line. 518 | # os.environ["CUDA_VISIBLE_DEVICES"] = f"{gpu}" 519 | # "device": torch.device(f"cuda:{gpu}" if torch.cuda.is_available() else "cpu"), 520 | # set device env variable externally 521 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 522 | 523 | tkwargs = { 524 | "dtype": dtype, 525 | "device": device, 526 | } 527 | 528 | warnings.filterwarnings("ignore", category=NumericalWarning) 529 | run_multi_batch_sim( 530 | problem_str=problem_str, 531 | noisy=noisy, 532 | init_seed=init_seed, 533 | kernel=kernel, 534 | comp_noise_type=comp_noise_type, 535 | tkwargs=tkwargs, 536 | ) 537 | 538 | 539 | if __name__ == "__main__": 540 | fire.Fire(main) 541 | -------------------------------------------------------------------------------- /test_functions.py: -------------------------------------------------------------------------------- 1 | import time 2 | from copy import deepcopy 3 | from typing import Dict, Optional 4 | 5 | import numpy as np 6 | import scipy 7 | import torch 8 | from botorch.distributions.distributions import Kumaraswamy 9 | from botorch.models.gp_regression import SingleTaskGP 10 | from botorch.optim.utils import sample_all_priors 11 | from botorch.test_functions.base import MultiObjectiveTestProblem 12 | from botorch.test_functions.multi_objective import DTLZ2, VehicleSafety 13 | from botorch.utils.gp_sampling import get_gp_samples 14 | from gpytorch.constraints import GreaterThan 15 | from gpytorch.kernels.matern_kernel import MaternKernel 16 | from gpytorch.kernels.scale_kernel import ScaleKernel 17 | from gpytorch.likelihoods.gaussian_likelihood import GaussianLikelihood 18 | from gpytorch.priors import GammaPrior 19 | from scipy.optimize import minimize 20 | from torch import Tensor, square 21 | # from torch.distributions import Beta 22 | from torch.quasirandom import SobolEngine 23 | 24 | # probit noise such that the DM makes 10% error for top 10% utilty using random X 25 | probit_noise_dict = { 26 | "vehiclesafety_5d3d_kumaraswamyproduct": 0.0203, 27 | "dtlz2_8d4d_negl1dist": 0.0467, 28 | "osy_6d8d_piecewiselinear": 2.4131, 29 | "carcabdesign_7d9d_piecewiselinear": 0.1151, 30 | "vehiclesafety_5d3d_piecewiselinear": 0.1587, 31 | "dtlz2_8d4d_piecewiselinear": 0.1872, 32 | "osy_6d8d_sigmodconstraints": 0.0299, 33 | "carcabdesign_7d9d_linear": 0.0439, 34 | } 35 | 36 | 37 | def gen_rand_points(n, dim, bounds): 38 | sobol = SobolEngine(dim, scramble=True) 39 | X = sobol.draw(n).to(bounds) 40 | X = X * (bounds[1, :] - bounds[0, :]) + bounds[0, :] 41 | X = X.to(bounds) 42 | return X 43 | 44 | 45 | def gen_rand_X(n, problem): 46 | return gen_rand_points(n, problem.dim, problem.bounds) 47 | 48 | 49 | def find_max(problem, get_util): 50 | problem_cpu = deepcopy(problem).cpu() 51 | util_cpu = deepcopy(get_util).cpu() 52 | 53 | def max_func_wrapper(x): 54 | Y = problem_cpu.evaluate_true(torch.tensor(x)).cpu() 55 | if len(Y.shape) == 1: 56 | Y = Y.unsqueeze(0) 57 | return -util_cpu(Y).numpy() 58 | 59 | real_max = -np.inf 60 | for i in range(10): 61 | n_sample = int(128) 62 | X = gen_rand_X(n_sample, problem_cpu) 63 | Y = problem.evaluate_true(X) 64 | util = get_util(Y) 65 | x0 = X[util.argmax(), :].numpy() 66 | 67 | res = minimize( 68 | max_func_wrapper, 69 | x0=x0, 70 | bounds=problem_cpu.bounds.numpy().T, 71 | ) 72 | proposed_max = util_cpu(problem_cpu.evaluate_true(torch.tensor(res.x))).item() 73 | # print(i, proposed_max, x0) 74 | if proposed_max > real_max: 75 | real_max = proposed_max 76 | real_max_X = torch.tensor(res.x) 77 | 78 | return real_max_X, real_max 79 | 80 | 81 | class AdaptedOSY(MultiObjectiveTestProblem): 82 | r""" 83 | Adapted OSY test problem from [Oszycka1995]_. 84 | This is adapted from botorch implementation. 85 | We negated the fs and treat gs a objectives so that the goal is to maximzie everything 86 | """ 87 | 88 | dim = 6 89 | num_objectives = 8 90 | _bounds = [ 91 | (0.0, 10.0), 92 | (0.0, 10.0), 93 | (1.0, 5.0), 94 | (0.0, 6.0), 95 | (1.0, 5.0), 96 | (0.0, 10.0), 97 | ] 98 | # Placeholder reference point 99 | _ref_point = [0.0] * 8 100 | 101 | def evaluate_true(self, X: Tensor) -> Tensor: 102 | f1 = ( 103 | 25 * (X[..., 0] - 2) ** 2 104 | + (X[..., 1] - 2) ** 2 105 | + (X[..., 2] - 1) ** 2 106 | + (X[..., 3] - 4) ** 2 107 | + (X[..., 4] - 1) ** 2 108 | ) 109 | f2 = -(X ** 2).sum(-1) 110 | g1 = X[..., 0] + X[..., 1] - 2.0 111 | g2 = 6.0 - X[..., 0] - X[..., 1] 112 | g3 = 2.0 - X[..., 1] + X[..., 0] 113 | g4 = 2.0 - X[..., 0] + 3.0 * X[..., 1] 114 | g5 = 4.0 - (X[..., 2] - 3.0) ** 2 - X[..., 3] 115 | g6 = (X[..., 4] - 3.0) ** 2 + X[..., 5] - 4.0 116 | return torch.stack([f1, f2, g1, g2, g3, g4, g5, g6], dim=-1) 117 | 118 | 119 | class NegativeVehicleSafety(VehicleSafety): 120 | def evaluate_true(self, X: Tensor) -> Tensor: 121 | f = -super().evaluate_true(X) 122 | Y_bounds = torch.tensor( 123 | [ 124 | [-1.7040e03, -1.1708e01, -2.6192e-01], 125 | [-1.6619e03, -6.2136e00, -4.2879e-02], 126 | ] 127 | ).to(X) 128 | f = (f - Y_bounds[0, :]) / (Y_bounds[1, :] - Y_bounds[0, :]) 129 | return f 130 | 131 | 132 | class CarCabDesign(MultiObjectiveTestProblem): 133 | r"""RE9-7-1 car cab design from Tanabe & Ishibuchi (2020)""" 134 | 135 | dim = 7 136 | num_objectives = 9 137 | _bounds = [ 138 | (0.5, 1.5), 139 | (0.45, 1.35), 140 | (0.5, 1.5), 141 | (0.5, 1.5), 142 | (0.875, 2.625), 143 | (0.4, 1.2), 144 | (0.4, 1.2), 145 | ] 146 | _ref_point = [0.0, 0.0] # TODO: Determine proper reference point 147 | 148 | def evaluate_true(self, X: Tensor) -> Tensor: 149 | f = torch.empty(X.shape[:-1] + (self.num_objectives,), dtype=X.dtype, device=X.device) 150 | 151 | X1 = X[..., 0] 152 | X2 = X[..., 1] 153 | X3 = X[..., 2] 154 | X4 = X[..., 3] 155 | X5 = X[..., 4] 156 | X6 = X[..., 5] 157 | X7 = X[..., 6] 158 | # # stochastic variables 159 | # X8 = 0.006 * (torch.randn_like(X1)) + 0.345 160 | # X9 = 0.006 * (torch.randn_like(X1)) + 0.192 161 | # X10 = 10 * (torch.randn_like(X1)) + 0.0 162 | # X11 = 10 * (torch.randn_like(X1)) + 0.0 163 | 164 | # not using stochastic variables for the real function 165 | X8 = torch.zeros_like(X1) 166 | X9 = torch.zeros_like(X1) 167 | X10 = torch.zeros_like(X1) 168 | X11 = torch.zeros_like(X1) 169 | 170 | # First function 171 | # negate the first function as we want minimize car weight 172 | f[..., 0] = -( 173 | 1.98 174 | + 4.9 * X1 175 | + 6.67 * X2 176 | + 6.98 * X3 177 | + 4.01 * X4 178 | + 1.75 * X5 179 | + 0.00001 * X6 180 | + 2.73 * X7 181 | ) 182 | # Second function 183 | f[..., 1] = 1 - ( 184 | 1.16 - 0.3717 * X2 * X4 - 0.00931 * X2 * X10 - 0.484 * X3 * X9 + 0.01343 * X6 * X10 185 | ) 186 | # Third function 187 | f[..., 2] = 0.32 - ( 188 | 0.261 189 | - 0.0159 * X1 * X2 190 | - 0.188 * X1 * X8 191 | - 0.019 * X2 * X7 192 | + 0.0144 * X3 * X5 193 | + 0.87570001 * X5 * X10 194 | + 0.08045 * X6 * X9 195 | + 0.00139 * X8 * X11 196 | + 0.00001575 * X10 * X11 197 | ) 198 | # Fourth function 199 | f[..., 3] = 0.32 - ( 200 | 0.214 201 | + 0.00817 * X5 202 | - 0.131 * X1 * X8 203 | - 0.0704 * X1 * X9 204 | + 0.03099 * X2 * X6 205 | - 0.018 * X2 * X7 206 | + 0.0208 * X3 * X8 207 | + 0.121 * X3 * X9 208 | - 0.00364 * X5 * X6 209 | + 0.0007715 * X5 * X10 210 | - 0.0005354 * X6 * X10 211 | + 0.00121 * X8 * X11 212 | + 0.00184 * X9 * X10 213 | - 0.018 * X2 * X2 214 | ) 215 | # Fifth function 216 | f[..., 4] = 0.32 - ( 217 | 0.74 218 | - 0.61 * X2 219 | - 0.163 * X3 * X8 220 | + 0.001232 * X3 * X10 221 | - 0.166 * X7 * X9 222 | + 0.227 * X2 * X2 223 | ) 224 | # SiXth function 225 | tmp = ( 226 | ( 227 | 28.98 228 | + 3.818 * X3 229 | - 4.2 * X1 * X2 230 | + 0.0207 * X5 * X10 231 | + 6.63 * X6 * X9 232 | - 7.77 * X7 * X8 233 | + 0.32 * X9 * X10 234 | ) 235 | + ( 236 | 33.86 237 | + 2.95 * X3 238 | + 0.1792 * X10 239 | - 5.057 * X1 * X2 240 | - 11 * X2 * X8 241 | - 0.0215 * X5 * X10 242 | - 9.98 * X7 * X8 243 | + 22 * X8 * X9 244 | ) 245 | + (46.36 - 9.9 * X2 - 12.9 * X1 * X8 + 0.1107 * X3 * X10) 246 | ) / 3 247 | f[..., 5] = 32 - tmp 248 | # Seventh function 249 | f[..., 6] = 32 - ( 250 | 4.72 251 | - 0.5 * X4 252 | - 0.19 * X2 * X3 253 | - 0.0122 * X4 * X10 254 | + 0.009325 * X6 * X10 255 | + 0.000191 * X11 * X11 256 | ) 257 | # EighthEighth function 258 | f[..., 7] = 4 - ( 259 | 10.58 260 | - 0.674 * X1 * X2 261 | - 1.95 * X2 * X8 262 | + 0.02054 * X3 * X10 263 | - 0.0198 * X4 * X10 264 | + 0.028 * X6 * X10 265 | ) 266 | # Ninth function 267 | f[..., 8] = 9.9 - ( 268 | 16.45 269 | - 0.489 * X3 * X7 270 | - 0.843 * X5 * X6 271 | + 0.0432 * X9 * X10 272 | - 0.0556 * X9 * X11 273 | - 0.000786 * X11 * X11 274 | ) 275 | 276 | Y_bounds = torch.tensor( 277 | [ 278 | [ 279 | -4.2150e01, 280 | -4.7829e-01, 281 | -1.1563e02, 282 | -7.2040e-03, 283 | -1.8255e-01, 284 | -1.0168e01, 285 | 2.7023e01, 286 | -8.0731e00, 287 | -6.4556e00, 288 | ], 289 | # Old upper bound from 1e8 points 290 | # [-16.0992, 0.9511, 112.7138, 0.2750, 0.1909, 14.4804, 28.9855, -2.4875, -0.8270], 291 | # make upper bounds of constraints to be something > 0 so that it's possible to not violate the constraints 292 | [-16.0992, 0.9511, 112.7138, 0.2750, 0.1909, 14.4804, 28.9855, 0.5, 0.5], 293 | ] 294 | ).to(f) 295 | f = (f - Y_bounds[0, :]) / (Y_bounds[1, :] - Y_bounds[0, :]) 296 | 297 | # normalize f to between 0 and 1 roughly so that we won't disadvantage ParEGO 298 | return f 299 | 300 | 301 | # ======= Utility functions ========== 302 | 303 | 304 | class OSYSigmoidConstraintsUtil(torch.nn.Module): 305 | def __init__(self, Y_bounds): 306 | super().__init__() 307 | self.register_buffer("Y_bounds", Y_bounds) 308 | 309 | def calc_raw_util_per_dim(self, Y): 310 | Y_bounds = self.Y_bounds 311 | obj_Y = Y[..., :2] 312 | constr_Y = Y[..., 2:] 313 | norm_obj_Y = (obj_Y - Y_bounds[0, :2]) / (Y_bounds[1, :2] - Y_bounds[0, :2]) 314 | 315 | obj_vals = norm_obj_Y.exp() 316 | constr_vals = torch.sigmoid( 317 | 50 318 | * constr_Y 319 | / torch.min(torch.stack((-Y_bounds[0, 2:], Y_bounds[1, 2:])), dim=0).values 320 | ) 321 | return torch.cat((obj_vals, constr_vals), dim=-1) 322 | 323 | def forward(self, Y, X=None): 324 | util_vals = self.calc_raw_util_per_dim(Y) 325 | constr_vals = util_vals[..., 2:] 326 | obj_vals = util_vals[..., :2] 327 | 328 | obj_sum = obj_vals.sum(-1) 329 | constr_prod = constr_vals.prod(dim=-1) 330 | 331 | util = obj_sum * constr_prod 332 | return util 333 | 334 | 335 | class NegDist(torch.nn.Module): 336 | def __init__(self, ideal_point, p, square=False): 337 | super().__init__() 338 | self.register_buffer("ideal_point", ideal_point) 339 | self.p = p 340 | self.square = square 341 | 342 | def forward(self, Y, X=None): 343 | if len(Y.shape) == 1: 344 | Y = Y.unsqueeze(0) 345 | expanded_ideal = self.ideal_point.expand(Y.shape[:-2] + (1, -1)).contiguous() 346 | dist = torch.cdist(Y, expanded_ideal, p=self.p).squeeze(-1) 347 | if self.square: 348 | return -dist.square() 349 | else: 350 | return -dist 351 | 352 | 353 | class LinearUtil(torch.nn.Module): 354 | def __init__(self, beta): 355 | super().__init__() 356 | self.register_buffer("beta", beta) 357 | 358 | def calc_raw_util_per_dim(self, Y): 359 | return Y * self.beta.to(Y) 360 | 361 | def forward(self, Y, X=None): 362 | return Y @ self.beta.to(Y) 363 | 364 | 365 | class PiecewiseLinear(torch.nn.Module): 366 | def __init__(self, beta1, beta2, thresholds): 367 | super().__init__() 368 | self.register_buffer("beta1", beta1) 369 | self.register_buffer("beta2", beta2) 370 | self.register_buffer("thresholds", thresholds) 371 | 372 | def calc_raw_util_per_dim(self, Y): 373 | # below thresholds 374 | bt = Y < self.thresholds 375 | b1 = self.beta1.expand(Y.shape) 376 | b2 = self.beta2.expand(Y.shape) 377 | shift = (b2 - b1) * self.thresholds 378 | util_val = torch.empty_like(Y) 379 | 380 | # util_val[bt] = Y[bt] * b1[bt] 381 | util_val[bt] = Y[bt] * b1[bt] + shift[bt] 382 | util_val[~bt] = Y[~bt] * b2[~bt] 383 | 384 | return util_val 385 | 386 | def forward(self, Y, X=None): 387 | util_val = self.calc_raw_util_per_dim(Y) 388 | util_val = util_val.sum(dim=-1) 389 | return util_val 390 | 391 | 392 | class KumaraswamyCDF(torch.nn.Module): 393 | def __init__(self, concentration1, concentration2, Y_bounds): 394 | super().__init__() 395 | self.register_buffer("concentration1", concentration1) 396 | self.register_buffer("concentration2", concentration2) 397 | self.register_buffer("Y_bounds", Y_bounds) 398 | self.kdist = Kumaraswamy(concentration1, concentration2) 399 | 400 | def calc_raw_util_per_dim(self, Y): 401 | Y_bounds = self.Y_bounds 402 | 403 | Y = (Y - Y_bounds[0, :]) / (Y_bounds[1, :] - Y_bounds[0, :]) 404 | eps = 1e-6 405 | Y = torch.clamp(Y, min=eps, max=1 - eps) 406 | 407 | util_val = self.kdist.cdf(Y) 408 | 409 | return util_val 410 | 411 | def forward(self, Y, X=None): 412 | util_val = self.calc_raw_util_per_dim(Y) 413 | util_val = util_val[..., ::2] * util_val[..., 1::2] 414 | util_val = util_val.sum(dim=-1) 415 | 416 | return util_val 417 | 418 | 419 | class KumaraswamyCDFProduct(KumaraswamyCDF): 420 | def forward(self, Y, X=None): 421 | util_val = self.calc_raw_util_per_dim(Y) 422 | util_val = torch.prod(util_val, dim=-1) 423 | 424 | return util_val 425 | 426 | 427 | class PiecewiseUtil(torch.nn.Module): 428 | def __init__(self, beta, thresholds, alphas, ymin, ymax): 429 | super().__init__() 430 | self.register_buffer("beta", beta) 431 | self.thresholds = thresholds.to(beta) 432 | self.alphas = alphas 433 | self.shift = 1 434 | self.pow_size = 4 435 | self.ymin = ymin 436 | 437 | n_max = ( 438 | self.calc_raw_util_per_dim( 439 | torch.full(size=(1, beta.shape[0]), fill_value=ymax).to(beta) 440 | ) 441 | .max() 442 | .item() 443 | ) 444 | n_min = ( 445 | self.calc_raw_util_per_dim( 446 | torch.full(size=(1, beta.shape[0]), fill_value=ymin).to(beta) 447 | ) 448 | .min() 449 | .item() 450 | ) 451 | self.norm_range = (n_min, n_max) 452 | 453 | def calc_raw_util_per_dim(self, Y): 454 | # assuming Y is generally between 0 and 1.5 455 | Y = torch.clamp(Y, min=self.ymin) 456 | shift = self.shift 457 | alphas = self.alphas 458 | thresholds = self.thresholds 459 | pow_size = self.pow_size 460 | beta_mat = self.beta.expand(Y.shape) 461 | thresholds_mat = thresholds.expand(Y.shape) 462 | alphas_mat = alphas.expand(Y.shape) 463 | below_threshold = Y < self.thresholds 464 | util_val = torch.empty_like(Y) 465 | 466 | util_val[below_threshold] = ( 467 | Y[below_threshold] - thresholds_mat[below_threshold] - shift 468 | ).pow(pow_size) 469 | util_val[below_threshold] = (-util_val[below_threshold] + shift ** pow_size) * alphas_mat[ 470 | below_threshold 471 | ] 472 | util_val[~below_threshold] = ( 473 | Y[~below_threshold] - thresholds_mat[~below_threshold] 474 | ) * beta_mat[~below_threshold] 475 | 476 | return util_val 477 | 478 | def forward(self, Y, X=None): 479 | if len(Y.shape) == 1: 480 | Y = Y.unsqueeze(0) 481 | 482 | util_val = self.calc_raw_util_per_dim(Y) 483 | util_val = (util_val - self.norm_range[0]) / (self.norm_range[1] - self.norm_range[0]) 484 | util_val_int = util_val 485 | util_val_int = util_val_int[..., ::2] * util_val_int[..., 1::2] 486 | 487 | util_val = util_val.sum(-1) + util_val_int.sum(-1) 488 | return util_val 489 | 490 | 491 | def problem_setup(problem_str, noisy=False, **tkwargs): 492 | """example problem_str: 493 | "vehiclesafety_5d3d_kumaraswamyproduct" 494 | "dtlz2_8d4d_negl1dist" 495 | "osy_6d8d_piecewiselinear" 496 | "carcabdesign_7d9d_piecewiselinear" 497 | 498 | "vehiclesafety_5d3d_piecewiselinear" 499 | "dtlz2_8d4d_piecewiselinear" 500 | "osy_6d8d_sigmodconstraints" 501 | "carcabdesign_7d9d_linear" 502 | """ 503 | problem_name, dims_str, util_type = problem_str.split("_") 504 | Y_bounds = None 505 | 506 | # dtlz 2 response surface 507 | if problem_name == "dtlz2": 508 | dims = dims_str.split("d") 509 | X_dim, Y_dim = int(dims[0]), int(dims[1]) 510 | if dims_str == "8d4d": 511 | # upper bound obatined using 1.2 * max 512 | Y_bounds = torch.tensor( 513 | [ 514 | [0.0000, 0.0000, 0.0000, 0.0000], 515 | [2.5366, 2.5237, 2.5996, 2.6484], 516 | ] 517 | ).to(**tkwargs) 518 | else: 519 | raise RuntimeError("Unsupported problem_str") 520 | if noisy: 521 | # lowered noise level 522 | noise_std = 0.05 523 | # noise_std = 0.1 524 | else: 525 | noise_std = 0 526 | 527 | problem = DTLZ2(dim=X_dim, num_objectives=Y_dim, noise_std=noise_std).to(**tkwargs) 528 | # min-max normalization range for creating interaction terms 529 | ymin, ymax = 0.0, 1.5 530 | 531 | # utility functions for dtlz2 532 | if util_type == "piecewiselinear": 533 | if Y_dim == 4: 534 | beta1 = torch.tensor([4, 3, 2, 1]).to(**tkwargs) 535 | beta2 = torch.tensor([0.4, 0.3, 0.2, 0.1]).to(**tkwargs) 536 | thresholds = torch.tensor([1.0] * Y_dim).to(**tkwargs) 537 | get_util = PiecewiseLinear(beta1=beta1, beta2=beta2, thresholds=thresholds) 538 | else: 539 | raise RuntimeError("Unsupported Y_dim for piecewise linear utility") 540 | elif util_type == "negl1dist": 541 | get_util = NegDist( 542 | problem.evaluate_true(torch.tensor([0.5] * X_dim, **tkwargs)), p=1, square=False 543 | ) 544 | else: 545 | raise RuntimeError("Unsupported utility!") 546 | elif problem_name == "vehiclesafety": 547 | if noisy: 548 | # lowered noise level 549 | noise_std = 0.05 550 | else: 551 | noise_std = 0 552 | # we wish to minimize all metrics in the original problems 553 | # hence we negate all values 554 | problem = NegativeVehicleSafety(noise_std=noise_std).to(**tkwargs) 555 | X_dim = problem.dim 556 | Y_dim = problem.num_objectives 557 | Y_bounds = torch.tensor( 558 | [ 559 | [0, 0, 0], 560 | [1, 1, 1], 561 | ] 562 | ).to(**tkwargs) 563 | if util_type == "piecewiselinear": 564 | beta1 = torch.tensor([2, 6, 8]).to(**tkwargs) 565 | beta2 = torch.tensor([1, 2, 2]).to(**tkwargs) 566 | thresholds = torch.tensor([0.5, 0.8, 0.8]).to(**tkwargs) 567 | get_util = PiecewiseLinear(beta1=beta1, beta2=beta2, thresholds=thresholds) 568 | elif util_type == "kumaraswamyproduct": 569 | concentration1 = torch.tensor([0.5, 1, 1.5]).to(**tkwargs) 570 | concentration2 = torch.tensor([1.0, 2.0, 3.0]).to(**tkwargs) 571 | get_util = KumaraswamyCDFProduct( 572 | concentration1=concentration1, concentration2=concentration2, Y_bounds=Y_bounds 573 | ) 574 | else: 575 | raise RuntimeError("Unsupported utility!") 576 | elif problem_name == "carcabdesign": 577 | if noisy: 578 | # lowered noise level 579 | noise_std = 0.02 580 | else: 581 | noise_std = 0 582 | problem = CarCabDesign(noise_std=noise_std).to(**tkwargs) 583 | X_dim = problem.dim 584 | Y_dim = problem.num_objectives 585 | Y_bounds = torch.tensor( 586 | [ 587 | [0, 0, 0, 0, 0, 0, 0, 0, 0], 588 | [1, 1, 1, 1, 1, 1, 1, 1, 1], 589 | ] 590 | ).to(**tkwargs) 591 | 592 | if util_type == "linear": 593 | beta = torch.tensor([2.25, 2, 1.75, 1.5, 1.25, 1, 0.75, 0.5, 0.25]).to(**tkwargs) 594 | get_util = LinearUtil(beta=beta) 595 | elif util_type == "piecewiselinear": 596 | beta1 = torch.tensor([7.0, 6.75, 6.5, 6.25, 6.0, 5.75, 5.5, 5.25, 5.0]).to(**tkwargs) 597 | beta2 = torch.tensor([0.5, 0.4, 0.375, 0.35, 0.325, 0.3, 0.275, 0.25, 0.225]).to( 598 | **tkwargs 599 | ) 600 | thresholds = torch.tensor([0.55, 0.54, 0.53, 0.52, 0.51, 0.5, 0.49, 0.48, 0.47]).to( 601 | **tkwargs 602 | ) 603 | get_util = PiecewiseLinear(beta1=beta1, beta2=beta2, thresholds=thresholds) 604 | else: 605 | raise RuntimeError("Unsupported utility!") 606 | elif problem_name == "osy": 607 | if noisy: 608 | raise NotImplementedError("Noise level not yet determined!") 609 | else: 610 | noise_std = 0 611 | if dims_str == "6d8d": 612 | # Scale the empirical bounds by 1.1 to make sure we can include extreme values 613 | Y_bounds = torch.tensor( 614 | [ 615 | [ 616 | 4.2358e-02, 617 | -3.7138e02, 618 | -1.9988e00, 619 | -1.3999e01, 620 | -7.9987e00, 621 | -7.9990e00, 622 | -5.9989e00, 623 | -4.0000e00, 624 | ], 625 | [1707.5742, -2.6934, 17.9988, 5.9988, 11.9993, 31.9968, 3.9999, 9.9983], 626 | ] 627 | ).to(**tkwargs) 628 | problem = AdaptedOSY(noise_std=noise_std).to(**tkwargs) 629 | X_dim = problem.dim 630 | Y_dim = problem.num_objectives 631 | 632 | if util_type == "piecewiselinear": 633 | if Y_dim == 8: 634 | beta1 = torch.tensor([0.02, 0.2, 10, 10, 10, 10, 10, 10]).to(**tkwargs) 635 | beta2 = torch.tensor([0.01, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]).to(**tkwargs) 636 | thresholds = torch.tensor([1000, -100] + [0.0] * (Y_dim - 2)).to(**tkwargs) 637 | else: 638 | raise RuntimeError("Unsupported Y_dim for betacdf utility") 639 | get_util = PiecewiseLinear(beta1=beta1, beta2=beta2, thresholds=thresholds) 640 | elif util_type == "sigmodconstraints": 641 | get_util = OSYSigmoidConstraintsUtil(Y_bounds=Y_bounds) 642 | else: 643 | raise RuntimeError("Unsupported problem!") 644 | 645 | if problem_str in probit_noise_dict: 646 | probit_noise = probit_noise_dict[problem_str] 647 | else: 648 | probit_noise = None 649 | print(f"{problem_str}, noisy: {noisy}, noise_std: {problem.noise_std}") 650 | return X_dim, Y_dim, problem, util_type, get_util, Y_bounds, probit_noise 651 | -------------------------------------------------------------------------------- /sim_helpers.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import random 3 | import time 4 | from copy import deepcopy 5 | 6 | import cma 7 | import torch 8 | from botorch.acquisition import GenericMCObjective 9 | from botorch.acquisition.monte_carlo import (qNoisyExpectedImprovement, 10 | qSimpleRegret) 11 | from botorch.acquisition.utils import prune_inferior_points 12 | from botorch.exceptions.errors import UnsupportedError 13 | from botorch.fit import fit_gpytorch_model 14 | from botorch.models.gp_regression import SingleTaskGP 15 | from botorch.models.pairwise_gp import (PairwiseGP, 16 | PairwiseLaplaceMarginalLogLikelihood) 17 | from botorch.models.transforms.input import (ChainedInputTransform, Normalize, 18 | Warp) 19 | from botorch.models.transforms.outcome import Standardize 20 | from botorch.optim.optimize import optimize_acqf 21 | from botorch.sampling.samplers import IIDNormalSampler, SobolQMCNormalSampler 22 | from botorch.utils.multi_objective.scalarization import \ 23 | get_chebyshev_scalarization 24 | from botorch.utils.sampling import sample_simplex 25 | from gpytorch.kernels import (AdditiveStructureKernel, LinearKernel, 26 | PolynomialKernel, ScaleKernel) 27 | from gpytorch.kernels.rbf_kernel import RBFKernel 28 | from gpytorch.mlls.exact_marginal_log_likelihood import \ 29 | ExactMarginalLogLikelihood 30 | from gpytorch.priors.smoothed_box_prior import SmoothedBoxPrior 31 | from gpytorch.priors.torch_priors import LogNormalPrior 32 | from scipy.stats import kendalltau 33 | 34 | from acquisition_functions import BALD, ExpectedUtility 35 | from constants import * 36 | from helper_classes import LearnedPrefereceObjective, PosteriorMeanDummySampler 37 | from test_functions import gen_rand_points, gen_rand_X, problem_setup 38 | 39 | 40 | def fit_outcome_model(X, Y, X_bounds): 41 | # fit outcome model 42 | input_tf = Normalize(d=X.shape[-1], bounds=X_bounds) 43 | outcome_model = SingleTaskGP( 44 | train_X=X, 45 | train_Y=Y, 46 | outcome_transform=Standardize(m=Y.shape[-1]), 47 | input_transform=input_tf, 48 | ) 49 | mll = ExactMarginalLogLikelihood(outcome_model.likelihood, outcome_model) 50 | fit_gpytorch_model(mll) 51 | return outcome_model.to(device=X.device, dtype=X.dtype) 52 | 53 | 54 | def fit_pref_model(Y, comps, kernel, transform_input=True, Y_bounds=None, jitter=1e-4): 55 | """Preference model fitting helper function""" 56 | Y_dim = Y.shape[-1] 57 | if Y_bounds is None or not transform_input: 58 | chained_transform = None 59 | else: 60 | normalize_tf = Normalize(d=Y_dim, bounds=Y_bounds) 61 | warp_tf = Warp( 62 | indices=list(range(Y_dim)), 63 | # use a prior with median at 1. 64 | # when a=1 and b=1, the Kumaraswamy CDF is the identity function 65 | concentration1_prior=LogNormalPrior(0.0, 0.75 ** 0.5), 66 | concentration0_prior=LogNormalPrior(0.0, 0.75 ** 0.5), 67 | ) 68 | chained_transform = ChainedInputTransform(normalize_tf=normalize_tf, warp_tf=warp_tf) 69 | 70 | if kernel == "default": 71 | model = PairwiseGP( 72 | Y.double().cpu(), 73 | comps.double().cpu(), 74 | jitter=jitter, 75 | input_transform=chained_transform, 76 | ) 77 | elif kernel == "linear": 78 | covar_module = ScaleKernel( 79 | LinearKernel(num_dimensions=Y.shape[-1]), 80 | outputscale_prior=SmoothedBoxPrior(a=1, b=4), 81 | ) 82 | model = PairwiseGP( 83 | Y.double().cpu(), 84 | comps.double().cpu(), 85 | covar_module=covar_module, 86 | jitter=jitter, 87 | input_transform=chained_transform, 88 | ) 89 | elif kernel == "additive": 90 | covar_module = AdditiveStructureKernel(base_kernel=RBFKernel(), num_dims=Y.shape[1]) 91 | model = PairwiseGP( 92 | Y.double().cpu(), 93 | comps.double().cpu(), 94 | covar_module=covar_module, 95 | jitter=jitter, 96 | input_transform=chained_transform, 97 | ) 98 | elif kernel == "polynomial": 99 | covar_module = ScaleKernel( 100 | PolynomialKernel(power=2), 101 | outputscale_prior=SmoothedBoxPrior(a=1, b=2), 102 | ) 103 | model = PairwiseGP( 104 | Y.double().cpu(), 105 | comps.double().cpu(), 106 | covar_module=covar_module, 107 | jitter=jitter, 108 | input_transform=chained_transform, 109 | ) 110 | else: 111 | raise RuntimeError("Unsupported kernel") 112 | mll = PairwiseLaplaceMarginalLogLikelihood(model) 113 | fit_gpytorch_model(mll) 114 | model = model.to(device=Y.device, dtype=Y.dtype) 115 | 116 | return model 117 | 118 | 119 | def inject_comp_error(comp, util_diff, comp_noise_type, comp_noise): 120 | std_norm = torch.distributions.normal.Normal( 121 | torch.zeros(1, dtype=util_diff.dtype, device=util_diff.device), 122 | torch.ones(1, dtype=util_diff.dtype, device=util_diff.device), 123 | ) 124 | 125 | if comp_noise_type == "constant": 126 | comp_error_p = comp_noise 127 | elif comp_noise_type == "probit": 128 | comp_error_p = 1 - std_norm.cdf(util_diff / comp_noise) 129 | else: 130 | raise UnsupportedError(f"Unsupported comp_noise_type: {comp_noise_type}") 131 | 132 | # with comp_error_p probability to make a comparison mistake 133 | flip_rand = torch.rand(util_diff.shape).to(util_diff) 134 | to_flip = flip_rand < comp_error_p 135 | flipped_comp = comp.clone() 136 | if len(flipped_comp.shape) > 1: 137 | assert (util_diff >= 0).all() 138 | # flip tensor 139 | flipped_comp[to_flip, 0], flipped_comp[to_flip, 1] = comp[to_flip, 1], comp[to_flip, 0] 140 | else: 141 | assert util_diff > 0 142 | # flip a single pair 143 | if to_flip: 144 | flipped_comp[[0, 1]] = flipped_comp[[1, 0]] 145 | return flipped_comp 146 | 147 | 148 | def organize_comparisons(utils, comps, comp_noise_type, comp_noise): 149 | """ 150 | Given utility and comparisons in arbitrary orders, 151 | re-order comparisons such that comparisons are in correct orders 152 | with comparison noise injected 153 | 154 | Args: 155 | utils ([type]): [description] 156 | comps ([type]): [description] 157 | comp_noise_type ([type]): [description] 158 | comp_noise ([type]): [description] 159 | 160 | Returns: 161 | [type]: [description] 162 | """ 163 | comps = deepcopy(comps) 164 | pair_utils = utils[comps] 165 | is_incorrect = pair_utils[..., 0] < pair_utils[..., 1] 166 | comps[is_incorrect, 0], comps[is_incorrect, 1] = ( 167 | comps[is_incorrect, 1], 168 | comps[is_incorrect, 0], 169 | ) 170 | 171 | # inject comparison error 172 | util_diff = utils[comps] 173 | util_diff = util_diff[..., 0] - util_diff[..., 1] 174 | comps = inject_comp_error( 175 | comps, util_diff, comp_noise_type=comp_noise_type, comp_noise=comp_noise 176 | ) 177 | comps = comps.to(device=utils.device) 178 | return comps 179 | 180 | 181 | def gen_comps(utility, comp_noise_type, comp_noise): 182 | """Create pairwise comparisons""" 183 | cpu_util = utility.cpu() 184 | 185 | comp_pairs = [] 186 | for i in range(cpu_util.shape[0] // 2): 187 | i1 = i * 2 188 | i2 = i * 2 + 1 189 | if cpu_util[i1] > cpu_util[i2]: 190 | new_comp = [i1, i2] 191 | util_diff = cpu_util[i1] - cpu_util[i2] 192 | else: 193 | new_comp = [i2, i1] 194 | util_diff = cpu_util[i2] - cpu_util[i1] 195 | 196 | new_comp = torch.tensor(new_comp, device=utility.device, dtype=torch.long) 197 | new_comp = inject_comp_error(new_comp, util_diff, comp_noise_type, comp_noise) 198 | comp_pairs.append(new_comp) 199 | 200 | comp_pairs = torch.stack(comp_pairs) 201 | 202 | return comp_pairs 203 | 204 | 205 | def gen_initial_real_data(n, problem, get_util): 206 | # generate (noisy) ground truth data 207 | X = gen_rand_X(n, problem) 208 | Y = problem(X) 209 | util = get_util(Y) 210 | comps = gen_comps(util, comp_noise_type="constant", comp_noise=0) 211 | 212 | return X, Y, util, comps 213 | 214 | 215 | def gen_exp_comps(X, model, get_util, comp_noise_type, comp_noise, sample_outcome): 216 | # generate posterior dras and make simulated comparisons based on that 217 | if sample_outcome: 218 | cand_Y = model.posterior(X).sample().squeeze(0) 219 | else: 220 | cand_Y = model.posterior(X).mean.clone().detach() 221 | cand_util = get_util(cand_Y) 222 | cand_comps = gen_comps(cand_util, comp_noise_type, comp_noise) 223 | return cand_Y, cand_util, cand_comps 224 | 225 | 226 | def gen_uncorrelated_candidates(q, Y_bounds, get_util, comp_noise_type, comp_noise): 227 | # randomly selected points in Y space 228 | cand_X = torch.tensor([]).to(Y_bounds) 229 | cand_Y = gen_rand_points(q, Y_bounds.shape[-1], Y_bounds) 230 | cand_util = get_util(cand_Y) 231 | cand_comps = gen_comps(cand_util, comp_noise_type, comp_noise) 232 | 233 | return cand_X, cand_Y, cand_util, cand_comps 234 | 235 | 236 | def gen_random_candidates(model, q, problem, get_util, comp_noise_type, comp_noise, sample_outcome): 237 | # generate training data 238 | cand_X = gen_rand_X(q, problem=problem) 239 | cand_Y, cand_util, cand_comps = gen_exp_comps( 240 | cand_X, model, get_util, comp_noise_type, comp_noise, sample_outcome 241 | ) 242 | return cand_X, cand_Y, cand_util, cand_comps 243 | 244 | 245 | def gen_observed_candidates( 246 | outcome_X, outcome_Y, selected_pairs, get_util, comp_noise_type, comp_noise 247 | ): 248 | # randomly selected observed points 249 | 250 | # all combination of index pairs 251 | all_combo = list(itertools.combinations(range(outcome_Y.shape[0]), 2)) 252 | new_comp = all_combo[random.randrange(len(all_combo))] 253 | 254 | # put the new pair in right order 255 | new_util = get_util(outcome_Y[new_comp, :]) 256 | if new_util[1] > new_util[0]: 257 | new_comp = (new_comp[1], new_comp[0]) 258 | util_diff = new_util[1] - new_util[0] 259 | else: 260 | util_diff = new_util[0] - new_util[1] 261 | 262 | new_comp = inject_comp_error(new_comp, util_diff, comp_noise_type, comp_noise) 263 | 264 | selected_pairs.append(new_comp) 265 | unique_ids = torch.tensor(selected_pairs).flatten().unique().tolist() 266 | id_map = dict(zip(unique_ids, range(len(unique_ids)))) 267 | 268 | # construct all candidate values (instead of only new ones) 269 | all_cand_X = outcome_X[unique_ids, :] 270 | all_cand_Y = outcome_Y[unique_ids, :] 271 | all_cand_util = get_util(all_cand_Y) 272 | all_cand_comps = torch.tensor([(id_map[p1], id_map[p2]) for p1, p2 in selected_pairs]) 273 | 274 | return all_cand_X, all_cand_Y, all_cand_util, all_cand_comps, selected_pairs 275 | 276 | 277 | def gen_parego_candidates( 278 | model, X, Y, q, problem, get_util, comp_noise_type, comp_noise, sample_outcome, gen_method 279 | ): 280 | cand_X = [] 281 | for _ in range(q): 282 | weights = sample_simplex(problem.num_objectives).squeeze().to(Y) 283 | objective = GenericMCObjective(get_chebyshev_scalarization(weights=weights, Y=Y)) 284 | 285 | if gen_method == "ts": 286 | n_sample = 1024 287 | 288 | rand_X = gen_rand_X(n_sample, problem) 289 | outcome_post = model.posterior(rand_X).sample().squeeze(0) 290 | post_util = objective(outcome_post) 291 | cand_X.append(rand_X[torch.argmax(post_util), :]) 292 | else: 293 | try: 294 | sampler = SobolQMCNormalSampler(num_samples=NUM_PAREGO_SAMPLES) 295 | if gen_method == "qnei": 296 | acq_func = qNoisyExpectedImprovement( 297 | model=model, 298 | X_baseline=X, 299 | sampler=sampler, 300 | objective=objective, 301 | prune_baseline=True, 302 | ) 303 | else: 304 | raise RuntimeError 305 | 306 | # optimize 307 | # generate 1 candidate at a time, repeat q times 308 | single_cand_X, _ = optimize_acqf( 309 | acq_function=acq_func, 310 | q=1, 311 | bounds=problem.bounds, 312 | num_restarts=NUM_RESTARTS, 313 | raw_samples=RAW_SAMPLES, # used for intialization heuristic 314 | options={"batch_limit": BATCH_LIMIT, "ftol": FTOL}, 315 | ) 316 | cand_X.append(single_cand_X.squeeze(0)) 317 | except UnsupportedError: 318 | sampler = IIDNormalSampler(num_samples=NUM_PAREGO_SAMPLES) 319 | if gen_method == "qnei": 320 | acq_func = qNoisyExpectedImprovement( 321 | model=model, 322 | X_baseline=X, 323 | sampler=sampler, 324 | objective=objective, 325 | prune_baseline=True, 326 | ) 327 | else: 328 | raise RuntimeError 329 | 330 | # optimize 331 | # generate 1 candidate at a time, repeat q times 332 | single_cand_X, _ = optimize_acqf( 333 | acq_function=acq_func, 334 | q=1, 335 | bounds=problem.bounds, 336 | num_restarts=NUM_RESTARTS, 337 | raw_samples=RAW_SAMPLES, # used for intialization heuristic 338 | options={"batch_limit": BATCH_LIMIT, "ftol": FTOL}, 339 | ) 340 | cand_X.append(single_cand_X.squeeze(0)) 341 | 342 | cand_X = torch.stack(cand_X) 343 | # "observe" new values from outcome model 344 | cand_Y, cand_util, cand_comps = gen_exp_comps( 345 | cand_X, model, get_util, comp_noise_type, comp_noise, sample_outcome 346 | ) 347 | 348 | return cand_X, cand_Y, cand_util, cand_comps 349 | 350 | 351 | def gen_true_util_data(model, X, Y, q, problem, get_util, comp_noise_type, comp_noise, gen_method): 352 | sampler = SobolQMCNormalSampler(num_samples=NUM_TRUE_UTIL_SAMPLES) 353 | true_obj = GenericMCObjective(get_util) 354 | 355 | if gen_method == "ts": 356 | cand_X = [] 357 | n_sample = 1024 358 | 359 | for i in range(q): 360 | rand_X = gen_rand_X(n_sample, problem) 361 | outcome_post = model.posterior(rand_X).sample().squeeze(0) 362 | post_util = true_obj(outcome_post) 363 | cand_X.append(rand_X[torch.argmax(post_util), :]) 364 | 365 | cand_X = torch.stack(cand_X) 366 | 367 | else: 368 | if gen_method == "qnei": 369 | acq_func = qNoisyExpectedImprovement( 370 | model=model, 371 | X_baseline=X[:1], 372 | sampler=sampler, 373 | objective=true_obj, 374 | prune_baseline=False, 375 | ) 376 | else: 377 | raise RuntimeError 378 | cand_X, _ = optimize_acqf( 379 | acq_function=acq_func, 380 | q=q, 381 | bounds=problem.bounds, 382 | num_restarts=NUM_RESTARTS, 383 | raw_samples=RAW_SAMPLES, # used for intialization heuristic 384 | options={"batch_limit": BATCH_LIMIT, "ftol": FTOL}, 385 | ) 386 | 387 | # "observe" new values from outcome model 388 | cand_Y, cand_util, cand_comps = gen_exp_comps( 389 | cand_X, 390 | model, 391 | get_util, 392 | comp_noise_type, 393 | comp_noise, 394 | sample_outcome=False, 395 | ) 396 | return cand_X, cand_Y, cand_util, cand_comps 397 | 398 | 399 | def get_pref_acqf( 400 | outcome_model, 401 | pref_model, 402 | X_baseline, 403 | problem, 404 | sampler_constructor, 405 | gen_method, 406 | **kwargs, 407 | ): 408 | prune_pref_sample_num = kwargs.get("prune_pref_sample_num", 64) 409 | prune_outcome_sample_num = kwargs.get("prune_outcome_sample_num", 64) 410 | pref_mean = kwargs.get("pref_mean", False) 411 | pref_sample_num = kwargs.get("pref_sample_num", NUM_LEARN_PREF_SAMPLES_UNEIPM) 412 | outcome_mean = kwargs.get("outcome_mean", True) 413 | outcome_sample_num = kwargs.get("outcome_sample_num", 1) 414 | device = kwargs.get("device", torch.device("cpu")) 415 | dtype = kwargs.get("dtype", torch.float) 416 | 417 | print(f"Inside pref_mean: {pref_mean}, {pref_sample_num}, {outcome_mean}, {outcome_sample_num}") 418 | 419 | prune_obj = LearnedPrefereceObjective( 420 | pref_model=pref_model, 421 | sampler=sampler_constructor(num_samples=prune_pref_sample_num), 422 | use_mean=False, 423 | ).to(device=device, dtype=dtype) 424 | 425 | if outcome_mean: 426 | prune_sampler = PosteriorMeanDummySampler(model=outcome_model) 427 | else: 428 | prune_sampler = sampler_constructor(num_samples=prune_outcome_sample_num) 429 | 430 | pruned_train_X = prune_inferior_points( 431 | model=outcome_model, 432 | X=X_baseline, 433 | objective=prune_obj, 434 | sampler=prune_sampler, 435 | ) 436 | 437 | pref_obj = LearnedPrefereceObjective( 438 | pref_model=pref_model, 439 | sampler=sampler_constructor(num_samples=pref_sample_num), 440 | use_mean=pref_mean, 441 | ) 442 | if outcome_mean: 443 | outcome_sampler = PosteriorMeanDummySampler(model=outcome_model) 444 | else: 445 | outcome_sampler = sampler_constructor(num_samples=outcome_sample_num) 446 | 447 | if gen_method == "qnei": 448 | acq_func = qNoisyExpectedImprovement( 449 | model=outcome_model, 450 | X_baseline=pruned_train_X, 451 | sampler=outcome_sampler, 452 | objective=pref_obj, 453 | prune_baseline=False, 454 | ) 455 | else: 456 | raise RuntimeError(f"unsupported qnei gen method {gen_method}") 457 | 458 | return acq_func 459 | 460 | 461 | def gen_bald_candidates( 462 | outcome_model, pref_model, problem, gen_method, Y_bounds, search_space_type, **kwargs 463 | ): 464 | q = kwargs.get("q", 1) 465 | num_restarts = kwargs.get("num_restarts", NUM_RESTARTS) 466 | raw_samples = kwargs.get("raw_samples", RAW_SAMPLES) 467 | batch_limit = kwargs.get("batch_limit", BATCH_LIMIT) 468 | sequential = kwargs.get("sequential", False) 469 | 470 | print(f"BALD q={q}, search_space_type={search_space_type}") 471 | if gen_method == "ts": 472 | raise RuntimeError("Can't do TS!") 473 | 474 | if search_space_type == "y": 475 | bounds = Y_bounds 476 | else: 477 | bounds = problem.bounds 478 | 479 | acqf = BALD( 480 | outcome_model=outcome_model, pref_model=pref_model, search_space_type=search_space_type 481 | ) 482 | 483 | cand_X, acqf_val = optimize_acqf( 484 | acq_function=acqf, 485 | bounds=bounds, 486 | q=q, 487 | num_restarts=num_restarts, 488 | raw_samples=raw_samples, 489 | options={"batch_limit": batch_limit, "ftol": FTOL}, 490 | sequential=sequential, 491 | ) 492 | 493 | if search_space_type == "rff": 494 | cand_Y = acqf.gp_samples.posterior(cand_X).mean.squeeze(0).clone().detach() 495 | elif search_space_type == "f_mean": 496 | cand_Y = outcome_model.posterior(cand_X).mean.clone().detach() 497 | elif search_space_type == "y": 498 | cand_Y = cand_X 499 | # create empty tensor so that it won't trigger issues when we append it to train_X 500 | cand_X = torch.empty(0).to(cand_Y) 501 | else: 502 | raise UnsupportedError("Unsupported search_space_type!") 503 | 504 | return cand_X, cand_Y, acqf_val 505 | 506 | 507 | def gen_expected_util_candidates( 508 | outcome_model, pref_model, problem, previous_winner, search_space_type, **kwargs 509 | ): 510 | """Analytical EUBO""" 511 | q = 2 if previous_winner is None else 1 512 | num_restarts = kwargs.get("num_restarts", NUM_RESTARTS) 513 | raw_samples = kwargs.get("raw_samples", RAW_SAMPLES) 514 | batch_limit = kwargs.get("batch_limit", BATCH_LIMIT) 515 | sequential = kwargs.get("sequential", False) 516 | Y_bounds = kwargs.get("Y_bounds", False) 517 | return_acqf = kwargs.get("return_acqf", False) 518 | 519 | if search_space_type == "y": 520 | bounds = Y_bounds 521 | else: 522 | bounds = problem.bounds 523 | 524 | acqf = ExpectedUtility( 525 | preference_model=pref_model, 526 | outcome_model=outcome_model, 527 | previous_winner=previous_winner, 528 | search_space_type=search_space_type, 529 | ) 530 | 531 | cand_X, acqf_val = optimize_acqf( 532 | acq_function=acqf, 533 | bounds=bounds, 534 | q=q, 535 | num_restarts=num_restarts, 536 | raw_samples=raw_samples, 537 | options={"batch_limit": batch_limit, "ftol": FTOL}, 538 | sequential=sequential, 539 | ) 540 | 541 | if search_space_type == "rff": 542 | cand_Y = acqf.gp_samples.posterior(cand_X).mean.squeeze(0).clone().detach() 543 | elif search_space_type == "f_mean": 544 | cand_Y = outcome_model.posterior(cand_X).mean.clone().detach() 545 | elif search_space_type == "one_sample": 546 | post = outcome_model.posterior(cand_X) 547 | cand_Y = (post.mean + post.variance.sqrt() * acqf.w).clone().detach() 548 | elif search_space_type == "y": 549 | cand_Y = cand_X 550 | # create empty tensor so that it won't trigger issues when we append it to train_X 551 | cand_X = torch.empty(0).to(cand_Y) 552 | else: 553 | raise UnsupportedError("Unsupported search_space_type!") 554 | if return_acqf: 555 | return cand_X, cand_Y, acqf_val, acqf 556 | else: 557 | return cand_X, cand_Y, acqf_val 558 | 559 | 560 | def gen_pref_candidates(outcome_model, pref_model, X_baseline, problem, gen_method, **kwargs): 561 | q = kwargs.get("q", 1) 562 | num_restarts = kwargs.get("num_restarts", NUM_RESTARTS) 563 | raw_samples = kwargs.get("raw_samples", RAW_SAMPLES) 564 | batch_limit = kwargs.get("batch_limit", BATCH_LIMIT) 565 | sequential = kwargs.get("sequential", False) 566 | 567 | try: 568 | sampler_constructor = SobolQMCNormalSampler 569 | acqf = get_pref_acqf( 570 | outcome_model, 571 | pref_model, 572 | X_baseline, 573 | problem, 574 | gen_method=gen_method, 575 | sampler_constructor=sampler_constructor, 576 | **kwargs, 577 | ) 578 | cand_X, acqf_val = optimize_acqf( 579 | acq_function=acqf, 580 | bounds=problem.bounds, 581 | q=q, 582 | num_restarts=num_restarts, 583 | raw_samples=raw_samples, 584 | options={"batch_limit": batch_limit, "ftol": FTOL}, 585 | sequential=sequential, 586 | ) 587 | except UnsupportedError: 588 | "Switch to IID normal sampler if sobol fails" 589 | sampler_constructor = IIDNormalSampler 590 | acqf = get_pref_acqf( 591 | outcome_model, 592 | pref_model, 593 | X_baseline, 594 | problem, 595 | gen_method=gen_method, 596 | sampler_constructor=sampler_constructor, 597 | **kwargs, 598 | ) 599 | cand_X, acqf_val = optimize_acqf( 600 | acq_function=acqf, 601 | bounds=problem.bounds, 602 | q=q, 603 | num_restarts=num_restarts, 604 | raw_samples=raw_samples, 605 | options={"batch_limit": batch_limit, "ftol": FTOL}, 606 | sequential=sequential, 607 | ) 608 | 609 | return cand_X, acqf_val 610 | 611 | 612 | def gen_pref_candidates_eval( 613 | outcome_model, pref_model, X_baseline, problem, q, gen_method, tkwargs 614 | ): 615 | return gen_pref_candidates( 616 | outcome_model=outcome_model, 617 | pref_model=pref_model, 618 | X_baseline=X_baseline, 619 | problem=problem, 620 | gen_method=gen_method, 621 | q=q, 622 | num_restarts=NUM_RESTARTS, 623 | batch_limit=BATCH_LIMIT, 624 | raw_samples=RAW_SAMPLES, 625 | sequential=True, 626 | pref_sample_num=NUM_EVAL_PREF_SAMPLES, 627 | pref_mean=False, 628 | outcome_sample_num=NUM_EVAL_OUTCOME_SAMPLES, 629 | outcome_mean=False, 630 | **tkwargs, 631 | ) 632 | 633 | 634 | def gen_post_mean(outcome_model, pref_model, problem, **kwargs): 635 | pref_sample_num = kwargs.get("pref_sample_num", 64) 636 | outcome_sample_num = kwargs.get("outcome_sample_num", 64) 637 | num_restarts = kwargs.get("num_restarts", NUM_RESTARTS) 638 | batch_limit = kwargs.get("batch_limit", BATCH_LIMIT) 639 | raw_samples = kwargs.get("raw_samples", RAW_SAMPLES) 640 | use_mean = kwargs.get("use_mean", False) 641 | 642 | pref_obj = LearnedPrefereceObjective( 643 | pref_model=pref_model, 644 | sampler=SobolQMCNormalSampler(num_samples=pref_sample_num), 645 | use_mean=use_mean, 646 | ) 647 | outcome_sampler = SobolQMCNormalSampler(num_samples=outcome_sample_num) 648 | post_mean = qSimpleRegret( 649 | outcome_model, 650 | sampler=outcome_sampler, 651 | objective=pref_obj, 652 | ) 653 | cand_X, _ = optimize_acqf( 654 | acq_function=post_mean, 655 | bounds=problem.bounds, 656 | q=1, 657 | num_restarts=num_restarts, 658 | raw_samples=raw_samples, 659 | options={"batch_limit": batch_limit, "ftol": FTOL}, 660 | ) 661 | return cand_X 662 | 663 | 664 | def gen_learn_candidates( 665 | q, 666 | problem, 667 | get_util, 668 | Y_bounds, 669 | learn_strategy, 670 | outcome_model, 671 | pref_model, 672 | gen_method, 673 | X_baseline, 674 | outcome_Y, 675 | train_Y, 676 | comp_noise_type, 677 | comp_noise, 678 | sample_outcome, 679 | previous_winner_idx, 680 | kernel, 681 | **kwargs, 682 | ): 683 | 684 | cand_X = None 685 | cand_Y = None 686 | 687 | # import pdb; pdb.set_trace() 688 | for _ in range(2): 689 | try: 690 | if learn_strategy == "qnei": 691 | # uNEI-PM 692 | cand_X, _ = gen_pref_candidates( 693 | outcome_model=outcome_model, 694 | pref_model=pref_model, 695 | X_baseline=X_baseline, 696 | problem=problem, 697 | gen_method=gen_method, 698 | q=q, 699 | **kwargs, 700 | ) 701 | elif learn_strategy in ("bald_correct", "bald_yspace", "bald_rff"): 702 | if learn_strategy == "bald_correct": 703 | search_space_type = "f_mean" 704 | elif learn_strategy == "bald_yspace": 705 | search_space_type = "y" 706 | elif learn_strategy == "bald_rff": 707 | search_space_type = "rff" 708 | else: 709 | raise UnsupportedError("Uknown BALD search_space_type!") 710 | 711 | cand_X, cand_Y, _ = gen_bald_candidates( 712 | outcome_model=outcome_model, 713 | pref_model=pref_model, 714 | problem=problem, 715 | gen_method=gen_method, 716 | q=q, 717 | Y_bounds=Y_bounds, 718 | search_space_type=search_space_type, 719 | **kwargs, 720 | ) 721 | elif learn_strategy == "eubo_rff": 722 | # EUBO-PS 723 | cand_X, cand_Y, _ = gen_expected_util_candidates( 724 | outcome_model=outcome_model, 725 | pref_model=pref_model, 726 | problem=problem, 727 | previous_winner=None, 728 | search_space_type="rff", 729 | ) 730 | elif learn_strategy == "eubo_one_sample": 731 | # EUBO-OPS 732 | cand_X, cand_Y, _ = gen_expected_util_candidates( 733 | outcome_model=outcome_model, 734 | pref_model=pref_model, 735 | problem=problem, 736 | previous_winner=None, 737 | search_space_type="one_sample", 738 | ) 739 | elif learn_strategy == "eubo_y": 740 | # EUBO-PS 741 | cand_X, cand_Y, _ = gen_expected_util_candidates( 742 | outcome_model=outcome_model, 743 | pref_model=pref_model, 744 | problem=problem, 745 | previous_winner=None, 746 | search_space_type="y", 747 | Y_bounds=Y_bounds, 748 | ) 749 | elif learn_strategy in ("random", "random_ps"): 750 | # Surrogate(-ps) random 751 | if learn_strategy == "random_ps": 752 | sample_outcome = True 753 | cand_X, cand_Y, _, _ = gen_random_candidates( 754 | model=outcome_model, 755 | q=q, 756 | problem=problem, 757 | get_util=get_util, 758 | comp_noise_type=comp_noise_type, 759 | comp_noise=comp_noise, 760 | sample_outcome=sample_outcome, 761 | ) 762 | elif learn_strategy == "uncorrelated": 763 | # Uniform random 764 | cand_X, cand_Y, _, _ = gen_uncorrelated_candidates( 765 | q=q, 766 | Y_bounds=Y_bounds, 767 | get_util=get_util, 768 | comp_noise_type=comp_noise_type, 769 | comp_noise=comp_noise, 770 | ) 771 | else: 772 | raise RuntimeError("Unsupported learning strategy") 773 | except Exception as e: 774 | print(e) 775 | print("Encounter exceptions... try again...") 776 | continue 777 | break 778 | 779 | if cand_Y is None: 780 | if sample_outcome: 781 | cand_Y = outcome_model.posterior(cand_X).sample().squeeze(0).clone().detach() 782 | else: 783 | cand_Y = outcome_model.posterior(cand_X).mean.clone().detach() 784 | 785 | return cand_X, cand_Y 786 | 787 | 788 | def run_one_round_sim( 789 | total_training_round, 790 | init_round, 791 | problem_str, 792 | noisy, 793 | comp_noise_type, 794 | comp_noise, 795 | outcome_model, 796 | outcome_X, 797 | outcome_Y, 798 | train_X, 799 | train_Y, 800 | train_comps, 801 | init_strategy, 802 | learn_strategy, 803 | gen_method, 804 | keep_winner_prob, 805 | sample_outcome, 806 | kernel, 807 | check_post_mean, 808 | check_post_mean_every_k, 809 | tkwargs, 810 | selected_pairs, # for "observed" init/learn strategy only, set to [] by default 811 | ): 812 | 813 | ( 814 | X_dim, 815 | Y_dim, 816 | problem, 817 | util_type, 818 | get_util, 819 | Y_bounds, 820 | probit_noise, 821 | ) = problem_setup(problem_str, noisy=noisy, **tkwargs) 822 | 823 | pref_model = None 824 | last_winner_idx = None 825 | post_mean_X = [] 826 | post_mean_idx = [] 827 | run_times = [] 828 | acq_run_times = [] 829 | 830 | # if started with previous train_Y, initialize the pref model 831 | if train_Y is not None: 832 | pref_model = fit_pref_model( 833 | train_Y, train_comps, kernel=kernel, transform_input=True, Y_bounds=Y_bounds 834 | ) 835 | 836 | for i in range(total_training_round): 837 | start_time = time.time() 838 | if i < init_round or keep_winner_prob is None or learn_strategy == "observed": 839 | # Init phase 840 | current_strategy = init_strategy 841 | # using initialization strategy 842 | if init_strategy in ("random", "random_ps"): 843 | if init_strategy == "random_ps": 844 | sample_outcome = True 845 | (pref_init_X, pref_init_Y, _, pref_init_comps,) = gen_random_candidates( 846 | model=outcome_model, 847 | q=2, 848 | problem=problem, 849 | get_util=get_util, 850 | comp_noise_type=comp_noise_type, 851 | comp_noise=comp_noise, 852 | sample_outcome=sample_outcome, 853 | ) 854 | elif init_strategy == "parego": 855 | if train_X is None: 856 | X_baseline = outcome_X 857 | else: 858 | X_baseline = torch.cat((train_X, outcome_X)).to(**tkwargs) 859 | (pref_init_X, pref_init_Y, _, pref_init_comps,) = gen_parego_candidates( 860 | model=outcome_model, 861 | X=X_baseline, 862 | Y=outcome_Y, 863 | q=2, 864 | problem=problem, 865 | get_util=get_util, 866 | comp_noise_type=comp_noise_type, 867 | comp_noise=comp_noise, 868 | sample_outcome=sample_outcome, 869 | gen_method=gen_method, 870 | ) 871 | elif init_strategy == "uncorrelated": 872 | # not using sample_outcome 873 | (pref_init_X, pref_init_Y, _, pref_init_comps,) = gen_uncorrelated_candidates( 874 | q=2, 875 | Y_bounds=Y_bounds, 876 | get_util=get_util, 877 | comp_noise_type=comp_noise_type, 878 | comp_noise=comp_noise, 879 | ) 880 | elif init_strategy == "observed": 881 | # not using sample_outcome 882 | ( 883 | pref_init_X, 884 | pref_init_Y, 885 | _, 886 | pref_init_comps, 887 | selected_pairs, 888 | ) = gen_observed_candidates( 889 | outcome_X, outcome_Y, selected_pairs, get_util, comp_noise_type, comp_noise 890 | ) 891 | # manually set train_X to be None so that we can update the whole training data 892 | train_X = None 893 | else: 894 | raise RuntimeError 895 | 896 | if train_X is None: 897 | train_X = pref_init_X 898 | train_Y = pref_init_Y 899 | train_comps = pref_init_comps 900 | else: 901 | comps_shifted = pref_init_comps + train_Y.shape[0] 902 | train_X = torch.cat((train_X, pref_init_X), dim=0) 903 | train_Y = torch.cat((train_Y, pref_init_Y), dim=0) 904 | train_comps = torch.cat((train_comps, comps_shifted), dim=0) 905 | else: 906 | # Learning phase 907 | current_strategy = learn_strategy 908 | X_baseline = torch.cat((train_X, outcome_X)).to(**tkwargs) 909 | if last_winner_idx is not None and random.random() < keep_winner_prob: 910 | keep_winner = True 911 | q = 1 912 | else: 913 | keep_winner = False 914 | q = 2 915 | 916 | # generate candidate(s) 917 | cand_X, cand_Y = gen_learn_candidates( 918 | q=q, 919 | problem=problem, 920 | get_util=get_util, 921 | Y_bounds=Y_bounds, 922 | learn_strategy=learn_strategy, 923 | outcome_model=outcome_model, 924 | pref_model=pref_model, 925 | gen_method=gen_method, 926 | X_baseline=X_baseline, 927 | outcome_Y=outcome_Y, 928 | train_Y=train_Y, 929 | comp_noise_type=comp_noise_type, 930 | comp_noise=comp_noise, 931 | sample_outcome=sample_outcome, 932 | previous_winner_idx=last_winner_idx if keep_winner else None, 933 | kernel=kernel, 934 | **tkwargs, 935 | ) 936 | 937 | if keep_winner: 938 | new_util = get_util(cand_Y)[0] 939 | last_winner_util = get_util(train_Y[[last_winner_idx], :])[0] 940 | new_idx = train_Y.shape[0] 941 | 942 | if new_util > last_winner_util: 943 | new_comp = [new_idx, last_winner_idx] 944 | util_diff = new_util - last_winner_util 945 | else: 946 | new_comp = [last_winner_idx, new_idx] 947 | util_diff = last_winner_util - new_util 948 | 949 | new_comp = torch.tensor(new_comp, device=train_Y.device, dtype=torch.long) 950 | cand_comps = inject_comp_error(new_comp, util_diff, comp_noise_type, comp_noise) 951 | cand_comps = cand_comps.unsqueeze(0) 952 | else: 953 | cand_util = get_util(cand_Y) 954 | assert cand_util.shape[0] == 2 955 | cand_comps = gen_comps(cand_util, comp_noise_type, comp_noise) + train_Y.shape[0] 956 | 957 | train_X = torch.cat((train_X, cand_X)).to(**tkwargs) 958 | train_Y = torch.cat((train_Y, cand_Y)).to(**tkwargs) 959 | train_comps = torch.cat((train_comps, cand_comps)) 960 | last_winner_idx = train_comps[-1, 0] 961 | 962 | acq_run_time = time.time() - start_time 963 | acq_run_times.append(acq_run_time) 964 | 965 | pref_model = fit_pref_model( 966 | train_Y, train_comps, kernel=kernel, transform_input=True, Y_bounds=Y_bounds 967 | ) 968 | 969 | if check_post_mean and ( 970 | (i % check_post_mean_every_k == 0) or (i == total_training_round - 1) 971 | ): 972 | if current_strategy in ["uncorrelated"]: 973 | use_mean = True 974 | else: 975 | use_mean = False 976 | # evaluate posterior mean after each iteration 977 | single_post_mean_X = gen_post_mean( 978 | outcome_model, pref_model, problem, use_mean=use_mean 979 | ) 980 | post_mean_X.append(single_post_mean_X) 981 | post_mean_idx.append(i) 982 | 983 | run_time = time.time() - start_time 984 | run_times.append(run_time) 985 | print( 986 | f"iteration {i}: acquisition takes {acq_run_time:.1f}s; " 987 | f"total runtime: {run_time:.1f}s" 988 | ) 989 | if check_post_mean: 990 | post_mean_X = torch.cat(post_mean_X, dim=0) 991 | 992 | return ( 993 | train_X, 994 | train_Y, 995 | train_comps, 996 | acq_run_times, 997 | run_times, 998 | post_mean_X, 999 | post_mean_idx, 1000 | selected_pairs, 1001 | ) 1002 | --------------------------------------------------------------------------------