├── .gitignore ├── HyperSphere ├── BO │ ├── __init__.py │ ├── acquisition │ │ ├── __init__.py │ │ ├── acquisition_functions.py │ │ └── acquisition_maximization.py │ ├── axv │ │ ├── __init__.py │ │ ├── cubeard_BO.py │ │ ├── cubeardboundary_BO.py │ │ ├── cubeboundary_BO.py │ │ ├── radialboth_BO.py │ │ ├── radialboundary_BO.py │ │ ├── radialnone_BO.py │ │ ├── radialorigin_BO.py │ │ ├── sphereboundary_BO.py │ │ ├── spherenone_BO.py │ │ ├── spherewarpingboth_BO.py │ │ ├── spherewarpingboundary_BO.py │ │ ├── spherewarpingnone_BO.py │ │ └── spherewarpingorigin_BO.py │ ├── run_BO.py │ ├── run_BO_multicore.py │ ├── run_experiments.py │ ├── shadow_inference │ │ ├── __init__.py │ │ ├── inference_slide_both.py │ │ ├── inference_slide_boundary.py │ │ ├── inference_slide_origin.py │ │ ├── inference_sphere_origin.py │ │ ├── inference_sphere_origin_satellite.py │ │ └── inference_sphere_satellite.py │ └── utils │ │ ├── __init__.py │ │ ├── normal_cdf.py │ │ ├── sobol.py │ │ └── sobol_c │ │ └── sobol ├── GP │ ├── __init__.py │ ├── inference │ │ ├── __init__.py │ │ ├── inference.py │ │ ├── inverse_bilinear_form.py │ │ └── log_determinant.py │ ├── kernels │ │ ├── __init__.py │ │ ├── functions │ │ │ ├── __init__.py │ │ │ ├── inner_product.py │ │ │ ├── matern52.py │ │ │ └── squared_exponential.py │ │ └── modules │ │ │ ├── __init__.py │ │ │ ├── inner_product.py │ │ │ ├── kernel.py │ │ │ ├── matern52.py │ │ │ ├── radialization.py │ │ │ ├── sphere_radial.py │ │ │ ├── squared_exponential.py │ │ │ └── stationary.py │ ├── likelihoods │ │ ├── __init__.py │ │ ├── functions │ │ │ └── __init__.py │ │ └── modules │ │ │ ├── __init__.py │ │ │ ├── gaussian.py │ │ │ └── likelihood.py │ ├── means │ │ ├── __init__.py │ │ ├── functions │ │ │ ├── __init__.py │ │ │ ├── constant.py │ │ │ └── quadratic.py │ │ └── modules │ │ │ ├── __init__.py │ │ │ ├── constant.py │ │ │ ├── mean.py │ │ │ └── quadratic.py │ ├── models │ │ ├── __init__.py │ │ ├── gp.py │ │ └── gp_regression.py │ └── modules │ │ ├── __init__.py │ │ └── gp_modules.py ├── __init__.py ├── blackboxfunction_optimizer.py ├── coordinate │ ├── __init__.py │ └── transformation.py ├── dummy │ ├── __init__.py │ ├── batch_run.sh │ ├── error_reproduce.py │ ├── experiment_info.py │ ├── model_info.py │ ├── paper_scripts │ │ ├── __init__.py │ │ ├── accuracy_time.py │ │ ├── acquisition_function.py │ │ ├── mnist_weight_train_valid_test.py │ │ ├── nondecreasing_concave.py │ │ ├── p_comparison.py │ │ ├── run_time_data.py │ │ └── stochastic_depth_resnet_result.py │ ├── plotting │ │ ├── __init__.py │ │ ├── datafile_utils.py │ │ ├── get_data_from_file.py │ │ ├── plot_color.py │ │ └── plotting.py │ ├── setup_conda.sh │ ├── split_srun.sh │ ├── srun_execution.sh │ ├── tester.py │ └── torque_run_script_generator.py ├── feature_map │ ├── __init__.py │ ├── functionals.py │ ├── functions │ │ ├── __init__.py │ │ ├── reduce_lp.py │ │ └── reduce_threshold.py │ └── modules │ │ ├── __init__.py │ │ ├── kumaraswamy.py │ │ ├── radial_threshold.py │ │ ├── reduce_lp.py │ │ ├── reduce_threshold.py │ │ ├── reflection_lp.py │ │ ├── reflection_threshold.py │ │ └── smooth_lp.py ├── interface │ ├── __init__.py │ ├── hyperparameter_search_method.py │ ├── main.py │ ├── random_search.py │ └── simple_GP.py └── test_functions │ ├── __init__.py │ ├── benchmarks.py │ ├── cifar10_weight.py │ ├── mnist_weight.py │ └── stochastic_depth_resnet.py ├── LICENSE.txt ├── README.md ├── requirements.txt ├── setup.cfg ├── setup.py └── setup_pip.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # conda virtualenv 91 | /bin 92 | /compiler_compat 93 | /conda-meta 94 | /include 95 | /lib 96 | /share 97 | /ssl 98 | /x86_64-conda_cos6-linux-gnu 99 | 100 | # Spyder project settings 101 | .spyderproject 102 | .spyproject 103 | 104 | # Rope project settings 105 | .ropeproject 106 | 107 | # PyCharm settings 108 | .idea 109 | 110 | # mkdocs documentation 111 | /site 112 | 113 | # mypy 114 | .mypy_cache/ 115 | 116 | HyperSphere/data 117 | -------------------------------------------------------------------------------- /HyperSphere/BO/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/BO/__init__.py -------------------------------------------------------------------------------- /HyperSphere/BO/acquisition/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/BO/acquisition/__init__.py -------------------------------------------------------------------------------- /HyperSphere/BO/acquisition/acquisition_functions.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | 5 | from HyperSphere.BO.utils.normal_cdf import norm_cdf 6 | 7 | 8 | def norm_pdf(x, mu=0.0, var=1.0): 9 | return torch.exp(-0.5 * (x-mu) ** 2 / var)/(2 * math.pi * var)**0.5 10 | 11 | 12 | def expected_improvement(mean, var, reference): 13 | std = torch.sqrt(var) 14 | standardized = (-mean + reference) / std 15 | return (std * norm_pdf(standardized) + (-mean + reference) * norm_cdf(standardized)).clamp(min=0) 16 | 17 | 18 | if __name__ == '__main__': 19 | import matplotlib.pyplot as plt 20 | from scipy.stats import norm 21 | x = torch.linspace(2, 3, 200) 22 | y1 = norm_cdf(x) 23 | y2 = norm.cdf(x.numpy()) 24 | plt.plot(x.numpy(), y1.numpy(), label='approximate') 25 | plt.plot(x.numpy(), y2, ':', label='exact') 26 | z1 = norm_pdf(x) 27 | z2 = norm.pdf(x.numpy()) 28 | plt.plot(x.numpy(), z1.numpy(), label='approximate') 29 | plt.plot(x.numpy(), z2, ':', label='exact') 30 | plt.legend() 31 | plt.show() 32 | -------------------------------------------------------------------------------- /HyperSphere/BO/axv/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/BO/axv/__init__.py -------------------------------------------------------------------------------- /HyperSphere/BO/axv/cubeard_BO.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | import pickle 4 | import sys 5 | import time 6 | from datetime import datetime 7 | 8 | from HyperSphere.BO.acquisition.acquisition_maximization import suggest, optimization_candidates, \ 9 | optimization_init_points, deepcopy_inference 10 | from HyperSphere.GP.inference.inference import Inference 11 | from HyperSphere.GP.kernels.modules.matern52 import Matern52 12 | from HyperSphere.GP.models.gp_regression import GPRegression 13 | from HyperSphere.feature_map.functionals import * 14 | from HyperSphere.dummy.plotting import EXPERIMENT_DIR 15 | from HyperSphere.test_functions.benchmarks import * 16 | 17 | exp_str = __file__.split('/')[-1].split('_')[0] 18 | 19 | 20 | def BO(n_eval=200, path=None, func=None, ndim=None): 21 | assert (path is None) != (func is None) 22 | if path is not None: 23 | if not os.path.exists(path): 24 | path = os.path.join(EXPERIMENT_DIR, path) 25 | model_filename = os.path.join(path, 'model.pt') 26 | data_config_filename = os.path.join(path, 'data_config.pkl') 27 | 28 | model = torch.load(model_filename) 29 | data_config_file = open(data_config_filename, 'r') 30 | for key, value in pickle.load(data_config_file).iteritems(): 31 | exec(key + '=value') 32 | data_config_file.close() 33 | 34 | inference = Inference((x_input, output), model) 35 | else: 36 | assert (func.dim == 0) != (ndim is None) 37 | if ndim is None: 38 | ndim = func.dim 39 | dir_list = [elm for elm in os.listdir(EXPERIMENT_DIR) if os.path.isdir(os.path.join(EXPERIMENT_DIR, elm))] 40 | folder_name = func.__name__ + '_D' + str(ndim) + '_' + exp_str + '_' + datetime.now().strftime('%Y%m%d-%H:%M:%S:%f') 41 | os.makedirs(os.path.join(EXPERIMENT_DIR, folder_name)) 42 | model_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'model.pt') 43 | data_config_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'data_config.pkl') 44 | 45 | x_input = Variable(torch.ger(-torch.arange(0, 2), torch.ones(ndim))) 46 | output = Variable(torch.zeros(x_input.size(0), 1)) 47 | for i in range(x_input.size(0)): 48 | output[i] = func(x_input[i]) 49 | 50 | kernel_input_map = id_transform 51 | model = GPRegression(kernel=Matern52(ndim=kernel_input_map.dim_change(ndim), ard=True, input_map=kernel_input_map)) 52 | 53 | time_list = [time.time()] * 2 54 | elapse_list = [0, 0] 55 | pred_mean_list = [0, 0] 56 | pred_std_list = [0, 0] 57 | pred_var_list = [0, 0] 58 | pred_stdmax_list = [1, 1] 59 | pred_varmax_list = [1, 1] 60 | reference_list = [output.data.squeeze()[0]] * 2 61 | refind_list = [1, 1] 62 | dist_to_ref_list = [0, 0] 63 | sample_info_list = [(10, 0, 10)] * 2 64 | 65 | inference = Inference((x_input, output), model) 66 | inference.init_parameters() 67 | inference.sampling(n_sample=1, n_burnin=99, n_thin=1) 68 | 69 | stored_variable_names = locals().keys() 70 | ignored_variable_names = ['n_eval', 'path', 'data_config_file', 'dir_list', 'folder_name', 71 | 'next_ind', 'model_filename', 'data_config_filename', 'i', 72 | 'kernel_input_map', 'model', 'inference'] 73 | stored_variable_names = set(stored_variable_names).difference(set(ignored_variable_names)) 74 | 75 | for _ in range(3): 76 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 77 | 78 | for _ in range(n_eval): 79 | inference = Inference((x_input, output), model) 80 | 81 | reference, ref_ind = torch.min(output, 0) 82 | reference = reference.data.squeeze()[0] 83 | gp_hyper_params = inference.sampling(n_sample=10, n_burnin=0, n_thin=1) 84 | inferences = deepcopy_inference(inference, gp_hyper_params) 85 | 86 | x0_cand = optimization_candidates(x_input, output, -1, 1) 87 | x0, sample_info = optimization_init_points(x0_cand, inferences, reference=reference) 88 | next_x_point, pred_mean, pred_std, pred_var, pred_stdmax, pred_varmax = suggest(inferences, x0=x0, bounds=(-1, 1), reference=reference) 89 | 90 | time_list.append(time.time()) 91 | elapse_list.append(time_list[-1] - time_list[-2]) 92 | 93 | x_input = torch.cat([x_input, next_x_point], 0) 94 | output = torch.cat([output, func(x_input[-1])]) 95 | pred_mean_list.append(pred_mean.squeeze()[0]) 96 | pred_std_list.append(pred_std.squeeze()[0]) 97 | pred_var_list.append(pred_var.squeeze()[0]) 98 | pred_stdmax_list.append(pred_stdmax.squeeze()[0]) 99 | pred_varmax_list.append(pred_varmax.squeeze()[0]) 100 | reference_list.append(reference) 101 | refind_list.append(ref_ind.data.squeeze()[0] + 1) 102 | dist_to_ref_list.append(torch.sum((next_x_point - x_input[ref_ind].data) ** 2) ** 0.5) 103 | sample_info_list.append(sample_info) 104 | 105 | min_ind = torch.min(output, 0)[1] 106 | min_loc = x_input[min_ind] 107 | min_val = output[min_ind] 108 | dist_to_suggest = torch.sum((x_input - x_input[-1]).data ** 2, 1) ** 0.5 109 | dist_to_min = torch.sum((x_input - min_loc).data ** 2, 1) ** 0.5 110 | out_of_box = torch.sum((torch.abs(x_input.data) > 1), 1) 111 | print('') 112 | for i in range(x_input.size(0)): 113 | time_str = time.strftime('%H:%M:%S', time.gmtime(time_list[i])) + '(' + time.strftime('%H:%M:%S', time.gmtime(elapse_list[i])) + ') ' 114 | data_str = ('%3d-th : %+12.4f(R:%8.4f[%4d]/ref:[%3d]%8.4f), sample([%2d] best:%2d/worst:%2d), ' 115 | 'mean : %+.4E, std : %.4E(%5.4f), var : %.4E(%5.4f), ' 116 | '2ownMIN : %8.4f, 2curMIN : %8.4f, 2new : %8.4f' % 117 | (i + 1, output.data.squeeze()[i], torch.sum(x_input.data[i] ** 2) ** 0.5, out_of_box[i], refind_list[i], reference_list[i], 118 | sample_info_list[i][2], sample_info_list[i][0], sample_info_list[i][1], 119 | pred_mean_list[i], pred_std_list[i], pred_std_list[i] / pred_stdmax_list[i], pred_var_list[i], pred_var_list[i] / pred_varmax_list[i], 120 | dist_to_ref_list[i], dist_to_min[i], dist_to_suggest[i])) 121 | min_str = ' <========= MIN' if i == min_ind.data.squeeze()[0] else '' 122 | print(time_str + data_str + min_str) 123 | print(model.kernel.__class__.__name__) 124 | 125 | torch.save(model, model_filename) 126 | stored_variable = dict() 127 | for key in stored_variable_names: 128 | stored_variable[key] = locals()[key] 129 | f = open(data_config_filename, 'w') 130 | pickle.dump(stored_variable, f) 131 | f.close() 132 | 133 | for _ in range(3): 134 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 135 | 136 | return os.path.split(model_filename)[0] 137 | 138 | 139 | if __name__ == '__main__': 140 | run_new = False 141 | path, suffix = os.path.split(sys.argv[1]) 142 | if path == '' and not ('_D' in suffix): 143 | run_new = True 144 | if run_new: 145 | func = locals()[sys.argv[1]] 146 | if func.dim == 0: 147 | n_eval = int(sys.argv[3]) if len(sys.argv) > 3 else 100 148 | BO(n_eval=n_eval, func=func, dim=int(sys.argv[2])) 149 | else: 150 | BO(n_eval=int(sys.argv[2]), func=func) 151 | else: 152 | BO(n_eval=int(sys.argv[2]), path=sys.argv[1]) 153 | -------------------------------------------------------------------------------- /HyperSphere/BO/axv/cubeardboundary_BO.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import pickle 3 | import sys 4 | import time 5 | from datetime import datetime 6 | 7 | # ShadowInference version should coincide with the one used in acquisition_maximization 8 | from HyperSphere.BO.acquisition.acquisition_maximization import suggest, optimization_candidates, \ 9 | optimization_init_points, deepcopy_inference 10 | from HyperSphere.BO.shadow_inference.inference_sphere_satellite import ShadowInference 11 | from HyperSphere.GP.kernels.modules.matern52 import Matern52 12 | from HyperSphere.GP.models.gp_regression import GPRegression 13 | from HyperSphere.feature_map.functionals import id_transform 14 | from HyperSphere.dummy.plotting import EXPERIMENT_DIR 15 | from HyperSphere.test_functions.benchmarks import * 16 | 17 | exp_str = __file__.split('/')[-1].split('_')[0] 18 | 19 | 20 | def BO(n_eval=200, path=None, func=None, ndim=None): 21 | assert (path is None) != (func is None) 22 | if path is not None: 23 | if not os.path.exists(path): 24 | path = os.path.join(EXPERIMENT_DIR, path) 25 | model_filename = os.path.join(path, 'model.pt') 26 | data_config_filename = os.path.join(path, 'data_config.pkl') 27 | 28 | model = torch.load(model_filename) 29 | data_config_file = open(data_config_filename, 'r') 30 | for key, value in pickle.load(data_config_file).iteritems(): 31 | exec(key + '=value') 32 | data_config_file.close() 33 | 34 | inference = ShadowInference((x_input, output), model) 35 | else: 36 | assert (func.dim == 0) != (ndim is None) 37 | if ndim is None: 38 | ndim = func.dim 39 | dir_list = [elm for elm in os.listdir(EXPERIMENT_DIR) if os.path.isdir(os.path.join(EXPERIMENT_DIR, elm))] 40 | folder_name = func.__name__ + '_D' + str(ndim) + '_' + exp_str + '_' + datetime.now().strftime('%Y%m%d-%H:%M:%S:%f') 41 | os.makedirs(os.path.join(EXPERIMENT_DIR, folder_name)) 42 | model_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'model.pt') 43 | data_config_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'data_config.pkl') 44 | 45 | x_input = Variable(torch.ger(-torch.arange(0, 2), torch.ones(ndim))) 46 | output = Variable(torch.zeros(x_input.size(0), 1)) 47 | for i in range(x_input.size(0)): 48 | output[i] = func(x_input[i]) 49 | 50 | kernel_input_map = id_transform 51 | model = GPRegression(kernel=Matern52(ndim=kernel_input_map.dim_change(ndim), ard=True, input_map=kernel_input_map)) 52 | 53 | time_list = [time.time()] * 2 54 | elapse_list = [0, 0] 55 | pred_mean_list = [0, 0] 56 | pred_std_list = [0, 0] 57 | pred_var_list = [0, 0] 58 | pred_stdmax_list = [1, 1] 59 | pred_varmax_list = [1, 1] 60 | reference_list = [output.data.squeeze()[0]] * 2 61 | refind_list = [1, 1] 62 | dist_to_ref_list = [0, 0] 63 | sample_info_list = [(10, 0, 10)] * 2 64 | 65 | inference = ShadowInference((x_input, output), model) 66 | inference.init_parameters() 67 | inference.sampling(n_sample=1, n_burnin=99, n_thin=1) 68 | 69 | stored_variable_names = locals().keys() 70 | ignored_variable_names = ['n_eval', 'path', 'data_config_file', 'dir_list', 'folder_name', 71 | 'next_ind', 'model_filename', 'data_config_filename', 'i', 72 | 'kernel_input_map', 'model', 'inference'] 73 | stored_variable_names = set(stored_variable_names).difference(set(ignored_variable_names)) 74 | 75 | for _ in range(3): 76 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 77 | 78 | for _ in range(n_eval): 79 | inference = ShadowInference((x_input, output), model) 80 | 81 | reference, ref_ind = torch.min(output, 0) 82 | reference = reference.data.squeeze()[0] 83 | gp_hyper_params = inference.sampling(n_sample=10, n_burnin=0, n_thin=1) 84 | inferences = deepcopy_inference(inference, gp_hyper_params) 85 | 86 | x0_cand = optimization_candidates(x_input, output, -1, 1) 87 | x0, sample_info = optimization_init_points(x0_cand, inferences, reference=reference) 88 | next_x_point, pred_mean, pred_std, pred_var, pred_stdmax, pred_varmax = suggest(inferences, x0=x0, bounds=(-1, 1), reference=reference) 89 | 90 | time_list.append(time.time()) 91 | elapse_list.append(time_list[-1] - time_list[-2]) 92 | pred_mean_list.append(pred_mean.squeeze()[0]) 93 | pred_std_list.append(pred_std.squeeze()[0]) 94 | pred_var_list.append(pred_var.squeeze()[0]) 95 | pred_stdmax_list.append(pred_stdmax.squeeze()[0]) 96 | pred_varmax_list.append(pred_varmax.squeeze()[0]) 97 | reference_list.append(reference) 98 | refind_list.append(ref_ind.data.squeeze()[0] + 1) 99 | dist_to_ref_list.append(torch.sum((next_x_point - x_input[ref_ind].data) ** 2) ** 0.5) 100 | sample_info_list.append(sample_info) 101 | 102 | x_input = torch.cat([x_input, next_x_point], 0) 103 | output = torch.cat([output, func(x_input[-1])]) 104 | 105 | min_ind = torch.min(output, 0)[1] 106 | min_loc = x_input[min_ind] 107 | min_val = output[min_ind] 108 | dist_to_suggest = torch.sum((x_input - x_input[-1]).data ** 2, 1) ** 0.5 109 | dist_to_min = torch.sum((x_input - min_loc).data ** 2, 1) ** 0.5 110 | out_of_box = torch.sum((torch.abs(x_input.data) > 1), 1) 111 | print('') 112 | for i in range(x_input.size(0)): 113 | time_str = time.strftime('%H:%M:%S', time.gmtime(time_list[i])) + '(' + time.strftime('%H:%M:%S', time.gmtime(elapse_list[i])) + ') ' 114 | data_str = ('%3d-th : %+12.4f(R:%8.4f[%4d]/ref:[%3d]%8.4f), sample([%2d] best:%2d/worst:%2d), ' 115 | 'mean : %+.4E, std : %.4E(%5.4f), var : %.4E(%5.4f), ' 116 | '2ownMIN : %8.4f, 2curMIN : %8.4f, 2new : %8.4f' % 117 | (i + 1, output.data.squeeze()[i], torch.sum(x_input.data[i] ** 2) ** 0.5, out_of_box[i], refind_list[i], reference_list[i], 118 | sample_info_list[i][2], sample_info_list[i][0], sample_info_list[i][1], 119 | pred_mean_list[i], pred_std_list[i], pred_std_list[i] / pred_stdmax_list[i], pred_var_list[i], pred_var_list[i] / pred_varmax_list[i], 120 | dist_to_ref_list[i], dist_to_min[i], dist_to_suggest[i])) 121 | min_str = ' <========= MIN' if i == min_ind.data.squeeze()[0] else '' 122 | print(time_str + data_str + min_str) 123 | print(model.kernel.__class__.__name__) 124 | 125 | torch.save(model, model_filename) 126 | stored_variable = dict() 127 | for key in stored_variable_names: 128 | stored_variable[key] = locals()[key] 129 | f = open(data_config_filename, 'w') 130 | pickle.dump(stored_variable, f) 131 | f.close() 132 | 133 | for _ in range(3): 134 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 135 | 136 | return os.path.split(model_filename)[0] 137 | 138 | 139 | if __name__ == '__main__': 140 | run_new = False 141 | path, suffix = os.path.split(sys.argv[1]) 142 | if path == '' and not ('_D' in suffix): 143 | run_new = True 144 | if run_new: 145 | func = locals()[sys.argv[1]] 146 | if func.dim == 0: 147 | n_eval = int(sys.argv[3]) if len(sys.argv) > 3 else 100 148 | BO(n_eval=n_eval, func=func, dim=int(sys.argv[2])) 149 | else: 150 | BO(n_eval=int(sys.argv[2]), func=func) 151 | else: 152 | BO(n_eval=int(sys.argv[2]), path=sys.argv[1]) 153 | -------------------------------------------------------------------------------- /HyperSphere/BO/axv/cubeboundary_BO.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import pickle 3 | import sys 4 | import time 5 | from datetime import datetime 6 | 7 | # ShadowInference version should coincide with the one used in acquisition_maximization 8 | from HyperSphere.BO.acquisition.acquisition_maximization import suggest, optimization_candidates, \ 9 | optimization_init_points, deepcopy_inference 10 | from HyperSphere.BO.shadow_inference.inference_sphere_satellite import ShadowInference 11 | from HyperSphere.GP.kernels.modules.matern52 import Matern52 12 | from HyperSphere.GP.models.gp_regression import GPRegression 13 | from HyperSphere.feature_map.functionals import * 14 | from HyperSphere.dummy.plotting import EXPERIMENT_DIR 15 | from HyperSphere.test_functions.benchmarks import * 16 | 17 | exp_str = __file__.split('/')[-1].split('_')[0] 18 | 19 | 20 | def BO(n_eval=200, path=None, func=None, ndim=None): 21 | assert (path is None) != (func is None) 22 | if path is not None: 23 | if not os.path.exists(path): 24 | path = os.path.join(EXPERIMENT_DIR, path) 25 | model_filename = os.path.join(path, 'model.pt') 26 | data_config_filename = os.path.join(path, 'data_config.pkl') 27 | 28 | model = torch.load(model_filename) 29 | data_config_file = open(data_config_filename, 'r') 30 | for key, value in pickle.load(data_config_file).iteritems(): 31 | exec(key + '=value') 32 | data_config_file.close() 33 | 34 | inference = ShadowInference((x_input, output), model) 35 | else: 36 | assert (func.dim == 0) != (ndim is None) 37 | if ndim is None: 38 | ndim = func.dim 39 | dir_list = [elm for elm in os.listdir(EXPERIMENT_DIR) if os.path.isdir(os.path.join(EXPERIMENT_DIR, elm))] 40 | folder_name = func.__name__ + '_D' + str(ndim) + '_' + exp_str + '_' + datetime.now().strftime('%Y%m%d-%H:%M:%S:%f') 41 | os.makedirs(os.path.join(EXPERIMENT_DIR, folder_name)) 42 | model_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'model.pt') 43 | data_config_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'data_config.pkl') 44 | 45 | x_input = Variable(torch.ger(-torch.arange(0, 2), torch.ones(ndim))) 46 | output = Variable(torch.zeros(x_input.size(0), 1)) 47 | for i in range(x_input.size(0)): 48 | output[i] = func(x_input[i]) 49 | 50 | kernel_input_map = id_transform 51 | model = GPRegression(kernel=Matern52(ndim=kernel_input_map.dim_change(ndim), ard=False, input_map=kernel_input_map)) 52 | 53 | time_list = [time.time()] * 2 54 | elapse_list = [0, 0] 55 | pred_mean_list = [0, 0] 56 | pred_std_list = [0, 0] 57 | pred_var_list = [0, 0] 58 | pred_stdmax_list = [1, 1] 59 | pred_varmax_list = [1, 1] 60 | reference_list = [output.data.squeeze()[0]] * 2 61 | refind_list = [1, 1] 62 | dist_to_ref_list = [0, 0] 63 | sample_info_list = [(10, 0, 10)] * 2 64 | 65 | inference = ShadowInference((x_input, output), model) 66 | inference.init_parameters() 67 | inference.sampling(n_sample=1, n_burnin=99, n_thin=1) 68 | 69 | stored_variable_names = locals().keys() 70 | ignored_variable_names = ['n_eval', 'path', 'data_config_file', 'dir_list', 'folder_name', 71 | 'next_ind', 'model_filename', 'data_config_filename', 'i', 72 | 'kernel_input_map', 'model', 'inference'] 73 | stored_variable_names = set(stored_variable_names).difference(set(ignored_variable_names)) 74 | 75 | for _ in range(3): 76 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 77 | 78 | for _ in range(n_eval): 79 | inference = ShadowInference((x_input, output), model) 80 | 81 | reference, ref_ind = torch.min(output, 0) 82 | reference = reference.data.squeeze()[0] 83 | gp_hyper_params = inference.sampling(n_sample=10, n_burnin=0, n_thin=1) 84 | inferences = deepcopy_inference(inference, gp_hyper_params) 85 | 86 | x0_cand = optimization_candidates(x_input, output, -1, 1) 87 | x0, sample_info = optimization_init_points(x0_cand, inferences, reference=reference) 88 | next_x_point, pred_mean, pred_std, pred_var, pred_stdmax, pred_varmax = suggest(inferences, x0=x0, bounds=(-1, 1), reference=reference) 89 | 90 | time_list.append(time.time()) 91 | elapse_list.append(time_list[-1] - time_list[-2]) 92 | pred_mean_list.append(pred_mean.squeeze()[0]) 93 | pred_std_list.append(pred_std.squeeze()[0]) 94 | pred_var_list.append(pred_var.squeeze()[0]) 95 | pred_stdmax_list.append(pred_stdmax.squeeze()[0]) 96 | pred_varmax_list.append(pred_varmax.squeeze()[0]) 97 | reference_list.append(reference) 98 | refind_list.append(ref_ind.data.squeeze()[0] + 1) 99 | dist_to_ref_list.append(torch.sum((next_x_point - x_input[ref_ind].data) ** 2) ** 0.5) 100 | sample_info_list.append(sample_info) 101 | 102 | x_input = torch.cat([x_input, next_x_point], 0) 103 | output = torch.cat([output, func(x_input[-1])]) 104 | 105 | min_ind = torch.min(output, 0)[1] 106 | min_loc = x_input[min_ind] 107 | min_val = output[min_ind] 108 | dist_to_suggest = torch.sum((x_input - x_input[-1]).data ** 2, 1) ** 0.5 109 | dist_to_min = torch.sum((x_input - min_loc).data ** 2, 1) ** 0.5 110 | out_of_box = torch.sum((torch.abs(x_input.data) > 1), 1) 111 | print('') 112 | for i in range(x_input.size(0)): 113 | time_str = time.strftime('%H:%M:%S', time.gmtime(time_list[i])) + '(' + time.strftime('%H:%M:%S', time.gmtime(elapse_list[i])) + ') ' 114 | data_str = ('%3d-th : %+12.4f(R:%8.4f[%4d]/ref:[%3d]%8.4f), sample([%2d] best:%2d/worst:%2d), ' 115 | 'mean : %+.4E, std : %.4E(%5.4f), var : %.4E(%5.4f), ' 116 | '2ownMIN : %8.4f, 2curMIN : %8.4f, 2new : %8.4f' % 117 | (i + 1, output.data.squeeze()[i], torch.sum(x_input.data[i] ** 2) ** 0.5, out_of_box[i], refind_list[i], reference_list[i], 118 | sample_info_list[i][2], sample_info_list[i][0], sample_info_list[i][1], 119 | pred_mean_list[i], pred_std_list[i], pred_std_list[i] / pred_stdmax_list[i], pred_var_list[i], pred_var_list[i] / pred_varmax_list[i], 120 | dist_to_ref_list[i], dist_to_min[i], dist_to_suggest[i])) 121 | min_str = ' <========= MIN' if i == min_ind.data.squeeze()[0] else '' 122 | print(time_str + data_str + min_str) 123 | print(model.kernel.__class__.__name__) 124 | torch.save(model, model_filename) 125 | stored_variable = dict() 126 | for key in stored_variable_names: 127 | stored_variable[key] = locals()[key] 128 | f = open(data_config_filename, 'w') 129 | pickle.dump(stored_variable, f) 130 | f.close() 131 | 132 | for _ in range(3): 133 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 134 | 135 | return os.path.split(model_filename)[0] 136 | 137 | 138 | if __name__ == '__main__': 139 | run_new = False 140 | path, suffix = os.path.split(sys.argv[1]) 141 | if path == '' and not ('_D' in suffix): 142 | run_new = True 143 | if run_new: 144 | func = locals()[sys.argv[1]] 145 | if func.dim == 0: 146 | n_eval = int(sys.argv[3]) if len(sys.argv) > 3 else 100 147 | BO(n_eval=n_eval, func=func, dim=int(sys.argv[2])) 148 | else: 149 | BO(n_eval=int(sys.argv[2]), func=func) 150 | else: 151 | BO(n_eval=int(sys.argv[2]), path=sys.argv[1]) 152 | -------------------------------------------------------------------------------- /HyperSphere/BO/axv/radialboth_BO.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import pickle 3 | import sys 4 | import time 5 | from datetime import datetime 6 | 7 | # ShadowInference version should coincide with the one used in acquisition_maximization 8 | from HyperSphere.BO.acquisition.acquisition_maximization import suggest, optimization_candidates, \ 9 | optimization_init_points, deepcopy_inference 10 | from HyperSphere.BO.shadow_inference.inference_slide_both import ShadowInference 11 | from HyperSphere.GP.kernels.modules.matern52 import Matern52 12 | from HyperSphere.GP.models.gp_regression import GPRegression 13 | from HyperSphere.feature_map.functionals import x2radial, radial_bound 14 | from HyperSphere.dummy.plotting import EXPERIMENT_DIR 15 | from HyperSphere.test_functions.benchmarks import * 16 | 17 | exp_str = __file__.split('/')[-1].split('_')[0] 18 | 19 | 20 | def BO(n_eval=200, path=None, func=None, ndim=None): 21 | assert (path is None) != (func is None) 22 | if path is not None: 23 | if not os.path.exists(path): 24 | path = os.path.join(EXPERIMENT_DIR, path) 25 | model_filename = os.path.join(path, 'model.pt') 26 | data_config_filename = os.path.join(path, 'data_config.pkl') 27 | 28 | model = torch.load(model_filename) 29 | data_config_file = open(data_config_filename, 'r') 30 | for key, value in pickle.load(data_config_file).iteritems(): 31 | exec(key + '=value') 32 | data_config_file.close() 33 | 34 | inference = ShadowInference((x_input, output), model) 35 | else: 36 | assert (func.dim == 0) != (ndim is None) 37 | if ndim is None: 38 | ndim = func.dim 39 | dir_list = [elm for elm in os.listdir(EXPERIMENT_DIR) if os.path.isdir(os.path.join(EXPERIMENT_DIR, elm))] 40 | folder_name = func.__name__ + '_D' + str(ndim) + '_' + exp_str + '_' + datetime.now().strftime('%Y%m%d-%H:%M:%S:%f') 41 | os.makedirs(os.path.join(EXPERIMENT_DIR, folder_name)) 42 | model_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'model.pt') 43 | data_config_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'data_config.pkl') 44 | search_sphere_radius = ndim ** 0.5 45 | 46 | x_input = Variable(torch.ger(-torch.arange(0, 2), torch.ones(ndim))) 47 | output = Variable(torch.zeros(x_input.size(0), 1)) 48 | for i in range(x_input.size(0)): 49 | output[i] = func(x_input[i]) 50 | 51 | kernel_input_map = x2radial 52 | model = GPRegression(kernel=Matern52(ndim=kernel_input_map.dim_change(ndim), input_map=kernel_input_map)) 53 | 54 | time_list = [time.time()] * 2 55 | elapse_list = [0, 0] 56 | pred_mean_list = [0, 0] 57 | pred_std_list = [0, 0] 58 | pred_var_list = [0, 0] 59 | pred_stdmax_list = [1, 1] 60 | pred_varmax_list = [1, 1] 61 | reference_list = [output.data.squeeze()[0]] * 2 62 | refind_list = [1, 1] 63 | dist_to_ref_list = [0, 0] 64 | 65 | inference = ShadowInference((x_input, output), model) 66 | inference.init_parameters() 67 | inference.sampling(n_sample=1, n_burnin=99, n_thin=1) 68 | 69 | stored_variable_names = locals().keys() 70 | ignored_variable_names = ['n_eval', 'path', 'data_config_file', 'dir_list', 'folder_name', 71 | 'next_ind', 'model_filename', 'data_config_filename', 'i', 72 | 'kernel_input_map', 'model', 'inference'] 73 | stored_variable_names = set(stored_variable_names).difference(set(ignored_variable_names)) 74 | 75 | bnd = radial_bound(search_sphere_radius) 76 | 77 | for _ in range(3): 78 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 79 | 80 | for _ in range(n_eval): 81 | inference = ShadowInference((x_input, output), model) 82 | 83 | reference, ref_ind = torch.min(output, 0) 84 | reference = reference.data.squeeze()[0] 85 | gp_hyper_params = inference.sampling(n_sample=10, n_burnin=0, n_thin=1) 86 | inferences = deepcopy_inference(inference, gp_hyper_params) 87 | 88 | x0_cand = optimization_candidates(x_input, output, -1, 1) 89 | x0, sample_info = optimization_init_points(x0_cand, inferences, reference=reference) 90 | next_x_point, pred_mean, pred_std, pred_var, pred_stdmax, pred_varmax = suggest(inferences, x0=x0, bounds=bnd, reference=reference) 91 | 92 | time_list.append(time.time()) 93 | elapse_list.append(time_list[-1] - time_list[-2]) 94 | pred_mean_list.append(pred_mean.squeeze()[0]) 95 | pred_std_list.append(pred_std.squeeze()[0]) 96 | pred_var_list.append(pred_var.squeeze()[0]) 97 | pred_stdmax_list.append(pred_stdmax.squeeze()[0]) 98 | pred_varmax_list.append(pred_varmax.squeeze()[0]) 99 | reference_list.append(reference) 100 | refind_list.append(ref_ind.data.squeeze()[0] + 1) 101 | dist_to_ref_list.append(torch.sum((next_x_point - x_input[ref_ind].data) ** 2) ** 0.5) 102 | 103 | x_input = torch.cat([x_input, Variable(next_x_point)], 0) 104 | output = torch.cat([output, func(x_input[-1])]) 105 | 106 | min_ind = torch.min(output, 0)[1] 107 | min_loc = x_input[min_ind] 108 | min_val = output[min_ind] 109 | dist_to_suggest = torch.sum((x_input - x_input[-1]).data ** 2, 1) ** 0.5 110 | dist_to_min = torch.sum((x_input - min_loc).data ** 2, 1) ** 0.5 111 | out_of_box = torch.sum((torch.abs(x_input.data) > 1), 1) 112 | print('') 113 | for i in range(x_input.size(0)): 114 | time_str = time.strftime('%H:%M:%S', time.gmtime(time_list[i])) + '(' + time.strftime('%H:%M:%S', time.gmtime(elapse_list[i])) + ') ' 115 | data_str = ('%3d-th : %+14.4f(R:%8.4f[%4d]/ref:[%3d]%8.4f), ' 116 | 'mean : %+.4E, std : %.4E, var : %.4E(%5.4f), ' 117 | '2ownMIN : %8.4f, 2curMIN : %8.4f, 2new : %8.4f' % 118 | (i + 1, output.data.squeeze()[i], torch.sum(x_input.data[i] ** 2) ** 0.5, out_of_box[i], refind_list[i], reference_list[i], 119 | pred_mean_list[i], pred_std_list[i], pred_std_list[i] / pred_stdmax_list[i], pred_var_list[i], pred_var_list[i] / pred_varmax_list[i], 120 | dist_to_ref_list[i], dist_to_min[i], dist_to_suggest[i])) 121 | min_str = ' <========= MIN' if i == min_ind.data.squeeze()[0] else '' 122 | print(time_str + data_str + min_str) 123 | print(model.kernel.__class__.__name__) 124 | 125 | torch.save(model, model_filename) 126 | stored_variable = dict() 127 | for key in stored_variable_names: 128 | stored_variable[key] = locals()[key] 129 | f = open(data_config_filename, 'w') 130 | pickle.dump(stored_variable, f) 131 | f.close() 132 | 133 | for _ in range(3): 134 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 135 | 136 | return os.path.split(model_filename)[0] 137 | 138 | 139 | if __name__ == '__main__': 140 | run_new = False 141 | path, suffix = os.path.split(sys.argv[1]) 142 | if path == '' and not ('_D' in suffix): 143 | run_new = True 144 | if run_new: 145 | func = locals()[sys.argv[1]] 146 | if func.dim == 0: 147 | n_eval = int(sys.argv[3]) if len(sys.argv) > 3 else 100 148 | BO(n_eval=n_eval, func=func, dim=int(sys.argv[2])) 149 | else: 150 | BO(n_eval=int(sys.argv[2]), func=func) 151 | else: 152 | BO(n_eval=int(sys.argv[2]), path=sys.argv[1]) 153 | -------------------------------------------------------------------------------- /HyperSphere/BO/axv/radialboundary_BO.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import pickle 3 | import sys 4 | import time 5 | from datetime import datetime 6 | 7 | # ShadowInference version should coincide with the one used in acquisition_maximization 8 | from HyperSphere.BO.acquisition.acquisition_maximization import suggest, optimization_candidates, \ 9 | optimization_init_points, deepcopy_inference 10 | from HyperSphere.BO.shadow_inference.inference_slide_boundary import ShadowInference 11 | from HyperSphere.GP.kernels.modules.matern52 import Matern52 12 | from HyperSphere.GP.models.gp_regression import GPRegression 13 | from HyperSphere.feature_map.functionals import x2radial, radial_bound 14 | from HyperSphere.dummy.plotting import EXPERIMENT_DIR 15 | from HyperSphere.test_functions.benchmarks import * 16 | 17 | exp_str = __file__.split('/')[-1].split('_')[0] 18 | 19 | 20 | def BO(n_eval=200, path=None, func=None, ndim=None): 21 | assert (path is None) != (func is None) 22 | if path is not None: 23 | if not os.path.exists(path): 24 | path = os.path.join(EXPERIMENT_DIR, path) 25 | model_filename = os.path.join(path, 'model.pt') 26 | data_config_filename = os.path.join(path, 'data_config.pkl') 27 | 28 | model = torch.load(model_filename) 29 | data_config_file = open(data_config_filename, 'r') 30 | for key, value in pickle.load(data_config_file).iteritems(): 31 | exec(key + '=value') 32 | data_config_file.close() 33 | 34 | inference = ShadowInference((x_input, output), model) 35 | else: 36 | assert (func.dim == 0) != (ndim is None) 37 | if ndim is None: 38 | ndim = func.dim 39 | dir_list = [elm for elm in os.listdir(EXPERIMENT_DIR) if os.path.isdir(os.path.join(EXPERIMENT_DIR, elm))] 40 | folder_name = func.__name__ + '_D' + str(ndim) + '_' + exp_str + '_' + datetime.now().strftime('%Y%m%d-%H:%M:%S:%f') 41 | os.makedirs(os.path.join(EXPERIMENT_DIR, folder_name)) 42 | model_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'model.pt') 43 | data_config_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'data_config.pkl') 44 | 45 | search_sphere_radius = ndim ** 0.5 46 | 47 | x_input = Variable(torch.ger(-torch.arange(0, 2), torch.ones(ndim))) 48 | output = Variable(torch.zeros(x_input.size(0), 1)) 49 | for i in range(x_input.size(0)): 50 | output[i] = func(x_input[i]) 51 | 52 | kernel_input_map = x2radial 53 | model = GPRegression(kernel=Matern52(ndim=kernel_input_map.dim_change(ndim), input_map=kernel_input_map)) 54 | 55 | time_list = [time.time()] * 2 56 | elapse_list = [0, 0] 57 | pred_mean_list = [0, 0] 58 | pred_std_list = [0, 0] 59 | pred_var_list = [0, 0] 60 | pred_stdmax_list = [1, 1] 61 | pred_varmax_list = [1, 1] 62 | reference_list = [output.data.squeeze()[0]] * 2 63 | refind_list = [1, 1] 64 | dist_to_ref_list = [0, 0] 65 | 66 | inference = ShadowInference((x_input, output), model) 67 | inference.init_parameters() 68 | inference.sampling(n_sample=1, n_burnin=99, n_thin=1) 69 | 70 | stored_variable_names = locals().keys() 71 | ignored_variable_names = ['n_eval', 'path', 'data_config_file', 'dir_list', 'folder_name', 72 | 'next_ind', 'model_filename', 'data_config_filename', 'i', 73 | 'kernel_input_map', 'model', 'inference'] 74 | stored_variable_names = set(stored_variable_names).difference(set(ignored_variable_names)) 75 | 76 | bnd = radial_bound(search_sphere_radius) 77 | 78 | for _ in range(3): 79 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 80 | 81 | for _ in range(n_eval): 82 | inference = ShadowInference((x_input, output), model) 83 | 84 | reference, ref_ind = torch.min(output, 0) 85 | reference = reference.data.squeeze()[0] 86 | gp_hyper_params = inference.sampling(n_sample=10, n_burnin=0, n_thin=1) 87 | inferences = deepcopy_inference(inference, gp_hyper_params) 88 | 89 | x0_cand = optimization_candidates(x_input, output, -1, 1) 90 | x0, sample_info = optimization_init_points(x0_cand, inferences, reference=reference) 91 | next_x_point, pred_mean, pred_std, pred_var, pred_stdmax, pred_varmax = suggest(inferences, x0=x0, bounds=bnd, reference=reference) 92 | 93 | time_list.append(time.time()) 94 | elapse_list.append(time_list[-1] - time_list[-2]) 95 | pred_mean_list.append(pred_mean.squeeze()[0]) 96 | pred_std_list.append(pred_std.squeeze()[0]) 97 | pred_var_list.append(pred_var.squeeze()[0]) 98 | pred_stdmax_list.append(pred_stdmax.squeeze()[0]) 99 | pred_varmax_list.append(pred_varmax.squeeze()[0]) 100 | reference_list.append(reference) 101 | refind_list.append(ref_ind.data.squeeze()[0] + 1) 102 | dist_to_ref_list.append(torch.sum((next_x_point - x_input[ref_ind].data) ** 2) ** 0.5) 103 | 104 | x_input = torch.cat([x_input, Variable(next_x_point)], 0) 105 | output = torch.cat([output, func(x_input[-1])]) 106 | 107 | min_ind = torch.min(output, 0)[1] 108 | min_loc = x_input[min_ind] 109 | min_val = output[min_ind] 110 | dist_to_suggest = torch.sum((x_input - x_input[-1]).data ** 2, 1) ** 0.5 111 | dist_to_min = torch.sum((x_input - min_loc).data ** 2, 1) ** 0.5 112 | out_of_box = torch.sum((torch.abs(x_input.data) > 1), 1) 113 | print('') 114 | for i in range(x_input.size(0)): 115 | time_str = time.strftime('%H:%M:%S', time.gmtime(time_list[i])) + '(' + time.strftime('%H:%M:%S', time.gmtime(elapse_list[i])) + ') ' 116 | data_str = ('%3d-th : %+14.4f(R:%8.4f[%4d]/ref:[%3d]%8.4f), ' 117 | 'mean : %+.4E, std : %.4E(%5.4f), var : %.4E(%5.4f), ' 118 | '2ownMIN : %8.4f, 2curMIN : %8.4f, 2new : %8.4f' % 119 | (i + 1, output.data.squeeze()[i], torch.sum(x_input.data[i] ** 2) ** 0.5, out_of_box[i], refind_list[i], reference_list[i], 120 | pred_mean_list[i], pred_std_list[i], pred_std_list[i] / pred_stdmax_list[i], pred_var_list[i], pred_var_list[i] / pred_varmax_list[i], 121 | dist_to_ref_list[i], dist_to_min[i], dist_to_suggest[i])) 122 | min_str = ' <========= MIN' if i == min_ind.data.squeeze()[0] else '' 123 | print(time_str + data_str + min_str) 124 | print(model.kernel.__class__.__name__) 125 | 126 | torch.save(model, model_filename) 127 | stored_variable = dict() 128 | for key in stored_variable_names: 129 | stored_variable[key] = locals()[key] 130 | f = open(data_config_filename, 'w') 131 | pickle.dump(stored_variable, f) 132 | f.close() 133 | 134 | for _ in range(3): 135 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 136 | 137 | return os.path.split(model_filename)[0] 138 | 139 | 140 | if __name__ == '__main__': 141 | run_new = False 142 | path, suffix = os.path.split(sys.argv[1]) 143 | if path == '' and not ('_D' in suffix): 144 | run_new = True 145 | if run_new: 146 | func = locals()[sys.argv[1]] 147 | if func.dim == 0: 148 | n_eval = int(sys.argv[3]) if len(sys.argv) > 3 else 100 149 | BO(n_eval=n_eval, func=func, dim=int(sys.argv[2])) 150 | else: 151 | BO(n_eval=int(sys.argv[2]), func=func) 152 | else: 153 | BO(n_eval=int(sys.argv[2]), path=sys.argv[1]) 154 | -------------------------------------------------------------------------------- /HyperSphere/BO/axv/radialnone_BO.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import pickle 3 | import sys 4 | import time 5 | from datetime import datetime 6 | 7 | # ShadowInference version should coincide with the one used in acquisition_maximization 8 | from HyperSphere.BO.acquisition.acquisition_maximization import suggest, optimization_candidates, \ 9 | optimization_init_points, deepcopy_inference 10 | from HyperSphere.GP.inference.inference import Inference 11 | from HyperSphere.GP.kernels.modules.matern52 import Matern52 12 | from HyperSphere.GP.models.gp_regression import GPRegression 13 | from HyperSphere.feature_map.functionals import x2radial, radial_bound 14 | from HyperSphere.dummy.plotting import EXPERIMENT_DIR 15 | from HyperSphere.test_functions.benchmarks import * 16 | 17 | exp_str = __file__.split('/')[-1].split('_')[0] 18 | 19 | 20 | def BO(n_eval=200, path=None, func=None, ndim=None): 21 | assert (path is None) != (func is None) 22 | if path is not None: 23 | if not os.path.exists(path): 24 | path = os.path.join(EXPERIMENT_DIR, path) 25 | model_filename = os.path.join(path, 'model.pt') 26 | data_config_filename = os.path.join(path, 'data_config.pkl') 27 | 28 | model = torch.load(model_filename) 29 | data_config_file = open(data_config_filename, 'r') 30 | for key, value in pickle.load(data_config_file).iteritems(): 31 | exec(key + '=value') 32 | data_config_file.close() 33 | 34 | inference = Inference((x_input, output), model) 35 | else: 36 | assert (func.dim == 0) != (ndim is None) 37 | if ndim is None: 38 | ndim = func.dim 39 | dir_list = [elm for elm in os.listdir(EXPERIMENT_DIR) if os.path.isdir(os.path.join(EXPERIMENT_DIR, elm))] 40 | folder_name = func.__name__ + '_D' + str(ndim) + '_' + exp_str + '_' + datetime.now().strftime('%Y%m%d-%H:%M:%S:%f') 41 | os.makedirs(os.path.join(EXPERIMENT_DIR, folder_name)) 42 | model_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'model.pt') 43 | data_config_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'data_config.pkl') 44 | 45 | search_sphere_radius = ndim ** 0.5 46 | 47 | x_input = Variable(torch.ger(-torch.arange(0, 2), torch.ones(ndim))) 48 | output = Variable(torch.zeros(x_input.size(0), 1)) 49 | for i in range(x_input.size(0)): 50 | output[i] = func(x_input[i]) 51 | 52 | kernel_input_map = x2radial 53 | model = GPRegression(kernel=Matern52(ndim=kernel_input_map.dim_change(ndim), input_map=kernel_input_map)) 54 | 55 | time_list = [time.time()] * 2 56 | elapse_list = [0, 0] 57 | pred_mean_list = [0, 0] 58 | pred_std_list = [0, 0] 59 | pred_var_list = [0, 0] 60 | pred_stdmax_list = [1, 1] 61 | pred_varmax_list = [1, 1] 62 | reference_list = [output.data.squeeze()[0]] * 2 63 | refind_list = [1, 1] 64 | dist_to_ref_list = [0, 0] 65 | 66 | inference = Inference((x_input, output), model) 67 | inference.init_parameters() 68 | inference.sampling(n_sample=1, n_burnin=99, n_thin=1) 69 | 70 | stored_variable_names = locals().keys() 71 | ignored_variable_names = ['n_eval', 'path', 'data_config_file', 'dir_list', 'folder_name', 72 | 'next_ind', 'model_filename', 'data_config_filename', 'i', 73 | 'kernel_input_map', 'model', 'inference'] 74 | stored_variable_names = set(stored_variable_names).difference(set(ignored_variable_names)) 75 | 76 | bnd = radial_bound(search_sphere_radius) 77 | 78 | for _ in range(3): 79 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 80 | 81 | for _ in range(n_eval): 82 | inference = Inference((x_input, output), model) 83 | 84 | reference, ref_ind = torch.min(output, 0) 85 | reference = reference.data.squeeze()[0] 86 | gp_hyper_params = inference.sampling(n_sample=10, n_burnin=0, n_thin=1) 87 | inferences = deepcopy_inference(inference, gp_hyper_params) 88 | 89 | x0_cand = optimization_candidates(x_input, output, -1, 1) 90 | x0, sample_info = optimization_init_points(x0_cand, inferences, reference=reference) 91 | next_x_point, pred_mean, pred_std, pred_var, pred_stdmax, pred_varmax = suggest(inferences, x0=x0, bounds=bnd, reference=reference) 92 | 93 | time_list.append(time.time()) 94 | elapse_list.append(time_list[-1] - time_list[-2]) 95 | pred_mean_list.append(pred_mean.squeeze()[0]) 96 | pred_std_list.append(pred_std.squeeze()[0]) 97 | pred_var_list.append(pred_var.squeeze()[0]) 98 | pred_stdmax_list.append(pred_stdmax.squeeze()[0]) 99 | pred_varmax_list.append(pred_varmax.squeeze()[0]) 100 | reference_list.append(reference) 101 | refind_list.append(ref_ind.data.squeeze()[0] + 1) 102 | dist_to_ref_list.append(torch.sum((next_x_point - x_input[ref_ind].data) ** 2) ** 0.5) 103 | 104 | x_input = torch.cat([x_input, Variable(next_x_point)], 0) 105 | output = torch.cat([output, func(x_input[-1])]) 106 | 107 | min_ind = torch.min(output, 0)[1] 108 | min_loc = x_input[min_ind] 109 | min_val = output[min_ind] 110 | dist_to_suggest = torch.sum((x_input - x_input[-1]).data ** 2, 1) ** 0.5 111 | dist_to_min = torch.sum((x_input - min_loc).data ** 2, 1) ** 0.5 112 | out_of_box = torch.sum((torch.abs(x_input.data) > 1), 1) 113 | print('') 114 | for i in range(x_input.size(0)): 115 | time_str = time.strftime('%H:%M:%S', time.gmtime(time_list[i])) + '(' + time.strftime('%H:%M:%S', time.gmtime(elapse_list[i])) + ') ' 116 | data_str = ('%3d-th : %+14.4f(R:%8.4f[%4d]/ref:[%3d]%8.4f), ' 117 | 'mean : %+.4E, std : %.4E(%5.4f), var : %.4E(%5.4f), ' 118 | '2ownMIN : %8.4f, 2curMIN : %8.4f, 2new : %8.4f' % 119 | (i + 1, output.data.squeeze()[i], torch.sum(x_input.data[i] ** 2) ** 0.5, out_of_box[i], refind_list[i], reference_list[i], 120 | pred_mean_list[i], pred_std_list[i], pred_std_list[i] / pred_stdmax_list[i], pred_var_list[i], pred_var_list[i] / pred_varmax_list[i], 121 | dist_to_ref_list[i], dist_to_min[i], dist_to_suggest[i])) 122 | min_str = ' <========= MIN' if i == min_ind.data.squeeze()[0] else '' 123 | print(time_str + data_str + min_str) 124 | print(model.kernel.__class__.__name__) 125 | 126 | torch.save(model, model_filename) 127 | stored_variable = dict() 128 | for key in stored_variable_names: 129 | stored_variable[key] = locals()[key] 130 | f = open(data_config_filename, 'w') 131 | pickle.dump(stored_variable, f) 132 | f.close() 133 | 134 | for _ in range(3): 135 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 136 | 137 | return os.path.split(model_filename)[0] 138 | 139 | 140 | if __name__ == '__main__': 141 | run_new = False 142 | path, suffix = os.path.split(sys.argv[1]) 143 | if path == '' and not ('_D' in suffix): 144 | run_new = True 145 | if run_new: 146 | func = locals()[sys.argv[1]] 147 | if func.dim == 0: 148 | n_eval = int(sys.argv[3]) if len(sys.argv) > 3 else 100 149 | BO(n_eval=n_eval, func=func, dim=int(sys.argv[2])) 150 | else: 151 | BO(n_eval=int(sys.argv[2]), func=func) 152 | else: 153 | BO(n_eval=int(sys.argv[2]), path=sys.argv[1]) 154 | -------------------------------------------------------------------------------- /HyperSphere/BO/axv/radialorigin_BO.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import pickle 3 | import sys 4 | import time 5 | from datetime import datetime 6 | 7 | # ShadowInference version should coincide with the one used in acquisition_maximization 8 | from HyperSphere.BO.acquisition.acquisition_maximization import suggest, optimization_candidates, \ 9 | optimization_init_points, deepcopy_inference 10 | from HyperSphere.BO.shadow_inference.inference_slide_origin import ShadowInference 11 | from HyperSphere.GP.kernels.modules.matern52 import Matern52 12 | from HyperSphere.GP.models.gp_regression import GPRegression 13 | from HyperSphere.feature_map.functionals import x2radial, radial_bound 14 | from HyperSphere.dummy.plotting import EXPERIMENT_DIR 15 | from HyperSphere.test_functions.benchmarks import * 16 | 17 | exp_str = __file__.split('/')[-1].split('_')[0] 18 | 19 | 20 | def BO(n_eval=200, path=None, func=None, ndim=None): 21 | assert (path is None) != (func is None) 22 | if path is not None: 23 | if not os.path.exists(path): 24 | path = os.path.join(EXPERIMENT_DIR, path) 25 | model_filename = os.path.join(path, 'model.pt') 26 | data_config_filename = os.path.join(path, 'data_config.pkl') 27 | 28 | model = torch.load(model_filename) 29 | data_config_file = open(data_config_filename, 'r') 30 | for key, value in pickle.load(data_config_file).iteritems(): 31 | exec(key + '=value') 32 | data_config_file.close() 33 | 34 | inference = ShadowInference((x_input, output), model) 35 | else: 36 | assert (func.dim == 0) != (ndim is None) 37 | if ndim is None: 38 | ndim = func.dim 39 | dir_list = [elm for elm in os.listdir(EXPERIMENT_DIR) if os.path.isdir(os.path.join(EXPERIMENT_DIR, elm))] 40 | folder_name = func.__name__ + '_D' + str(ndim) + '_' + exp_str + '_' + datetime.now().strftime('%Y%m%d-%H:%M:%S:%f') 41 | os.makedirs(os.path.join(EXPERIMENT_DIR, folder_name)) 42 | model_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'model.pt') 43 | data_config_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'data_config.pkl') 44 | 45 | search_sphere_radius = ndim ** 0.5 46 | 47 | x_input = Variable(torch.ger(-torch.arange(0, 2), torch.ones(ndim))) 48 | output = Variable(torch.zeros(x_input.size(0), 1)) 49 | for i in range(x_input.size(0)): 50 | output[i] = func(x_input[i]) 51 | 52 | kernel_input_map = x2radial 53 | model = GPRegression(kernel=Matern52(ndim=kernel_input_map.dim_change(ndim), input_map=kernel_input_map)) 54 | 55 | time_list = [time.time()] * 2 56 | elapse_list = [0, 0] 57 | pred_mean_list = [0, 0] 58 | pred_std_list = [0, 0] 59 | pred_var_list = [0, 0] 60 | pred_stdmax_list = [1, 1] 61 | pred_varmax_list = [1, 1] 62 | reference_list = [output.data.squeeze()[0]] * 2 63 | refind_list = [1, 1] 64 | dist_to_ref_list = [0, 0] 65 | 66 | inference = ShadowInference((x_input, output), model) 67 | inference.init_parameters() 68 | inference.sampling(n_sample=1, n_burnin=99, n_thin=1) 69 | 70 | stored_variable_names = locals().keys() 71 | ignored_variable_names = ['n_eval', 'path', 'data_config_file', 'dir_list', 'folder_name', 72 | 'next_ind', 'model_filename', 'data_config_filename', 'i', 73 | 'kernel_input_map', 'model', 'inference'] 74 | stored_variable_names = set(stored_variable_names).difference(set(ignored_variable_names)) 75 | 76 | bnd = radial_bound(search_sphere_radius) 77 | 78 | for _ in range(3): 79 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 80 | 81 | for _ in range(n_eval): 82 | inference = ShadowInference((x_input, output), model) 83 | 84 | reference, ref_ind = torch.min(output, 0) 85 | reference = reference.data.squeeze()[0] 86 | gp_hyper_params = inference.sampling(n_sample=10, n_burnin=0, n_thin=1) 87 | inferences = deepcopy_inference(inference, gp_hyper_params) 88 | 89 | x0_cand = optimization_candidates(x_input, output, -1, 1) 90 | x0, sample_info = optimization_init_points(x0_cand, inferences, reference=reference) 91 | next_x_point, pred_mean, pred_std, pred_var, pred_stdmax, pred_varmax = suggest(inferences, x0=x0, bounds=bnd, reference=reference) 92 | 93 | time_list.append(time.time()) 94 | elapse_list.append(time_list[-1] - time_list[-2]) 95 | pred_mean_list.append(pred_mean.squeeze()[0]) 96 | pred_std_list.append(pred_std.squeeze()[0]) 97 | pred_var_list.append(pred_var.squeeze()[0]) 98 | pred_stdmax_list.append(pred_stdmax.squeeze()[0]) 99 | pred_varmax_list.append(pred_varmax.squeeze()[0]) 100 | reference_list.append(reference) 101 | refind_list.append(ref_ind.data.squeeze()[0] + 1) 102 | dist_to_ref_list.append(torch.sum((next_x_point - x_input[ref_ind].data) ** 2) ** 0.5) 103 | 104 | x_input = torch.cat([x_input, Variable(next_x_point)], 0) 105 | output = torch.cat([output, func(x_input[-1])]) 106 | 107 | min_ind = torch.min(output, 0)[1] 108 | min_loc = x_input[min_ind] 109 | min_val = output[min_ind] 110 | dist_to_suggest = torch.sum((x_input - x_input[-1]).data ** 2, 1) ** 0.5 111 | dist_to_min = torch.sum((x_input - min_loc).data ** 2, 1) ** 0.5 112 | out_of_box = torch.sum((torch.abs(x_input.data) > 1), 1) 113 | print('') 114 | for i in range(x_input.size(0)): 115 | time_str = time.strftime('%H:%M:%S', time.gmtime(time_list[i])) + '(' + time.strftime('%H:%M:%S', time.gmtime(elapse_list[i])) + ') ' 116 | data_str = ('%3d-th : %+14.4f(R:%8.4f[%4d]/ref:[%3d]%8.4f), ' 117 | 'mean : %+.4E, std : %.4E(%5.4f), var : %.4E(%5.4f), ' 118 | '2ownMIN : %8.4f, 2curMIN : %8.4f, 2new : %8.4f' % 119 | (i + 1, output.data.squeeze()[i], torch.sum(x_input.data[i] ** 2) ** 0.5, out_of_box[i], refind_list[i], reference_list[i], 120 | pred_mean_list[i], pred_std_list[i], pred_std_list[i] / pred_stdmax_list[i], pred_var_list[i], pred_var_list[i] / pred_varmax_list[i], 121 | dist_to_ref_list[i], dist_to_min[i], dist_to_suggest[i])) 122 | min_str = ' <========= MIN' if i == min_ind.data.squeeze()[0] else '' 123 | print(time_str + data_str + min_str) 124 | print(model.kernel.__class__.__name__) 125 | 126 | torch.save(model, model_filename) 127 | stored_variable = dict() 128 | for key in stored_variable_names: 129 | stored_variable[key] = locals()[key] 130 | f = open(data_config_filename, 'w') 131 | pickle.dump(stored_variable, f) 132 | f.close() 133 | 134 | for _ in range(3): 135 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 136 | 137 | return os.path.split(model_filename)[0] 138 | 139 | 140 | if __name__ == '__main__': 141 | run_new = False 142 | path, suffix = os.path.split(sys.argv[1]) 143 | if path == '' and not ('_D' in suffix): 144 | run_new = True 145 | if run_new: 146 | func = locals()[sys.argv[1]] 147 | if func.dim == 0: 148 | n_eval = int(sys.argv[3]) if len(sys.argv) > 3 else 100 149 | BO(n_eval=n_eval, func=func, dim=int(sys.argv[2])) 150 | else: 151 | BO(n_eval=int(sys.argv[2]), func=func) 152 | else: 153 | BO(n_eval=int(sys.argv[2]), path=sys.argv[1]) 154 | -------------------------------------------------------------------------------- /HyperSphere/BO/axv/spherenone_BO.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import pickle 3 | import sys 4 | import time 5 | from datetime import datetime 6 | 7 | # ShadowInference version should coincide with the one used in acquisition_maximization 8 | from HyperSphere.BO.acquisition.acquisition_maximization import suggest, optimization_candidates, \ 9 | optimization_init_points, deepcopy_inference 10 | from HyperSphere.GP.inference.inference import Inference 11 | from HyperSphere.GP.kernels.modules.radialization import RadializationKernel 12 | from HyperSphere.GP.models.gp_regression import GPRegression 13 | from HyperSphere.feature_map.functionals import radial_bound 14 | from HyperSphere.dummy.plotting import EXPERIMENT_DIR 15 | from HyperSphere.test_functions.benchmarks import * 16 | 17 | exp_str = __file__.split('/')[-1].split('_')[0] 18 | 19 | 20 | def BO(n_eval=200, path=None, func=None, ndim=None): 21 | assert (path is None) != (func is None) 22 | if path is not None: 23 | if not os.path.exists(path): 24 | path = os.path.join(EXPERIMENT_DIR, path) 25 | model_filename = os.path.join(path, 'model.pt') 26 | data_config_filename = os.path.join(path, 'data_config.pkl') 27 | 28 | model = torch.load(model_filename) 29 | data_config_file = open(data_config_filename, 'r') 30 | for key, value in pickle.load(data_config_file).iteritems(): 31 | exec(key + '=value') 32 | data_config_file.close() 33 | 34 | inference = Inference((x_input, output), model) 35 | else: 36 | assert (func.dim == 0) != (ndim is None) 37 | if ndim is None: 38 | ndim = func.dim 39 | dir_list = [elm for elm in os.listdir(EXPERIMENT_DIR) if os.path.isdir(os.path.join(EXPERIMENT_DIR, elm))] 40 | folder_name = func.__name__ + '_D' + str(ndim) + '_' + exp_str + '_' + datetime.now().strftime('%Y%m%d-%H:%M:%S:%f') 41 | os.makedirs(os.path.join(EXPERIMENT_DIR, folder_name)) 42 | model_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'model.pt') 43 | data_config_filename = os.path.join(EXPERIMENT_DIR, folder_name, 'data_config.pkl') 44 | 45 | search_sphere_radius = ndim ** 0.5 46 | 47 | x_input = Variable(torch.ger(-torch.arange(0, 2), torch.ones(ndim))) 48 | output = Variable(torch.zeros(x_input.size(0), 1)) 49 | for i in range(x_input.size(0)): 50 | output[i] = func(x_input[i]) 51 | 52 | model = GPRegression(kernel=RadializationKernel(max_power=3, search_radius=search_sphere_radius)) 53 | 54 | time_list = [time.time()] * 2 55 | elapse_list = [0, 0] 56 | pred_mean_list = [0, 0] 57 | pred_std_list = [0, 0] 58 | pred_var_list = [0, 0] 59 | pred_stdmax_list = [1, 1] 60 | pred_varmax_list = [1, 1] 61 | reference_list = [output.data.squeeze()[0]] * 2 62 | refind_list = [1, 1] 63 | dist_to_ref_list = [0, 0] 64 | sample_info_list = [(10, 0, 10)] * 2 65 | 66 | inference = Inference((x_input, output), model) 67 | inference.init_parameters() 68 | inference.sampling(n_sample=1, n_burnin=99, n_thin=1) 69 | 70 | stored_variable_names = locals().keys() 71 | ignored_variable_names = ['n_eval', 'path', 'data_config_file', 'dir_list', 'folder_name', 72 | 'next_ind', 'model_filename', 'data_config_filename', 'i', 73 | 'kernel_input_map', 'model', 'inference'] 74 | stored_variable_names = set(stored_variable_names).difference(set(ignored_variable_names)) 75 | 76 | bnd = radial_bound(search_sphere_radius) 77 | 78 | for _ in range(3): 79 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 80 | 81 | for _ in range(n_eval): 82 | inference = Inference((x_input, output), model) 83 | 84 | reference, ref_ind = torch.min(output, 0) 85 | reference = reference.data.squeeze()[0] 86 | gp_hyper_params = inference.sampling(n_sample=10, n_burnin=0, n_thin=1) 87 | inferences = deepcopy_inference(inference, gp_hyper_params) 88 | 89 | x0_cand = optimization_candidates(x_input, output, -1, 1) 90 | x0, sample_info = optimization_init_points(x0_cand, inferences, reference=reference) 91 | next_x_point, pred_mean, pred_std, pred_var, pred_stdmax, pred_varmax = suggest(inferences, x0=x0, bounds=bnd, reference=reference) 92 | 93 | time_list.append(time.time()) 94 | elapse_list.append(time_list[-1] - time_list[-2]) 95 | pred_mean_list.append(pred_mean.squeeze()[0]) 96 | pred_std_list.append(pred_std.squeeze()[0]) 97 | pred_var_list.append(pred_var.squeeze()[0]) 98 | pred_stdmax_list.append(pred_stdmax.squeeze()[0]) 99 | pred_varmax_list.append(pred_varmax.squeeze()[0]) 100 | reference_list.append(reference) 101 | refind_list.append(ref_ind.data.squeeze()[0] + 1) 102 | dist_to_ref_list.append(torch.sum((next_x_point - x_input[ref_ind].data) ** 2) ** 0.5) 103 | sample_info_list.append(sample_info) 104 | 105 | x_input = torch.cat([x_input, Variable(next_x_point)], 0) 106 | output = torch.cat([output, func(x_input[-1])]) 107 | 108 | min_ind = torch.min(output, 0)[1] 109 | min_loc = x_input[min_ind] 110 | min_val = output[min_ind] 111 | dist_to_suggest = torch.sum((x_input - x_input[-1]).data ** 2, 1) ** 0.5 112 | dist_to_min = torch.sum((x_input - min_loc).data ** 2, 1) ** 0.5 113 | out_of_box = torch.sum((torch.abs(x_input.data) > 1), 1) 114 | print('') 115 | for i in range(x_input.size(0)): 116 | time_str = time.strftime('%H:%M:%S', time.gmtime(time_list[i])) + '(' + time.strftime('%H:%M:%S', time.gmtime(elapse_list[i])) + ') ' 117 | data_str = ('%3d-th : %+12.4f(R:%8.4f[%4d]/ref:[%3d]%8.4f), sample([%2d] best:%2d/worst:%2d), ' 118 | 'mean : %+.4E, std : %.4E(%5.4f), var : %.4E(%5.4f), ' 119 | '2ownMIN : %8.4f, 2curMIN : %8.4f, 2new : %8.4f' % 120 | (i + 1, output.data.squeeze()[i], torch.sum(x_input.data[i] ** 2) ** 0.5, out_of_box[i], refind_list[i], reference_list[i], 121 | sample_info_list[i][2], sample_info_list[i][0], sample_info_list[i][1], 122 | pred_mean_list[i], pred_std_list[i], pred_std_list[i] / pred_stdmax_list[i], pred_var_list[i], pred_var_list[i] / pred_varmax_list[i], 123 | dist_to_ref_list[i], dist_to_min[i], dist_to_suggest[i])) 124 | min_str = ' <========= MIN' if i == min_ind.data.squeeze()[0] else '' 125 | print(time_str + data_str + min_str) 126 | print(model.kernel.__class__.__name__) 127 | 128 | torch.save(model, model_filename) 129 | stored_variable = dict() 130 | for key in stored_variable_names: 131 | stored_variable[key] = locals()[key] 132 | f = open(data_config_filename, 'w') 133 | pickle.dump(stored_variable, f) 134 | f.close() 135 | 136 | for _ in range(3): 137 | print('Experiment based on data in ' + os.path.split(model_filename)[0]) 138 | 139 | return os.path.split(model_filename)[0] 140 | 141 | 142 | if __name__ == '__main__': 143 | run_new = False 144 | path, suffix = os.path.split(sys.argv[1]) 145 | if path == '' and not ('_D' in suffix): 146 | run_new = True 147 | if run_new: 148 | func = locals()[sys.argv[1]] 149 | if func.dim == 0: 150 | n_eval = int(sys.argv[3]) if len(sys.argv) > 3 else 100 151 | BO(n_eval=n_eval, func=func, dim=int(sys.argv[2])) 152 | else: 153 | BO(n_eval=int(sys.argv[2]), func=func) 154 | else: 155 | BO(n_eval=int(sys.argv[2]), path=sys.argv[1]) 156 | -------------------------------------------------------------------------------- /HyperSphere/BO/run_experiments.py: -------------------------------------------------------------------------------- 1 | import multiprocessing 2 | import sys 3 | from functools import partial 4 | 5 | from HyperSphere.BO.spherewarpingboundary_BO import BO as BO_boundary 6 | from HyperSphere.BO.spherewarpingnone_BO import BO as BO_none 7 | from HyperSphere.BO.spherewarpingorigin_BO import BO as BO_origin 8 | 9 | from HyperSphere.BO.axv.spherewarpingboth_BO import BO as BO_both 10 | 11 | if __name__ == '__main__': 12 | optimizer_str = sys.argv[1] 13 | optimizer_name_list = optimizer_str.split(',') 14 | optimizer_list = [] 15 | for optimizer_name in optimizer_name_list: 16 | if optimizer_name == 'spherewarpingnone': 17 | optimizer = BO_none 18 | elif optimizer_name == 'spherewarpingorigin': 19 | optimizer = BO_origin 20 | elif optimizer_name == 'spherewarpingboundary': 21 | optimizer = BO_boundary 22 | elif optimizer_name == 'spherewarpingboth': 23 | optimizer = BO_both 24 | optimizer_list.append(optimizer) 25 | n_dim = int(sys.argv[2]) 26 | n_eval = int(sys.argv[3]) 27 | func_str = sys.argv[4] 28 | func_name_list = sys.argv[4].split(',') 29 | n_optimizer = len(optimizer_list) 30 | n_func = len(func_name_list) 31 | n_cores = multiprocessing.cpu_count() 32 | assert n_cores > n_func * n_optimizer * 5 33 | func_list = [] 34 | for i in range(n_func): 35 | exec('func_list.append(' + func_name_list[i] + ')') 36 | 37 | pool_dict = {} 38 | for optimizer in optimizer_list: 39 | for func in func_list: 40 | pool_dict[(optimizer, func)] = multiprocessing.Pool(5) 41 | exp_result_dir = pool_dict[(optimizer, func)].map(partial(optimizer, kwargs={'func': func, 'dim': n_dim}), [n_eval] * 5) 42 | print(exp_result_dir) 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /HyperSphere/BO/shadow_inference/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/BO/shadow_inference/__init__.py -------------------------------------------------------------------------------- /HyperSphere/BO/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/BO/utils/__init__.py -------------------------------------------------------------------------------- /HyperSphere/BO/utils/normal_cdf.py: -------------------------------------------------------------------------------- 1 | from scipy.stats import norm 2 | 3 | import torch 4 | from torch.autograd import Function, Variable, gradcheck 5 | 6 | 7 | class NormalCDF(Function): 8 | 9 | @staticmethod 10 | def forward(ctx, input): 11 | ctx.save_for_backward(input) 12 | input_np = input.data if isinstance(input, Variable) else input 13 | input_np = (input_np.cpu() if input_np.is_cuda else input_np).numpy() 14 | 15 | output_np = norm.cdf(input_np) 16 | output = torch.from_numpy(output_np).type_as(input) 17 | if isinstance(input, Variable): 18 | output = Variable(output) 19 | 20 | return output 21 | 22 | @staticmethod 23 | def backward(ctx, grad_output): 24 | input, = ctx.saved_variables 25 | grad_input = None 26 | 27 | input_np = input.data if isinstance(input, Variable) else input 28 | input_np = (input_np.cpu() if input_np.is_cuda else input_np).numpy() 29 | 30 | grad_np = norm.pdf(input_np) 31 | grad = Variable(torch.from_numpy(grad_np).type_as(input.data)) 32 | 33 | if ctx.needs_input_grad[0]: 34 | grad_input = grad_output * grad 35 | 36 | return grad_input 37 | 38 | 39 | def norm_cdf(input): 40 | if isinstance(input, Variable): 41 | return NormalCDF.apply(input) 42 | else: 43 | return NormalCDF.apply(input).data 44 | 45 | 46 | if __name__ == '__main__': 47 | ndata = 10 48 | input = Variable(torch.randn(ndata), requires_grad=True) 49 | test = gradcheck(NormalCDF.apply, (input, ), eps=1e-4, atol=1e-3, rtol=1e-2) 50 | print(test) -------------------------------------------------------------------------------- /HyperSphere/BO/utils/sobol.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | import numpy as np 4 | import torch 5 | 6 | 7 | def sobol_generate(n_dim, n_point, n_skip=0): 8 | if n_dim > 1111: 9 | raise Exception('This program supports sobol sequence of dimension up to 1111') 10 | while True: 11 | try: 12 | sequence_file = tempfile.NamedTemporaryFile('r') 13 | filename = sequence_file.name 14 | cmd = os.path.join(os.path.split(os.path.abspath(os.path.realpath(__file__)))[0], 'sobol_c/sobol') 15 | cmd += ' ' + str(int(n_dim)) + ' ' + str(int(n_point)) + ' ' + str(int(n_skip)) + ' ' + filename 16 | os.system(cmd) 17 | sequence = np.fromfile(sequence_file.file, dtype=np.float64).astype(np.float32) 18 | sequence = sequence.reshape([n_point, n_dim]) 19 | sequence_file.close() 20 | return torch.from_numpy(sequence) 21 | except ValueError: 22 | print('%d data in file, but reshape is in (%d, %d)' % (sequence.size, n_point, n_dim)) 23 | continue 24 | 25 | 26 | if __name__ == '__main__': 27 | print sobol_generate(1111, 10001, 0)[-1] -------------------------------------------------------------------------------- /HyperSphere/BO/utils/sobol_c/sobol: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/BO/utils/sobol_c/sobol -------------------------------------------------------------------------------- /HyperSphere/GP/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/GP/__init__.py -------------------------------------------------------------------------------- /HyperSphere/GP/inference/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/GP/inference/__init__.py -------------------------------------------------------------------------------- /HyperSphere/GP/inference/inverse_bilinear_form.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Function, Variable, gradcheck 3 | 4 | 5 | class InverseBilinearForm(Function): 6 | 7 | @staticmethod 8 | def forward(ctx, vec_left, matrix, vec_right=None): 9 | if vec_right is None: 10 | vec_right = vec_left 11 | 12 | ctx.save_for_backward(vec_left, matrix, vec_right) 13 | 14 | jitter = 0 15 | eye_mat = torch.diag(matrix[:1, :1].clone().repeat(matrix.size(1), 1).view(-1) * 0 + 1) 16 | while True: 17 | try: 18 | vec_right_sol = torch.gesv(vec_right, matrix + eye_mat * jitter)[0] 19 | break 20 | except RuntimeError: 21 | jitter = torch.mean(torch.diag(matrix)) * 1e-6 if jitter == 0 else jitter * 10 22 | 23 | return torch.mm(vec_left.t(), vec_right_sol) 24 | 25 | @staticmethod 26 | def backward(ctx, grad_output): 27 | """ 28 | :param ctx: 29 | :param grad_output: grad_output is assumed to be d[Scalar]/dK and size() is n1 x n2 30 | :return: 31 | """ 32 | vec_left, matrix, vec_right = ctx.saved_variables 33 | grad_vec_left = grad_matrix = grad_vec_right = None 34 | 35 | linear_solver = torch.gesv(torch.cat([vec_left, vec_right], 1), matrix)[0] 36 | vec_left_sol = linear_solver[:, :vec_left.size(1)] 37 | vec_right_sol = linear_solver[:, vec_left.size(1):] 38 | if ctx.needs_input_grad[0]: 39 | grad_vec_left = grad_output * vec_right_sol 40 | if ctx.needs_input_grad[1]: 41 | grad_matrix = grad_output * -torch.mm(vec_left_sol, vec_right_sol.t()) 42 | if ctx.needs_input_grad[2]: 43 | grad_vec_right = grad_output * vec_left_sol 44 | 45 | return grad_vec_left, grad_matrix, grad_vec_right 46 | 47 | 48 | if __name__ == '__main__': 49 | ndim = 5 50 | vec_left = Variable(torch.randn(ndim, 1), requires_grad=True) 51 | A = torch.randn(ndim, ndim) 52 | matrix = Variable(A.mm(A.t()) + torch.eye(ndim), requires_grad=True) 53 | vec_right = Variable(torch.randn(ndim, 1), requires_grad=True) 54 | 55 | # gradcheck doesn't have to pass all the time. 56 | test = gradcheck(InverseBilinearForm.apply, (vec_left, matrix, vec_right), eps=1e-4, atol=1e-3, rtol=1e-2) 57 | print(test) 58 | -------------------------------------------------------------------------------- /HyperSphere/GP/inference/log_determinant.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import torch 4 | from torch.autograd import Function, Variable, gradcheck 5 | 6 | 7 | class LogDeterminant(Function): 8 | 9 | @staticmethod 10 | def forward(ctx, matrix): 11 | ctx.save_for_backward(matrix) 12 | 13 | try: 14 | chol_from_upper = torch.potrf(matrix, True) 15 | chol_from_lower = torch.potrf(matrix, False) 16 | return (torch.sum(torch.log(torch.diag(chol_from_upper)), 0, keepdim=True) + torch.sum(torch.log(torch.diag(chol_from_lower)), 0, keepdim=True)).view(1, 1) 17 | except RuntimeError: 18 | eigvals = torch.symeig(matrix)[0] 19 | return torch.sum(torch.log(eigvals[eigvals > 0]), 0, keepdim=True) 20 | 21 | @staticmethod 22 | def backward(ctx, grad_output): 23 | """ 24 | :param ctx: 25 | :param grad_output: grad_output is assumed to be d[Scalar]/dK and size() is n1 x n2 26 | :return: 27 | """ 28 | matrix, = ctx.saved_variables 29 | grad_matrix = None 30 | 31 | if ctx.needs_input_grad[0]: 32 | logdet_grad = torch.gesv(torch.diag(Variable(matrix.data.new(matrix.size(0)).fill_(1))), matrix)[0] 33 | grad_matrix = grad_output * logdet_grad 34 | 35 | return grad_matrix 36 | 37 | 38 | if __name__ == '__main__': 39 | ndim = 10 40 | A = torch.randn(ndim, ndim) 41 | A = A.mm(A.t()) 42 | eig, _ = torch.symeig(A) 43 | print(torch.min(eig)) 44 | matrix = Variable(A + (1e-4 - torch.min(eig)) * torch.eye(ndim), requires_grad=True) 45 | _, logdet = np.linalg.slogdet(matrix.data.numpy()) 46 | 47 | eps = 1e-4 48 | 49 | fdm_deriv = torch.zeros(ndim, ndim) 50 | for i in range(ndim): 51 | for j in range(ndim): 52 | matrix_perturb = Variable(matrix.data).clone() 53 | matrix_perturb.data[i, j] += eps 54 | _, logdet_perturb = np.linalg.slogdet(matrix_perturb.data.numpy()) 55 | fdm_deriv[i, j] = (logdet_perturb - logdet) / eps 56 | 57 | output = LogDeterminant.apply(matrix) 58 | output.backward() 59 | print(torch.abs((matrix.grad.data - fdm_deriv)/matrix.grad.data)) 60 | 61 | 62 | # gradcheck doesn't have to pass all the time. 63 | test = gradcheck(LogDeterminant.apply, (matrix, ), eps=eps, atol=1e-3, rtol=1e-2) 64 | print(test) 65 | -------------------------------------------------------------------------------- /HyperSphere/GP/kernels/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/GP/kernels/__init__.py -------------------------------------------------------------------------------- /HyperSphere/GP/kernels/functions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/GP/kernels/functions/__init__.py -------------------------------------------------------------------------------- /HyperSphere/GP/kernels/functions/inner_product.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Function, Variable, gradcheck 3 | 4 | 5 | def innerProductKernel(input1, input2, log_amp, sigma_sqrt): 6 | n1, ndim = input1.size() 7 | n2 = input2.size(0) 8 | 9 | if sigma_sqrt.dim() == 1: 10 | transformed_input1 = input1 * sigma_sqrt 11 | transformed_input2 = input2 * sigma_sqrt 12 | elif sigma_sqrt.dim() == 2: 13 | transformed_input1 = torch.mm(input1, sigma_sqrt) 14 | transformed_input2 = torch.mm(input2, sigma_sqrt) 15 | 16 | output = log_amp.exp() * (transformed_input1.unsqueeze(1).repeat(1, n2, 1) * transformed_input2.unsqueeze(0).repeat(n1, 1, 1)).sum(2) 17 | return output 18 | 19 | 20 | class InnerProductKernel(Function): 21 | 22 | @staticmethod 23 | def forward(ctx, input1, input2, log_amp, sigma_sqrt): 24 | n1, ndim = input1.size() 25 | n2 = input2.size(0) 26 | 27 | if sigma_sqrt.dim() == 1: 28 | transformed_input1 = input1 * sigma_sqrt 29 | transformed_input2 = input2 * sigma_sqrt 30 | elif sigma_sqrt.dim() == 2: 31 | transformed_input1 = torch.mm(input1, sigma_sqrt) 32 | transformed_input2 = torch.mm(input2, sigma_sqrt) 33 | 34 | output = log_amp.exp() * (transformed_input1.unsqueeze(1).repeat(1, n2, 1) * transformed_input2.unsqueeze(0).repeat(n1, 1, 1)).sum(2) 35 | ctx.save_for_backward(input1, input2, log_amp, sigma_sqrt, output) 36 | return output 37 | 38 | @staticmethod 39 | def backward(ctx, grad_output): 40 | """ 41 | :param ctx: 42 | :param grad_output: grad_output is assumed to be d[Scalar]/dK and size() is n1 x n2 43 | :return: 44 | """ 45 | input1, input2, log_amp, sigma_sqrt, output = ctx.saved_variables 46 | grad_input1 = grad_input2 = grad_log_amp = grad_sigma_sqrt = None 47 | 48 | n1, ndim = input1.size() 49 | n2 = input2.size(0) 50 | 51 | if sigma_sqrt.dim() == 1: 52 | sigma = sigma_sqrt ** 2 53 | mat_input1 = input1 * sigma 54 | mat_input2 = input2 * sigma 55 | elif sigma_sqrt.dim() == 2: 56 | sigma = sigma_sqrt.mm(sigma_sqrt.t()) 57 | mat_input1 = torch.mm(input1, sigma) 58 | mat_input2 = torch.mm(input2, sigma) 59 | 60 | if ctx.needs_input_grad[0]: 61 | grad_input1 = grad_output.mm(mat_input2) * log_amp.exp() 62 | if ctx.needs_input_grad[1]: 63 | grad_input2 = grad_output.t().mm(mat_input1) * log_amp.exp() 64 | if ctx.needs_input_grad[2]: 65 | grad_log_amp = (grad_output * output).sum() 66 | if ctx.needs_input_grad[3]: 67 | if sigma_sqrt.dim() == 1: 68 | all_prod = grad_output.unsqueeze(2).repeat(1, 1, ndim) * input1.unsqueeze(1).repeat(1, n2, 1) * input2.unsqueeze(0).repeat(n1, 1, 1) * sigma_sqrt.view(1, 1, -1).repeat(n1, n2, 1) 69 | grad_sigma_sqrt = log_amp.exp() * 2.0 * all_prod.sum(0).sum(0) 70 | elif sigma_sqrt.dim() == 2: 71 | y_xT = torch.bmm(input2.unsqueeze(2), (input1.unsqueeze(1).repeat(1, n2, 1) * grad_output.unsqueeze(2).repeat(1, 1, ndim)).sum(0, keepdim=True).transpose(0, 1)).sum(0) 72 | input_outer = y_xT + y_xT.t() 73 | grad_sigma_sqrt = log_amp.exp() * input_outer.mm(sigma_sqrt) 74 | 75 | return grad_input1, grad_input2, grad_log_amp, grad_sigma_sqrt 76 | 77 | 78 | if __name__ == '__main__': 79 | n1 = 3 80 | n2 = 4 81 | ndim = 5 82 | input_grad = False 83 | param_grad = not input_grad 84 | input1 = Variable(torch.randn(n1, ndim), requires_grad=input_grad) 85 | input2 = Variable(torch.randn(n2, ndim), requires_grad=input_grad) 86 | log_amp = Variable(torch.randn(1), requires_grad=False) 87 | sigma_sqrt = Variable(torch.tril(torch.randn(ndim, ndim)), requires_grad=param_grad) 88 | # sigma_sqrt = Variable(torch.randn(ndim), requires_grad=param_grad) 89 | # gradcheck doesn't have to pass all the time. 90 | # test = gradcheck(InnerProductKernel.apply, (input1, input2, log_amp, sigma_sqrt), eps=1e-4, atol=1e-3, rtol=1e-2) 91 | # print(test) 92 | 93 | if sigma_sqrt.grad is not None: 94 | sigma_sqrt.grad.data.zero_() 95 | output1 = InnerProductKernel.apply(input1, input2, log_amp, sigma_sqrt) 96 | output1.backward(torch.ones(n1, n2)) 97 | print(sigma_sqrt.grad) 98 | sigma_sqrt.grad.data.zero_() 99 | output2 = innerProductKernel(input1, input2, log_amp, sigma_sqrt) 100 | output2.backward(torch.ones(n1, n2)) 101 | print(sigma_sqrt.grad) 102 | -------------------------------------------------------------------------------- /HyperSphere/GP/kernels/functions/matern52.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Function, Variable, gradcheck 3 | 4 | root_five = 5.0 ** 0.5 5 | 6 | 7 | class Matern52(Function): 8 | 9 | @staticmethod 10 | def forward(ctx, input1, input2, log_amp, log_ls): 11 | n1, ndim = input1.size() 12 | n2 = input2.size(0) 13 | 14 | diff = input1.unsqueeze(1).repeat(1, n2, 1) - input2.unsqueeze(0).repeat(n1, 1, 1) 15 | scaling = log_ls.exp().view(1, 1, log_ls.numel()).repeat(n1, n2, 1) 16 | 17 | r_l_sq = torch.sum(diff**2 / scaling, dim=2) 18 | r_l = torch.sqrt(r_l_sq) 19 | output = log_amp.exp() * (1 + root_five * r_l + 5.0 / 3.0 * r_l_sq) * torch.exp(-root_five * r_l) 20 | ctx.save_for_backward(input1, input2, log_amp, log_ls, output) 21 | return output 22 | 23 | @staticmethod 24 | def backward(ctx, grad_output): 25 | """ 26 | :param ctx: 27 | :param grad_output: grad_output is assumed to be d[Scalar]/dK and size() is n1 x n2 28 | :return: 29 | """ 30 | input1, input2, log_amp, log_ls, output = ctx.saved_variables 31 | grad_input1 = grad_input2 = grad_log_amp = grad_log_ls = None 32 | 33 | n1, ndim = input1.size() 34 | n2 = input2.size(0) 35 | 36 | diff = input1.unsqueeze(1).repeat(1, n2, 1) - input2.unsqueeze(0).repeat(n1, 1, 1) 37 | scaling = log_ls.exp().view(1, 1, log_ls.numel()).repeat(n1, n2, 1) 38 | 39 | r_l_sq = torch.sum(diff ** 2 / scaling, dim=2) 40 | r_l = torch.sqrt(r_l_sq) 41 | 42 | intermediate_grad = log_amp.exp() * -5.0 / 6.0 * (1 + root_five * r_l) * torch.exp(-root_five * r_l) 43 | 44 | if ctx.needs_input_grad[0]: 45 | kernel_grad_input1 = 2.0 * diff / scaling * intermediate_grad.unsqueeze(2).repeat(1, 1, ndim) 46 | grad_input1 = (grad_output.unsqueeze(2).repeat(1, 1, ndim) * kernel_grad_input1).sum(1) 47 | if ctx.needs_input_grad[1]: 48 | kernel_grad_input2 = -2.0 * diff / scaling * intermediate_grad.unsqueeze(2).repeat(1, 1, ndim) 49 | grad_input2 = (grad_output.unsqueeze(2).repeat(1, 1, ndim) * kernel_grad_input2).sum(0) 50 | if ctx.needs_input_grad[2]: 51 | kernel_grad_log_amp = output 52 | grad_log_amp = (grad_output * kernel_grad_log_amp).sum() 53 | if ctx.needs_input_grad[3]: 54 | kernel_grad_log_ls = -diff ** 2/scaling * intermediate_grad.unsqueeze(2).repeat(1, 1, ndim) 55 | grad_log_ls = (grad_output.unsqueeze(2).repeat(1, 1, ndim) * kernel_grad_log_ls).sum(0).sum(0) 56 | 57 | return grad_input1, grad_input2, grad_log_amp, grad_log_ls 58 | 59 | 60 | if __name__ == '__main__': 61 | n1 = 3 62 | n2 = 4 63 | ndim = 5 64 | input_grad = False 65 | param_grad = not input_grad 66 | input1 = Variable(torch.randn(n1, ndim), requires_grad=input_grad) 67 | input2 = Variable(torch.randn(n2, ndim), requires_grad=input_grad) 68 | log_amp = Variable(torch.randn(1), requires_grad=param_grad) 69 | log_ls = Variable(torch.randn(ndim), requires_grad=param_grad) 70 | # gradcheck doesn't have to pass all the time. 71 | test = gradcheck(Matern52.apply, (input1, input2, log_amp, log_ls), eps=1e-4, atol=1e-3, rtol=1e-2) 72 | print(test) -------------------------------------------------------------------------------- /HyperSphere/GP/kernels/functions/squared_exponential.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Function, Variable, gradcheck 3 | 4 | 5 | class SquaredExponentialKernel(Function): 6 | 7 | @staticmethod 8 | def forward(ctx, input1, input2, log_amp, log_ls): 9 | n1, ndim = input1.size() 10 | n2 = input2.size(0) 11 | 12 | diff = input1.unsqueeze(1).repeat(1, n2, 1) - input2.unsqueeze(0).repeat(n1, 1, 1) 13 | scaling = log_ls.exp().view(1, 1, log_ls.numel()).repeat(n1, n2, 1) 14 | 15 | output = log_amp.exp() * torch.exp(-torch.sum(0.5 * diff**2 / scaling, dim=2)) 16 | ctx.save_for_backward(input1, input2, log_amp, log_ls, output) 17 | return output 18 | 19 | @staticmethod 20 | def backward(ctx, grad_output): 21 | """ 22 | :param ctx: 23 | :param grad_output: grad_output is assumed to be d[Scalar]/dK and size() is n1 x n2 24 | :return: 25 | """ 26 | input1, input2, log_amp, log_ls, output = ctx.saved_variables 27 | grad_input1 = grad_input2 = grad_log_amp = grad_log_ls = None 28 | 29 | n1, ndim = input1.size() 30 | n2 = input2.size(0) 31 | 32 | diff = input1.unsqueeze(1).repeat(1, n2, 1) - input2.unsqueeze(0).repeat(n1, 1, 1) 33 | scaling = log_ls.exp().view(1, 1, log_ls.numel()).repeat(n1, n2, 1) 34 | 35 | if ctx.needs_input_grad[0]: 36 | kernel_grad_input1 = -diff / scaling * output.unsqueeze(2).repeat(1, 1, ndim) 37 | grad_input1 = (grad_output.unsqueeze(2).repeat(1, 1, ndim) * kernel_grad_input1).sum(1) 38 | if ctx.needs_input_grad[1]: 39 | kernel_grad_input2 = diff / scaling * output.unsqueeze(2).repeat(1, 1, ndim) 40 | grad_input2 = (grad_output.unsqueeze(2).repeat(1, 1, ndim) * kernel_grad_input2).sum(0) 41 | if ctx.needs_input_grad[2]: 42 | kernel_grad_log_amp = output 43 | grad_log_amp = (grad_output * kernel_grad_log_amp).sum() 44 | if ctx.needs_input_grad[3]: 45 | kernel_grad_log_ls = 0.5 * diff ** 2/scaling * output.unsqueeze(2).repeat(1, 1, ndim) 46 | grad_log_ls = (grad_output.unsqueeze(2).repeat(1, 1, ndim) * kernel_grad_log_ls).sum(0).sum(0) 47 | 48 | return grad_input1, grad_input2, grad_log_amp, grad_log_ls 49 | 50 | 51 | if __name__ == '__main__': 52 | n1 = 3 53 | n2 = 4 54 | ndim = 5 55 | input_grad = True 56 | param_grad = not input_grad 57 | input1 = Variable(torch.randn(n1, ndim), requires_grad=input_grad) 58 | input2 = Variable(torch.randn(n2, ndim), requires_grad=input_grad) 59 | log_amp = Variable(torch.randn(1), requires_grad=param_grad) 60 | log_ls = Variable(torch.randn(ndim), requires_grad=param_grad) 61 | # gradcheck doesn't have to pass all the time. 62 | test = gradcheck(SquaredExponentialKernel.apply, (input1, input2, log_amp, log_ls), eps=1e-4, atol=1e-3, rtol=1e-2) 63 | print(test) -------------------------------------------------------------------------------- /HyperSphere/GP/kernels/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/GP/kernels/modules/__init__.py -------------------------------------------------------------------------------- /HyperSphere/GP/kernels/modules/inner_product.py: -------------------------------------------------------------------------------- 1 | import math 2 | import sampyl as smp 3 | 4 | import torch 5 | from torch.autograd import Variable 6 | from torch.nn.parameter import Parameter 7 | from HyperSphere.GP.kernels.modules.kernel import Kernel 8 | from HyperSphere.GP.kernels.functions import inner_product 9 | 10 | 11 | class InnerProductKernel(Kernel): 12 | 13 | def __init__(self, ndim, input_map=lambda x: x, diagonal=True): 14 | super(InnerProductKernel, self).__init__(input_map) 15 | self.diag = diagonal 16 | if diagonal: 17 | self.sigma_sqrt = Parameter(torch.FloatTensor(ndim)) 18 | else: 19 | self.sigma_sqrt = Parameter(torch.FloatTensor(ndim, ndim)) 20 | 21 | def reset_parameters(self): 22 | super(InnerProductKernel, self).reset_parameters() 23 | self.sigma_sqrt.data.normal_() 24 | 25 | def out_of_bounds(self, vec=None): 26 | if vec is None: 27 | return super(InnerProductKernel, self).out_of_bounds(self.log_amp) 28 | else: 29 | return not super(InnerProductKernel, self).out_of_bounds(vec[:1]) 30 | 31 | def n_params(self): 32 | return super(InnerProductKernel, self).n_params() + self.sigma_chol.numel() 33 | 34 | def param_to_vec(self): 35 | return torch.cat([self.log_amp.data, self.sigma_chol.data]) 36 | 37 | def vec_to_param(self, vec): 38 | self.log_amp.data = vec[0:1] 39 | self.sigma_sqrt.data = vec[1:] 40 | 41 | def prior(self, vec): 42 | return super(InnerProductKernel, self).prior(vec[:1]) + smp.normal(vec[1:]) 43 | 44 | def forward(self, input1, input2=None): 45 | stabilizer = 0 46 | if input2 is None: 47 | input2 = input1 48 | stabilizer = Variable(torch.diag(input1.data.new(input1.size(0)).fill_(1e-6 * math.exp(self.log_amp.data[0])))) 49 | gram_mat = inner_product.InnerProductKernel.apply(self.input_map(input1), self.input_map(input2), self.log_amp, self.sigma_sqrt if self.diag else self.sigma_sqrt.view(int(self.sigma_sqrt.numel() ** 0.5), -1)) 50 | return gram_mat + stabilizer 51 | 52 | def __repr__(self): 53 | return self.__class__.__name__ + ' (diag=' + ('True' if self.sigma_sqrt.dim()==1 else 'False') + ')' 54 | -------------------------------------------------------------------------------- /HyperSphere/GP/kernels/modules/kernel.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import torch 4 | from torch.nn.parameter import Parameter 5 | from HyperSphere.GP.modules.gp_modules import GPModule, log_lower_bnd, log_upper_bnd 6 | from HyperSphere.feature_map.functionals import id_transform 7 | 8 | 9 | class Kernel(GPModule): 10 | 11 | def __init__(self, input_map=None): 12 | super(Kernel, self).__init__() 13 | self.log_amp = Parameter(torch.FloatTensor(1)) 14 | if input_map is not None: 15 | self.input_map = input_map 16 | else: 17 | self.input_map = id_transform 18 | 19 | def reset_parameters(self): 20 | self.log_amp.data.normal_() 21 | if isinstance(self.input_map, GPModule): 22 | self.input_map.reset_parameters() 23 | 24 | def init_parameters(self, amp): 25 | self.log_amp.data.fill_(np.log(amp)) 26 | if isinstance(self.input_map, GPModule): 27 | self.input_map.init_parameters() 28 | 29 | def log_kernel_amp(self): 30 | return self.log_amp 31 | 32 | def out_of_bounds(self, vec=None): 33 | if vec is None: 34 | if (log_lower_bnd <= self.log_amp.data).all() and (self.log_amp.data <= log_upper_bnd).all(): 35 | if isinstance(self.input_map, GPModule): 36 | return self.input_map.out_of_bounds() 37 | return False 38 | else: 39 | if log_lower_bnd <= vec[0] and vec[0] <= log_upper_bnd: 40 | if isinstance(self.input_map, GPModule): 41 | return self.input_map.out_of_bounds(vec[1:]) 42 | return False 43 | return True 44 | 45 | def n_params(self): 46 | cnt = 1 47 | if isinstance(self.input_map, GPModule): 48 | for p in self.input_map.parameters(): 49 | cnt += p.numel() 50 | return cnt 51 | 52 | def param_to_vec(self): 53 | flat_param_list = [self.log_amp.data.clone()] 54 | if isinstance(self.input_map, GPModule): 55 | flat_param_list.append(self.input_map.param_to_vec()) 56 | return torch.cat(flat_param_list) 57 | 58 | def vec_to_param(self, vec): 59 | self.log_amp.data = vec[:1] 60 | if isinstance(self.input_map, GPModule): 61 | self.input_map.vec_to_param(vec[1:]) 62 | 63 | def prior(self, vec): 64 | if isinstance(self.input_map, GPModule): 65 | return self.input_map.prior(vec[1:]) 66 | return 0 67 | 68 | def forward(self, input1, input2=None): 69 | raise NotImplementedError 70 | -------------------------------------------------------------------------------- /HyperSphere/GP/kernels/modules/matern52.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | 4 | import torch 5 | from torch.autograd import Variable 6 | from HyperSphere.GP.kernels.modules.stationary import Stationary 7 | from HyperSphere.GP.kernels.functions import matern52 8 | 9 | 10 | class Matern52(Stationary): 11 | 12 | def __init__(self, ndim, ard=True, input_map=None, max_ls=None): 13 | super(Matern52, self).__init__(ndim, ard, input_map, max_ls) 14 | 15 | def forward(self, input1, input2=None): 16 | stabilizer = 0 17 | if input2 is None: 18 | input2 = input1 19 | stabilizer = Variable(torch.diag(input1.data.new(input1.size(0)).fill_(1e-6 * math.exp(self.log_amp.data[0])))) 20 | gram_mat = matern52.Matern52.apply(self.input_map(input1), self.input_map(input2), self.log_amp, self.log_ls) 21 | return gram_mat + stabilizer 22 | 23 | def __repr__(self): 24 | return self.__class__.__name__ + ' (' + 'dim=' + str(self.ndim) + ')' 25 | 26 | 27 | if __name__ == '__main__': 28 | kernel = Matern52(ndim=5) 29 | print(list(kernel.parameters())) 30 | -------------------------------------------------------------------------------- /HyperSphere/GP/kernels/modules/radialization.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Variable 3 | from HyperSphere.GP.modules.gp_modules import GPModule, log_upper_bnd 4 | from HyperSphere.GP.kernels.modules.sphere_radial import SphereRadialKernel 5 | from HyperSphere.GP.kernels.modules.matern52 import Matern52 6 | from HyperSphere.feature_map.functionals import x2radial 7 | 8 | 9 | class RadializationKernel(GPModule): 10 | 11 | def __init__(self, max_power, search_radius, radius_input_map=None): 12 | super(RadializationKernel, self).__init__() 13 | self.search_radius = search_radius 14 | self.radius_kernel = Matern52(ndim=1, max_ls=2.0 * search_radius, input_map=radius_input_map) 15 | self.sphere_kernel = SphereRadialKernel(max_power) 16 | 17 | def reset_parameters(self): 18 | self.radius_kernel.reset_parameters() 19 | self.sphere_kernel.reset_parameters() 20 | 21 | def init_parameters(self, amp): 22 | self.radius_kernel.init_parameters(amp) 23 | self.sphere_kernel.init_parameters() 24 | 25 | def log_kernel_amp(self): 26 | return self.radius_kernel.log_amp 27 | 28 | def out_of_bounds(self, vec=None): 29 | if vec is None: 30 | return self.radius_kernel.out_of_bounds() or self.sphere_kernel.out_of_bounds() 31 | else: 32 | n_param_super = self.radius_kernel.n_params() 33 | return self.radius_kernel.out_of_bounds(vec[:n_param_super]) or self.sphere_kernel.out_of_bounds(vec[n_param_super:]) 34 | 35 | def n_params(self): 36 | return self.radius_kernel.n_params() + self.sphere_kernel.n_params() 37 | 38 | def param_to_vec(self): 39 | vec = torch.cat([self.radius_kernel.param_to_vec(), self.sphere_kernel.param_to_vec()]) 40 | return vec 41 | 42 | def vec_to_param(self, vec): 43 | n_param_radius = self.radius_kernel.n_params() 44 | self.radius_kernel.vec_to_param(vec[:n_param_radius]) 45 | self.sphere_kernel.vec_to_param(vec[n_param_radius:]) 46 | 47 | def prior(self, vec): 48 | n_param_radius = self.radius_kernel.n_params() 49 | return self.radius_kernel.prior(vec[:n_param_radius]) + self.sphere_kernel.prior(vec[n_param_radius:]) 50 | 51 | def forward_on_identical(self): 52 | return torch.exp(self.radius_kernel.log_amp) * (1 + 1e-6) 53 | 54 | def forward(self, input1, input2=None): 55 | radial1 = x2radial(input1) 56 | stabilizer = 0 57 | if input2 is None: 58 | input2 = input1 59 | stabilizer = Variable(torch.diag(input1.data.new(input1.size(0)).fill_(1e-6 * torch.exp(self.log_kernel_amp()).data.squeeze()[0]))) 60 | radial2 = x2radial(input2) 61 | r1 = radial1[:, :1] 62 | d1 = radial1[:, 1:] 63 | r2 = radial2[:, :1] 64 | d2 = radial2[:, 1:] 65 | radial_gram = self.radius_kernel(r1, r2) 66 | sphere_gram = self.sphere_kernel(d1, d2) 67 | # only_radial_ind = ((r1 == 0) + (r2 == 0).t()).clamp(max=1) 68 | # sphere_gram[only_radial_ind] = 1.0 69 | return radial_gram * sphere_gram + stabilizer 70 | 71 | def __repr__(self): 72 | return self.__class__.__name__ + ' (max_power=' + str(self.sphere_kernel.max_power) + ')' 73 | -------------------------------------------------------------------------------- /HyperSphere/GP/kernels/modules/sphere_radial.py: -------------------------------------------------------------------------------- 1 | import sampyl as smp 2 | 3 | import torch 4 | from torch.autograd import Variable 5 | from torch.nn.parameter import Parameter 6 | from HyperSphere.GP.modules.gp_modules import GPModule, log_lower_bnd, log_upper_bnd 7 | from HyperSphere.feature_map.functionals import id_transform 8 | 9 | 10 | class SphereRadialKernel(GPModule): 11 | 12 | def __init__(self, max_power, input_map=None): 13 | super(SphereRadialKernel, self).__init__() 14 | self.max_power = max_power 15 | self.log_amp_const = Parameter(torch.FloatTensor(1)) 16 | self.log_amp_power = Parameter(torch.FloatTensor(max_power)) 17 | if input_map is not None: 18 | self.input_map = input_map 19 | else: 20 | self.input_map = id_transform 21 | 22 | def reset_parameters(self): 23 | self.log_amp_const.data.normal_() 24 | self.log_amp_power.data.normal_() 25 | if isinstance(self.input_map, GPModule): 26 | self.input_map.reset_parameters() 27 | 28 | def init_parameters(self): 29 | self.log_amp_const.data.fill_(0) 30 | self.log_amp_power.data.fill_(0) 31 | if isinstance(self.input_map, GPModule): 32 | self.input_map.init_parameters() 33 | 34 | def out_of_bounds(self, vec=None): 35 | if vec is None: 36 | if (self.log_amp_power.data < log_lower_bnd).any() or (self.log_amp_power.data > log_upper_bnd).any(): 37 | return True 38 | if isinstance(self.input_map, GPModule): 39 | return self.input_map.out_of_bounds() 40 | return (self.log_amp_const.data < log_lower_bnd).any() or (self.log_amp_const.data > log_upper_bnd).any() 41 | else: 42 | if isinstance(self.input_map, GPModule): 43 | return self.input_map.out_of_bounds(vec[1+self.max_power:]) 44 | return (vec[:1+self.max_power] < log_lower_bnd).any() or (vec[:1+self.max_power] > log_upper_bnd).any() 45 | 46 | def n_params(self): 47 | cnt = self.max_power + 1 48 | if isinstance(self.input_map, GPModule): 49 | for p in self.input_map.parameters(): 50 | cnt += p.numel() 51 | return cnt 52 | 53 | def param_to_vec(self): 54 | flat_param_list = [self.log_amp_const.data, self.log_amp_power.data] 55 | if isinstance(self.input_map, GPModule): 56 | flat_param_list.append(self.input_map.param_to_vec()) 57 | return torch.cat(flat_param_list) 58 | 59 | def vec_to_param(self, vec): 60 | self.log_amp_const.data = vec[:1] 61 | self.log_amp_power.data = vec[1:1+self.max_power] 62 | if isinstance(self.input_map, GPModule): 63 | self.input_map.vec_to_param(vec[1+self.max_power:]) 64 | 65 | def prior(self, vec): 66 | log_lik = smp.normal(vec[:1], 0, 2) + smp.normal(vec[1:], 0, 2) 67 | if isinstance(self.input_map, GPModule): 68 | log_lik += self.input_map.prior(vec[1:]) 69 | return log_lik 70 | 71 | def forward(self, input1, input2=None): 72 | stabilizer = 0 73 | if input2 is None: 74 | input2 = input1 75 | stabilizer = Variable(torch.diag(input1.data.new(input1.size(0)).fill_(1e-6))) 76 | inner_prod = input1.mm(input2.t()) 77 | sum_exp = torch.exp(self.log_amp_const) + torch.sum(torch.exp(self.log_amp_power)) 78 | gram_mat = (torch.exp(self.log_amp_const) / sum_exp) + (torch.exp(self.log_amp_power[0]) / sum_exp) * inner_prod 79 | for p in range(1, self.max_power): 80 | gram_mat += (torch.exp(self.log_amp_power[p]) / sum_exp) * inner_prod ** (p + 1) 81 | return gram_mat + stabilizer 82 | 83 | def __repr__(self): 84 | return self.__class__.__name__ + ' (max_power=' + str(self.max_power) + ')' 85 | 86 | 87 | if __name__ == '__main__': 88 | kernel = SphereRadialKernel(max_power=3) 89 | vec = torch.ones(4) 90 | vec[2:] *= 80 91 | print(kernel.out_of_bounds(vec)) -------------------------------------------------------------------------------- /HyperSphere/GP/kernels/modules/squared_exponential.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | from torch.autograd import Variable 5 | from HyperSphere.GP.kernels.modules.stationary import Stationary 6 | from HyperSphere.GP.kernels.functions import squared_exponential 7 | 8 | 9 | class SquaredExponentialKernel(Stationary): 10 | 11 | def __init__(self, ndim, input_map=None): 12 | super(SquaredExponentialKernel, self).__init__(ndim, input_map) 13 | 14 | def reset_parameters(self): 15 | super(SquaredExponentialKernel, self).reset_parameters() 16 | 17 | def forward(self, input1, input2=None): 18 | stabilizer = 0 19 | if input2 is None: 20 | input2 = input1 21 | stabilizer = Variable(torch.diag(input1.data.new(input1.size(0)).fill_(1e-6 * math.exp(self.log_amp.data[0])))) 22 | gram_mat = squared_exponential.SquaredExponentialKernel.apply(self.input_map(input1), self.input_map(input2), self.log_amp, self.log_ls) 23 | return gram_mat + stabilizer 24 | 25 | def __repr__(self): 26 | return self.__class__.__name__ + ' (' + 'dim=' + str(self.ndim) + ')' 27 | 28 | 29 | if __name__ == '__main__': 30 | kernel = SquaredExponentialKernel(ndim=5) 31 | print(list(kernel.parameters())) 32 | -------------------------------------------------------------------------------- /HyperSphere/GP/kernels/modules/stationary.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sampyl as smp 3 | 4 | import torch 5 | from torch.nn.parameter import Parameter 6 | from HyperSphere.GP.kernels.modules.kernel import Kernel, log_lower_bnd 7 | 8 | 9 | class Stationary(Kernel): 10 | 11 | def __init__(self, ndim, ard=True, input_map=None, max_ls=None): 12 | super(Stationary, self).__init__(input_map) 13 | self.max_log_ls = np.log(2.0 * ndim ** 0.5) if max_ls is None else np.log(max_ls) 14 | self.ndim = ndim 15 | self.ard = ard 16 | if ard: 17 | self.log_ls = Parameter(torch.FloatTensor(ndim)) 18 | else: 19 | self.log_ls = Parameter(torch.FloatTensor(1)) 20 | 21 | def reset_parameters(self): 22 | super(Stationary, self).reset_parameters() 23 | self.log_ls.data.uniform_().mul_(np.exp(self.max_log_ls)).log_() 24 | 25 | def init_parameters(self, amp): 26 | super(Stationary, self).init_parameters(amp) 27 | self.log_ls.data.fill_(self.max_log_ls - np.log(2.0)) 28 | 29 | def out_of_bounds(self, vec=None): 30 | if vec is None: 31 | if not super(Stationary, self).out_of_bounds(): 32 | return (self.log_ls.data > self.max_log_ls).any() or (self.log_ls.data < log_lower_bnd).any() 33 | else: 34 | n_param_super = super(Stationary, self).n_params() 35 | if not super(Stationary, self).out_of_bounds(vec[:n_param_super]): 36 | return (vec[n_param_super:] > self.max_log_ls).any() or (vec[n_param_super:] < log_lower_bnd).any() 37 | return True 38 | 39 | def n_params(self): 40 | cnt = super(Stationary, self).n_params() + (self.ndim if self.ard else 1) 41 | return cnt 42 | 43 | def param_to_vec(self): 44 | return torch.cat([super(Stationary, self).param_to_vec(), self.log_ls.data]) 45 | 46 | def vec_to_param(self, vec): 47 | n_param_super = super(Stationary, self).n_params() 48 | super(Stationary, self).vec_to_param(vec[:n_param_super]) 49 | self.log_ls.data = vec[n_param_super:] 50 | 51 | def prior(self, vec): 52 | n_param_super = super(Stationary, self).n_params() 53 | return super(Stationary, self).prior(vec[:n_param_super]) + smp.uniform(np.exp(vec[n_param_super:]), lower=np.exp(log_lower_bnd), upper=np.exp(self.max_log_ls)) 54 | 55 | def forward_on_identical(self): 56 | return torch.exp(self.log_amp) * (1 + 1e-6) 57 | 58 | def __repr__(self): 59 | return self.__class__.__name__ + ' (dim=' + str(self.ndim) + ', ARD=' + ('TRUE' if self.ard else 'FALSE') + ')' 60 | -------------------------------------------------------------------------------- /HyperSphere/GP/likelihoods/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/GP/likelihoods/__init__.py -------------------------------------------------------------------------------- /HyperSphere/GP/likelihoods/functions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/GP/likelihoods/functions/__init__.py -------------------------------------------------------------------------------- /HyperSphere/GP/likelihoods/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/GP/likelihoods/modules/__init__.py -------------------------------------------------------------------------------- /HyperSphere/GP/likelihoods/modules/gaussian.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import torch 4 | from torch.nn.parameter import Parameter 5 | from HyperSphere.GP.likelihoods.modules.likelihood import Likelihood, log_lower_bnd, log_upper_bnd 6 | 7 | 8 | class GaussianLikelihood(Likelihood): 9 | 10 | def __init__(self): 11 | super(GaussianLikelihood, self).__init__() 12 | self.log_noise_var = Parameter(torch.FloatTensor(1)) 13 | self.noise_scale = 0.1 14 | 15 | def reset_parameters(self): 16 | self.log_noise_var.data.normal_(std=np.abs(np.random.standard_cauchy()) * self.noise_scale).pow_(2).log_() 17 | 18 | def out_of_bounds(self, vec=None): 19 | if vec is None: 20 | return (self.log_noise_var.data < log_lower_bnd).any() or (self.log_noise_var.data > 16 + np.log(self.noise_scale/0.1)).any() 21 | else: 22 | return (vec < log_lower_bnd).any() or (vec > 16 + np.log(self.noise_scale / 0.1)).any() 23 | 24 | def n_params(self): 25 | return 1 26 | 27 | def param_to_vec(self): 28 | return self.log_noise_var.data.clone() 29 | 30 | def vec_to_param(self, vec): 31 | self.log_noise_var.data = vec 32 | 33 | def prior(self, vec): 34 | if (vec < log_lower_bnd).any() or (vec > 16 + np.log(self.noise_scale/0.1)).any(): 35 | return -np.inf 36 | return np.log(np.log(1 + 2 * (self.noise_scale / np.exp(vec)) ** 2)).sum() 37 | 38 | def forward(self, input): 39 | return torch.exp(self.log_noise_var).repeat(input.size(0), 1) 40 | 41 | def __repr__(self): 42 | return self.__class__.__name__ 43 | 44 | 45 | if __name__ == '__main__': 46 | likelihood = GaussianLikelihood() 47 | print(list(likelihood.parameters())) 48 | -------------------------------------------------------------------------------- /HyperSphere/GP/likelihoods/modules/likelihood.py: -------------------------------------------------------------------------------- 1 | from HyperSphere.GP.modules.gp_modules import GPModule, log_lower_bnd, log_upper_bnd 2 | 3 | 4 | class Likelihood(GPModule): 5 | 6 | def __init__(self): 7 | super(Likelihood, self).__init__() 8 | -------------------------------------------------------------------------------- /HyperSphere/GP/means/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/GP/means/__init__.py -------------------------------------------------------------------------------- /HyperSphere/GP/means/functions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/GP/means/functions/__init__.py -------------------------------------------------------------------------------- /HyperSphere/GP/means/functions/constant.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Function, Variable, gradcheck 3 | 4 | 5 | class ConstantMean(Function): 6 | 7 | @staticmethod 8 | def forward(ctx, input, const_mean): 9 | ctx.save_for_backward(input, const_mean) 10 | 11 | return const_mean.repeat(input.size(0), 1) 12 | 13 | @staticmethod 14 | def backward(ctx, grad_output): 15 | """ 16 | :param ctx: 17 | :param grad_output: grad_output is assumed to be d[Scalar]/dK and size() is n_input 18 | :return: 19 | """ 20 | input, const_mean = ctx.saved_variables 21 | grad_input = grad_const_mean = None 22 | 23 | if ctx.needs_input_grad[0]: 24 | grad_input = input.clone() 25 | grad_input.data.zero_() 26 | if ctx.needs_input_grad[1]: 27 | grad_const_mean = torch.sum(grad_output) 28 | 29 | return grad_input, grad_const_mean 30 | 31 | 32 | if __name__ == '__main__': 33 | n1 = 3 34 | ndim = 5 35 | input_grad = False 36 | param_grad = not input_grad 37 | input = Variable(torch.randn(n1, ndim), requires_grad=input_grad) 38 | const_mean = Variable(torch.randn(1) + 1.5, requires_grad=param_grad) 39 | eps = 1e-4 40 | # gradcheck doesn't have to pass all the time. 41 | test = gradcheck(ConstantMean.apply, (input, const_mean), eps=eps) 42 | print(test) 43 | # eval0 = torch.sum(ConstantMean.apply(input, const_mean)) 44 | # const_mean_perturb = Variable(const_mean.data + eps) 45 | # eval1 = torch.sum(ConstantMean.apply(input, const_mean_perturb)) 46 | # print((eval1-eval0)/eps) 47 | # eval0.backward(gradient=torch.ones(n1)) 48 | # print(const_mean.grad.data) 49 | -------------------------------------------------------------------------------- /HyperSphere/GP/means/functions/quadratic.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Function, Variable, gradcheck 3 | 4 | 5 | class QuadraticMean(Function): 6 | 7 | @staticmethod 8 | def forward(ctx, input, a, b): 9 | ctx.save_for_backward(input, a, c) 10 | 11 | return a * input[:, :1] ** 2 + c 12 | 13 | @staticmethod 14 | def backward(ctx, grad_output): 15 | """ 16 | :param ctx: 17 | :param grad_output: grad_output is assumed to be d[Scalar]/dK and size() is n_input 18 | :return: 19 | """ 20 | input, a, c = ctx.saved_variables 21 | grad_input = grad_a = grad_c = None 22 | 23 | if ctx.needs_input_grad[0]: 24 | grad_input = input.clone() 25 | grad_input.data.zero_() 26 | grad_input[:, :1] = 2 * a * input[:, :1] * grad_output 27 | if ctx.needs_input_grad[1]: 28 | grad_a = torch.sum(grad_output * input[:, :1] ** 2) 29 | if ctx.needs_input_grad[2]: 30 | grad_c = torch.sum(grad_output) 31 | 32 | return grad_input, grad_a, grad_c 33 | 34 | 35 | if __name__ == '__main__': 36 | n1 = 3 37 | ndim = 5 38 | input_grad = False 39 | param_grad = not input_grad 40 | input = Variable(torch.randn(n1, ndim), requires_grad=input_grad) 41 | a = Variable(torch.abs(torch.randn(1)), requires_grad=param_grad) 42 | b = Variable(torch.randn(1) + 1.5, requires_grad=param_grad) 43 | eps = 1e-4 44 | # gradcheck doesn't have to pass all the time. 45 | test = gradcheck(QuadraticMean.apply, (input, a, b), eps=eps, atol=1e-3, rtol=1e-2) 46 | print(test) 47 | # eval0 = torch.sum(ConstantMean.apply(input, const_mean)) 48 | # const_mean_perturb = Variable(const_mean.data + eps) 49 | # eval1 = torch.sum(ConstantMean.apply(input, const_mean_perturb)) 50 | # print((eval1-eval0)/eps) 51 | # eval0.backward(gradient=torch.ones(n1)) 52 | # print(const_mean.grad.data) 53 | -------------------------------------------------------------------------------- /HyperSphere/GP/means/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/GP/means/modules/__init__.py -------------------------------------------------------------------------------- /HyperSphere/GP/means/modules/constant.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.nn.parameter import Parameter 3 | from HyperSphere.GP.means.modules.mean import Mean 4 | from HyperSphere.GP.means.functions import constant 5 | 6 | 7 | class ConstantMean(Mean): 8 | 9 | def __init__(self): 10 | super(ConstantMean, self).__init__() 11 | self.const_mean = Parameter(torch.FloatTensor(1)) 12 | 13 | def reset_parameters(self): 14 | self.const_mean.data.normal_(std=10.0) # approximation to uniform 15 | 16 | def out_of_bounds(self, vec=None): 17 | if vec is None: 18 | return False 19 | else: 20 | return False 21 | 22 | def n_params(self): 23 | return 1 24 | 25 | def param_to_vec(self): 26 | return self.const_mean.data.clone() 27 | 28 | def vec_to_param(self, vec): 29 | self.const_mean.data = vec 30 | 31 | def prior(self, vec): 32 | return 0 33 | 34 | def forward(self, input): 35 | return constant.ConstantMean.apply(input, self.const_mean) 36 | 37 | def __repr__(self): 38 | return self.__class__.__name__ 39 | 40 | 41 | if __name__ == '__main__': 42 | likelihood = ConstantMean() 43 | print(list(likelihood.parameters())) -------------------------------------------------------------------------------- /HyperSphere/GP/means/modules/mean.py: -------------------------------------------------------------------------------- 1 | from HyperSphere.GP.modules.gp_modules import GPModule 2 | 3 | 4 | class Mean(GPModule): 5 | 6 | def __init__(self): 7 | super(Mean, self).__init__() 8 | -------------------------------------------------------------------------------- /HyperSphere/GP/means/modules/quadratic.py: -------------------------------------------------------------------------------- 1 | import sampyl as smp 2 | 3 | import torch 4 | from torch.nn.parameter import Parameter 5 | from HyperSphere.GP.means.modules.mean import Mean 6 | from HyperSphere.GP.means.functions import quadratic 7 | 8 | 9 | class QuadraticMean(Mean): 10 | 11 | def __init__(self): 12 | super(QuadraticMean, self).__init__() 13 | self.log_a = Parameter(torch.FloatTensor(1)) 14 | self.c = Parameter(torch.FloatTensor(1)) 15 | 16 | def reset_parameters(self): 17 | self.log_a.data.normal_() # a => roughly to 0 18 | self.c.data.normal_(std=10.0) # approximation to uniform 19 | 20 | def out_of_bounds(self, vec=None): 21 | if vec is None: 22 | return False 23 | else: 24 | return False 25 | 26 | def n_params(self): 27 | return 2 28 | 29 | def param_to_vec(self): 30 | return torch.cat([self.log_a.data, self.c.data]) 31 | 32 | def vec_to_param(self, vec): 33 | self.log_a.data = vec[0] 34 | self.c.data = vec[1] 35 | 36 | def prior(self, vec): 37 | return smp.normal(torch.exp(self.log_a).data.squeeze()[0]) 38 | 39 | def forward(self, input): 40 | return quadratic.QuadraticMean.apply(input, torch.exp(self.log_a), self.c) 41 | 42 | def __repr__(self): 43 | return self.__class__.__name__ 44 | 45 | 46 | if __name__ == '__main__': 47 | likelihood = ConstantMean() 48 | print(list(likelihood.parameters())) -------------------------------------------------------------------------------- /HyperSphere/GP/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/GP/models/__init__.py -------------------------------------------------------------------------------- /HyperSphere/GP/models/gp.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from HyperSphere.GP.modules.gp_modules import GPModule 3 | 4 | 5 | class GP(GPModule): 6 | def __init__(self, **kwargs): 7 | super(GP, self).__init__() 8 | 9 | def reset_parameters(self): 10 | for m in self.children(): 11 | if hasattr(m, 'reset_parameters'): 12 | m.reset_parameters() 13 | 14 | def log_kernel_amp(self): 15 | return self.kernel.log_kernel_amp() 16 | 17 | def out_of_bounds(self, vec=None): 18 | if vec is None: 19 | for m in self.children(): 20 | if m.out_of_bounds(): 21 | return True 22 | return False 23 | else: 24 | ind = 0 25 | for m in self.children(): 26 | jump = m.n_params() 27 | if m.out_of_bounds(vec[ind:ind + jump]): 28 | return True 29 | ind += jump 30 | return False 31 | 32 | def n_params(self): 33 | cnt = 0 34 | for param in self.parameters(): 35 | cnt += param.numel() 36 | return cnt 37 | 38 | def param_to_vec(self): 39 | flat_param_list = [] 40 | for m in self.children(): 41 | flat_param_list.append(m.param_to_vec()) 42 | return torch.cat(flat_param_list) 43 | 44 | def vec_to_param(self, vec): 45 | ind = 0 46 | for m in self.children(): 47 | jump = m.n_params() 48 | m.vec_to_param(vec[ind:ind+jump]) 49 | ind += jump 50 | 51 | def elastic_vec_to_param(self, vec, func): 52 | ind = 0 53 | for name, m in self.named_children(): 54 | jump = m.n_params() 55 | if name == 'kernel': 56 | m.elastic_vec_to_param(vec[ind:ind + jump], func) 57 | else: 58 | m.vec_to_param(vec[ind:ind + jump]) 59 | ind += jump 60 | 61 | def prior(self, vec): 62 | prior_lik = 0 63 | ind = 0 64 | for m in self.children(): 65 | jump = m.n_params() 66 | prior_lik += m.prior(vec[ind:ind + jump]) 67 | ind += jump 68 | return prior_lik 69 | -------------------------------------------------------------------------------- /HyperSphere/GP/models/gp_regression.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Variable 3 | 4 | from HyperSphere.GP.models.gp import GP 5 | from HyperSphere.GP.likelihoods.modules.gaussian import GaussianLikelihood 6 | from HyperSphere.GP.means.modules.constant import ConstantMean 7 | 8 | 9 | class GPRegression(GP): 10 | 11 | def __init__(self, kernel, mean=ConstantMean()): 12 | super(GPRegression, self).__init__() 13 | self.kernel = kernel 14 | self.mean = mean 15 | self.likelihood = GaussianLikelihood() 16 | 17 | 18 | if __name__ == '__main__': 19 | from HyperSphere.GP.kernels.modules.squared_exponential import SquaredExponentialKernel 20 | GP = GPRegression(kernel=SquaredExponentialKernel(5)) 21 | print(list(GP.named_parameters())) 22 | -------------------------------------------------------------------------------- /HyperSphere/GP/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/GP/modules/__init__.py -------------------------------------------------------------------------------- /HyperSphere/GP/modules/gp_modules.py: -------------------------------------------------------------------------------- 1 | from torch.nn.modules.module import Module 2 | 3 | log_lower_bnd = -12.0 4 | log_upper_bnd = 20.0 5 | 6 | 7 | class GPModule(Module): 8 | 9 | def __init__(self): 10 | super(GPModule, self).__init__() 11 | 12 | def reset_parameters(self): 13 | raise NotImplementedError 14 | 15 | def init_parameters(self): 16 | raise NotImplementedError 17 | 18 | def out_of_bounds(self, vec=None): 19 | raise NotImplementedError 20 | 21 | def n_params(self): 22 | raise NotImplementedError 23 | 24 | def param_to_vec(self): 25 | raise NotImplementedError 26 | 27 | def vec_to_param(self, vec): 28 | raise NotImplementedError 29 | 30 | def prior(self, vec): 31 | raise NotImplementedError 32 | 33 | def __repr__(self): 34 | return self.__class__.__name__ 35 | -------------------------------------------------------------------------------- /HyperSphere/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/__init__.py -------------------------------------------------------------------------------- /HyperSphere/blackboxfunction_optimizer.py: -------------------------------------------------------------------------------- 1 | 2 | import torch 3 | import torch.multiprocessing as multiprocessing 4 | 5 | 6 | from HyperSphere.GP.models.gp_regression import GPRegression 7 | from HyperSphere.GP.kernels.modules.matern52 import Matern52 8 | from HyperSphere.GP.kernels.modules.radialization import RadializationKernel 9 | from HyperSphere.GP.inference.inference import Inference 10 | from HyperSphere.BO.shadow_inference.inference_sphere_origin import ShadowInference as origin_ShadowInference 11 | from HyperSphere.BO.acquisition.acquisition_functions import expected_improvement 12 | from HyperSphere.BO.acquisition.acquisition_maximization import suggest, optimization_candidates, optimization_init_points, deepcopy_inference 13 | from HyperSphere.feature_map.modules.kumaraswamy import Kumaraswamy 14 | from HyperSphere.feature_map.functionals import sphere_bound 15 | 16 | 17 | class BlackBoxFunctionOptimization(): 18 | def __init__(self, *args): 19 | pass 20 | 21 | def next_suggestion(self, input_configuration, output_evaluation): 22 | raise NotImplementedError('This is a base method, not implemented') 23 | 24 | 25 | class BayesianOptimization(BlackBoxFunctionOptimization): 26 | def __init__(self, *args): 27 | super(BayesianOptimization, self).__init__(*args) 28 | self.surrogate_model = None 29 | self.surrogate_inference = None 30 | self.acquisition_function = None 31 | 32 | def restore_surrogate_model(self, model_path): 33 | raise NotImplementedError('This is not implemented in General BayesianOptimization Class') 34 | 35 | 36 | class SpearmintTypeBayesianOptimization(BayesianOptimization): 37 | def __init__(self, original2cube=None, cube2original=None): 38 | """ 39 | Sometimes original search space is not hypercube [-1, 1]^D 40 | You can specify this and its inverse transformation 41 | :param original2cube: transformation from original search space to hypercube [-1, 1]^D 42 | :param cube2original: transformation from hypercube [-1, 1]^D to original search space 43 | """ 44 | super(SpearmintTypeBayesianOptimization, self).__init__() 45 | self.original2cube = lambda x: x if original2cube is None else original2cube 46 | self.cube2original = lambda x: x if cube2original is None else cube2original 47 | n_acq_optim_cand = 20 48 | self.pool = multiprocessing.Pool(n_acq_optim_cand) 49 | 50 | def save_model(self, model_path): 51 | """ 52 | In Spearmint, surrogate model is trained using MCMC sampling, for stability it keeps track of samples 53 | this is the information that is requires to be saved. 54 | :param model_path: 55 | """ 56 | torch.save(self.surrogate_model, model_path) 57 | 58 | def restore_model(self, model_path): 59 | """ 60 | In Spearmint, surrogate model is trained using MCMC sampling, for stability it keeps track of samples 61 | this is the information that is requires to be retrieved from the file 62 | :param model_path: 63 | """ 64 | self.surrogate_model = torch.load(model_path) 65 | 66 | def next_suggestion(self, input_configuration, output_evaluation): 67 | """ 68 | According to the information you need, you can change return value 69 | :param input_configuration: n by p tensor 70 | :param output_evaluation: n by 1 tensor 71 | :return: 72 | """ 73 | inference = self.surrogate_inference((self.original2cube(input_configuration), output_evaluation)) 74 | reference, ref_ind = torch.min(output_evaluation, 0) 75 | reference = reference.data.squeeze()[0] 76 | gp_hyper_params = inference.sampling(n_sample=10, n_burnin=0, n_thin=1) 77 | inferences = deepcopy_inference(inference, gp_hyper_params) 78 | x0_cand = optimization_candidates(input_configuration, output_evaluation, -1, 1) 79 | x0, sample_info = optimization_init_points(x0_cand, reference=reference, inferences=inferences) 80 | next_x_point, pred_mean, pred_std, pred_var, pred_stdmax, pred_varmax = suggest(x0=x0, reference=reference, 81 | inferences=inferences, 82 | acquisition_function=self.acquisition_function, 83 | bounds=sphere_bound(self.radius), pool=self.pool) 84 | return self.cube2original(next_x_point) 85 | 86 | 87 | class NonArdSpearmintBayesianOptimization(BayesianOptimization): 88 | def __init__(self, ndim, original2cube=None, cube2original=None): 89 | super(NonArdSpearmintBayesianOptimization, self).__init__(original2cube, cube2original) 90 | self.surrogate_model = GPRegression(kernel=Matern52(ndim=ndim, ard=False)) 91 | self.surrogate_inference = Inference 92 | self.acquisition_function = expected_improvement 93 | 94 | 95 | class ArdSpearmintBayesianOptimization(BayesianOptimization): 96 | def __init__(self, ndim, original2cube=None, cube2original=None): 97 | super(NonArdSpearmintBayesianOptimization, self).__init__(original2cube, cube2original) 98 | self.surrogate_model = GPRegression(kernel=Matern52(ndim=ndim, ard=True)) 99 | self.surrogate_inference = Inference 100 | self.acquisition_function = expected_improvement 101 | 102 | 103 | class CylindricalKernelBayesianOptimization(BayesianOptimization): 104 | def __init__(self, ndim, original2cube=None, cube2original=None): 105 | super(CylindricalKernelBayesianOptimization, self).__init__(original2cube, cube2original) 106 | self.radius = ndim ** 0.5 107 | radius_input_map = Kumaraswamy(ndim=1, max_input=self.radius) 108 | self.surrogate_model = GPRegression(kernel=RadializationKernel(max_power=3, search_radius=self.radius, radius_input_map=radius_input_map)) 109 | self.surrogate_inference = origin_ShadowInference 110 | self.acquisition_function = expected_improvement 111 | -------------------------------------------------------------------------------- /HyperSphere/coordinate/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/coordinate/__init__.py -------------------------------------------------------------------------------- /HyperSphere/coordinate/transformation.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | from torch.autograd import Variable 5 | 6 | 7 | def rect2grass_radius(x, threshold_radius=1.0): 8 | radius = torch.sqrt(torch.sum(x ** 2, dim=1, keepdim=True)) 9 | large_radius_mask = radius.squeeze() > threshold_radius 10 | n_large_radius = torch.sum(large_radius_mask.data) 11 | direction = x.clone() 12 | if n_large_radius > 0: 13 | large_radius_ind = torch.sort(large_radius_mask.data, 0, descending=True)[1][:n_large_radius] 14 | direction[large_radius_ind] = direction[large_radius_ind] / radius[large_radius_ind] 15 | return torch.cat([(radius - threshold_radius).clamp(min=0), direction], 1) 16 | 17 | 18 | # def rect2grass_radius(x): 19 | # radius = torch.sqrt(torch.sum(x ** 2, dim=1, keepdim=True)) 20 | # nonzero_radius_mask = radius.squeeze() == 0 21 | # n_nonzero_radius = torch.sum(nonzero_radius_mask.data) 22 | # if n_nonzero_radius > 0: 23 | # nonzero_radius_ind = torch.sort(nonzero_radius_mask.data, 0, descending=True)[1][:n_nonzero_radius] 24 | # radius[nonzero_radius_ind] = 1.0 25 | # direction = x / radius 26 | # return torch.cat([radius, direction], 1) 27 | 28 | rect2grass_radius.dim_change = lambda x: x+ 1 29 | 30 | 31 | def grass_radius2rect(rd, threshold_radius): 32 | return rd[:, :1] * rd[:, 1:] 33 | 34 | 35 | def rect2grass_angle(x, search_radius): 36 | radius = torch.sqrt(torch.sum(x ** 2, dim=1, keepdim=True)) 37 | angle = torch.acos(1 - 2 * radius / search_radius) / math.pi 38 | nonzero_radius_mask = radius.squeeze() == 0 39 | n_nonzero_radius = torch.sum(nonzero_radius_mask.data) 40 | if n_nonzero_radius > 0: 41 | nonzero_radius_ind = torch.sort(nonzero_radius_mask.data, 0, descending=True)[1][:n_nonzero_radius] 42 | radius[nonzero_radius_ind] = 1.0 43 | direction = x / radius 44 | return torch.cat([angle, direction], 1) 45 | 46 | rect2grass_angle.dim_change = lambda x: x+ 1 47 | 48 | 49 | def grass_angle2rect(ad, search_radius): 50 | radius = 0.5 * (1 - torch.cos(ad[:, :1] * math.pi)) * search_radius 51 | return radius * ad[:, 1] 52 | 53 | 54 | def rect2spherical(x, rotation_mat=None): 55 | _, n_dim = x.size() 56 | if rotation_mat is None: 57 | if hasattr(x, 'data'): 58 | rotation_mat = Variable(torch.eye(n_dim)).type_as(x) 59 | else: 60 | rotation_mat = torch.eye(n_dim).type_as(x) 61 | reverse_ind = torch.arange(n_dim-1, -1, -1).type_as(x.data if hasattr(x, 'data') else x).long() 62 | x_rotated = x.mm(rotation_mat.t()) 63 | x_rotated_sq_accum = torch.sqrt(torch.cumsum((x_rotated**2)[:, reverse_ind], dim=1)[:, reverse_ind]) 64 | rphi = torch.cat((x_rotated_sq_accum[:, [0]], x_rotated[:, 0:n_dim-1]), dim=1) 65 | if n_dim > 2: 66 | rphi[:, 1:n_dim-1] = torch.acos(x_rotated[:, 0:n_dim-2]/x_rotated_sq_accum[:, 0:n_dim-2]) 67 | rphi[:, -1] = torch.acos(x_rotated[:, -2] / x_rotated_sq_accum[:, -2]) 68 | if hasattr(x, 'data'): 69 | rphi.data[:, -1][x_rotated.data[:, -1] < 0] = 2 * math.pi - rphi.data[:, -1][x_rotated.data[:, -1] < 0] 70 | zero_radius_mask = rphi.data[:, 0] == 0 71 | n_zero_radius = torch.sum(zero_radius_mask) 72 | if n_zero_radius > 0: 73 | _, zero_radius = torch.sort(zero_radius_mask, 0, descending=True) 74 | zero_radius = zero_radius[:n_zero_radius] 75 | rphi.data[zero_radius[0]] = 0.5 * math.pi 76 | if n_zero_radius > 1: 77 | rphi.data[zero_radius[1:]] = rphi.data.new(n_zero_radius - 1, n_dim).uniform_() * math.pi 78 | rphi.data[zero_radius, zero_radius.new(1).zero_()] = rphi.data.new(n_zero_radius).fill_(0) 79 | rphi.data[zero_radius, zero_radius.new(1).fill_(n_dim-1)] *= 2 80 | else: 81 | rphi[:, -1][x_rotated[:, -1] < 0] = 2 * math.pi - rphi[:, -1][x_rotated[:, -1] < 0] 82 | zero_radius_mask = rphi[:, 0] == 0 83 | n_zero_radius = torch.sum(zero_radius_mask) 84 | if n_zero_radius > 0: 85 | _, zero_radius = torch.sort(zero_radius_mask, 0, descending=True) 86 | zero_radius = zero_radius[:n_zero_radius] 87 | rphi[zero_radius[0]] = 0.5 * math.pi 88 | if n_zero_radius > 1: 89 | rphi[zero_radius[1:]] = rphi.new(n_zero_radius - 1, n_dim).uniform_() * math.pi 90 | rphi[zero_radius, zero_radius.new(1).zero_()] = rphi.new(n_zero_radius).fill_(0) 91 | rphi[zero_radius, zero_radius.new(1).fill_(n_dim - 1)] *= 2 92 | return rphi 93 | 94 | 95 | def spherical2rect(rphi, rotation_mat=None): 96 | """ 97 | 98 | :param rphi: r in [0, radius], phi0 in [0, pi], phi1 in [0, pi], ..., phi(n-1) in [0, 2pi] 99 | :param shuffle_ind: 100 | :return: 101 | """ 102 | _, n_dim = rphi.size() 103 | if rotation_mat is None: 104 | rotation_mat = torch.eye(n_dim).type_as(rphi.data if hasattr(rphi, 'data') else rphi) 105 | x = torch.cumprod(torch.cat((rphi[:, [0]], torch.sin(rphi[:, 1:n_dim])), dim=1), dim=1) 106 | x[:, 0:n_dim-1] = x[:, 0:n_dim-1] * torch.cos(rphi[:, 1:n_dim]) 107 | return x.mm(rotation_mat) 108 | 109 | 110 | def rphi2phi(rphi, radius): 111 | """ 112 | :param rphi: r in [0, radius], phi_i in [0, pi] for i in [1, d-2], phi_i in [0, 2pi] for i = d-1 113 | :param radius 114 | :return: point in cube [0, 1] ^ d 115 | """ 116 | phi0 = torch.acos(1 - 2 * rphi[:, :1] / radius) / math.pi 117 | if rphi.size(1) > 2: 118 | phi = torch.cat([phi0, rphi[:, 1:-1] / math.pi, rphi[:, -1:] / (2.0 * math.pi)], 1) 119 | else: 120 | phi = torch.cat([phi0, rphi[:, -1:] / (2.0 * math.pi)], 1) 121 | return phi 122 | 123 | 124 | def phi2rphi(phi, radius): 125 | """ 126 | :param phi: in cube [0, 1] ^ d 127 | :param radius: R, pi, ..., pi, 2pi 128 | :return: r in [0, R], phi_i in [0, pi] for i in [1, d-2], phi_i in [0, 2pi] for i = d-1 129 | """ 130 | r = 0.5 * (1 - torch.cos(phi[:, 0:1] * math.pi)) * radius 131 | if phi.size(1) > 2: 132 | rphi = torch.cat([r, phi[:, 1:-1] * math.pi, phi[:, -1:] * 2 * math.pi], 1) 133 | else: 134 | rphi = torch.cat([r, phi[:, -1:] * 2 * math.pi], 1) 135 | return rphi 136 | 137 | 138 | def check_rphi(rphi): 139 | assert torch.sum(rphi < 0) == 0 140 | assert torch.sum(rphi[:, 1:-1] > math.pi) == 0 141 | assert torch.sum(rphi[:, -1] > 2 * math.pi) == 0 142 | 143 | 144 | if __name__ == '__main__': 145 | n_dim = 5 146 | n_data = 5 147 | x = torch.randn(n_data, n_dim) 148 | _, shuffle_ind = torch.sort(torch.randn(n_dim), 0) 149 | # shuffle_ind = torch.arange(0, n_dim).long() 150 | rphi = rect2spherical(x, shuffle_ind) 151 | x_recovered = spherical2rect(rphi, shuffle_ind) 152 | print(x) 153 | print(x_recovered) 154 | print(torch.dist(x, x_recovered)) 155 | # phi = torch.FloatTensor(n_data, n_dim-1).uniform_(0, 2*math.pi) 156 | # r_phi = torch.cat((torch.ones(n_data, 1) * 1.0, phi), dim=1) 157 | # r_phi_domain = rect2spherical(spherical2rect(r_phi)) 158 | # in_domain = (r_phi[:, 1:-1] <= math.pi).type_as(r_phi) 159 | # in_domain_accum = torch.cumsum(in_domain, dim=1) 160 | # not_in_domain = (r_phi[:, 1:-1] > math.pi).type_as(r_phi) 161 | # not_in_domain_accum = torch.cumsum(not_in_domain, dim=1) 162 | # diff = (r_phi[:, 1:-1] - r_phi_domain[:, 1:-1]) / math.pi 163 | # sum = (r_phi[:, 1:-1] + r_phi_domain[:, 1:-1]) / math.pi 164 | # for i in range(n_data): 165 | # print(torch.stack((in_domain[i], in_domain_accum[i], not_in_domain_accum[i], sum[i], diff[i]), dim=0)) 166 | 167 | -------------------------------------------------------------------------------- /HyperSphere/dummy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/dummy/__init__.py -------------------------------------------------------------------------------- /HyperSphere/dummy/batch_run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python HyperSphere/BO/sphere_bayesian_optimization.py rosenbrock 5 100 4 | python HyperSphere/BO/cube_bayesian_optimization.py rosenbrock 5 100 5 | python HyperSphere/BO/sphere_bayesian_optimization.py rosenbrock 10 200 6 | python HyperSphere/BO/cube_bayesian_optimization.py rosenbrock 10 200 7 | 8 | python HyperSphere/BO/sphere_bayesian_optimization.py levy 5 100 9 | python HyperSphere/BO/cube_bayesian_optimization.py levy 5 100 10 | python HyperSphere/BO/sphere_bayesian_optimization.py levy 10 200 11 | python HyperSphere/BO/cube_bayesian_optimization.py levy 10 200 12 | -------------------------------------------------------------------------------- /HyperSphere/dummy/error_reproduce.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Variable, grad 3 | import torch.optim as optim 4 | import time 5 | 6 | 7 | def optimize(max_step, x0, b, A): 8 | x = Variable(x0.clone(), requires_grad=True) 9 | optimizer = optim.Adam([x], lr=0.01) 10 | for s in range(max_step): 11 | optimizer.zero_grad() 12 | loss = torch.mean((A.mm(x) - b) ** 2) 13 | x.grad = grad([loss], [x], retain_graph=True)[0] 14 | optimizer.step() 15 | optimum_loc = x.clone() 16 | optimum_value = torch.sum((A.mm(x) - b) ** 2).data.squeeze()[0] 17 | return optimum_loc, optimum_value 18 | 19 | 20 | def error_reproduce_pool(): 21 | ndim = 1000 22 | n_init = 20 23 | 24 | # Calling pool BEFORE any pytorch linear algebra operation does not cause any problem 25 | # pool = torch.multiprocessing.Pool(n_init) 26 | 27 | b = Variable(torch.randn(ndim, 1)) 28 | A = Variable(torch.randn(ndim, ndim)) 29 | A = A.mm(A.t()) # pytorch Linear Algebra operation 30 | x0 = torch.randn(ndim, n_init) 31 | 32 | # Calling pool AFTER any pytorch linear algebra operation causes hanging 33 | pool = torch.multiprocessing.Pool(n_init) 34 | 35 | result_list = [] 36 | for i in range(n_init): 37 | result_list.append(pool.apply_async(optimize, args=(100, x0[:, i:i+1], b, A))) 38 | while [not p.ready() for p in result_list].count(True) > 0: 39 | time.sleep(1) 40 | pool.close() 41 | return [r.get() for r in result_list] 42 | 43 | 44 | def error_reproduce_process(): 45 | n_data = 500 46 | ndim = 1000 47 | 48 | b = Variable(torch.randn(ndim, n_data)) 49 | A = Variable(torch.randn(ndim, ndim)) 50 | 51 | def gesv_wrapper(return_dict, i, *args): 52 | return_dict[i] = torch.gesv(*args)[0] 53 | 54 | return_dict = torch.multiprocessing.Manager().dict() 55 | 56 | process = torch.multiprocessing.Process(target=gesv_wrapper, args=(return_dict, 0, b, A)) 57 | process.start() 58 | process.join() 59 | return return_dict 60 | 61 | 62 | def no_error_without_blas(): 63 | n_data = 500 64 | ndim = 1000 65 | 66 | b = Variable(torch.randn(ndim, n_data)) 67 | A = Variable(torch.randn(ndim, ndim)) 68 | 69 | pool = torch.multiprocessing.Pool(1) 70 | res = pool.apply_async(torch.gesv, args=(b, A)) 71 | pool.close() 72 | pool.join() 73 | return res.get() 74 | 75 | 76 | def no_error_by_calling_pool_first(): 77 | n_data = 500 78 | ndim = 1000 79 | 80 | b = Variable(torch.randn(ndim, n_data)) 81 | A = Variable(torch.randn(ndim, ndim)) 82 | 83 | pool = torch.multiprocessing.Pool(1) 84 | 85 | res = pool.apply_async(torch.gesv, args=(b, A)) 86 | pool.close() 87 | pool.join() 88 | return res.get() 89 | 90 | 91 | def pool_reuse_check(): 92 | n_batch = 5 93 | pool = torch.multiprocessing.Pool(n_batch) 94 | x = range(20) 95 | 96 | for i in range(4): 97 | result_list = [] 98 | for j in range(n_batch): 99 | result_list.append(pool.apply_async(test_func, args=(x[n_batch * i + j],))) 100 | return_values = [res.get() for res in result_list] 101 | print(return_values) 102 | 103 | 104 | def test_func(x): 105 | return x ** 2 106 | 107 | 108 | if __name__ == '__main__': 109 | print(pool_reuse_check()) 110 | -------------------------------------------------------------------------------- /HyperSphere/dummy/experiment_info.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def how_many_evaluations(): 5 | exp_dir = os.path.join('/'.join(os.path.realpath(__file__).split('/')[:-3]), 'Experiments', 'Hypersphere') 6 | exp_list = os.listdir(exp_dir) 7 | for exp in sorted(exp_list): 8 | eval_logs = os.listdir(os.path.join(exp_dir, exp, 'log')) 9 | if len(eval_logs) > 0: 10 | print(exp, int(max(eval_logs).split('.')[0])) 11 | else: 12 | print(exp, 0) 13 | 14 | 15 | if __name__ == '__main__': 16 | how_many_evaluations() -------------------------------------------------------------------------------- /HyperSphere/dummy/model_info.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | 4 | 5 | def model_info(): 6 | exp_data_dir = '/home/coh1/Experiments/Hypersphere_ALL' 7 | dir_list = sorted(os.listdir(exp_data_dir)) 8 | for elm in dir_list: 9 | sub_dir = os.path.join(exp_data_dir, elm) 10 | if os.path.isdir(sub_dir) and 'sphere' in elm and 'cifar' in elm: 11 | try: 12 | kernel = torch.load(os.path.join(sub_dir, 'model.pt')).kernel 13 | print kernel.sphere_kernel.max_power, sub_dir 14 | except ImportError: 15 | print('Import Error %s' % (sub_dir)) 16 | except EOFError: 17 | print('EOF Error %s' % (sub_dir)) 18 | except IOError: 19 | print('IO Error %s' % (sub_dir)) 20 | 21 | if __name__ == '__main__': 22 | model_info() -------------------------------------------------------------------------------- /HyperSphere/dummy/paper_scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/dummy/paper_scripts/__init__.py -------------------------------------------------------------------------------- /HyperSphere/dummy/paper_scripts/accuracy_time.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | from HyperSphere.dummy.plotting.plot_color import algorithm_color 4 | 5 | from HyperSphere.dummy.plotting.get_data_from_file import get_data_sphere, get_data_HPOlib, get_data_elastic, \ 6 | get_data_additive, get_data_warping 7 | from run_time_data import run_time_hyper, run_time_spearmint, run_time_elastic, run_time_additive, run_time_warping, \ 8 | run_time_smac, run_time_tpe 9 | 10 | FUNC_NAME = 'rosenbrock' 11 | 12 | 13 | def accuarcy_vs_time(dims, suffix='_center-random', P_setting='_P=3'): 14 | sphere_dir = '/home/coh1/Experiments/Hypersphere_ALL' + suffix + P_setting + '/' 15 | cube_dir = '/home/coh1/Experiments/Cube_ALL' + suffix + '/' 16 | 17 | result = {} 18 | result['smac'] = run_time_smac() 19 | result['hyperopt'] = run_time_tpe() 20 | result['additive'] = run_time_additive() 21 | result['elastic'] = run_time_elastic() 22 | result['spearmint_warping'] = run_time_warping() 23 | result['spearmint'] = run_time_spearmint() 24 | result['cube'] = run_time_hyper('cube', cube_dir) 25 | result['sphereboth'] = run_time_hyper('sphereboth', sphere_dir) 26 | result['sphereorigin'] = run_time_hyper('sphereorigin', sphere_dir) 27 | result['spherewarpingboth'] = run_time_hyper('spherewarpingboth', sphere_dir) 28 | result['spherewarpingorigin'] = run_time_hyper('spherewarpingorigin', sphere_dir) 29 | 30 | for algo, all_func in result.iteritems(): 31 | for exp in all_func.keys(): 32 | if FUNC_NAME + '_' + str(dims) != exp: 33 | del all_func[exp] 34 | for algo, data in result.iteritems(): 35 | try: 36 | result[algo] = data[FUNC_NAME + '_' + str(dims)][0] 37 | except KeyError: 38 | print(algo, FUNC_NAME, dims) 39 | print(data.keys()) 40 | assert 1 == 0 41 | runtime_dict = result 42 | 43 | data_list = [] 44 | data_list += get_data_HPOlib('/home/coh1/Experiments/spearmint_ALL' + suffix + '/' + FUNC_NAME + '_' + str(dims)) 45 | data_list += get_data_HPOlib('/home/coh1/Experiments/smac_ALL' + suffix + '/' + FUNC_NAME + '_' + str(dims), optimizer_name='smac_2_10_00-dev') 46 | data_list += get_data_HPOlib('/home/coh1/Experiments/tpe_ALL' + suffix + '/' + FUNC_NAME + '_' + str(dims), optimizer_name='hyperopt_august2013_mod') 47 | data_list += get_data_warping('/home/coh1/Experiments/Warping_ALL' + suffix + '/', FUNC_NAME, dims) 48 | multiple_additive_date = get_data_additive('/home/coh1/Experiments/Additive_BO_mat_ALL' + suffix + '/', FUNC_NAME, dims) 49 | additive_optimum_dict = {} 50 | for elm in multiple_additive_date: 51 | try: 52 | additive_optimum_dict[elm['algorithm']] += elm['optimum'][-1] 53 | except KeyError: 54 | additive_optimum_dict[elm['algorithm']] = elm['optimum'][-1] 55 | additive_data = [elm for elm in multiple_additive_date if elm['algorithm']==min(additive_optimum_dict, key=additive_optimum_dict.get)] 56 | for elm in additive_data: 57 | elm['algorithm'] = 'additive' 58 | data_list += additive_data 59 | data_list += get_data_elastic('/home/coh1/Experiments/elastic_BO_mat_center-random/', FUNC_NAME, dims) 60 | data_list += get_data_sphere(cube_dir, ['cube', 'cubeard'], FUNC_NAME, dims) 61 | data_list += get_data_sphere(sphere_dir, ['sphereboth', 'sphereorigin', 'spherewarpingboth', 'spherewarpingorigin'], FUNC_NAME, dims) 62 | 63 | 64 | optimum_dict = {} 65 | for elm in data_list: 66 | try: 67 | optimum_dict[elm['algorithm']] += [elm['optimum'][-1]] 68 | except KeyError: 69 | optimum_dict[elm['algorithm']] = [elm['optimum'][-1]] 70 | 71 | plt.rc('pdf', fonttype=42) 72 | plt.rc('ps', fonttype=42) 73 | 74 | plt.figure() 75 | for key in runtime_dict.keys(): 76 | if key == 'sphereboth': 77 | continue 78 | label = 'BOCK B' 79 | elif key == 'sphereorigin': 80 | continue 81 | label = 'BOCK-W' 82 | elif key == 'spherewarpingboth': 83 | continue 84 | label = 'BOCK+B' 85 | elif key == 'spherewarpingorigin': 86 | label = 'BOCK' 87 | elif key == 'cube': 88 | label = 'Matern' 89 | elif key == 'cubeard': 90 | continue 91 | label = 'MaternARD' 92 | elif key == 'spearmint': 93 | label = 'Spearmint' 94 | elif key == 'spearmint_warping': 95 | label = 'Spearmint+' 96 | elif key == 'additive': 97 | label = 'AdditiveBO' 98 | elif key == 'elastic': 99 | continue 100 | label = 'ElasticGP' 101 | elif key == 'hyperopt': 102 | label = 'TPE' 103 | elif key == 'smac': 104 | label = 'SMAC' 105 | 106 | x_mean = np.mean(runtime_dict[key]) / 3600.0 107 | x_std = np.std(runtime_dict[key]) / 3600.0 108 | y_mean = np.mean(optimum_dict[key]) 109 | y_std = np.std(optimum_dict[key]) 110 | mew = 4 111 | if label=='SMAC': 112 | mew = 3.0 113 | elif label=='TPE': 114 | mew = 6.0 115 | 116 | plt.plot([x_mean], [y_mean], '+' if label!='SMAC' else 'x', color=algorithm_color(key) if label!='SMAC' else 'k', label=label, markersize=16, mew=mew) 117 | # plt.axvline(x=x_mean, ymin=y_mean - y_std, ymax=y_mean + y_std, color=algorithm_color(key)) 118 | # plt.axhline(y=y_mean, xmin=x_mean - x_std, xmax=x_mean + x_std, color=algorithm_color(key)) 119 | print('%20s %10.4f %10.4f %10.4fvs %10.4f %10.4f %10.4f' % (key, x_mean + x_std, x_mean, x_mean + x_std, y_mean - y_std, y_mean, y_mean + y_std)) 120 | plt.legend(fontsize=16, loc=1 if dims == 20 else 2) 121 | plt.ylabel('Discovered minimum', fontsize=16) 122 | plt.xlabel('Run time(hours)', fontsize=16) 123 | plt.yticks(fontsize=16) 124 | plt.xticks(fontsize=16) 125 | plt.tight_layout(rect=[0.0, 0.0, 1.0, 1.0]) 126 | plt.show() 127 | 128 | 129 | if __name__ == '__main__': 130 | accuarcy_vs_time(dims=20) -------------------------------------------------------------------------------- /HyperSphere/dummy/paper_scripts/mnist_weight_train_valid_test.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | from datetime import datetime 3 | 4 | import numpy as np 5 | import torch 6 | 7 | from HyperSphere.dummy.plotting.get_data_from_file import get_data_HPOlib, get_data_sphere 8 | from HyperSphere.test_functions.mnist_weight import mnist_weight 9 | 10 | 11 | def get_data_mnist_weight(ndim): 12 | func_name = 'mnist_weight' 13 | data_list = [] 14 | spearmint_dir_name = '/home/coh1/Experiments/spearmint_mnist_weight_train_valid_test/' + func_name + '_' + str(ndim) 15 | data_list += get_data_HPOlib(spearmint_dir_name, 'spearmint_april2013_mod') 16 | sphere_dir_name = '/home/coh1/Experiments/Hypersphere_mnist_weight_train_valid_test/' 17 | data_list += get_data_sphere(sphere_dir_name, ['spherewarpingorigin'], func_name, ndim) 18 | cube_dir_name = '/home/coh1/Experiments/Cube_mnist_weight_train_valid_test/' 19 | data_list += get_data_sphere(cube_dir_name, ['cube'], func_name, ndim) 20 | 21 | for elm in data_list: 22 | elm['optimum_ind'] = np.argmin(elm['y']) 23 | 24 | return data_list 25 | 26 | 27 | def mnist_weight_SGD_comparison(data_list): 28 | n_repeat = 5 29 | trainvalid_test_eval_list = [] 30 | 31 | optimum_weight_list = [] 32 | for elm in data_list: 33 | optimum_ind = elm['optimum_ind'] 34 | optimum = dict() 35 | optimum['x'] = torch.from_numpy(elm['x'][optimum_ind].astype(np.float32)) 36 | optimum['y'] = elm['y'][optimum_ind] 37 | optimum['algorithm'] = elm['algorithm'] 38 | optimum_weight_list.append(optimum) 39 | 40 | for elm in optimum_weight_list: 41 | trainvalid_test_eval_elm = dict() 42 | trainvalid_test_eval_elm['algorithm'] = elm['algorithm'] 43 | evaluation_result = [] 44 | print('Optimum from %s algorithm is trained on train+validation and evaluated on test' % elm['algorithm']) 45 | print(' Due to the randomness in optimizer(SGD:ADAM) multiple training and testing is done.') 46 | for r in range(n_repeat): 47 | print(' %s %d/%d has been started. Validation score : %f' % (datetime.now().strftime('%H:%M:%S:%f'), r + 1, n_repeat, elm['y'])) 48 | evaluation_result.append(mnist_weight(elm['x'], use_BO=True, use_validation=False).squeeze()[0]) 49 | print(evaluation_result) 50 | trainvalid_test_eval_elm['trainvalid_test_eval'] = evaluation_result[:] 51 | trainvalid_test_eval_list.append(trainvalid_test_eval_elm) 52 | 53 | trainvalid_test_eval_elm = dict() 54 | trainvalid_test_eval_elm['algorithm'] = 'SGD' 55 | evaluation_result = [] 56 | print('Optimum from %s algorithm is trained on train+validation and evaluated on test' % 'SGD') 57 | print(' Due to the randomness in optimizer(SGD:ADAM) multiple training and testing is done.') 58 | for r in range(n_repeat): 59 | print(' %s %d/%d has been started.' % (datetime.now().strftime('%H:%M:%S:%f'), r + 1, n_repeat)) 60 | # random elm['x'] is fine as long as size is correct when use_BO=False 61 | evaluation, _ = mnist_weight(elm['x'], use_BO=False, use_validation=False) 62 | evaluation_result.append(evaluation.squeeze()[0]) 63 | print(evaluation_result) 64 | trainvalid_test_eval_elm['trainvalid_test_eval'] = evaluation_result[:] 65 | trainvalid_test_eval_list.append(trainvalid_test_eval_elm) 66 | 67 | for elm in trainvalid_test_eval_list: 68 | print(elm['algorithm']) 69 | eval_data = elm['trainvalid_test_eval'] 70 | print(' ' + ('Average:%f(%f)' % (np.mean(eval_data), np.std(eval_data))) + (' '.join(['%+12.6f' % e for e in eval_data]))) 71 | 72 | return trainvalid_test_eval_list 73 | 74 | 75 | def mnist_weight_SGD_only(ndim): 76 | n_repeat = 5 77 | trainvalid_test_eval_list = [] 78 | 79 | trainvalid_test_eval_elm = dict() 80 | trainvalid_test_eval_elm['algorithm'] = 'SGD' 81 | evaluation_result = [] 82 | radius_result = [] 83 | print('Optimum from %s algorithm is trained on train+validation and evaluated on test' % 'SGD') 84 | print(' Due to the randomness in optimizer(SGD:ADAM) multiple training and testing is done.') 85 | for r in range(n_repeat): 86 | print(' %s %d/%d has been started.' % (datetime.now().strftime('%H:%M:%S:%f'), r + 1, n_repeat)) 87 | dummy = torch.FloatTensor(ndim) 88 | evaluation, radius = mnist_weight(dummy, use_BO=False, use_validation=False) 89 | evaluation_result.append(evaluation.squeeze()[0]) 90 | radius_result.append(radius) 91 | print(evaluation_result) 92 | print(radius_result) 93 | trainvalid_test_eval_elm['trainvalid_test_eval'] = evaluation_result[:] 94 | trainvalid_test_eval_elm['radius'] = radius_result[:] 95 | trainvalid_test_eval_list.append(trainvalid_test_eval_elm) 96 | 97 | for elm in trainvalid_test_eval_list: 98 | print(elm['algorithm']) 99 | eval_data = elm['trainvalid_test_eval'] 100 | print(' ' + ('Average:%f(%f)' % (np.mean(eval_data), np.std(eval_data))) + (' '.join(['%+12.6f' % e for e in eval_data]))) 101 | 102 | return trainvalid_test_eval_list 103 | 104 | 105 | def print_result(filename): 106 | data_file = open(filename) 107 | data = pickle.load(data_file) 108 | data_file.close() 109 | dim = int(filename[-10:-7]) 110 | 111 | print('----------%d----------%d----------%d----------' % (dim, dim, dim)) 112 | for elm in data: 113 | algo_str = '%-19s ' % elm['algorithm'] 114 | eval_data_list = elm['trainvalid_test_eval'] 115 | mean_eval = np.mean(eval_data_list) 116 | std_eval = np.std(eval_data_list) 117 | eval_data_str = algo_str + ('%8.6f(%8.6f)' % (mean_eval, std_eval)) 118 | eval_data_str += ' '.join(['%8.6f' % elm for elm in eval_data_list]) 119 | print(eval_data_str) 120 | 121 | 122 | if __name__ == '__main__': 123 | # for ndim in [200, 500]: 124 | # mnist_weight_data_list = get_data_mnist_weight(ndim=ndim) 125 | # result = mnist_weight_SGD_comparison(mnist_weight_data_list) 126 | # save_file_name = '/home/coh1/OCY_docs/publication/icml2017/trainvalid_test_eval_' + str(ndim) + 'dim.pkl' 127 | # save_file = open(save_file_name, 'w') 128 | # pickle.dump(result, save_file) 129 | # save_file.close() 130 | # print(result) 131 | # dim = 500 132 | # filename = '/home/coh1/OCY_docs/publication/ICML2018/trainvalid_test_eval_' + str(dim) + 'dim.pkl' 133 | # print_result(filename) 134 | 135 | # mnist_weight_data_list = get_data_mnist_weight(ndim=dim) 136 | # for elm in mnist_weight_data_list: 137 | # optimum_x = elm['x'][elm['optimum_ind']] 138 | # print(elm['algorithm'], np.sum(optimum_x ** 2) ** 0.5) 139 | # optimum_y = elm['y'][elm['optimum_ind']] 140 | # outside_of_cube = np.sum(np.abs(optimum_x) > 1) 141 | # print('%-20s %8.6f(%d)' % (elm['algorithm'], optimum_y, outside_of_cube)) 142 | # if outside_of_cube > 0: 143 | # print optimum_x[np.abs(optimum_x) > 1] 144 | 145 | for ndim in [100, 200, 500]: 146 | result = mnist_weight_SGD_only(ndim) 147 | print(result) 148 | -------------------------------------------------------------------------------- /HyperSphere/dummy/paper_scripts/nondecreasing_concave.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import pickle 4 | import time 5 | import numpy as np 6 | import scipy.stats as stats 7 | import matplotlib.pyplot as plt 8 | 9 | 10 | def warping_sample(): 11 | x = np.linspace(0, 1, 100) 12 | y = np.log(50 * x + 1) 13 | expand_begin = 3 14 | expand_end = 13 15 | shrink_begin = 75 16 | shrink_end = 90 17 | x_max = np.max(x) 18 | y_max = np.max(y) 19 | y = y / y_max 20 | plt.xlim(0, 1) 21 | plt.ylim(0, 1) 22 | plt.plot(x, y) 23 | plt.axhline(y[expand_begin], xmin=0, xmax=x[expand_begin], color='g', ls='--') 24 | plt.axhline(y[expand_end], xmin=0, xmax=x[expand_end], color='g', ls='--') 25 | plt.axvline(x[expand_begin], ymin=0, ymax=y[expand_begin], color='g', ls='--') 26 | plt.axvline(x[expand_end], ymin=0, ymax=y[expand_end], color='g', ls='--') 27 | 28 | plt.axhline(y[shrink_begin], xmin=0, xmax=x[shrink_begin], color='y', ls='--') 29 | plt.axhline(y[shrink_end], xmin=0, xmax=x[shrink_end], color='y', ls='--') 30 | plt.axvline(x[shrink_begin], ymin=0, ymax=y[shrink_begin], color='y', ls='--') 31 | plt.axvline(x[shrink_end], ymin=0, ymax=y[shrink_end], color='y', ls='--') 32 | plt.tight_layout() 33 | plt.show() 34 | 35 | 36 | def kumaraswamy_cdf(): 37 | x = np.linspace(0, 1, 100) 38 | a_list = [1.0/4, 1.0/2, 1.0] 39 | b_list = [1, 2, 4] 40 | for a in a_list: 41 | for b in b_list: 42 | plt.plot(x, (1 - (1 - x ** a) ** b), label=('a=%.2f' % a) + ',b=' + str(b)) 43 | plt.legend() 44 | plt.tight_layout() 45 | plt.show() 46 | 47 | 48 | if __name__ == '__main__': 49 | kumaraswamy_cdf() -------------------------------------------------------------------------------- /HyperSphere/dummy/paper_scripts/p_comparison.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import pickle 4 | import time 5 | import numpy as np 6 | import pandas as pd 7 | import torch 8 | import matplotlib.pyplot as plt 9 | from matplotlib.ticker import MultipleLocator 10 | import matplotlib.gridspec as gridspec 11 | from matplotlib.ticker import FormatStrFormatter 12 | 13 | 14 | EXPS = ['rosenbrock_D20'] 15 | ALGORITHMS = ['sphereboth', 'sphereorigin', 'spherewarpingboth', 'spherewarpingorigin'] 16 | ALGO_NAMES = {} 17 | ALGO_NAMES['sphereboth'] = 'BOCK B' 18 | ALGO_NAMES['sphereorigin'] = 'BOCK' 19 | ALGO_NAMES['spherewarpingboth'] = 'BOCK W B' 20 | ALGO_NAMES['spherewarpingorigin'] = 'BOCK W' 21 | DATA_DIR = '/home/coh1/Experiments/Hypersphere_P_comparison' 22 | P = [3, 5, 7, 9] 23 | 24 | 25 | def algorithm_color(algorithm): 26 | if algorithm == 'sphereboth': 27 | return 'green' 28 | if algorithm == 'sphereorigin': 29 | return 'limegreen' 30 | if algorithm == 'spherewarpingboth': 31 | return 'olive' 32 | if algorithm == 'spherewarpingorigin': 33 | return 'darkcyan' 34 | 35 | 36 | def get_data(): 37 | result_dict = {} 38 | for exp in EXPS: 39 | for algo in ALGORITHMS: 40 | comparison_dict = {} 41 | for p in P: 42 | p_dir = os.path.join(DATA_DIR, 'P=' + str(p)) 43 | optimum_list = [] 44 | for instance in os.listdir(p_dir): 45 | if '_'.join([exp, algo]) in instance: 46 | instance_dir = os.path.join(p_dir, instance) 47 | kernel = torch.load(os.path.join(instance_dir, 'model.pt')).kernel 48 | assert int(str(kernel)[-2:-1]) == p 49 | data_file = open(os.path.join(instance_dir, 'data_config.pkl')) 50 | output = pickle.load(data_file)['output'] 51 | optimum = np.array([torch.min(output[:i].data) for i in range(1, output.numel() + 1)]) 52 | data_file.close() 53 | optimum_list.append(optimum) 54 | comparison_dict[p] = optimum_list 55 | result_dict['_'.join([exp, algo])] = comparison_dict 56 | return result_dict 57 | 58 | 59 | def plot_comparison(): 60 | comparison_data = get_data() 61 | summary_data = {} 62 | for key in comparison_data.keys(): 63 | p_data = comparison_data[key] 64 | stacked_data = {} 65 | for p in P: 66 | n_min_eval = np.min([len(elm) for elm in p_data[p]]) 67 | stacked_data[p] = np.stack([elm[:n_min_eval] for elm in p_data[p]]) 68 | summary_data[key] = stacked_data 69 | for key in sorted(summary_data.keys()): 70 | exp = '_'.join(key.split('_')[:2]) 71 | algo = key.split('_')[-1] 72 | print(exp, ALGO_NAMES[algo]) 73 | for p in sorted(P): 74 | optimum_data = summary_data[key][p] 75 | result_samples = '/'.join(['%12.4f' % elm for elm in optimum_data[:, -1]]) 76 | middle_three = sorted(optimum_data[:, -1])[1:-1] 77 | middle_three_str = '%12.4f(%10.4f) %8.2f$\pm$%-8.2f' % (np.mean(middle_three), np.std(middle_three), np.mean(middle_three), np.std(middle_three)) 78 | print(('%d %12.4f(%10.4f) %8.2f$\pm$%-8.2f' % (p, np.mean(optimum_data[:, -1]), np.std(optimum_data[:, -1]), np.mean(optimum_data[:, -1]), np.std(optimum_data[:, -1]))) + result_samples + middle_three_str) 79 | 80 | 81 | if __name__ == '__main__': 82 | plot_comparison() 83 | 84 | 85 | -------------------------------------------------------------------------------- /HyperSphere/dummy/paper_scripts/stochastic_depth_resnet_result.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import matplotlib.pyplot as plt 4 | import numpy as np 5 | 6 | p_2490_2536_filename = os.path.join('/home/coh1/Experiments/StochasticDepthP', 'stochastic_depth_death_rate_cifar100+_20180125-11:42:54:345544_2490&2536.pkl') 7 | p_2494_2489_filename = os.path.join('/home/coh1/Experiments/StochasticDepthP', 'stochastic_depth_death_rate_cifar100+_20180125-11:51:59:879674_2494&2489.pkl') 8 | p_2474_2497_filename = os.path.join('/home/coh1/Experiments/StochasticDepthP', 'stochastic_depth_death_rate_cifar100+_20180130-13:15:35:724102_2474&2497.pkl') 9 | 10 | 11 | def death_rate_plot(p_filename): 12 | valid_err = float((os.path.split(p_filename)[1])[-13:-9]) / 100 13 | test_err = float((os.path.split(p_filename)[1])[-8:-4]) / 100 14 | f = open(p_filename) 15 | p_data = pickle.load(f) 16 | f.close() 17 | plt.figure(figsize=(10, 8)) 18 | plt.bar(range(1, len(p_data) + 1), p_data) 19 | plt.xticks(size=24) 20 | plt.xlabel('n-th residual block', fontsize=24) 21 | plt.yticks(size=24) 22 | plt.ylabel('Death rate', fontsize=24) 23 | plt.title('Validation Error : %5.2f%% / Test Error : %5.2f%%' % (valid_err, test_err), fontsize=24) 24 | plt.subplots_adjust(left=0.12, right=0.98, top=0.95, bottom=0.1) 25 | print(np.sum(p_data)) 26 | plt.show() 27 | 28 | 29 | if __name__ == '__main__': 30 | death_rate_plot(p_2490_2536_filename) -------------------------------------------------------------------------------- /HyperSphere/dummy/plotting/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/dummy/plotting/__init__.py -------------------------------------------------------------------------------- /HyperSphere/dummy/plotting/datafile_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | import pickle 4 | import sys 5 | 6 | import numpy as np 7 | from torch.autograd import Variable 8 | 9 | 10 | file_path = os.path.realpath(__file__) 11 | if 'anaconda' in file_path.lower(): 12 | EXPERIMENT_DIR = os.path.join('/'.join(file_path.split('/')[:-7]), 'Experiments/Hypersphere') 13 | else: 14 | EXPERIMENT_DIR = os.path.join('/'.join(file_path.split('/')[:-6]), 'Experiments/Hypersphere') 15 | 16 | 17 | def remove_last_evaluation(path): 18 | if not os.path.exists(path): 19 | path = os.path.join(EXPERIMENT_DIR, path) 20 | data_config_filename = os.path.join(path, 'data_config.pkl') 21 | data_config_file = open(data_config_filename, 'r') 22 | data_config = pickle.load(data_config_file) 23 | data_config_file.close() 24 | for key, value in data_config.iteritems(): 25 | if isinstance(value, Variable) and value.dim() == 2: 26 | print('%12s : %4d-th evaluation %4d dimensional data whose sum is %.6E' % (key, value.size(0), value.size(1), (value.data if hasattr(value, 'data') else value).sum())) 27 | while True: 28 | sys.stdout.write('Want to remove this last evaluation?(YES/NO) : ') 29 | decision = sys.stdin.readline()[:-1] 30 | if decision == 'YES': 31 | for key, value in data_config.iteritems(): 32 | if isinstance(value, Variable) and value.dim() == 2: 33 | data_config[key] = value[:-1] 34 | data_config_file = open(data_config_filename, 'w') 35 | pickle.dump(data_config, data_config_file) 36 | data_config_file.close() 37 | break 38 | elif decision == 'NO': 39 | break 40 | else: 41 | sys.stdout.write('Input YES or NO\n') 42 | 43 | 44 | def folder_name_list(path): 45 | parent_dir, prefix = os.path.split(path) 46 | if not os.path.exists(parent_dir): 47 | parent_dir = os.path.join(EXPERIMENT_DIR, parent_dir) 48 | search_sub_dir = not np.any([os.path.isfile(os.path.join(parent_dir, elm)) for elm in os.listdir(parent_dir)]) 49 | if search_sub_dir: 50 | experiment_type = set() 51 | sub_dir_list = [elm for elm in os.listdir(parent_dir) if os.path.isdir(os.path.join(parent_dir, elm))] 52 | for sub_folder in sub_dir_list: 53 | sub_folder_folder_list = [elm for elm in os.listdir(os.path.join(parent_dir, sub_folder)) if os.path.isdir(os.path.join(parent_dir, sub_folder, elm)) and prefix == elm[:len(prefix)]] 54 | experiment_type = experiment_type.union([remove_entailing_number(elm.split('_')[-1]) for elm in sub_folder_folder_list]) 55 | result_dict = dict() 56 | for exp_type in experiment_type: 57 | result_dict[exp_type] = [] 58 | for sub_folder in sub_dir_list: 59 | sub_folder_folder_list = [elm for elm in os.listdir(os.path.join(parent_dir, sub_folder)) if os.path.isdir(os.path.join(parent_dir, sub_folder, elm)) and prefix == elm[:len(prefix)]] 60 | for exp_type in experiment_type: 61 | result_dict[exp_type] += [os.path.join(parent_dir, sub_folder, elm) for elm in sub_folder_folder_list if exp_type == elm.split('_')[-1][:len(exp_type)]] 62 | else: 63 | folder_list = [elm for elm in os.listdir(parent_dir) if os.path.isdir(os.path.join(parent_dir, elm)) and prefix == elm[:len(prefix)]] 64 | experiment_type = set([remove_entailing_number(elm.split('_')[-1]) for elm in folder_list]) 65 | if len(folder_list) == 0: 66 | print('No experimental result') 67 | return 68 | result_dict = dict() 69 | for exp_type in experiment_type: 70 | result_dict[exp_type] = [] 71 | for exp_type in experiment_type: 72 | result_dict[exp_type] += [os.path.join(parent_dir, elm) for elm in folder_list if exp_type in elm] 73 | return result_dict 74 | 75 | 76 | def remove_entailing_number(str): 77 | result = str[:] 78 | for i in range(len(result)-1, -1, -1): 79 | if not result[i].isdigit(): 80 | result = result[:i+1] 81 | break 82 | return result 83 | 84 | 85 | def how_many_evaluations(path): 86 | folder_category = folder_name_list(path) 87 | grassmanian_folder_list = folder_category['grassmanian'] 88 | sphere_folder_list = folder_category['sphere'] 89 | cube_folder_list = folder_category['cube'] 90 | for folder in grassmanian_folder_list + sphere_folder_list + cube_folder_list: 91 | f = open(os.path.join(folder, 'data_config.pkl')) 92 | n_eval = pickle.load(f)['output'].numel() 93 | f.close() 94 | print('%4d evaluation in Experiment %s' % (n_eval, folder)) 95 | 96 | 97 | if __name__ == '__main__': 98 | print(' 1: The number of evaluations') 99 | print(' 2: The nan value check(remove last evaluation)') 100 | sys.stdout.write('Choose command : ') 101 | command = int(sys.stdin.readline()[:-1]) 102 | if command == 1: 103 | path_info = '/'.join(os.path.abspath(__file__).split('/')[:3] + ['Experiments']) 104 | sys.stdout.write('path(%s) : ' % path_info) 105 | path = sys.stdin.readline()[:-1] 106 | if not os.path.exists(os.path.split(path)[0]): 107 | path = os.path.join(path_info, path) 108 | how_many_evaluations(path) 109 | elif command == 2: 110 | path_info = '/'.join(os.path.abspath(__file__).split('/')[:3] + ['Experiments']) 111 | sys.stdout.write('path(%s) : ' % path_info) 112 | path = sys.stdin.readline()[:-1] 113 | if not os.path.exists(os.path.split(path)[0]): 114 | path = os.path.join(path_info, path) 115 | remove_last_evaluation(path) 116 | 117 | -------------------------------------------------------------------------------- /HyperSphere/dummy/plotting/get_data_from_file.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import pickle 4 | import numpy as np 5 | from scipy.io import loadmat 6 | import pandas as pd 7 | 8 | 9 | def get_data_sphere(dir_name, algorithms, func_name, ndim): 10 | if isinstance(algorithms, str): 11 | algorithms = [algorithms] 12 | 13 | data_list = [] 14 | for algorithm in algorithms: 15 | result_dir_name_list = [os.path.join(dir_name, elm) for elm in os.listdir(dir_name) if '_'.join(elm.split('_')[:-1]) == func_name + '_D' + str(ndim) + '_' + algorithm] 16 | for result_dir_name in result_dir_name_list: 17 | result_file = open(os.path.join(result_dir_name, 'data_config.pkl')) 18 | unpickled_data = pickle.load(result_file) 19 | data = dict() 20 | data['algorithm'] = algorithm 21 | data['n_eval'] = unpickled_data['output'].numel() 22 | data['x'] = unpickled_data['x_input'].data.numpy() 23 | data['y'] = unpickled_data['output'].data.numpy() 24 | data['optimum'] = np.array([np.min(data['y'][:i]) for i in range(1, data['n_eval'] + 1)]) 25 | data_list.append(data) 26 | return data_list 27 | 28 | 29 | def get_data_HPOlib(dir_name, optimizer_name='spearmint_april2013_mod'): 30 | folder_name = [os.path.join(dir_name, elm) for elm in os.listdir(dir_name) if os.path.isdir(os.path.join(dir_name, elm)) and elm != 'Plots'] 31 | file_name_list = [os.path.join(elm, optimizer_name + '.pkl') for elm in folder_name if os.path.exists(os.path.join(elm, optimizer_name + '.pkl'))] 32 | 33 | n_data = len(file_name_list) 34 | data_list = [] 35 | for i in range(n_data): 36 | data_file = open(file_name_list[i]) 37 | unpickled_data = pickle.load(data_file) 38 | data = {} 39 | data['algorithm'] = optimizer_name.split('_')[0] 40 | data['n_eval'] = len(unpickled_data['trials']) 41 | data['x'] = np.array([HPOlib_params2np(unpickled_data['trials'][i]['params']) for i in range(data['n_eval'])]) 42 | data['y'] = np.array([unpickled_data['trials'][i]['result'] for i in range(data['n_eval'])]) 43 | data['optimum'] = np.array([np.min(data['y'][:i]) for i in range(1, data['n_eval'] + 1)]) 44 | data_list.append(data) 45 | return data_list 46 | 47 | 48 | def HPOlib_params2np(params): 49 | x = np.array([[int(key[1:]), float(value)] for key, value in params.iteritems()]) 50 | sort_ind = np.argsort(x[:, 0]) 51 | return x[sort_ind, 1] 52 | 53 | 54 | def get_data_additive(dir_name, func_name, ndim, simpler=False): 55 | exp_id = func_name + '_D' + str(ndim) 56 | file_pool = [elm for elm in os.listdir(dir_name) if elm[:len(exp_id)] == exp_id] 57 | additive_ids = np.unique(['_'.join(elm.split('_')[2:4]) for elm in file_pool]) 58 | data_list = [] 59 | for additive_id in additive_ids: 60 | data_id = '_'.join([exp_id, additive_id]) 61 | data_x_list = [elm for elm in os.listdir(dir_name) if data_id == elm[:len(data_id)] and elm[-6:] == '_x.mat'] 62 | instance_ids = [elm.split('_')[4] for elm in data_x_list] 63 | for instance_id in instance_ids: 64 | data_batch_x = loadmat(os.path.join(dir_name, '_'.join([data_id, instance_id, 'x.mat'])))['queries'] 65 | data_batch_y = loadmat(os.path.join(dir_name, '_'.join([data_id, instance_id, 'y.mat'])))['neg_values'] 66 | data_batch_optimum = loadmat(os.path.join(dir_name, '_'.join([data_id, instance_id, 'optimum.mat'])))['neg_optima'] 67 | for s in range(data_batch_x.shape[0]): 68 | data = {} 69 | if simpler: 70 | data['algorithm'] = 'additive' 71 | else: 72 | data['algorithm'] = 'additiveBO_' + additive_id[5:] 73 | data['x'] = data_batch_x[s] 74 | data['y'] = data_batch_y[s] 75 | data['optimum'] = data_batch_optimum[s] 76 | data['n_eval'] = data['y'].size 77 | data_list.append(data) 78 | return data_list 79 | 80 | 81 | def get_data_warping(dir_name, func_name, ndim): 82 | data_list = [] 83 | folder_list = [os.path.join(dir_name, elm) for elm in os.listdir(dir_name) if func_name + '_' + str(ndim) == elm[:len(func_name + str(ndim)) + 1]] 84 | for folder in folder_list: 85 | df = pd.read_pickle(os.path.join(folder, 'query_eval_data.pkl')) 86 | df.sort_index(inplace=True) 87 | df = df.applymap(float) 88 | n_eval = df.shape[0] 89 | data = {} 90 | data['algorithm'] = 'spearmint_warping' 91 | x_data = np.empty((n_eval, 0)) 92 | for i in range(1, ndim + 1): 93 | x_data = np.hstack((x_data, np.array(df['x' + str(i)]).reshape((n_eval, 1)))) 94 | data['x'] = x_data 95 | data['y'] = np.array(df['value']) 96 | data['optimum'] = np.array([np.min(data['y'][:i]) for i in range(1, n_eval + 1)]) 97 | data['n_eval'] = n_eval 98 | data_list.append(data) 99 | return data_list 100 | 101 | 102 | def get_data_elastic(dir_name, func_name, ndim): 103 | data_list = [] 104 | exp_id = func_name + str(ndim) 105 | filename_list = [os.path.join(dir_name, elm) for elm in os.listdir(dir_name) if exp_id == elm[:len(exp_id)] and '.mat' in elm] 106 | for filename in filename_list: 107 | mat_data = loadmat(filename) 108 | data = {} 109 | data['algorithm'] = 'elastic' 110 | data['x'] = mat_data['x'] 111 | data['y'] = -mat_data['y'].flatten() 112 | data['optimum'] = -mat_data['ybest'].flatten() 113 | data['n_eval'] = mat_data['y'].size 114 | data_list.append(data) 115 | return data_list 116 | 117 | 118 | def get_data(func_name, ndim, suffix='_center-random', P_setting='_P=9'): 119 | # suffix = '_center-corner' 120 | spearmint_dir_name = '/home/coh1/Experiments/spearmint_mnist_weight' + suffix + '/' + func_name + '_' + str(ndim) 121 | smac_dir_name = '/home/coh1/Experiments/smac_ALL' + suffix + '/' + func_name + '_' + str(ndim) 122 | tpe_dir_name = '/home/coh1/Experiments/tpe_ALL' + suffix + '/' + func_name + '_' + str(ndim) 123 | additive_dir_name = '/home/coh1/Experiments/Additive_BO_mat_ALL' + suffix + '/' 124 | warping_dir_name = '/home/coh1/Experiments/Warping_ALL' + suffix + '123/' 125 | elastic_dir_name = '/home/coh1/Experiments/elastic_BO_mat' + suffix + '/' 126 | sphere_dir_name = '/home/coh1/Experiments/Hypersphere_mnist_weight' + suffix + P_setting + '/' 127 | cube_dir_name = '/home/coh1/Experiments/Cube_mnist_weight' + suffix + '/' 128 | data_list = [] 129 | try: 130 | data_list += get_data_HPOlib(spearmint_dir_name, 'spearmint_april2013_mod') 131 | except OSError: 132 | pass 133 | try: 134 | data_list += get_data_HPOlib(tpe_dir_name, 'hyperopt_august2013_mod') 135 | except OSError: 136 | pass 137 | try: 138 | data_list += get_data_HPOlib(smac_dir_name, 'smac_2_10_00-dev') 139 | except OSError: 140 | pass 141 | try: 142 | data_list += get_data_additive(additive_dir_name, func_name, ndim) 143 | except OSError: 144 | pass 145 | try: 146 | data_list += get_data_warping(warping_dir_name, func_name, ndim) 147 | except OSError: 148 | pass 149 | try: 150 | data_list += get_data_elastic(elastic_dir_name, func_name, ndim) 151 | except OSError: 152 | pass 153 | data_list += get_data_sphere(cube_dir_name, ['cube', 'cubeard'], func_name, ndim) 154 | data_list += get_data_sphere(sphere_dir_name, ['sphereboth', 'sphereorigin', 'spherewarpingboth', 'spherewarpingorigin'], func_name, ndim) 155 | return data_list 156 | 157 | 158 | if __name__ == '__main__': 159 | get_data_sphere(*sys.argv[1:]) -------------------------------------------------------------------------------- /HyperSphere/dummy/plotting/plot_color.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | def algorithm_color(algorithm): 4 | if algorithm == 'hyperopt': 5 | return 'r' 6 | if algorithm == 'smac': 7 | return 'dodgerblue' 8 | if algorithm == 'spearmint': 9 | return 'darkorchid' 10 | if algorithm == 'spearmint_warping': 11 | return 'indigo' 12 | if algorithm == 'cube': 13 | return 'salmon' 14 | if algorithm == 'cubeard': 15 | return 'r' 16 | if algorithm[:10] == 'additiveBO': 17 | return 'g' 18 | if algorithm == 'elasticGP': 19 | return 'darkslategray' 20 | if algorithm == 'sphereboth': 21 | return 'green' 22 | if algorithm == 'sphereorigin': 23 | return 'limegreen' 24 | if algorithm == 'spherewarpingboth': 25 | return 'lime' 26 | if algorithm == 'spherewarpingorigin': 27 | return 'lime' -------------------------------------------------------------------------------- /HyperSphere/dummy/setup_conda.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | VENV_ROOT_DIR="`which python | xargs dirname | xargs dirname`/envs/HyperSphere" 4 | if [ -d "$VENV_ROOT_DIR" ]; then 5 | cd "$VENV_ROOT_DIR" 6 | if [ ! -d "$VENV_ROOT_DIR/.git" ]; then 7 | echo "Data in HyperSphere is moved to virtual environment root directory." 8 | mv HyperSphere HyperSphere_TBR 9 | cp -a HyperSphere_TBR/. ./ 10 | rm -rf HyperSphere_TBR 11 | else 12 | echo "Data has been moved" 13 | fi 14 | source activate HyperSphere 15 | conda install --yes pytorch torchvision -c soumith -n HyperSphere 16 | pip install -r requirements.txt 17 | else 18 | echo "Already in virtual environment" 19 | conda install --yes pytorch torchvision -c soumith -n HyperSphere 20 | pip install -r requirements.tx 21 | fi 22 | -------------------------------------------------------------------------------- /HyperSphere/dummy/split_srun.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | run_command="srun python ./HyperSphere/BO/run_BO.py" 4 | 5 | case $1 in 6 | 'cube') 7 | exp_config="-g cube";; 8 | 'cubeard') 9 | exp_config="-g cube --ard";; 10 | 'sphere') 11 | exp_config="-g sphere --origin";; 12 | 'sphereboundary') 13 | exp_config="-g sphere --origin --boundary";; 14 | 'spherewarping') 15 | exp_config="-g sphere --origin --warping";; 16 | 'spherewarpingboundary') 17 | exp_config="-g sphere --origin --warping --boundary";; 18 | esac 19 | 20 | case $2 in 21 | 20) 22 | n_eval=200;; 23 | 50) 24 | n_eval=400;; 25 | 100) 26 | n_eval=600;; 27 | 200) 28 | n_eval=800;; 29 | 1000) 30 | n_eval=1000;; 31 | esac 32 | 33 | for s in {1..5} 34 | do 35 | dir_name=$($run_command $exp_config -d $2 -f $3 | tail -1) 36 | for (( i=1; i<=$n_eval; i++ )) 37 | do 38 | $run_command -p $dir_name 39 | done 40 | done -------------------------------------------------------------------------------- /HyperSphere/dummy/srun_execution.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | ALGORITHM=$1 4 | 5 | INIT_COMMAND_CUBE_ROOT="srun python HyperSphere/BO/run_BO.py --parallel -g cube" 6 | INIT_COMMAND_SPHERE_ROOT="srun python HyperSphere/BO/run_BO.py --parallel -g sphere --origin" 7 | 8 | case "$ALGORITHM" in 9 | cube) INIT_COMMAND_PREFIX="$INIT_COMMAND_CUBE_ROOT" 10 | ;; 11 | cubeard) INIT_COMMAND_PREFIX="$INIT_COMMAND_CUBE_ROOT --ard" 12 | ;; 13 | none) INIT_COMMAND_PREFIX="$INIT_COMMAND_SPHERE_ROOT" 14 | ;; 15 | boundary) INIT_COMMAND_PREFIX="$INIT_COMMAND_SPHERE_ROOT --boundary" 16 | ;; 17 | warping) INIT_COMMAND_PREFIX="$INIT_COMMAND_SPHERE_ROOT --warping" 18 | ;; 19 | warpingboundary) INIT_COMMAND_PREFIX="$INIT_COMMAND_SPHERE_ROOT --warping --boundary" 20 | ;; 21 | esac 22 | 23 | 24 | for DIM in 20 50 100; do 25 | case "$DIM" in 26 | 20) NEVAL=200 27 | ;; 28 | 50) NEVAL=400 29 | ;; 30 | 100) NEVAL=600 31 | ;; 32 | esac 33 | for TASK in branin hartmann6 rosenbrock levy; do 34 | INIT_COMMAND="$INIT_COMMAND_PREFIX -f $TASK -d $DIM" 35 | echo "====================COMMAND====================" 36 | echo "$INIT_COMMAND" 37 | EVALSTDOUT=$(eval "$INIT_COMMAND") 38 | echo "$EVALSTDOUT" 39 | EXPPATH="${EVALSTDOUT##*$'\n'}" 40 | for ((i=1;i<=$NEVAL;i++)); do 41 | echo "====================COMMAND====================" 42 | CONTINUED_COMMAND="srun -C cpunode python HyperSphere/BO/run_BO.py -p $EXPPATH" 43 | echo "$CONTINUED_COMMAND" 44 | eval "$CONTINUED_COMMAND" 45 | done 46 | done 47 | done 48 | -------------------------------------------------------------------------------- /HyperSphere/dummy/torque_run_script_generator.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | 5 | HEAD = "" 6 | HEAD += "#Set job requirements\n" 7 | HEAD += "#PBS -S /bin/bash\n" 8 | HEAD += "#PBS -lnodes=1:ppn=16\n" 9 | HEAD += "#PBS -lwalltime=16:00:00\n" 10 | HEAD += "\n" 11 | HEAD += "#Loading modules\n" 12 | HEAD += "module load python\n" 13 | HEAD += "cd \"$HOME/git_repositories/HyperSphere\"\n" 14 | HEAD += "\n" 15 | 16 | BODY_INITIATING = "" 17 | BODY_INITIATING += "INIT_COMMAND_CUBE_ROOT=\"python HyperSphere/BO/run_BO.py --parallel -g cube\"\n" 18 | BODY_INITIATING += "INIT_COMMAND_SPHERE_ROOT=\"python HyperSphere/BO/run_BO.py --parallel -g sphere --origin\"\n" 19 | BODY_INITIATING += "case \"$ALGORITHM\" in\n" 20 | BODY_INITIATING += " cube) INIT_COMMAND_PREFIX=\"$INIT_COMMAND_CUBE_ROOT\"\n" 21 | BODY_INITIATING += " ;;\n" 22 | BODY_INITIATING += " cubeard) INIT_COMMAND_PREFIX=\"$INIT_COMMAND_CUBE_ROOT --ard\"\n" 23 | BODY_INITIATING += " ;;\n" 24 | BODY_INITIATING += " none) INIT_COMMAND_PREFIX=\"$INIT_COMMAND_SPHERE_ROOT\"\n" 25 | BODY_INITIATING += " ;;\n" 26 | BODY_INITIATING += " boundary) INIT_COMMAND_PREFIX=\"$INIT_COMMAND_SPHERE_ROOT --boundary\"\n" 27 | BODY_INITIATING += " ;;\n" 28 | BODY_INITIATING += " warping) INIT_COMMAND_PREFIX=\"$INIT_COMMAND_SPHERE_ROOT --warping\"\n" 29 | BODY_INITIATING += " ;;\n" 30 | BODY_INITIATING += " warpingboundary) INIT_COMMAND_PREFIX=\"$INIT_COMMAND_SPHERE_ROOT --warping --boundary\"\n" 31 | BODY_INITIATING += " ;;\n" 32 | BODY_INITIATING += "esac\n" 33 | BODY_INITIATING += "\n" 34 | BODY_INITIATING += "INIT_COMMAND=\"$INIT_COMMAND_PREFIX -f $TASK -d $DIM\"\n" 35 | BODY_INITIATING += "EVALSTDOUT=$(eval \"$INIT_COMMAND\")\n" 36 | BODY_INITIATING += "EXPPATH=\"${EVALSTDOUT##*$'\\n'}\"\n" 37 | 38 | BODY_CONTINUING = "" 39 | BODY_CONTINUING += "CONTINUED_COMMAND=\"python HyperSphere/BO/run_BO.py -p $EXPPATH\"\n" 40 | BODY_CONTINUING += "for ((i=1;i<=$NEVAL;i++)); do\n" 41 | BODY_CONTINUING += " eval \"$CONTINUED_COMMAND\"\n" 42 | BODY_CONTINUING += "done\n" 43 | 44 | 45 | def generate_initiating_script(algorithm, func_name, dim, n_eval): 46 | script = HEAD 47 | script += "ALGORITHM=\"" + algorithm + "\"\n" 48 | script += "TASK=\"" + func_name + "\"\n" 49 | script += "DIM=" + str(dim) + "\n" 50 | script += "NEVAL=" + str(n_eval) + "\n" 51 | script += "\n" 52 | script += BODY_INITIATING 53 | script += BODY_CONTINUING 54 | 55 | filename = '_'.join([algorithm, func_name, 'D' + str(dim), 'E' + str(n_eval) + '.sh']) 56 | sh_file = open(filename, 'wt') 57 | sh_file.write(script) 58 | sh_file.close() 59 | 60 | 61 | def generate_continuing_script(pathname, n_eval): 62 | script = HEAD 63 | script += "EXPPATH=\"" + os.path.realpath(pathname) + "\"\n" 64 | script += "NEVAL=" + str(n_eval) + "\n" 65 | script += "\n" 66 | script += BODY_CONTINUING 67 | 68 | filename = '_'.join([os.path.split(pathname)[1], 'E' + str(n_eval) + '.sh']) 69 | sh_file = open(filename, 'wt') 70 | sh_file.write(script) 71 | sh_file.close() 72 | 73 | 74 | if __name__ == '__main__': 75 | if len(sys.argv) == 5: 76 | generate_initiating_script(*sys.argv[1:]) 77 | elif len(sys.argv) == 3: 78 | generate_continuing_script(*sys.argv[1:]) 79 | else: 80 | print("Argument should be") 81 | print("1 : algorithm(none, boudanry, warping, warpingboundary, cube, cubeard") 82 | print("2 : task(branin, hartmann6, rosenbrock, levy, styblinskitang, schwefel, michalewicz, camelback, qing, bird") 83 | print("3 : dim(20, 50, 100)") 84 | print("4 : n_eval") 85 | print("OR") 86 | print("1 : path") 87 | print("2 : n_eval") -------------------------------------------------------------------------------- /HyperSphere/feature_map/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/feature_map/__init__.py -------------------------------------------------------------------------------- /HyperSphere/feature_map/functionals.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | import torch 4 | from functools import partial 5 | 6 | 7 | def id_transform(phi): 8 | return phi 9 | 10 | id_transform.dim_change = lambda x: x 11 | 12 | 13 | def x2radial(x): 14 | radius = torch.sqrt(torch.sum(x ** 2, dim=1, keepdim=True)) 15 | normalizer = radius.clone() 16 | normalizer[(radius == 0).detach()] = 1 17 | return torch.cat([radius, x/normalizer], 1) 18 | 19 | x2radial.dim_change = lambda x: x + 1 20 | 21 | 22 | def in_sphere(x, radius): 23 | return torch.sum(x.data ** 2) > radius ** 2 24 | 25 | 26 | def sphere_bound(radius): 27 | return partial(in_sphere, radius=radius) 28 | 29 | 30 | def phi_reflection(phi): 31 | f_phi0 = torch.cos(phi[:, :1] * math.pi) 32 | f_phi_rest = torch.cat([torch.cos(phi[:, -1:] * 2 * math.pi), torch.sin(phi[:, -1:] * 2 * math.pi)], 1) 33 | if phi.size(1) > 2: 34 | f_phi_rest = torch.cat([torch.cos(phi[:, 1:-1] * math.pi), f_phi_rest], 1) 35 | return torch.cat([f_phi0, f_phi_rest], 1) 36 | 37 | phi_reflection.dim_change = lambda x: x+1 38 | 39 | 40 | def phi_smooth(phi): 41 | f_phi0 = torch.abs(torch.sin(phi[:, :1] * math.pi * 0.25)) 42 | sin_part = torch.cumprod(torch.cat([torch.sin(phi[:, 1:-1] * math.pi), torch.sin(phi[:, -1:] * math.pi * 2.0)], 1), 1) 43 | cos_part = torch.cat([torch.cos(phi[:, 1:-1] * math.pi), torch.cos(phi[:, -1:] * math.pi * 2)], 1) 44 | f_phi_rest = torch.cat([cos_part[:, :1], sin_part[:, :-1] * cos_part[:, 1:], sin_part[:, -1:]], 1) 45 | return torch.cat([f_phi0, f_phi_rest * f_phi0.view(-1, 1)], 1) 46 | 47 | phi_smooth.dim_change = lambda x: x+1 48 | 49 | 50 | def phi_reflection_lp(phi, p=3): 51 | f_phi0 = torch.cos(phi[:, :1] * math.pi) 52 | ratio = 0.5 * (1 - f_phi0) 53 | reduction = (1 - (1 - ratio) ** p) ** (1.0 / p) 54 | f_phi_rest = torch.cat([torch.cos(phi[:, 1:-1] * math.pi), torch.cos(phi[:, -1:] * 2 * math.pi), torch.sin(phi[:, -1:] * 2 * math.pi)], 1) 55 | return torch.cat([f_phi0, f_phi_rest * reduction.view(-1, 1)], 1) 56 | 57 | phi_reflection_lp.dim_change = lambda x: x+1 58 | 59 | 60 | def phi_reflection_threshold(phi, threshold=0.1): 61 | f_phi0 = torch.cos(phi[:, :1] * math.pi) 62 | ratio = 0.5 * (1 - f_phi0) 63 | reduction = ratio.clone() * 0 + 1 64 | ind_small = ratio < threshold 65 | # reduction[ind_small] = 0.5 * (1 - torch.cos(ratio[ind_small] * math.pi / threshold)) 66 | reduction[ind_small] = torch.sin(ratio[ind_small] * 0.25 * math.pi / threshold) 67 | f_phi_rest = torch.cat([torch.cos(phi[:, -1:] * 2 * math.pi), torch.sin(phi[:, -1:] * 2 * math.pi)], 1) 68 | if phi.size(1) > 2: 69 | f_phi_rest = torch.cat([torch.cos(phi[:, 1:-1] * math.pi), f_phi_rest], 1) 70 | return torch.cat([f_phi0, f_phi_rest * reduction.view(-1, 1)], 1) 71 | 72 | 73 | phi_reflection_threshold.dim_change = lambda x: x+1 74 | 75 | 76 | def sigmoid_numpy(x): 77 | return 1.0 / (1.0 + np.exp(-x)) 78 | 79 | 80 | def sigmoid_inv_numpy(x): 81 | return 1.0 - np.log(1.0/x - 1) 82 | 83 | 84 | def sigmoid_inv(x): 85 | return 1.0 - torch.log(1.0/x - 1) -------------------------------------------------------------------------------- /HyperSphere/feature_map/functions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/feature_map/functions/__init__.py -------------------------------------------------------------------------------- /HyperSphere/feature_map/functions/reduce_lp.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | from torch.autograd import Function, Variable, gradcheck 5 | 6 | 7 | class ReduceLp(Function): 8 | 9 | @staticmethod 10 | def forward(ctx, input, p): 11 | ctx.save_for_backward(input, p) 12 | ratio = 0.5 * (1.0 - input[:, 0:1]) 13 | reduction = (1.0 - (1.0 - ratio) ** p) ** (1.0 / p) 14 | return torch.cat([input[:, 0:1], input[:, 1:] * reduction.view(-1, 1)], 1) 15 | 16 | @staticmethod 17 | def backward(ctx, grad_output): 18 | input, p = ctx.saved_variables 19 | grad_input = grad_threshold = None 20 | 21 | ratio = 0.5 * (1.0 - input[:, 0:1]) 22 | ratio_derivative = -0.5 23 | reduction = (1.0 - (1.0 - ratio) ** p) ** (1.0 / p) 24 | 25 | if ctx.needs_input_grad[0]: 26 | grad_f_phi0 = ((1.0 - ratio) / reduction) ** (p - 1.0) * ratio_derivative 27 | grad_phi0 = grad_output[:, 0:1] + (input[:, 1:] * grad_output[:, 1:] * grad_f_phi0.view(-1, 1)).sum(1, keepdim=True) 28 | grad_input = torch.cat([grad_phi0, reduction.view(-1, 1).repeat(1, input.size(1) - 1) * grad_output[:, 1:]], 1) 29 | if ctx.needs_input_grad[1]: 30 | p_power = (1.0 - ratio) ** p 31 | grad_reduction = -(p_power * torch.log(p_power) + (1 - p_power) * torch.log(1.0 - p_power)) / (p ** 2 * reduction ** (p - 1)) 32 | grad_threshold = (grad_output[:, 1:] * input[:, 1:] * grad_reduction.view(-1, 1)).sum() 33 | 34 | return grad_input, grad_threshold 35 | 36 | 37 | if __name__ == '__main__': 38 | n = 1 39 | ndim = 5 40 | input_grad = True 41 | param_grad = not input_grad 42 | input = Variable(torch.FloatTensor(n, ndim).uniform_(-1, 1), requires_grad=input_grad) 43 | p = Variable(torch.FloatTensor(1).normal_().abs().exp(), requires_grad=param_grad) 44 | eps = 1e-4 45 | 46 | # d = 3 47 | # output = (ReduceThreshold.apply(input, threshold))[:, d] 48 | # fdm_grad = torch.zeros(n, ndim) 49 | # for i in range(n): 50 | # for j in range(ndim): 51 | # input_perturb = Variable(input.data.clone()) 52 | # input_perturb.data[i, j] += eps 53 | # output_perturb = (ReduceThreshold.apply(input_perturb, threshold))[:, d] 54 | # fdm_grad[i, j] = (output_perturb - output).data.squeeze()[0] / eps 55 | # print(input.data) 56 | # ratio = 0.5 * (1 - torch.cos(input.data[:, 0:1] * math.pi)) 57 | # reduction = ratio.clone() * 0 + 1 58 | # ind_small = ratio < threshold.data 59 | # if torch.sum(ind_small) > 0: 60 | # reduction[ind_small] = 0.5 * (1 - torch.cos(ratio[ind_small] * math.pi / threshold.data)) 61 | # print('reduction', reduction.squeeze()[0]) 62 | # print(fdm_grad) 63 | # output.backward() 64 | # print(input.grad.data) 65 | 66 | # gradcheck doesn't have to pass all the time. 67 | test = gradcheck(ReduceLp.apply, (input, p), eps=eps, atol=1e-3, rtol=1e-2) 68 | print(test) -------------------------------------------------------------------------------- /HyperSphere/feature_map/functions/reduce_threshold.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | from torch.autograd import Function, Variable, gradcheck 5 | 6 | 7 | class ReduceThreshold(Function): 8 | 9 | @staticmethod 10 | def forward(ctx, input, threshold): 11 | ctx.save_for_backward(input, threshold) 12 | ratio = 0.5 * (1 - input[:, 0:1]) 13 | reduction = ratio.clone() * 0 + 1 14 | ind_small = ratio < threshold 15 | if torch.sum(ind_small) > 0: 16 | # reduction[ind_small] = 0.5 * (1 - torch.cos(ratio[ind_small] * math.pi / threshold)) 17 | reduction[ind_small] = torch.sin(ratio[ind_small] * 0.25 * math.pi / threshold) 18 | return torch.cat([input[:, :1], input[:, 1:] * reduction.view(-1, 1)], 1) 19 | 20 | @staticmethod 21 | def backward(ctx, grad_output): 22 | input, threshold = ctx.saved_variables 23 | grad_input = grad_threshold = None 24 | 25 | ratio = 0.5 * (1 - input[:, 0:1]) 26 | ratio_derivative = -0.5 27 | reduction = ratio.clone() * 0 + 1 28 | ind_small = ratio < threshold 29 | if ind_small.data.any(): 30 | # reduction[ind_small] = 0.5 * (1 - torch.cos(ratio[ind_small] * math.pi / threshold)) 31 | reduction[ind_small] = torch.sin(ratio[ind_small] * 0.25 * math.pi / threshold) 32 | 33 | if ctx.needs_input_grad[0]: 34 | grad_f_phi0 = ratio.clone() * 0 35 | if ind_small.data.any(): 36 | grad_f_phi0[ind_small] = 0.25 * math.pi / threshold * torch.cos(ratio[ind_small] * 0.25 * math.pi / threshold) * ratio_derivative 37 | grad_phi0 = grad_output[:, 0:1] + (input[:, 1:] * grad_f_phi0.view(-1, 1) * grad_output[:, 1:]).sum(1, keepdim=True) 38 | grad_input = torch.cat([grad_phi0, grad_output[:, 1:] * reduction.view(-1, 1)], 1) 39 | if ctx.needs_input_grad[1]: 40 | grad_reduction = ratio.clone() * 0 41 | if ind_small.data.any(): 42 | grad_reduction[ind_small] = -0.25 * ratio[ind_small] * math.pi / threshold**2 * torch.cos(ratio[ind_small] * 0.25 * math.pi / threshold) 43 | grad_threshold = (grad_output[:, 1:] * input[:, 1:] * grad_reduction.view(-1, 1)).sum() 44 | 45 | return grad_input, grad_threshold 46 | 47 | 48 | if __name__ == '__main__': 49 | n = 1 50 | ndim = 5 51 | input_grad = False 52 | param_grad = not input_grad 53 | input = Variable(torch.FloatTensor(n, ndim).uniform_(-1, 1), requires_grad=input_grad) 54 | threshold = Variable(torch.FloatTensor(1).fill_(0.5), requires_grad=param_grad) 55 | eps = 1e-4 56 | 57 | # d = 3 58 | # output = (ReduceThreshold.apply(input, threshold))[:, d] 59 | # fdm_grad = torch.zeros(n, ndim) 60 | # for i in range(n): 61 | # for j in range(ndim): 62 | # input_perturb = Variable(input.data.clone()) 63 | # input_perturb.data[i, j] += eps 64 | # output_perturb = (ReduceThreshold.apply(input_perturb, threshold))[:, d] 65 | # fdm_grad[i, j] = (output_perturb - output).data.squeeze()[0] / eps 66 | # print(input.data) 67 | # ratio = 0.5 * (1 - torch.cos(input.data[:, 0:1] * math.pi)) 68 | # reduction = ratio.clone() * 0 + 1 69 | # ind_small = ratio < threshold.data 70 | # if torch.sum(ind_small) > 0: 71 | # reduction[ind_small] = 0.5 * (1 - torch.cos(ratio[ind_small] * math.pi / threshold.data)) 72 | # print('reduction', reduction.squeeze()[0]) 73 | # print(fdm_grad) 74 | # output.backward() 75 | # print(input.grad.data) 76 | 77 | # gradcheck doesn't have to pass all the time. 78 | test = gradcheck(ReduceThreshold.apply, (input, threshold), eps=eps, atol=1e-3, rtol=1e-2) 79 | print(test) -------------------------------------------------------------------------------- /HyperSphere/feature_map/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/feature_map/modules/__init__.py -------------------------------------------------------------------------------- /HyperSphere/feature_map/modules/kumaraswamy.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy import stats 3 | 4 | import torch 5 | from torch.autograd import Variable 6 | from torch.nn.parameter import Parameter 7 | from HyperSphere.GP.modules.gp_modules import GPModule, log_lower_bnd, log_upper_bnd 8 | 9 | 10 | def id_dim_change(x): 11 | return x 12 | 13 | 14 | class Kumaraswamy(GPModule): 15 | 16 | def __init__(self, ndim, max_input=None): 17 | super(Kumaraswamy, self).__init__() 18 | self.dim_change = id_dim_change 19 | self.ndim = ndim 20 | if max_input is None: 21 | max_input = Variable(torch.ones(ndim)) 22 | elif isinstance(max_input, float): 23 | max_input = Variable(torch.ones(ndim) * max_input) 24 | self.max_input = max_input 25 | self.log_a = Parameter(torch.FloatTensor(ndim)) 26 | self.log_b = Parameter(torch.FloatTensor(ndim)) 27 | self.log_a_max = np.log(2.0) 28 | self.log_b_max = np.log(2.0) 29 | 30 | def reset_parameters(self): 31 | if np.random.uniform() > 0.5: 32 | self.log_a.data.normal_(mean=0, std=0.01).clamp(min=-self.log_a_max, max=0) 33 | else: 34 | self.log_a.data.uniform_(-self.log_a_max, 0) 35 | if np.random.uniform() > 0.5: 36 | self.log_b.data.normal_(mean=0, std=0.01).clamp(min=0, max=self.log_b_max) 37 | else: 38 | self.log_b.data.uniform_(0, self.log_b_max) 39 | 40 | def init_parameters(self): 41 | self.log_a.data.fill_(0) 42 | self.log_b.data.fill_(0) 43 | 44 | def out_of_bounds(self, vec=None): 45 | if vec is None: 46 | return (self.log_a.data > 0).any() or (self.log_a.data < -self.log_a_max).any() or (self.log_b.data < 0).any() or (self.log_b.data > self.log_b_max).any() 47 | else: 48 | return (vec[:1] > 0).any() or (vec[:1] < -self.log_a_max).any() or (vec[1:] < 0).any() or (vec[1:] > self.log_b_max).any() 49 | 50 | def n_params(self): 51 | return 2 52 | 53 | def param_to_vec(self): 54 | return torch.cat([self.log_a.data, self.log_b.data]) 55 | 56 | def vec_to_param(self, vec): 57 | self.log_a.data = vec[:1] 58 | self.log_b.data = vec[1:] 59 | 60 | def prior(self, vec): 61 | # return smp.normal(vec[:1], 0, 0.25) + smp.normal(vec[1:], 0, 0.25) 62 | return np.sum(np.log(stats.norm.pdf(vec[:1], 0, 0.01) + 0.5 / self.log_a_max)) + np.sum(np.log(stats.norm.pdf(vec[1:], 0, 0.01) + 0.5 / self.log_b_max)) 63 | 64 | def forward(self, input): 65 | a = torch.exp(self.log_a) 66 | b = torch.exp(self.log_b) 67 | max_value = self.max_input.type_as(input) 68 | return self.max_input.type_as(input) * (1 - (1 - (input / max_value).clamp(min=0, max=1) ** a) ** b) 69 | 70 | 71 | if __name__ == '__main__': 72 | from HyperSphere.feature_map.functionals import phi_reflection_lp 73 | n = 10 74 | dim = 10 75 | input = Variable(torch.FloatTensor(n, dim).uniform_(-1, 1)) 76 | feature_map = Kumaraswamy() 77 | feature_map.reset_parameters() 78 | print(torch.exp(feature_map.log_p_minus_one.data)[0] + 1) 79 | output1 = feature_map(input) 80 | output2 = phi_reflection_lp(input, torch.exp(feature_map.log_p_minus_one.data)[0] + 1) 81 | print(torch.dist(output1, output2)) -------------------------------------------------------------------------------- /HyperSphere/feature_map/modules/radial_threshold.py: -------------------------------------------------------------------------------- 1 | import scipy.stats as stats 2 | import sampyl as smp 3 | 4 | import torch 5 | from torch.nn.parameter import Parameter 6 | from HyperSphere.GP.modules.gp_modules import GPModule 7 | from HyperSphere.feature_map.functions import reduce_threshold 8 | from HyperSphere.feature_map.functionals import sigmoid_numpy, sigmoid_inv_numpy 9 | 10 | 11 | class RadialThreshold(GPModule): 12 | 13 | def __init__(self): 14 | super(RadialThreshold, self).__init__() 15 | self.sigmoid_inv_threshold = Parameter(torch.FloatTensor(1)) 16 | self.alpha = 2.0 17 | self.beta = 10.0 18 | 19 | def reset_parameters(self): 20 | self.sigmoid_inv_threshold.data.fill_(sigmoid_inv_numpy(stats.beta.rvs(a=self.alpha, b=self.beta))) 21 | 22 | def init_parameters(self): 23 | self.sigmoid_inv_threshold.data.fill_(sigmoid_inv_numpy((self.alpha - 1.0)/(self.alpha + self.beta - 2.0))) 24 | 25 | def out_of_bounds(self, vec=None): 26 | if vec is None: 27 | return (self.sigmoid_inv_threshold.data < -10).any() 28 | else: 29 | return (vec < -10).any() 30 | 31 | def n_params(self): 32 | return 1 33 | 34 | def param_to_vec(self): 35 | return self.sigmoid_inv_threshold.data.clone() 36 | 37 | def vec_to_param(self, vec): 38 | self.sigmoid_inv_threshold.data = vec 39 | 40 | def prior(self, vec): 41 | return smp.beta(sigmoid_numpy(vec), alpha=self.alpha, beta=self.beta) 42 | 43 | def forward(self, input): 44 | return reduce_threshold.ReduceThreshold.apply(input, torch.sigmoid(self.sigmoid_inv_threshold)) 45 | 46 | 47 | 48 | 49 | if __name__ == '__main__': 50 | import torch 51 | from torch.autograd import Variable 52 | from HyperSphere.feature_map.functionals import phi_reflection_threshold 53 | n = 10 54 | dim = 10 55 | input = Variable(torch.FloatTensor(n, dim).uniform_(-1, 1)) 56 | feature_map = ReflectionThreshold() 57 | feature_map.reset_parameters() 58 | print(torch.sigmoid(feature_map.sigmoid_inv_threshold.data)[0]) 59 | output1 = feature_map(input) 60 | output2 = phi_reflection_threshold(input, torch.sigmoid(feature_map.sigmoid_inv_threshold.data)[0]) 61 | print(torch.dist(output1, output2)) -------------------------------------------------------------------------------- /HyperSphere/feature_map/modules/reduce_lp.py: -------------------------------------------------------------------------------- 1 | import sampyl as smp 2 | 3 | import torch 4 | from torch.nn.parameter import Parameter 5 | from HyperSphere.GP.modules.gp_modules import GPModule 6 | from HyperSphere.feature_map.functions import reduce_lp 7 | 8 | 9 | class ReduceLp(GPModule): 10 | 11 | def __init__(self): 12 | super(ReduceLp, self).__init__() 13 | self.log_p_minus_one = Parameter(torch.FloatTensor(1)) 14 | 15 | def reset_parameters(self): 16 | self.log_p_minus_one.data.normal_() 17 | 18 | def init_parameters(self): 19 | self.log_p_minus_one.data.fill_(0.0) 20 | 21 | def out_of_bounds(self, vec=None): 22 | if vec is None: 23 | return (self.log_p_minus_one.data < -10).any() or (self.log_p_minus_one.data > 5).any() 24 | else: 25 | return (vec < -10).any() or (vec > 5).any() 26 | 27 | def n_params(self): 28 | return 1 29 | 30 | def param_to_vec(self): 31 | return self.log_p_minus_one.data.clone() 32 | 33 | def vec_to_param(self, vec): 34 | self.log_p_minus_one.data = vec 35 | 36 | def prior(self, vec): 37 | return smp.normal(vec) 38 | 39 | def forward(self, input): 40 | return reduce_lp.ReduceLp.apply(input, 1 + torch.exp(self.log_p_minus_one)) 41 | 42 | -------------------------------------------------------------------------------- /HyperSphere/feature_map/modules/reduce_threshold.py: -------------------------------------------------------------------------------- 1 | import scipy.stats as stats 2 | import sampyl as smp 3 | 4 | import torch 5 | from torch.nn.parameter import Parameter 6 | from HyperSphere.GP.modules.gp_modules import GPModule 7 | from HyperSphere.feature_map.functions import reduce_threshold 8 | from HyperSphere.feature_map.functionals import sigmoid_numpy, sigmoid_inv_numpy 9 | 10 | 11 | class ReduceThreshold(GPModule): 12 | 13 | def __init__(self): 14 | super(ReduceThreshold, self).__init__() 15 | self.sigmoid_inv_threshold = Parameter(torch.FloatTensor(1)) 16 | self.alpha = 2.0 17 | self.beta = 10.0 18 | 19 | def reset_parameters(self): 20 | self.sigmoid_inv_threshold.data.fill_(sigmoid_inv_numpy(stats.beta.rvs(a=self.alpha, b=self.beta))) 21 | 22 | def init_parameters(self): 23 | self.sigmoid_inv_threshold.data.fill_(sigmoid_inv_numpy((self.alpha - 1.0)/(self.alpha + self.beta - 2.0))) 24 | 25 | def out_of_bounds(self, vec=None): 26 | if vec is None: 27 | return (self.sigmoid_inv_threshold.data < -10).any() 28 | else: 29 | return (vec < -10).any() 30 | 31 | def n_params(self): 32 | return 1 33 | 34 | def param_to_vec(self): 35 | return self.sigmoid_inv_threshold.data.clone() 36 | 37 | def vec_to_param(self, vec): 38 | self.sigmoid_inv_threshold.data = vec 39 | 40 | def prior(self, vec): 41 | return smp.beta(sigmoid_numpy(vec), alpha=self.alpha, beta=self.beta) 42 | 43 | def forward(self, input): 44 | return reduce_threshold.ReduceThreshold.apply(input, torch.sigmoid(self.sigmoid_inv_threshold)) 45 | 46 | -------------------------------------------------------------------------------- /HyperSphere/feature_map/modules/reflection_lp.py: -------------------------------------------------------------------------------- 1 | from HyperSphere.feature_map.functionals import phi_reflection 2 | from HyperSphere.feature_map.modules.reduce_lp import ReduceLp 3 | 4 | 5 | def reflection_lp_dim_change(x): 6 | return x + 1 7 | 8 | 9 | class ReflectionLp(ReduceLp): 10 | 11 | def __init__(self): 12 | super(ReflectionLp, self).__init__() 13 | self.dim_change = reflection_lp_dim_change 14 | 15 | def forward(self, input): 16 | return super(ReflectionLp, self).forward(phi_reflection(input)) 17 | 18 | 19 | if __name__ == '__main__': 20 | import torch 21 | from torch.autograd import Variable 22 | from HyperSphere.feature_map.functionals import phi_reflection_lp 23 | n = 10 24 | dim = 10 25 | input = Variable(torch.FloatTensor(n, dim).uniform_(-1, 1)) 26 | feature_map = ReflectionLp() 27 | feature_map.reset_parameters() 28 | print(torch.exp(feature_map.log_p_minus_one.data)[0] + 1) 29 | output1 = feature_map(input) 30 | output2 = phi_reflection_lp(input, torch.exp(feature_map.log_p_minus_one.data)[0] + 1) 31 | print(torch.dist(output1, output2)) -------------------------------------------------------------------------------- /HyperSphere/feature_map/modules/reflection_threshold.py: -------------------------------------------------------------------------------- 1 | from HyperSphere.feature_map.functionals import phi_reflection 2 | from HyperSphere.feature_map.modules.reduce_threshold import ReduceThreshold 3 | 4 | 5 | def reflection_threshold_dim_change(x): 6 | return x + 1 7 | 8 | 9 | class ReflectionThreshold(ReduceThreshold): 10 | 11 | def __init__(self): 12 | super(ReflectionThreshold, self).__init__() 13 | self.dim_change = reflection_threshold_dim_change 14 | 15 | def forward(self, input): 16 | return super(ReflectionThreshold, self).forward(phi_reflection(input)) 17 | 18 | 19 | if __name__ == '__main__': 20 | import torch 21 | from torch.autograd import Variable 22 | from HyperSphere.feature_map.functionals import phi_reflection_threshold 23 | n = 10 24 | dim = 10 25 | input = Variable(torch.FloatTensor(n, dim).uniform_(-1, 1)) 26 | feature_map = ReflectionThreshold() 27 | feature_map.reset_parameters() 28 | print(torch.sigmoid(feature_map.sigmoid_inv_threshold.data)[0]) 29 | output1 = feature_map(input) 30 | output2 = phi_reflection_threshold(input, torch.sigmoid(feature_map.sigmoid_inv_threshold.data)[0]) 31 | print(torch.dist(output1, output2)) -------------------------------------------------------------------------------- /HyperSphere/feature_map/modules/smooth_lp.py: -------------------------------------------------------------------------------- 1 | from HyperSphere.feature_map.functionals import phi_smooth 2 | from HyperSphere.feature_map.modules.reduce_lp import ReduceLp 3 | 4 | 5 | def smooth_lp_dim_change(x): 6 | return x + 1 7 | 8 | 9 | class SmoothLp(ReduceLp): 10 | 11 | def __init__(self): 12 | super(SmoothLp, self).__init__() 13 | self.dim_change = smooth_lp_dim_change 14 | 15 | def forward(self, input): 16 | return super(SmoothLp, self).forward(phi_smooth(input)) 17 | 18 | 19 | if __name__ == '__main__': 20 | import torch 21 | from torch.autograd import Variable 22 | from HyperSphere.feature_map.functionals import phi_reflection_lp 23 | n = 10 24 | dim = 10 25 | input = Variable(torch.FloatTensor(n, dim).uniform_(-1, 1)) 26 | feature_map = SmoothLp() 27 | feature_map.reset_parameters() 28 | print(torch.exp(feature_map.log_p_minus_one.data)[0] + 1) 29 | output1 = feature_map(input) 30 | output2 = phi_reflection_lp(input, torch.exp(feature_map.log_p_minus_one.data)[0] + 1) 31 | print(torch.dist(output1, output2)) -------------------------------------------------------------------------------- /HyperSphere/interface/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/interface/__init__.py -------------------------------------------------------------------------------- /HyperSphere/interface/hyperparameter_search_method.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by Tijmen Blankevoort 2018 | tijmen@qti.qualcomm.com 3 | 4 | # ============================================================================ 5 | # 6 | # @@-COPYRIGHT-START-@@ 7 | # 8 | # Copyright 2018 Qualcomm Technologies, Inc. All rights reserved. 9 | # Confidential & Proprietary - Qualcomm Technologies, Inc. ("QTI") 10 | # 11 | # The party receiving this software directly from QTI (the "Recipient") 12 | # may use this software as reasonably necessary solely for the purposes 13 | # set forth in the agreement between the Recipient and QTI (the 14 | # "Agreement"). The software may be used in source code form solely by 15 | # the Recipient's employees (if any) authorized by the Agreement. Unless 16 | # expressly authorized in the Agreement, the Recipient may not sublicense, 17 | # assign, transfer or otherwise provide the source code to any third 18 | # party. Qualcomm Technologies, Inc. retains all ownership rights in and 19 | # to the software 20 | # 21 | # This notice supersedes any other QTI notices contained within the software 22 | # except copyright notices indicating different years of publication for 23 | # different portions of the software. This notice does not supersede the 24 | # application of any third party copyright notice to that third party's 25 | # code. 26 | # 27 | # @@-COPYRIGHT-END-@@ 28 | # 29 | # ============================================================================ 30 | 31 | import numpy as np 32 | 33 | class HyperParameterSearchMethod(): 34 | def __init__(self, ranges): 35 | self.result_list = [] 36 | self.experiment_number = 0 37 | self.ranges = ranges 38 | assert type(self.ranges) == list 39 | for tup in self.ranges: 40 | assert type(tup) == tuple or type(tup) == list 41 | 42 | def get_new_setting(self): 43 | raise NotImplementedError('This is a base method, not implemented') 44 | 45 | def submit_result(self, setting, result): 46 | self.result_list.append([setting,result]) 47 | 48 | def get_best_found_setting(self): 49 | best_setting = None 50 | best_score = np.infty 51 | for pair in self.result_dict: 52 | if pair['result'] < best_score: 53 | best_score = pair['result'] 54 | best_setting = pair['setting'] 55 | 56 | return best_setting -------------------------------------------------------------------------------- /HyperSphere/interface/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by Tijmen Blankevoort 2018 | tijmen@qti.qualcomm.com 3 | 4 | # ============================================================================ 5 | # 6 | # @@-COPYRIGHT-START-@@ 7 | # 8 | # Copyright 2018 Qualcomm Technologies, Inc. All rights reserved. 9 | # Confidential & Proprietary - Qualcomm Technologies, Inc. ("QTI") 10 | # 11 | # The party receiving this software directly from QTI (the "Recipient") 12 | # may use this software as reasonably necessary solely for the purposes 13 | # set forth in the agreement between the Recipient and QTI (the 14 | # "Agreement"). The software may be used in source code form solely by 15 | # the Recipient's employees (if any) authorized by the Agreement. Unless 16 | # expressly authorized in the Agreement, the Recipient may not sublicense, 17 | # assign, transfer or otherwise provide the source code to any third 18 | # party. Qualcomm Technologies, Inc. retains all ownership rights in and 19 | # to the software 20 | # 21 | # This notice supersedes any other QTI notices contained within the software 22 | # except copyright notices indicating different years of publication for 23 | # different portions of the software. This notice does not supersede the 24 | # application of any third party copyright notice to that third party's 25 | # code. 26 | # 27 | # @@-COPYRIGHT-END-@@ 28 | # 29 | # ============================================================================ 30 | 31 | from tasks.Rosenbrock import rosenbrock 32 | from HyperSphere.interface.random_search import RandomSearch 33 | from HyperSphere.interface.simple_GP import SimpleGP 34 | import numpy as np 35 | 36 | if __name__ == "__main__": 37 | task = rosenbrock 38 | ranges = [(-1, 3), (-1, 3)] 39 | random_search = RandomSearch(ranges) 40 | optimizer = SimpleGP(ranges) 41 | iterations = 200 42 | num_init = 25 43 | 44 | min_found = np.infty 45 | best_setting = None 46 | 47 | for i in range(num_init): 48 | setting = random_search.get_new_setting() 49 | result = task(setting) 50 | 51 | optimizer.submit_result(setting, result) 52 | print("Random setting {}, gives score: {}".format(setting, result)) 53 | 54 | for i in range(iterations): 55 | setting = optimizer.get_new_setting() 56 | result = task(setting) 57 | 58 | optimizer.submit_result(setting, result) 59 | 60 | if result < min_found: 61 | min_found = result 62 | best_setting = setting 63 | 64 | print('iteration {}, current min found: {}'.format(i, min_found)) 65 | 66 | print("Best minimum found: {}, at {}, " 67 | "after {} iterations".format(min_found, best_setting, iterations)) 68 | 69 | -------------------------------------------------------------------------------- /HyperSphere/interface/random_search.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by Tijmen Blankevoort 2018 | tijmen@qti.qualcomm.com 3 | 4 | # ============================================================================ 5 | # 6 | # @@-COPYRIGHT-START-@@ 7 | # 8 | # Copyright 2018 Qualcomm Technologies, Inc. All rights reserved. 9 | # Confidential & Proprietary - Qualcomm Technologies, Inc. ("QTI") 10 | # 11 | # The party receiving this software directly from QTI (the "Recipient") 12 | # may use this software as reasonably necessary solely for the purposes 13 | # set forth in the agreement between the Recipient and QTI (the 14 | # "Agreement"). The software may be used in source code form solely by 15 | # the Recipient's employees (if any) authorized by the Agreement. Unless 16 | # expressly authorized in the Agreement, the Recipient may not sublicense, 17 | # assign, transfer or otherwise provide the source code to any third 18 | # party. Qualcomm Technologies, Inc. retains all ownership rights in and 19 | # to the software 20 | # 21 | # This notice supersedes any other QTI notices contained within the software 22 | # except copyright notices indicating different years of publication for 23 | # different portions of the software. This notice does not supersede the 24 | # application of any third party copyright notice to that third party's 25 | # code. 26 | # 27 | # @@-COPYRIGHT-END-@@ 28 | # 29 | # ============================================================================ 30 | 31 | from HyperSphere.interface.hyperparameter_search_method import HyperParameterSearchMethod 32 | 33 | import numpy as np 34 | 35 | 36 | class RandomSearch(HyperParameterSearchMethod): 37 | def __init__(self, *args): 38 | super(RandomSearch, self).__init__(*args) 39 | 40 | def get_new_setting(self): 41 | x = np.empty(len(self.ranges)) 42 | 43 | i = 0 44 | for param_range in self.ranges: 45 | x[i] = np.random.uniform(*param_range) 46 | i += 1 47 | 48 | return x -------------------------------------------------------------------------------- /HyperSphere/interface/simple_GP.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by Tijmen Blankevoort 2018 | tijmen@qti.qualcomm.com 3 | 4 | # ============================================================================ 5 | # 6 | # @@-COPYRIGHT-START-@@ 7 | # 8 | # Copyright 2018 Qualcomm Technologies, Inc. All rights reserved. 9 | # Confidential & Proprietary - Qualcomm Technologies, Inc. ("QTI") 10 | # 11 | # The party receiving this software directly from QTI (the "Recipient") 12 | # may use this software as reasonably necessary solely for the purposes 13 | # set forth in the agreement between the Recipient and QTI (the 14 | # "Agreement"). The software may be used in source code form solely by 15 | # the Recipient's employees (if any) authorized by the Agreement. Unless 16 | # expressly authorized in the Agreement, the Recipient may not sublicense, 17 | # assign, transfer or otherwise provide the source code to any third 18 | # party. Qualcomm Technologies, Inc. retains all ownership rights in and 19 | # to the software 20 | # 21 | # This notice supersedes any other QTI notices contained within the software 22 | # except copyright notices indicating different years of publication for 23 | # different portions of the software. This notice does not supersede the 24 | # application of any third party copyright notice to that third party's 25 | # code. 26 | # 27 | # @@-COPYRIGHT-END-@@ 28 | # 29 | # ============================================================================ 30 | 31 | import numpy as np 32 | 33 | from HyperSphere.interface.hyperparameter_search_method import HyperParameterSearchMethod 34 | from sklearn.gaussian_process import GaussianProcessRegressor 35 | from sklearn.gaussian_process.kernels import Matern 36 | from scipy.optimize import minimize 37 | from scipy.stats import norm 38 | 39 | 40 | class SimpleGP(HyperParameterSearchMethod): 41 | def __init__(self, *args): 42 | super(SimpleGP, self).__init__(*args) 43 | self.gp = GaussianProcessRegressor( 44 | kernel=Matern(nu=2.5), 45 | n_restarts_optimizer=25, 46 | alpha=1e-4, 47 | ) 48 | 49 | def make_X_Y(self): 50 | X = np.vstack([result[0] for result in self.result_list]) 51 | Y = np.vstack([result[1] for result in self.result_list]) 52 | return X, Y 53 | 54 | def expected_improvement(self, x, gaussian_process, evaluated_loss, n_params=1): 55 | """ expected_improvement 56 | Expected improvement acquisition function. 57 | Arguments: 58 | ---------- 59 | x: array-like, shape = [n_samples, n_hyperparams] 60 | The point for which the expected improvement needs to be computed. 61 | gaussian_process: GaussianProcessRegressor object. 62 | Gaussian process trained on previously evaluated hyperparameters. 63 | evaluated_loss: Numpy array. 64 | Numpy array that contains the values off the loss function for the previously 65 | evaluated hyperparameters. 66 | greater_is_better: Boolean. 67 | Boolean flag that indicates whether the loss function is to be maximised or minimised. 68 | n_params: int. 69 | Dimension of the hyperparameter space. 70 | """ 71 | 72 | x_to_predict = x.reshape(-1, n_params) 73 | 74 | mu, sigma = gaussian_process.predict(x_to_predict, return_std=True) 75 | 76 | loss_optimum = np.min(evaluated_loss) 77 | 78 | scaling_factor = -1 79 | 80 | # In case sigma equals zero 81 | with np.errstate(divide='ignore'): 82 | Z = scaling_factor * (mu - loss_optimum) / sigma 83 | expected_improvement = scaling_factor * (mu - loss_optimum) * norm.cdf(Z) + sigma * norm.pdf(Z) 84 | expected_improvement[sigma == 0.0] == 0.0 85 | 86 | return -1 * expected_improvement 87 | 88 | 89 | def sample_next_hyperparameter(self, 90 | evaluated_loss, 91 | bounds=(0, 10), n_restarts=25): 92 | """ sample_next_hyperparameter 93 | Proposes the next hyperparameter to sample the loss function for. 94 | Arguments: 95 | ---------- 96 | acquisition_func: function. 97 | Acquisition function to optimise. 98 | gaussian_process: GaussianProcessRegressor object. 99 | Gaussian process trained on previously evaluated hyperparameters. 100 | evaluated_loss: array-like, shape = [n_obs,] 101 | Numpy array that contains the values off the loss function for the previously 102 | evaluated hyperparameters. 103 | greater_is_better: Boolean. 104 | Boolean flag that indicates whether the loss function is to be maximised or minimised. 105 | bounds: Tuple. 106 | Bounds for the L-BFGS optimiser. 107 | n_restarts: integer. 108 | Number of times to run the minimiser with different starting points. 109 | """ 110 | best_x = None 111 | best_acquisition_value = 1 112 | n_params = bounds.shape[0] 113 | 114 | for starting_point in np.random.uniform(bounds[:, 0], bounds[:, 1], 115 | size=(n_restarts, n_params)): 116 | 117 | res = minimize(fun=self.expected_improvement, 118 | x0=starting_point.reshape(1, -1), 119 | bounds=bounds, 120 | method='L-BFGS-B', 121 | args=(self.gp, evaluated_loss,n_params)) 122 | 123 | if res.fun < best_acquisition_value: 124 | best_acquisition_value = res.fun 125 | best_x = res.x 126 | 127 | return best_x 128 | 129 | def get_new_setting(self): 130 | X, Y = self.make_X_Y() 131 | self.gp.fit(X, Y) 132 | 133 | next_sample = self.sample_next_hyperparameter(Y, bounds=np.array(self.ranges), n_restarts=100) 134 | 135 | return next_sample -------------------------------------------------------------------------------- /HyperSphere/test_functions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ChangYong-Oh/HyperSphere/ba4a849d05cd6696b7ba2c62531943ff1cb0fbb5/HyperSphere/test_functions/__init__.py -------------------------------------------------------------------------------- /HyperSphere/test_functions/cifar10_weight.py: -------------------------------------------------------------------------------- 1 | import os 2 | import math 3 | from datetime import datetime 4 | import pickle 5 | import subprocess 6 | import sys 7 | reload(sys) 8 | sys.setdefaultencoding('utf-8') 9 | 10 | import numpy as np 11 | import GPUtil 12 | import torch 13 | from torch.autograd import Variable 14 | NDIM = 54 15 | 16 | 17 | def stochastic_depth_resnet_cifar10(probability_tensor): 18 | return _stochastic_depth_resnet(probability_tensor, 'cifar10+') 19 | 20 | stochastic_depth_resnet_cifar10.dim = NDIM 21 | 22 | 23 | def stochastic_depth_resnet_cifar100(probability_tensor): 24 | return _stochastic_depth_resnet(probability_tensor, 'cifar100+') 25 | 26 | stochastic_depth_resnet_cifar100.dim = NDIM 27 | 28 | 29 | def transform_with_center(x, center_probability=0.5): 30 | if isinstance(center_probability, (float, int)): 31 | center_probability = x.data.clone() * 0 + center_probability 32 | assert x.numel() == center_probability.numel() 33 | assert torch.sum(center_probability > 1.0) == 0 34 | assert torch.sum(center_probability < 0.0) == 0 35 | 36 | shift = [] 37 | for i in range(center_probability.numel()): 38 | poly_d = center_probability.squeeze()[i] 39 | if poly_d == 0: 40 | shift.append(-1.0) 41 | elif poly_d == 1: 42 | shift.append(1.0) 43 | elif 0 < poly_d < 1: 44 | poly_zeros = np.roots([-0.25, 0, 0.75, 0.5 - poly_d]) 45 | shift.append(poly_zeros[np.argmin(np.abs(poly_zeros))]) 46 | if hasattr(x, 'data'): 47 | shift = Variable(torch.FloatTensor(shift)).type_as(x).resize_as(x) 48 | else: 49 | shift = torch.FloatTensor(shift).type_as(x).view_as(x) 50 | 51 | x = ((x + 1 + shift) * 0.5).clamp(min=0, max=1) 52 | y = 3 * x ** 2 - 2 * x ** 3 53 | 54 | return y 55 | 56 | 57 | def _stochastic_depth_resnet(probability_tensor, data_type): 58 | time_tag = datetime.now().strftime('%Y%m%d-%H:%M:%S:%f') 59 | 60 | stochastic_depth_dir = os.path.join(os.path.abspath(os.path.join(os.path.split(__file__)[0], '../../../')), 'img_classification_pk_pytorch') 61 | pretrain_dir = os.path.join(stochastic_depth_dir, 'save', 'cifar100+_warm-start') 62 | 63 | save_dir = os.path.join(stochastic_depth_dir, 'save', data_type + '_' + time_tag) 64 | 65 | probability_filename = os.path.join(stochastic_depth_dir, 'save', 'stochastic_depth_death_rate_' + data_type + '_' + time_tag + '.pkl') 66 | probability_file = open(probability_filename, 'w') 67 | # Use information given in stochastic resnet paper setting as the center point 68 | probability_list = transform_with_center(probability_tensor, 0.5) 69 | pickle.dump(list(probability_list.data if hasattr(probability_list, 'data') else probability_list), probability_file) 70 | probability_file.close() 71 | 72 | gpu_device = str(GPUtil.getAvailable(order='random', limit=1)[0]) 73 | 74 | cmd_str = 'cd ' + stochastic_depth_dir + ';' 75 | cmd_str += 'CUDA_VISIBLE_DEVICES=' + gpu_device + ' python main.py' 76 | cmd_str += ' --data ' + data_type + ' --normalized' 77 | cmd_str += ' --resume ' + os.path.join(pretrain_dir, 'model_best.pth.tar ') + ' --save ' + save_dir 78 | cmd_str += ' --death-mode chosen --death-rate-filename ' + probability_filename 79 | cmd_str += ' --decay_rate 0.1 --decay_epoch_ratio 0.5 --learning-rate 0.01 --epoch 250' 80 | print(('=' * 20) + 'COMMAND' + ('=' * 20)) 81 | print(cmd_str) 82 | process = subprocess.Popen(cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 83 | lastline = '' 84 | while True: 85 | nextline = process.stdout.readline() 86 | if nextline == '': 87 | if process.poll() is not None: 88 | break 89 | else: 90 | lastline = nextline 91 | sys.stdout.write(nextline) 92 | sys.stdout.flush() 93 | return torch.FloatTensor([[float(lastline)]]) 94 | 95 | 96 | if __name__ == '__main__': 97 | print('Return value : ', _stochastic_depth_resnet(torch.rand(54), 'cifar100+')) 98 | 99 | 100 | 101 | # CUDA_VISIBLE_DEVICES=0 python main.py --data cifar100+ --arch resnet --depth 110 --save save/cifar100+-resnet-110-batch256 --batch-size 256 --epochs 50 -------------------------------------------------------------------------------- /HyperSphere/test_functions/mnist_weight.py: -------------------------------------------------------------------------------- 1 | import sys 2 | reload(sys) 3 | sys.setdefaultencoding('utf-8') 4 | import progressbar 5 | from scipy.io import loadmat 6 | 7 | import torch 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | from torch.nn.parameter import Parameter 11 | from torch.autograd import Variable 12 | import torch.cuda as cuda 13 | import torch.optim as optim 14 | from torch.utils.data import DataLoader, sampler 15 | from torchvision import datasets, transforms 16 | 17 | 18 | BATCH_SIZE = 64 19 | EPOCH = 20 20 | USE_VALIDATION = True 21 | 22 | 23 | class Net(nn.Module): 24 | def __init__(self, n_hid, hid_weight=None): 25 | super(Net, self).__init__() 26 | self.fc1 = nn.Linear(784, n_hid) 27 | if hid_weight is None: 28 | self.hid_weight = None 29 | self.fc2 = nn.Linear(n_hid, 10) 30 | else: 31 | self.hid_weight = hid_weight if hasattr(hid_weight, 'data') else Variable(hid_weight) 32 | self.hid_bias = Parameter(torch.Tensor(10)) 33 | # hid_bias is also optimized by SGD 34 | # this is just for making problem 100, 200, 500 not 110, 210, 510 35 | 36 | def forward(self, x): 37 | x = x.view(-1, 784) 38 | x = F.relu(self.fc1(x)) 39 | if self.hid_weight is None: 40 | x = self.fc2(x) 41 | else: 42 | if x.is_cuda: 43 | self.hid_weight = self.hid_weight.cuda() 44 | x = F.linear(x, self.hid_weight, self.hid_bias) 45 | return F.log_softmax(x) 46 | 47 | 48 | def load_mnist(batch_size, use_cuda, use_validation): 49 | kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} 50 | transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]) 51 | mnist_train = datasets.MNIST('../data', train=True, download=True, transform=transform) 52 | mnist_test = datasets.MNIST('../data', train=False, transform=transform) 53 | if use_validation: 54 | train_sampler = sampler.SubsetRandomSampler(range(45000)) 55 | validation_sampler = sampler.SubsetRandomSampler(range(45000, 50000)) 56 | train_loader = DataLoader(mnist_train, batch_size=batch_size, shuffle=False, sampler=train_sampler, **kwargs) 57 | validation_loader = DataLoader(mnist_train, batch_size=batch_size, shuffle=False, sampler=validation_sampler, **kwargs) 58 | else: 59 | train_loader = DataLoader(mnist_train, batch_size=batch_size, shuffle=True, **kwargs) 60 | test_loader = DataLoader(mnist_test, batch_size=batch_size, shuffle=False, **kwargs) 61 | if use_validation: 62 | return train_loader, validation_loader, test_loader 63 | else: 64 | return train_loader, test_loader 65 | 66 | 67 | def train(train_loader, model, epoch, optimizer, use_cuda): 68 | model.train() 69 | progress = progressbar.ProgressBar(max_value=epoch) 70 | for e in range(epoch): 71 | for batch_idx, (data, target) in enumerate(train_loader): 72 | if use_cuda: 73 | data, target = data.cuda(), target.cuda() 74 | data, target = Variable(data), Variable(target) 75 | optimizer.zero_grad() 76 | output = model(data) 77 | loss = F.nll_loss(output, target) 78 | loss.backward() 79 | optimizer.step() 80 | progress.update(e) 81 | 82 | 83 | def evaluation(evaluation_loader, model, use_cuda): 84 | model.eval() 85 | evaluation_loss = 0 86 | correct = 0 87 | n_eval_data = 0 88 | for data, target in evaluation_loader: 89 | n_eval_data += data.size(0) 90 | if use_cuda: 91 | data, target = data.cuda(), target.cuda() 92 | data, target = Variable(data, volatile=True), Variable(target) 93 | output = model(data) 94 | evaluation_loss += F.nll_loss(output, target, size_average=False).data[0] # sum up batch loss 95 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 96 | correct += pred.eq(target.data.view_as(pred)).cpu().sum() 97 | 98 | evaluation_loss /= float(n_eval_data) 99 | evaluation_accuracy = correct / float(n_eval_data) 100 | return evaluation_loss, evaluation_accuracy 101 | 102 | 103 | def mnist_weight(weight_vector, use_BO=True, use_validation=USE_VALIDATION, use_cuda=True): 104 | use_cuda = cuda.is_available() and use_cuda 105 | if use_BO: 106 | model = Net(n_hid=weight_vector.numel() / 10, hid_weight=weight_vector.view(10, -1)) 107 | else: 108 | model = Net(n_hid=weight_vector.numel() / 10, hid_weight=None) 109 | for m in model.parameters(): 110 | if m.dim() == 2: 111 | nn.init.xavier_normal(m.data) 112 | else: 113 | m.data.normal_() 114 | if use_cuda: 115 | model.cuda() 116 | if use_validation: 117 | train_loader, validation_loader, test_loader = load_mnist(BATCH_SIZE, use_cuda, use_validation) 118 | else: 119 | train_loader, test_loader = load_mnist(BATCH_SIZE, use_cuda, use_validation) 120 | if use_BO: 121 | optimizer = optim.Adam(model.parameters()) 122 | else: 123 | optimizer = optim.Adam(model.parameters(), weight_decay=0.0001) 124 | train(train_loader, model, EPOCH, optimizer, use_cuda) 125 | if use_validation: 126 | loss, accuracy = evaluation(validation_loader, model, use_cuda) 127 | else: 128 | loss, accuracy = evaluation(test_loader, model, use_cuda) 129 | # if not use_BO: 130 | # print('Entirely with SGD(Adam)') 131 | # print(model.fc2.weight.data) 132 | print('\n%s loss : %f / Accuracy : %6.4f' % ('Validation' if use_validation else 'Test', loss, accuracy)) 133 | 134 | if use_BO: 135 | return torch.FloatTensor([[loss]]) 136 | if not use_BO: 137 | weight_radius = (torch.sum([elm[1] for elm in model.named_parameters() if elm[0]=='fc2.weight'][0] ** 2) ** 0.5).data[0] 138 | return torch.FloatTensor([[loss]]), weight_radius 139 | 140 | mnist_weight.dim = 0 141 | 142 | 143 | def mnist_weight_baseline(ndim, type='loss'): 144 | if ndim == 100: 145 | if type == 'loss': 146 | return [0.242009, 0.230133, 0.216998, 0.222007, 0.242975] 147 | elif type == 'accuracy': 148 | return [0.9322, 0.9349, 0.9388, 0.9406, 0.9321] 149 | elif ndim == 200: 150 | if type == 'loss': 151 | return [0.145960, 0.159507, 0.140117, 0.140135, 0.165476] 152 | elif type == 'accuracy': 153 | return [0.9585, 0.9559, 0.9605, 0.9619, 0.9545] 154 | elif ndim == 500: 155 | if type == 'loss': 156 | return [0.118356, 0.132987, 0.122135, 0.132618, 0.121591] 157 | elif type == 'accuracy': 158 | return [0.9726, 0.9729, 0.9732, 0.9728, 0.9729] 159 | 160 | if __name__ == '__main__': 161 | weight_vector = torch.randn(500) 162 | print(mnist_weight(weight_vector)) 163 | 164 | # 10 by 10 case 165 | # Loss : 0.242009 / Accuracy : 0.9322 166 | # Loss : 0.230133 / Accuracy : 0.9349 167 | # Loss : 0.216998 / Accuracy : 0.9388 168 | # Loss : 0.222007 / Accuracy : 0.9406 169 | # Loss : 0.242975 / Accuracy : 0.9321 170 | 171 | # 10 by 20 case 172 | # Loss : 0.145960 / Accuracy : 0.9585 173 | # Loss : 0.159507 / Accuracy : 0.9559 174 | # Loss : 0.140117 / Accuracy : 0.9605 175 | # Loss : 0.140135 / Accuracy : 0.9619 176 | # Loss : 0.165476 / Accuracy : 0.9545 177 | 178 | # 10 by 50 case 179 | # Loss : 0.118356 / Accuracy : 0.9726 180 | # Loss : 0.132987 / Accuracy : 0.9729 181 | # Loss : 0.122135 / Accuracy : 0.9732 182 | # Loss : 0.132618 / Accuracy : 0.9728 183 | # Loss : 0.121591 / Accuracy : 0.9729 184 | -------------------------------------------------------------------------------- /HyperSphere/test_functions/stochastic_depth_resnet.py: -------------------------------------------------------------------------------- 1 | import os 2 | import math 3 | from datetime import datetime 4 | import pickle 5 | import subprocess 6 | import sys 7 | reload(sys) 8 | sys.setdefaultencoding('utf-8') 9 | 10 | import numpy as np 11 | import GPUtil 12 | import torch 13 | from torch.autograd import Variable 14 | NDIM = 54 15 | 16 | 17 | def stochastic_depth_resnet_cifar10(probability_tensor): 18 | return _stochastic_depth_resnet(probability_tensor, 'cifar10+') 19 | 20 | stochastic_depth_resnet_cifar10.dim = NDIM 21 | 22 | 23 | def stochastic_depth_resnet_cifar100(probability_tensor): 24 | return _stochastic_depth_resnet(probability_tensor, 'cifar100+') 25 | 26 | stochastic_depth_resnet_cifar100.dim = NDIM 27 | 28 | 29 | def transform_with_center(x, center_probability=0.5): 30 | if isinstance(center_probability, (float, int)): 31 | center_probability = x.data.clone() * 0 + center_probability 32 | assert x.numel() == center_probability.numel() 33 | assert torch.sum(center_probability > 1.0) == 0 34 | assert torch.sum(center_probability < 0.0) == 0 35 | 36 | shift = [] 37 | for i in range(center_probability.numel()): 38 | poly_d = center_probability.squeeze()[i] 39 | if poly_d == 0: 40 | shift.append(-1.0) 41 | elif poly_d == 1: 42 | shift.append(1.0) 43 | elif 0 < poly_d < 1: 44 | poly_zeros = np.roots([-0.25, 0, 0.75, 0.5 - poly_d]) 45 | shift.append(poly_zeros[np.argmin(np.abs(poly_zeros))]) 46 | if hasattr(x, 'data'): 47 | shift = Variable(torch.FloatTensor(shift)).type_as(x).resize_as(x) 48 | else: 49 | shift = torch.FloatTensor(shift).type_as(x).view_as(x) 50 | 51 | x = ((x + 1 + shift) * 0.5).clamp(min=0, max=1) 52 | y = 3 * x ** 2 - 2 * x ** 3 53 | 54 | return y 55 | 56 | 57 | def _stochastic_depth_resnet(probability_tensor, data_type): 58 | time_tag = datetime.now().strftime('%Y%m%d-%H:%M:%S:%f') 59 | 60 | stochastic_depth_dir = os.path.join(os.path.abspath(os.path.join(os.path.split(__file__)[0], '../../../')), 'img_classification_pk_pytorch') 61 | pretrain_dir = os.path.join(stochastic_depth_dir, 'save', 'cifar100+_warm-start') 62 | 63 | save_dir = os.path.join(stochastic_depth_dir, 'save', data_type + '_' + time_tag) 64 | 65 | probability_filename = os.path.join(stochastic_depth_dir, 'save', 'stochastic_depth_death_rate_' + data_type + '_' + time_tag + '.pkl') 66 | probability_file = open(probability_filename, 'w') 67 | # Use information given in stochastic resnet paper setting as the center point 68 | probability_list = transform_with_center(probability_tensor, 0.5) 69 | pickle.dump(list(probability_list.data if hasattr(probability_list, 'data') else probability_list), probability_file) 70 | probability_file.close() 71 | 72 | gpu_device = str(GPUtil.getAvailable(order='random', limit=1)[0]) 73 | 74 | cmd_str = 'cd ' + stochastic_depth_dir + ';' 75 | cmd_str += 'CUDA_VISIBLE_DEVICES=' + gpu_device + ' python main.py' 76 | cmd_str += ' --data ' + data_type + ' --normalized' 77 | cmd_str += ' --resume ' + os.path.join(pretrain_dir, 'model_best.pth.tar ') + ' --save ' + save_dir 78 | cmd_str += ' --death-mode chosen --death-rate-filename ' + probability_filename 79 | cmd_str += ' --decay_rate 0.1 --decay_epoch_ratio 0.5 --learning-rate 0.01 --epoch 250' 80 | print(('=' * 20) + 'COMMAND' + ('=' * 20)) 81 | print(cmd_str) 82 | process = subprocess.Popen(cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 83 | lastline = '' 84 | while True: 85 | nextline = process.stdout.readline() 86 | if nextline == '': 87 | if process.poll() is not None: 88 | break 89 | else: 90 | lastline = nextline 91 | sys.stdout.write(nextline) 92 | sys.stdout.flush() 93 | return torch.FloatTensor([[float(lastline)]]) 94 | 95 | 96 | if __name__ == '__main__': 97 | print('Return value : ', _stochastic_depth_resnet(torch.rand(54), 'cifar100+')) 98 | 99 | 100 | 101 | # CUDA_VISIBLE_DEVICES=0 python main.py --data cifar100+ --arch resnet --depth 110 --save save/cifar100+-resnet-110-batch256 --batch-size 256 --epochs 50 -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018, QUVA-Lab, University of Amsterdam 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | The views and conclusions contained in the software and documentation are those 25 | of the authors and should not be interpreted as representing official policies, 26 | either expressed or implied, of the FreeBSD Project. -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | backports.weakref==1.0rc1 2 | bleach==1.5.0 3 | cycler==0.10.0 4 | dill 5 | enum34==1.1.6 6 | funcsigs==1.0.2 7 | functools32==3.2.3.post2 8 | html5lib==0.9999999 9 | Keras==2.0.5 10 | Markdown==2.2.0 11 | matplotlib==2.0.2 12 | mock==2.0.0 13 | numpy==1.13.0 14 | olefile==0.44 15 | pbr==3.1.1 16 | Pillow==4.1.1 17 | progressbar2 18 | protobuf==3.3.0 19 | psutil 20 | py==1.4.34 21 | pyparsing==2.2.0 22 | pytest==3.1.2 23 | python-dateutil==2.6.0 24 | pytz==2017.2 25 | PyYAML==3.12 26 | sampyl-mcmc 27 | scipy==0.19.1 28 | si-prefix==0.4.post10 29 | six==1.10.0 30 | subprocess32==3.2.7 31 | torch==0.3.1 32 | torchvision 33 | tabulate==0.7.7 34 | Werkzeug==0.12.2 35 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [pytest] 2 | norecursedirs = venv 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name = "HyperSphere", 5 | author = "ChangYong-Ohs", 6 | version = 0, 7 | dependency_links = (), 8 | install_requires = ['numpy'], 9 | scripts = [], 10 | packages=find_packages(), 11 | ) 12 | -------------------------------------------------------------------------------- /setup_pip.sh: -------------------------------------------------------------------------------- 1 | virtualenv venv 2 | source venv/bin/activate 3 | pip install --upgrade pip 4 | pip install -r requirements.txt 5 | pip install -e . 6 | --------------------------------------------------------------------------------