├── .gitignore ├── LICENSE ├── NOTICE ├── README.md ├── dso ├── dso │ ├── __init__.py │ ├── checkpoint.py │ ├── config │ │ ├── __init__.py │ │ ├── config_common.json │ │ ├── config_control.json │ │ ├── config_regression.json │ │ ├── config_regression_gp.json │ │ └── examples │ │ │ ├── control │ │ │ ├── BipedalWalker.json │ │ │ ├── CustomCartPoleContinuous-v0.json │ │ │ ├── CustomCartPoleContinuous-v0_DecisionTree.json │ │ │ ├── Hopper.json │ │ │ ├── LunarLander.json │ │ │ ├── LunarLanderMultiDiscrete.json │ │ │ └── MountainCar-v0.json │ │ │ └── regression │ │ │ ├── Nguyen-2-GPmeld.json │ │ │ ├── Nguyen-2.json │ │ │ ├── PiecewiseFunction-1.json │ │ │ └── Poly-1.json │ ├── const.py │ ├── core.py │ ├── cyfunc.pyx │ ├── execute.py │ ├── functions.py │ ├── gp │ │ ├── __init__.py │ │ ├── base.py │ │ ├── gp_controller.py │ │ └── utils.py │ ├── language_model │ │ ├── __init__.py │ │ ├── language_model_prior.py │ │ └── model │ │ │ ├── model_dyn_rnn.py │ │ │ └── saved_model │ │ │ ├── checkpoint │ │ │ ├── dynrnn.data-00000-of-00001 │ │ │ ├── dynrnn.index │ │ │ ├── dynrnn.meta │ │ │ └── word_dict.pkl │ ├── library.py │ ├── logeval.py │ ├── memory.py │ ├── policy │ │ ├── __init__.py │ │ ├── policy.py │ │ └── rnn_policy.py │ ├── policy_optimizer │ │ ├── __init__.py │ │ ├── pg_policy_optimizer.py │ │ ├── policy_optimizer.py │ │ ├── ppo_policy_optimizer.py │ │ └── pqt_policy_optimizer.py │ ├── prior.py │ ├── program.py │ ├── run.py │ ├── scripts │ │ └── search_space.py │ ├── subroutines.py │ ├── task │ │ ├── __init__.py │ │ ├── control │ │ │ ├── __init__.py │ │ │ ├── control.py │ │ │ ├── data │ │ │ │ ├── BipedalWalker-v2 │ │ │ │ │ └── model-td3.zip │ │ │ │ ├── CustomCartPoleContinuous-v0 │ │ │ │ │ └── model-td3.zip │ │ │ │ ├── HopperBulletEnv-v0 │ │ │ │ │ └── model-td3.pkl │ │ │ │ ├── InvertedDoublePendulumBulletEnv-v0 │ │ │ │ │ └── model-td3.pkl │ │ │ │ ├── InvertedPendulumSwingupBulletEnv-v0 │ │ │ │ │ └── model-sac.pkl │ │ │ │ ├── LunarLanderContinuous-v2 │ │ │ │ │ └── model-sac.pkl │ │ │ │ ├── MountainCarContinuous-v0 │ │ │ │ │ └── model-td3.pkl │ │ │ │ ├── Pendulum-v0 │ │ │ │ │ └── model-td3.pkl │ │ │ │ └── ReacherBulletEnv-v0 │ │ │ │ │ └── model-td3.pkl │ │ │ ├── envs │ │ │ │ ├── cartpole_bullet.py │ │ │ │ ├── continuous_cartpole.py │ │ │ │ ├── lander.py │ │ │ │ ├── pendulum.py │ │ │ │ └── test │ │ │ │ │ ├── test_envs.py │ │ │ │ │ └── test_lander.py │ │ │ ├── scripts │ │ │ │ ├── README.md │ │ │ │ ├── benchmark_zoo.py │ │ │ │ ├── policy_eval.py │ │ │ │ └── sample_zoo.py │ │ │ └── utils.py │ │ ├── regression │ │ │ ├── __init__.py │ │ │ ├── benchmarks.csv │ │ │ ├── data │ │ │ │ ├── Constant-1.csv │ │ │ │ ├── Constant-1_test.csv │ │ │ │ ├── Constant-2.csv │ │ │ │ ├── Constant-2_test.csv │ │ │ │ ├── Constant-3.csv │ │ │ │ ├── Constant-3_test.csv │ │ │ │ ├── Constant-4.csv │ │ │ │ ├── Constant-4_test.csv │ │ │ │ ├── Constant-5.csv │ │ │ │ ├── Constant-5_test.csv │ │ │ │ ├── Constant-6.csv │ │ │ │ ├── Constant-6_test.csv │ │ │ │ ├── Constant-7.csv │ │ │ │ ├── Constant-7_test.csv │ │ │ │ ├── Constant-8.csv │ │ │ │ ├── Constant-8_test.csv │ │ │ │ ├── Custom-1.csv │ │ │ │ ├── Custom-10.csv │ │ │ │ ├── Custom-10_test.csv │ │ │ │ ├── Custom-11.csv │ │ │ │ ├── Custom-11_test.csv │ │ │ │ ├── Custom-12.csv │ │ │ │ ├── Custom-12_test.csv │ │ │ │ ├── Custom-13.csv │ │ │ │ ├── Custom-13_test.csv │ │ │ │ ├── Custom-14.csv │ │ │ │ ├── Custom-14_test.csv │ │ │ │ ├── Custom-15.csv │ │ │ │ ├── Custom-15_test.csv │ │ │ │ ├── Custom-16.csv │ │ │ │ ├── Custom-16_test.csv │ │ │ │ ├── Custom-17.csv │ │ │ │ ├── Custom-17_test.csv │ │ │ │ ├── Custom-18.csv │ │ │ │ ├── Custom-18_test.csv │ │ │ │ ├── Custom-19.csv │ │ │ │ ├── Custom-19_test.csv │ │ │ │ ├── Custom-1_test.csv │ │ │ │ ├── Custom-2.csv │ │ │ │ ├── Custom-20.csv │ │ │ │ ├── Custom-20_test.csv │ │ │ │ ├── Custom-21.csv │ │ │ │ ├── Custom-21_test.csv │ │ │ │ ├── Custom-22.csv │ │ │ │ ├── Custom-22_test.csv │ │ │ │ ├── Custom-23.csv │ │ │ │ ├── Custom-23_test.csv │ │ │ │ ├── Custom-2_test.csv │ │ │ │ ├── Custom-3.csv │ │ │ │ ├── Custom-3_test.csv │ │ │ │ ├── Custom-4.csv │ │ │ │ ├── Custom-4_test.csv │ │ │ │ ├── Custom-5.csv │ │ │ │ ├── Custom-5_test.csv │ │ │ │ ├── Custom-6.csv │ │ │ │ ├── Custom-6_test.csv │ │ │ │ ├── Custom-7.csv │ │ │ │ ├── Custom-7_test.csv │ │ │ │ ├── Custom-8.csv │ │ │ │ ├── Custom-8_test.csv │ │ │ │ ├── Custom-9.csv │ │ │ │ ├── Custom-9_test.csv │ │ │ │ ├── Nguyen-1.csv │ │ │ │ ├── Nguyen-10.csv │ │ │ │ ├── Nguyen-10_test.csv │ │ │ │ ├── Nguyen-11.csv │ │ │ │ ├── Nguyen-11_test.csv │ │ │ │ ├── Nguyen-12.csv │ │ │ │ ├── Nguyen-12_test.csv │ │ │ │ ├── Nguyen-1_test.csv │ │ │ │ ├── Nguyen-2.csv │ │ │ │ ├── Nguyen-2_test.csv │ │ │ │ ├── Nguyen-3.csv │ │ │ │ ├── Nguyen-3_test.csv │ │ │ │ ├── Nguyen-4.csv │ │ │ │ ├── Nguyen-4_test.csv │ │ │ │ ├── Nguyen-5.csv │ │ │ │ ├── Nguyen-5_test.csv │ │ │ │ ├── Nguyen-6.csv │ │ │ │ ├── Nguyen-6_test.csv │ │ │ │ ├── Nguyen-7.csv │ │ │ │ ├── Nguyen-7_test.csv │ │ │ │ ├── Nguyen-8.csv │ │ │ │ ├── Nguyen-8_test.csv │ │ │ │ ├── Nguyen-9.csv │ │ │ │ ├── Nguyen-9_test.csv │ │ │ │ ├── PiecewiseFunction-1.csv │ │ │ │ ├── Test-1.csv │ │ │ │ └── Test-1_test.csv │ │ │ ├── dataset.py │ │ │ ├── function_sets.csv │ │ │ ├── mat_mult_benchmark.py │ │ │ ├── polyfit.py │ │ │ ├── regression.py │ │ │ ├── sklearn.py │ │ │ └── test_sklearn.py │ │ └── task.py │ ├── test │ │ ├── __init__.py │ │ ├── custom_tasks │ │ │ ├── custom_task_prior.py │ │ │ └── test_custom_task_prior.py │ │ ├── data │ │ │ ├── test_model.meta │ │ │ ├── test_model_strong.data-00000-of-00001 │ │ │ ├── test_model_strong.index │ │ │ ├── test_model_weak.data-00000-of-00001 │ │ │ └── test_model_weak.index │ │ ├── generate_test_data.py │ │ ├── test_checkpoint.py │ │ ├── test_constant.py │ │ ├── test_core.py │ │ ├── test_polynomial.py │ │ └── test_prior.py │ ├── tf_state_manager.py │ ├── train.py │ ├── train_stats.py │ ├── utils.py │ └── variance.py └── setup.py └── images ├── banner.png ├── code_map.png ├── srbench_accuracy-solution.png └── srbench_symbolic-solution.png /.gitignore: -------------------------------------------------------------------------------- 1 | *.DS_Store 2 | *.pyc 3 | *.egg* 4 | venv* 5 | dso/dso/summary* 6 | *log_* 7 | .ipynb_checkpoints 8 | ~$* 9 | *.vscode/ 10 | dso/build 11 | dso/dso/cyfunc.c 12 | dso/dso/cyfunc*.so 13 | **/log/ 14 | /tmp 15 | /demo-web/user 16 | /demo-web/static/image/user/ 17 | *videos 18 | *bsub 19 | dep-repos/ 20 | .* 21 | !/.gitignore 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Lawrence Livermore National Security, LLC 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | This work was produced under the auspices of the U.S. Department of 2 | Energy by Lawrence Livermore National Laboratory under Contract 3 | DE-AC52-07NA27344. 4 | 5 | This work was prepared as an account of work sponsored by an agency of 6 | the United States Government. Neither the United States Government nor 7 | Lawrence Livermore National Security, LLC, nor any of their employees 8 | makes any warranty, expressed or implied, or assumes any legal liability 9 | or responsibility for the accuracy, completeness, or usefulness of any 10 | information, apparatus, product, or process disclosed, or represents that 11 | its use would not infringe privately owned rights. 12 | 13 | Reference herein to any specific commercial product, process, or service 14 | by trade name, trademark, manufacturer, or otherwise does not necessarily 15 | constitute or imply its endorsement, recommendation, or favoring by the 16 | United States Government or Lawrence Livermore National Security, LLC. 17 | 18 | The views and opinions of authors expressed herein do not necessarily 19 | state or reflect those of the United States Government or Lawrence 20 | Livermore National Security, LLC, and shall not be used for advertising 21 | or product endorsement purposes. 22 | -------------------------------------------------------------------------------- /dso/dso/__init__.py: -------------------------------------------------------------------------------- 1 | from dso.core import DeepSymbolicOptimizer 2 | from dso.task.regression.sklearn import DeepSymbolicRegressor 3 | 4 | -------------------------------------------------------------------------------- /dso/dso/config/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import commentjson as json 4 | 5 | from dso.utils import safe_merge_dicts 6 | 7 | 8 | def get_base_config(task, language_prior): 9 | # Load base config 10 | with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "config_common.json"), encoding='utf-8') as f: 11 | base_config = json.load(f) 12 | 13 | # Load task specific config 14 | task_config_file = None 15 | if task in ["regression", None]: 16 | task_config_file = "config_regression.json" 17 | elif task in ["control"]: 18 | task_config_file = "config_control.json" 19 | else: 20 | # Custom tasks use config_common.json. 21 | task_config_file = "config_common.json" 22 | with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), task_config_file), encoding='utf-8') as f: 23 | task_config = json.load(f) 24 | 25 | # Load language prior config 26 | if language_prior: 27 | with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "config_language.json"), encoding='utf-8') as f: 28 | language_config = json.load(f) 29 | task_config = safe_merge_dicts(task_config, language_config) 30 | 31 | return safe_merge_dicts(base_config, task_config) 32 | 33 | 34 | def load_config(config=None): 35 | # Load user config 36 | if isinstance(config, str): 37 | with open(config, encoding='utf-8') as f: 38 | user_config = json.load(f) 39 | elif isinstance(config, dict): 40 | user_config = config 41 | else: 42 | assert config is None, "Config must be None, str, or dict." 43 | user_config = {} 44 | 45 | # Determine the task and language prior 46 | try: 47 | task = user_config["task"]["task_type"] 48 | except KeyError: 49 | task = "regression" 50 | print("WARNING: Task type not specified. Falling back to default task type '{}' to load config.".format(task)) 51 | try: 52 | language_prior = user_config["prior"]["language_model"]["on"] 53 | except KeyError: 54 | language_prior = False 55 | 56 | # Load task-specific base config 57 | base_config = get_base_config(task, language_prior) 58 | 59 | # Return combined configs 60 | return safe_merge_dicts(base_config, user_config) 61 | -------------------------------------------------------------------------------- /dso/dso/config/config_control.json: -------------------------------------------------------------------------------- 1 | { 2 | "task" : { 3 | // Deep Symbolic Policy 4 | "task_type" : "control", 5 | 6 | // Name of your environment. The argument is similar to what you'd use for 7 | // gym.make(). To use a custom environment, use "mymodule:MyEnv-v0", where 8 | // MyEnv-v0 is expected to be registered to gym upon importing mymodule. 9 | "env" : "CustomCartPoleContinuous-v0", 10 | 11 | // Path to a pre-trained stable-baselines "anchor". The environments used 12 | // in the ICML paper have default values, so you do not have to specify 13 | // one. You also don't have to specify an "anchor" for environments with 14 | // one-dimensional action spaces. 15 | "anchor" : null, 16 | 17 | // This list specifies which action is to be learned symbolically (null), 18 | // which ones should use the anchor model ("anchor"), and which ones 19 | // should use previously learned symbolic policies (str representing the 20 | // expression). 21 | "action_spec" : [null], 22 | 23 | // Number of episodes to average per reward computation. 24 | "n_episodes_train" : 10, 25 | 26 | // Number of test episodes to average to determine the best policies. 27 | "n_episodes_test" : 1000, 28 | 29 | // Early stopping is triggered if all test episode's episodic rewards are 30 | // >= success_score. 31 | "success_score": 999999.0, 32 | 33 | // Allowed functions. See functions.py for a list of supported functions. 34 | // Hard-coded constants can be combined to form new constants. The use of 35 | // the placeholder constant "const" is supported but is likely 36 | // prohibitively expensive for the control task, so is not recommended. 37 | "function_set" : ["add", "sub", "mul", "div", "sin", "cos", "exp", "log", 0.1, 1, 5], 38 | 39 | // With protected=false, floating-point errors (e.g. log of negative 40 | // number) may occur, in which case the action is set to 0 for that 41 | // timestep. With protected=true, "protected" functions will prevent 42 | // floating-point errors, but may introduce discontinuities in the learned 43 | // functions. 44 | "protected" : false, 45 | 46 | // If true, each reward computation will use the same set of seeds (via 47 | // env.seed()). This is useful because it renders the task deterministic. 48 | // However, it can introduce bias if the seeds aren't representative of 49 | // the whole environment. 50 | "fix_seeds" : true, 51 | 52 | // If fix_seeds=True, this shifts the seeds used each episode, which 53 | // changes the reward function from run to run. If fix_seeds=false, this 54 | // has no effect. 55 | "episode_seed_shift" : 0, 56 | 57 | // If your env has kwargs, they can go here. 58 | "env_kwargs" : {}, 59 | 60 | // A list of [r_min, r_max] for your environment, used to rescale rewards 61 | // to roughly [0, 1]. If true, it uses precomputed [r_min, r_max] for 62 | // environments supported in control.py. If false, reward scaling is not 63 | // used. 64 | "reward_scale" : true, 65 | 66 | // If the action space is multi-discrete, this is the default action 67 | // If null, ref_action will be set to 0 for all action dimensions 68 | "ref_action": null, 69 | 70 | // Set of thresholds (shared by all state variables) for building 71 | // decision trees. Note that no StateChecker will be added to Library 72 | // if decision_tree_threshold_set is an empty list or null. 73 | "decision_tree_threshold_set" : [] 74 | }, 75 | 76 | // Only the key training hyperparameters are listed here. See 77 | // config_common.json for the full list. 78 | "training" : { 79 | "n_samples": 200000, 80 | "batch_size": 200, 81 | "epsilon": 0.1, 82 | 83 | // Recommended to set this to as many cores as you can use! 84 | "n_cores_batch" : 1, 85 | "early_stopping" : false 86 | }, 87 | "logging" : { 88 | "hof" : null, 89 | "save_pareto_front" : false 90 | }, 91 | 92 | // Only the key RNN controller hyperparameters are listed here. See 93 | // config_common.json for the full list. 94 | "controller" : { 95 | "learning_rate" : 0.001, 96 | "entropy_weight" : 0.01, 97 | "entropy_gamma" : 0.85 98 | }, 99 | 100 | // Hyperparameters related to including in situ priors and constraints. Each 101 | // prior must explicitly be turned "on" or it will not be used. See 102 | // config_common.json for descriptions of each prior. 103 | "prior": { 104 | "length" : { 105 | "min_" : 4, 106 | "max_" : 30, 107 | "on" : true 108 | }, 109 | "inverse" : { 110 | "on" : true 111 | }, 112 | "trig" : { 113 | "on" : true 114 | }, 115 | "const" : { 116 | "on" : true 117 | }, 118 | "no_inputs" : { 119 | "on" : true 120 | }, 121 | "uniform_arity" : { 122 | "on" : true 123 | }, 124 | "soft_length" : { 125 | "loc" : 10, 126 | "scale" : 5, 127 | "on" : true 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /dso/dso/config/config_regression.json: -------------------------------------------------------------------------------- 1 | { 2 | "task" : { 3 | // Deep Symbolic Regression 4 | "task_type" : "regression", 5 | 6 | // This can either be (1) the name of the benchmark dataset (see 7 | // benchmarks.csv for a list of supported benchmarks) or (2) a path to a 8 | // CSV file containing the data. 9 | "dataset" : "Nguyen-1", 10 | 11 | // To customize a function set, edit this! See functions.py for a list of 12 | // supported functions. Note "const" will add placeholder constants that 13 | // will be optimized within the training loop. This will considerably 14 | // increase runtime. 15 | "function_set": ["add", "sub", "mul", "div", "sin", "cos", "exp", "log"], 16 | 17 | // Metric to be used for the reward function. See regression.py for 18 | // supported metrics. 19 | "metric" : "inv_nrmse", 20 | "metric_params" : [1.0], 21 | 22 | // Optional alternate metric to be used at evaluation time. 23 | "extra_metric_test" : null, 24 | "extra_metric_test_params" : [], 25 | 26 | // NRMSE threshold for early stopping. This is useful for noiseless 27 | // benchmark problems when DSO discovers the true solution. 28 | "threshold" : 1e-12, 29 | 30 | // With protected=false, floating-point errors (e.g. log of negative 31 | // number) will simply returns a minimal reward. With protected=true, 32 | // "protected" functions will prevent floating-point errors, but may 33 | // introduce discontinuities in the learned functions. 34 | "protected" : false, 35 | 36 | // You can add artificial reward noise directly to the reward function. 37 | // Note this does NOT add noise to the dataset. 38 | "reward_noise" : 0.0, 39 | "reward_noise_type" : "r", 40 | "normalize_variance" : false, 41 | 42 | // Set of thresholds (shared by all input variables) for building 43 | // decision trees. Note that no StateChecker will be added to Library 44 | // if decision_tree_threshold_set is an empty list or null. 45 | "decision_tree_threshold_set" : [], 46 | 47 | // Parameters for optimizing the "poly" token. 48 | // Note: poly_optimizer is turned on if and only if "poly" is in function_set. 49 | "poly_optimizer_params" : { 50 | // The (maximal) degree of the polynomials used to fit the data 51 | "degree": 3, 52 | // Cutoff value for the coefficients of polynomials. Coefficients 53 | // with magnitude less than this value will be regarded as 0. 54 | "coef_tol": 1e-6, 55 | // linear models from sklearn: linear_regression, lasso, 56 | // and ridge are currently supported, or our own implementation 57 | // of least squares regressor "dso_least_squares". 58 | "regressor": "dso_least_squares", 59 | "regressor_params": { 60 | // Cutoff value for p-value of coefficients. Coefficients with 61 | // larger p-values are forced to zero. 62 | "cutoff_p_value": 1.0, 63 | // Maximum number of terms in the polynomial. If more coefficients are nonzero, 64 | // coefficients with larger p-values will be forced to zero. 65 | "n_max_terms": null, 66 | // Cutoff value for the coefficients of polynomials. Coefficients 67 | // with magnitude less than this value will be regarded as 0. 68 | "coef_tol": 1e-6 69 | } 70 | } 71 | }, 72 | 73 | // Only the key training hyperparameters are listed here. See 74 | // config_common.json for the full list. 75 | "training" : { 76 | "n_samples" : 2000000, 77 | "batch_size" : 1000, 78 | "epsilon" : 0.05, 79 | 80 | // Recommended to set this to as many cores as you can use! Especially if 81 | // using the "const" token. 82 | "n_cores_batch" : 1 83 | }, 84 | 85 | // // Only the key Policy Optimizer hyperparameters are listed here. See 86 | // // config_common.json for the full list. 87 | "policy_optimizer" : { 88 | "learning_rate" : 0.0005, 89 | "entropy_weight" : 0.03, 90 | "entropy_gamma" : 0.7, 91 | 92 | // EXPERIMENTAL: Proximal policy optimization hyperparameters. 93 | // "policy_optimizer_type" : "ppo", 94 | // "ppo_clip_ratio" : 0.2, 95 | // "ppo_n_iters" : 10, 96 | // "ppo_n_mb" : 4, 97 | 98 | // EXPERIMENTAL: Priority queue training hyperparameters. 99 | // "policy_optimizer_type" : "pqt", 100 | // "pqt_k" : 10, 101 | // "pqt_batch_size" : 1, 102 | // "pqt_weight" : 200.0, 103 | // "pqt_use_pg" : false 104 | 105 | }, 106 | 107 | // Hyperparameters related to including in situ priors and constraints. Each 108 | // prior must explicitly be turned "on" or it will not be used. See 109 | // config_common.json for descriptions of each prior. 110 | "prior": { 111 | "length" : { 112 | "min_" : 4, 113 | "max_" : 64, 114 | "on" : true 115 | }, 116 | "repeat" : { 117 | "tokens" : "const", 118 | "min_" : null, 119 | "max_" : 3, 120 | "on" : true 121 | }, 122 | "inverse" : { 123 | "on" : true 124 | }, 125 | "trig" : { 126 | "on" : true 127 | }, 128 | "const" : { 129 | "on" : true 130 | }, 131 | "no_inputs" : { 132 | "on" : true 133 | }, 134 | "uniform_arity" : { 135 | "on" : true 136 | }, 137 | "soft_length" : { 138 | "loc" : 10, 139 | "scale" : 5, 140 | "on" : true 141 | }, 142 | "domain_range" : { 143 | "on" : false 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /dso/dso/config/config_regression_gp.json: -------------------------------------------------------------------------------- 1 | { 2 | "task" : { 3 | // Deep Symbolic Regression 4 | "task_type" : "regression", 5 | 6 | // This can either be (1) the name of the benchmark dataset (see 7 | // benchmarks.csv for a list of supported benchmarks) or (2) a path to a 8 | // CSV file containing the data. 9 | "dataset" : "Nguyen-1", 10 | 11 | // To customize a function set, edit this! See functions.py for a list of 12 | // supported functions. Note "const" will add placeholder constants that 13 | // will be optimized within the training loop. This will considerably 14 | // increase runtime. 15 | "function_set": ["add", "sub", "mul", "div", "sin", "cos", "exp", "log"], 16 | 17 | // Metric to be used for the reward function. See regression.py for 18 | // supported metrics. 19 | "metric" : "inv_nrmse", 20 | "metric_params" : [1.0], 21 | 22 | // Optional alternate metric to be used at evaluation time. 23 | "extra_metric_test" : null, 24 | "extra_metric_test_params" : [], 25 | 26 | // NRMSE threshold for early stopping. This is useful for noiseless 27 | // benchmark problems when DSO discovers the true solution. 28 | "threshold" : 1e-12, 29 | 30 | // With protected=false, floating-point errors (e.g. log of negative 31 | // number) will simply returns a minimal reward. With protected=true, 32 | // "protected" functions will prevent floating-point errors, but may 33 | // introduce discontinuities in the learned functions. 34 | "protected" : false, 35 | 36 | // You can add artificial reward noise directly to the reward function. 37 | // Note this does NOT add noise to the dataset. 38 | "reward_noise" : 0.0, 39 | "reward_noise_type" : "r", 40 | "normalize_variance" : false, 41 | 42 | // Set of thresholds (shared by all input variables) for building 43 | // decision trees. Note that no StateChecker will be added to Library 44 | // if decision_tree_threshold_set is an empty list or null. 45 | "decision_tree_threshold_set" : [], 46 | 47 | // Parameters for optimizing the "poly" token. 48 | // Note: poly_optimizer is turned on if and only if "poly" is in function_set. 49 | "poly_optimizer_params" : { 50 | // The (maximal) degree of the polynomials used to fit the data 51 | "degree": 3, 52 | // Cutoff value for the coefficients of polynomials. Coefficients 53 | // with magnitude less than this value will be regarded as 0. 54 | "coef_tol": 1e-6, 55 | // linear models from sklearn: linear_regression, lasso, 56 | // and ridge are currently supported, or our own implementation 57 | // of least squares regressor "dso_least_squares". 58 | "regressor": "dso_least_squares", 59 | "regressor_params": { 60 | // Cutoff value for p-value of coefficients. Coefficients with 61 | // larger p-values are forced to zero. 62 | "cutoff_p_value": 1.0, 63 | // Maximum number of terms in the polynomial. If more coefficients are nonzero, 64 | // coefficients with larger p-values will be forced to zero. 65 | "n_max_terms": null, 66 | // Cutoff value for the coefficients of polynomials. Coefficients 67 | // with magnitude less than this value will be regarded as 0. 68 | "coef_tol": 1e-6 69 | } 70 | } 71 | }, 72 | 73 | // Hyperparameters related to genetic programming hybrid methods. 74 | "gp_meld" : { 75 | "run_gp_meld" : true, 76 | "population_size" : 100, 77 | "generations" : 20, 78 | "crossover_operator" : "cxOnePoint", 79 | "p_crossover" : 0.5, 80 | "mutation_operator" : "multi_mutate", 81 | "p_mutate" : 0.5, 82 | "tournament_size" : 5, 83 | "train_n" : 50, 84 | "mutate_tree_max" : 3, 85 | "verbose" : false, 86 | // Speeds up processing when doing expensive evaluations. 87 | "parallel_eval" : true 88 | }, 89 | // Only the key training hyperparameters are listed here. See 90 | // config_common.json for the full list. 91 | "training" : { 92 | "n_samples" : 20000, 93 | "batch_size" : 500, 94 | "epsilon" : 0.02, 95 | // Recommended to set this to as many cores as you can use! Especially if 96 | // using the "const" token. 97 | "n_cores_batch" : -1 98 | }, 99 | 100 | // Only the key RNN controller hyperparameters are listed here. See 101 | // config_common.json for the full list. 102 | "controller" : { 103 | "learning_rate": 0.0025, 104 | "entropy_weight" : 0.03, 105 | "entropy_gamma" : 0.7, 106 | // EXPERIMENTAL: Priority queue training hyperparameters. 107 | "pqt" : true, 108 | "pqt_k" : 10, 109 | "pqt_batch_size" : 1, 110 | "pqt_weight" : 200.0, 111 | "pqt_use_pg" : false 112 | }, 113 | 114 | // Hyperparameters related to including in situ priors and constraints. Each 115 | // prior must explicitly be turned "on" or it will not be used. See 116 | // config_common.json for descriptions of each prior. 117 | "prior": { 118 | "length" : { 119 | "min_" : 4, 120 | "max_" : 100, 121 | "on" : true 122 | }, 123 | "inverse" : { 124 | "on" : true 125 | }, 126 | "trig" : { 127 | "on" : true 128 | }, 129 | "const" : { 130 | "on" : true 131 | }, 132 | "no_inputs" : { 133 | "on" : true 134 | }, 135 | "uniform_arity" : { 136 | "on" : true 137 | }, 138 | "soft_length" : { 139 | "loc" : 10, 140 | "scale" : 5, 141 | "on" : true 142 | }, 143 | "domain_range" : { 144 | "on" : true 145 | } 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /dso/dso/config/examples/control/BipedalWalker.json: -------------------------------------------------------------------------------- 1 | // This example contains the tuned entropy_weight and entropy_gamma 2 | // hyperparameters used to solve BipedalWalker-v2 3 | 4 | { 5 | "task" : { 6 | "task_type" : "control", 7 | "env" : "BipedalWalker-v2", 8 | "action_spec" : [null, "anchor", "anchor", "anchor"], 9 | }, 10 | "training" : { 11 | // Recommended to set this to as many cores as you can use! 12 | "n_cores_batch" : 16 13 | }, 14 | "controller" : { 15 | "entropy_weight" : 0.01, 16 | "entropy_gamma" : 0.85 17 | }, 18 | } 19 | -------------------------------------------------------------------------------- /dso/dso/config/examples/control/CustomCartPoleContinuous-v0.json: -------------------------------------------------------------------------------- 1 | { 2 | "task" : { 3 | "task_type" : "control", 4 | "env" : "CustomCartPoleContinuous-v0", 5 | "action_spec" : [null], 6 | "n_episodes_train" : 5, 7 | "n_episodes_test" : 10, 8 | "success_score": 800.0, 9 | "function_set" : ["add", "sub", "mul", "div", "sin", "cos", "exp", "log", 0.1, 1, 5], 10 | "fix_seeds" : false, 11 | "episode_seed_shift" : 0, 12 | "env_kwargs" : {"dt" : 0.02}, 13 | "reward_scale" : true 14 | }, 15 | "training" : { 16 | "n_samples" : 2000, 17 | "batch_size" : 20, 18 | "n_cores_batch": 2 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /dso/dso/config/examples/control/CustomCartPoleContinuous-v0_DecisionTree.json: -------------------------------------------------------------------------------- 1 | { 2 | "task" : { 3 | "task_type" : "control", 4 | "env" : "CustomCartPoleContinuous-v0", 5 | "action_spec" : [null], 6 | "n_episodes_train" : 5, 7 | "n_episodes_test" : 10, 8 | "success_score": 800.0, 9 | "function_set" : ["add", "sub", "mul", "div", "sin", "cos", "exp", "log", 0.1, 1, 5], 10 | "fix_seeds" : false, 11 | "episode_seed_shift" : 0, 12 | "env_kwargs" : {"dt" : 0.02}, 13 | "reward_scale" : true, 14 | "decision_tree_threshold_set" : [[-2.4, 0.0, 2.4], [0.0], [-0.21, 0.0, 0.21], [0.0]] 15 | }, 16 | "prior": { 17 | "state_checker" : { "on" : true } 18 | }, 19 | "training" : { 20 | "n_samples" : 2000, 21 | "batch_size" : 20, 22 | "save_pareto_front" : false, 23 | "n_cores_batch": 2 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dso/dso/config/examples/control/Hopper.json: -------------------------------------------------------------------------------- 1 | // This example contains the tuned entropy_weight and entropy_gamma 2 | // hyperparameters used to solve HopperBulletEnv-v0 3 | 4 | { 5 | "task" : { 6 | "task_type" : "control", 7 | "env" : "HopperBulletEnv-v0", 8 | "action_spec" : [null, "anchor", "anchor"], 9 | }, 10 | "training" : { 11 | // Recommended to set this to as many cores as you can use! 12 | "n_cores_batch" : 16 13 | }, 14 | "controller" : { 15 | "entropy_weight" : 0.01, 16 | "entropy_gamma" : 0.85 17 | }, 18 | } 19 | -------------------------------------------------------------------------------- /dso/dso/config/examples/control/LunarLander.json: -------------------------------------------------------------------------------- 1 | // This example contains the tuned entropy_weight and entropy_gamma 2 | // hyperparameters used to solve LunarLanderContinuous-v2 3 | 4 | { 5 | "task" : { 6 | "task_type" : "control", 7 | "env" : "LunarLanderContinuous-v2", 8 | "action_spec" : [null, "anchor"], 9 | }, 10 | "training" : { 11 | // Recommended to set this to as many cores as you can use! 12 | "n_cores_batch" : 16 13 | }, 14 | "controller" : { 15 | "entropy_weight" : 0.02, 16 | "entropy_gamma" : 0.85 17 | }, 18 | } 19 | -------------------------------------------------------------------------------- /dso/dso/config/examples/control/LunarLanderMultiDiscrete.json: -------------------------------------------------------------------------------- 1 | // This example contains an example of how to use MultiDiscreteAction to learn both actions of LunarLanderMultiDiscrete-v0 at once. 2 | 3 | { 4 | "task" : { 5 | "task_type" : "control", 6 | "env" : "LunarLanderMultiDiscrete-v0", 7 | "action_spec" : [null, null], 8 | "n_episodes_train" : 5, 9 | "n_episodes_test" : 10, 10 | "reward_scale": false, 11 | "decision_tree_threshold_set": [[-0.5, 0.0, 0.5], [-0.75, 0.0, 0.25, 0.5, 0.75, 1.0, 1.25], 12 | [0.0], [0.0], [0.0], [0.0], [0.5], [0.5]], 13 | "ref_action" : [0, 1], 14 | "success_score" : 215.0 15 | }, 16 | "training" : { 17 | "n_samples" : 2000, 18 | "batch_size" : 4, 19 | "n_cores_batch": 2 20 | }, 21 | "controller" : { 22 | "entropy_weight" : 0.01, 23 | "entropy_gamma" : 0.85 24 | }, 25 | "prior": { 26 | "length": { 27 | "min_": null, 28 | "max_": 128, 29 | "on": true 30 | }, 31 | "no_inputs" : { 32 | "on" : false 33 | }, 34 | "uniform_arity" : { 35 | "on" : false 36 | }, 37 | "soft_length" : { 38 | "loc" : 30, 39 | "scale" : 8, 40 | "on" : true 41 | }, 42 | "multi_discrete" : { 43 | "dense" : false, 44 | "ordered" : false, 45 | "on" : true 46 | } 47 | }, 48 | "logging": { 49 | "save_top_samples_per_batch": 0.01 50 | } 51 | } 52 | 53 | -------------------------------------------------------------------------------- /dso/dso/config/examples/control/MountainCar-v0.json: -------------------------------------------------------------------------------- 1 | { 2 | "task" : { 3 | "task_type" : "control", 4 | "env" : "MountainCar-v0", 5 | "action_spec" : [null], 6 | "n_episodes_train" : 10, 7 | "n_episodes_test" : 100, 8 | "success_score": -90.0, 9 | "function_set" : [0.0, 1.0, 2.0], 10 | "fix_seeds" : true, 11 | "episode_seed_shift" : 0, 12 | "env_kwargs" : {}, 13 | "reward_scale" : false, 14 | "decision_tree_threshold_set" : [[-0.9, -0.6, -0.3, 0.0, 0.3], 15 | [-0.06, -0.05, -0.04, -0.03, -0.02, -0.01, 0.0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06]] 16 | }, 17 | "prior": { 18 | "length": { 19 | "min_": 3, 20 | "max_": 30, 21 | "on": true 22 | }, 23 | "no_inputs" : { 24 | "on" : false 25 | } 26 | }, 27 | "training" : { 28 | "n_samples" : 20000, 29 | "batch_size" : 100, 30 | "n_cores_batch": 4 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dso/dso/config/examples/regression/Nguyen-2-GPmeld.json: -------------------------------------------------------------------------------- 1 | { 2 | "task" : { 3 | "dataset" : "Nguyen-2" 4 | }, 5 | // Hyperparameters related to genetic programming hybrid methods. 6 | "gp_meld" : { 7 | "run_gp_meld" : true, 8 | "population_size" : 100, 9 | "generations" : 10, 10 | "crossover_operator" : "cxOnePoint", 11 | "p_crossover" : 0.5, 12 | "mutation_operator" : "multi_mutate", 13 | "p_mutate" : 0.5, 14 | "tournament_size" : 5, 15 | "train_n" : 50, 16 | "mutate_tree_max" : 3, 17 | "verbose" : false, 18 | // Speeds up processing when doing expensive evaluations. 19 | "parallel_eval" : false 20 | }, 21 | "training": { 22 | "n_samples": 2000000, 23 | "batch_size": 1000 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dso/dso/config/examples/regression/Nguyen-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "task" : { 3 | "dataset" : "Nguyen-2" 4 | } 5 | } -------------------------------------------------------------------------------- /dso/dso/config/examples/regression/PiecewiseFunction-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "task": { 3 | "task_type" : "regression", 4 | "function_set": ["add", "sub", "mul", "div", "sin", "cos", "sqrt"], 5 | "dataset" : "task/regression/data/PiecewiseFunction-1.csv", 6 | "protected" : true, 7 | "decision_tree_threshold_set" : [-0.6, -0.3, 0.0, 0.3, 0.6] 8 | }, 9 | "training": { 10 | "n_samples": 2000000, 11 | "batch_size": 1000 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /dso/dso/config/examples/regression/Poly-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "task": { 3 | "dataset": "Poly-1" 4 | } 5 | } -------------------------------------------------------------------------------- /dso/dso/const.py: -------------------------------------------------------------------------------- 1 | """Constant optimizer used for deep symbolic optimization.""" 2 | 3 | from functools import partial 4 | 5 | import numpy as np 6 | from scipy.optimize import minimize 7 | 8 | 9 | def make_const_optimizer(name, **kwargs): 10 | """Returns a ConstOptimizer given a name and keyword arguments""" 11 | 12 | const_optimizers = { 13 | None : Dummy, 14 | "dummy" : Dummy, 15 | "scipy" : ScipyMinimize, 16 | } 17 | 18 | return const_optimizers[name](**kwargs) 19 | 20 | 21 | class ConstOptimizer(object): 22 | """Base class for constant optimizer""" 23 | 24 | def __init__(self, **kwargs): 25 | self.kwargs = kwargs 26 | 27 | 28 | def __call__(self, f, x0): 29 | """ 30 | Optimizes an objective function from an initial guess. 31 | 32 | The objective function is the negative of the base reward (reward 33 | without penalty) used for training. Optimization excludes any penalties 34 | because they are constant w.r.t. to the constants being optimized. 35 | 36 | Parameters 37 | ---------- 38 | f : function mapping np.ndarray to float 39 | Objective function (negative base reward). 40 | 41 | x0 : np.ndarray 42 | Initial guess for constant placeholders. 43 | 44 | Returns 45 | ------- 46 | x : np.ndarray 47 | Vector of optimized constants. 48 | """ 49 | raise NotImplementedError 50 | 51 | 52 | class Dummy(ConstOptimizer): 53 | """Dummy class that selects the initial guess for each constant""" 54 | 55 | def __init__(self, **kwargs): 56 | super(Dummy, self).__init__(**kwargs) 57 | 58 | 59 | def __call__(self, f, x0): 60 | return x0 61 | 62 | 63 | class ScipyMinimize(ConstOptimizer): 64 | """SciPy's non-linear optimizer""" 65 | 66 | def __init__(self, **kwargs): 67 | super(ScipyMinimize, self).__init__(**kwargs) 68 | 69 | 70 | def __call__(self, f, x0): 71 | with np.errstate(divide='ignore'): 72 | opt_result = partial(minimize, **self.kwargs)(f, x0) 73 | x = opt_result['x'] 74 | return x 75 | -------------------------------------------------------------------------------- /dso/dso/cyfunc.pyx: -------------------------------------------------------------------------------- 1 | ''' 2 | # cython: linetrace=True 3 | # distutils: define_macros=CYTHON_TRACE_NOGIL=1 4 | ''' 5 | # Uncomment the above lines for cProfile 6 | 7 | import numpy as np 8 | import array 9 | 10 | from dso.library import StateChecker, Polynomial 11 | 12 | # Cython specific C imports 13 | cimport numpy as np 14 | from cpython cimport array 15 | cimport cython 16 | from libc.stdlib cimport malloc, free 17 | from cpython.ref cimport PyObject 18 | 19 | # Static inits 20 | cdef list apply_stack = [[None for i in range(25)] for i in range(1024)] 21 | cdef int *stack_count = malloc(1024 * sizeof(int)) 22 | 23 | @cython.boundscheck(False) # turn off bounds-checking for entire function 24 | @cython.wraparound(False) # turn off negative index wrapping for entire function 25 | def execute(np.ndarray X, int len_traversal, list traversal, int[:] is_input_var): 26 | 27 | """Executes the program according to X. 28 | 29 | Parameters 30 | ---------- 31 | X : array-like, shape = [n_samples, n_features] 32 | Training vectors, where n_samples is the number of samples and 33 | n_features is the number of features. 34 | 35 | Returns 36 | ------- 37 | y_hats : array-like, shape = [n_samples] 38 | The result of executing the program on X. 39 | """ 40 | #sp = 0 # allow a dummy first row, requires a none type function with arity of -1 41 | 42 | # Init some ints 43 | cdef int sp = -1 # Stack pointer 44 | cdef int Xs = X.shape[0] 45 | 46 | # Give cdef hints for object types 47 | cdef int i 48 | cdef int n 49 | cdef int arity 50 | cdef np.ndarray intermediate_result 51 | cdef list stack_end 52 | cdef object stack_end_function 53 | 54 | for i in range(len_traversal): 55 | 56 | if not is_input_var[i]: 57 | sp += 1 58 | # Move this to the front with a memset call 59 | stack_count[sp] = 0 60 | # Store the reference to stack_count[sp] rather than keep calling 61 | apply_stack[sp][stack_count[sp]] = traversal[i] 62 | stack_end = apply_stack[sp] 63 | # The first element is the function itself 64 | stack_end_function = stack_end[0] 65 | arity = stack_end_function.arity 66 | else: 67 | # Not a function, so lazily evaluate later 68 | stack_count[sp] += 1 69 | stack_end[stack_count[sp]] = X[:, traversal[i].input_var] 70 | 71 | # Keep on doing this so long as arity matches up, we can 72 | # add in numbers above and complete the arity later. 73 | while stack_count[sp] == arity: 74 | # If stack_end_function is a StateChecker (xi < tj), which is associated with 75 | # the i-th state variable xi and threshold tj, then stack_end_function needs to know 76 | # the value of xi (through set_state_value) in order to evaluate the returned value. 77 | # The index i is specified by the attribute state_index of a StateChecker. 78 | if isinstance(stack_end_function, StateChecker): 79 | stack_end_function.set_state_value(X[:, stack_end_function.state_index]) 80 | if isinstance(stack_end_function, Polynomial): 81 | intermediate_result = stack_end_function(X) 82 | else: 83 | intermediate_result = stack_end_function(*stack_end[1:(stack_count[sp] + 1)]) # 85% of overhead 84 | 85 | # I think we can get rid of this line, but will require a major rewrite. 86 | if sp == 0: 87 | return intermediate_result 88 | 89 | sp -= 1 90 | # Adjust pointer at the end of the stack 91 | stack_end = apply_stack[sp] 92 | stack_count[sp] += 1 93 | stack_end[stack_count[sp]] = intermediate_result 94 | 95 | # The first element is the function itself 96 | stack_end_function = stack_end[0] 97 | arity = stack_end_function.arity 98 | 99 | # We should never get here 100 | assert False, "Function should never get here!" 101 | return None 102 | -------------------------------------------------------------------------------- /dso/dso/execute.py: -------------------------------------------------------------------------------- 1 | try: 2 | from dso import cyfunc 3 | except ImportError: 4 | cyfunc = None 5 | import array 6 | 7 | from dso.library import StateChecker, Polynomial 8 | 9 | 10 | def python_execute(traversal, X): 11 | """ 12 | Executes the program according to X using Python. 13 | 14 | Parameters 15 | ---------- 16 | X : array-like, shape = [n_samples, n_features] 17 | Training vectors, where n_samples is the number of samples and 18 | n_features is the number of features. 19 | 20 | Returns 21 | ------- 22 | y_hats : array-like, shape = [n_samples] 23 | The result of executing the program on X. 24 | """ 25 | 26 | apply_stack = [] 27 | 28 | for node in traversal: 29 | apply_stack.append([node]) 30 | 31 | while len(apply_stack[-1]) == apply_stack[-1][0].arity + 1: 32 | token = apply_stack[-1][0] 33 | terminals = apply_stack[-1][1:] 34 | 35 | if token.input_var is not None: 36 | intermediate_result = X[:, token.input_var] 37 | else: 38 | if isinstance(token, StateChecker): 39 | token.set_state_value(X[:, token.state_index]) 40 | if isinstance(token, Polynomial): 41 | intermediate_result = token(X) 42 | else: 43 | intermediate_result = token(*terminals) 44 | if len(apply_stack) != 1: 45 | apply_stack.pop() 46 | apply_stack[-1].append(intermediate_result) 47 | else: 48 | return intermediate_result 49 | 50 | assert False, "Function should never get here!" 51 | return None 52 | 53 | def cython_execute(traversal, X): 54 | """ 55 | Execute cython function using given traversal over input X. 56 | 57 | Parameters 58 | ---------- 59 | 60 | traversal : list 61 | A list of nodes representing the traversal over a Program. 62 | X : np.array 63 | The input values to execute the traversal over. 64 | 65 | Returns 66 | ------- 67 | 68 | result : float 69 | The result of executing the traversal. 70 | """ 71 | if len(traversal) > 1: 72 | is_input_var = array.array('i', [t.input_var is not None for t in traversal]) 73 | return cyfunc.execute(X, len(traversal), traversal, is_input_var) 74 | else: 75 | return python_execute(traversal, X) 76 | 77 | 78 | -------------------------------------------------------------------------------- /dso/dso/gp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/gp/__init__.py -------------------------------------------------------------------------------- /dso/dso/gp/base.py: -------------------------------------------------------------------------------- 1 | import random 2 | import time 3 | import copy 4 | 5 | import numpy as np 6 | from deap import tools 7 | import dso.gp.utils as U 8 | 9 | from dso.program import from_tokens, _finish_tokens, Program 10 | 11 | def _eval_step(tokens): 12 | """ 13 | Take in an action which is a numpy array/string and return the reward 14 | """ 15 | p = from_tokens(tokens, on_policy=False, finish_tokens=False, skip_cache=True) 16 | r = p.r # This calls the reward computation 17 | return p 18 | 19 | class RunOneStepAlgorithm: 20 | """ 21 | Top level class which runs the GP one generation at a time, replacing 22 | classes like eaSimple since we need more control over how it runs. 23 | """ 24 | 25 | def __init__(self, toolbox, p_crossover, verbose=__debug__): 26 | 27 | self.toolbox = toolbox 28 | self.verbose = verbose 29 | 30 | self.cxpb = p_crossover # TODO: use toolbox.mate.keywords['indpb'] 31 | 32 | self.logbook = tools.Logbook() 33 | self.logbook.header = ['gen', 'iter', 'pop_size', 'nevals', 34 | 'uncached_size', 'best_val', 'timer'] 35 | 36 | self.population = None # Must be explicitly set 37 | self.gen = 0 38 | 39 | def _eval(self, population): 40 | # Evaluate the individuals with an invalid fitness 41 | # This way we do not evaluate individuals that we have already seen. 42 | tokens_list = [] 43 | uncached_ind = [] 44 | invalid_ind = [ ind for ind in population if not ind.fitness.valid ] 45 | for ind in invalid_ind: 46 | # Get from deap to numpy array 47 | # Make sure token is equiv with program finished token used for cache key 48 | tokens = _finish_tokens(ind.tokenized_repr) 49 | 50 | try: 51 | # we've seen this one before, copy back the reward. 52 | # We use a try loop to avoid double look-ups in the dict 53 | p = Program.cache[tokens.tostring()] 54 | ind.fitness.values = (-p.r,) 55 | except (KeyError, TypeError): 56 | # We have not seen this one, we need to compute reward 57 | tokens_list.append(tokens) 58 | # Keep track of uncached inds for now 59 | uncached_ind.append(ind) 60 | # Deap demands this is a value and not None. positive inf should not 61 | # be viable since rewards are always negative (are penalties) in Deap. 62 | ind.fitness.values = (np.inf,) 63 | 64 | # Calls either map or pool.map 65 | programs = list(self.toolbox.cmap(_eval_step, tokens_list)) 66 | for i, ind in enumerate(population): 67 | if np.isposinf(ind.fitness.values[0]): 68 | p = programs.pop(0) 69 | ind.fitness.values = (-p.r,) 70 | Program.cache[p.str] = p 71 | 72 | return population, invalid_ind, uncached_ind 73 | 74 | # Would this benefit from using process pooling? 75 | def _var_and(self, population): 76 | 77 | offspring = [self.toolbox.clone(ind) for ind in population] 78 | 79 | # Apply crossover on the offspring 80 | for i in range(1, len(offspring), 2): 81 | if random.random() < self.cxpb: 82 | offspring[i - 1], offspring[i] = self.toolbox.mate(offspring[i - 1], offspring[i]) 83 | 84 | del offspring[i - 1].fitness.values, offspring[i].fitness.values 85 | 86 | # Apply mutation on the offspring 87 | for i in range(len(offspring)): 88 | offspring[i], = self.toolbox.mutate(offspring[i]) 89 | del offspring[i].fitness.values 90 | 91 | return offspring 92 | 93 | def __call__(self, hof=None, iter=None): 94 | 95 | t1 = time.perf_counter() 96 | 97 | # Select the next generation individuals 98 | offspring = self.toolbox.select(self.population, len(self.population)) 99 | 100 | # Vary the pool of individuals 101 | offspring = self._var_and(offspring) 102 | 103 | # Evaluate the individuals with an invalid fitness 104 | population, invalid_ind, uncached_ind = self._eval(offspring) 105 | 106 | pop_size = len(population) # Total population size 107 | nevals = len(invalid_ind) # New individuals which are Deap not valid (Not yet evaluated) 108 | uncached_size = len(uncached_ind) # Individuals which are not in cache 109 | 110 | # Replace the current population by the offspring 111 | self.population[:] = offspring 112 | 113 | # Update hall of fame 114 | if hof is not None: 115 | hof.update(self.population) 116 | best_val = best_val=hof[0].fitness.values 117 | else: 118 | best_val = None 119 | 120 | timer = time.perf_counter() - t1 121 | 122 | self.logbook.record(gen=self.gen, iter=iter, pop_size=pop_size, 123 | nevals=nevals, uncached_size=uncached_size, 124 | best_val=best_val, timer=timer) 125 | 126 | if self.verbose: 127 | print(self.logbook.stream) 128 | 129 | self.gen += 1 130 | 131 | return nevals 132 | 133 | def set_population(self, population): 134 | 135 | self.population = population 136 | -------------------------------------------------------------------------------- /dso/dso/language_model/__init__.py: -------------------------------------------------------------------------------- 1 | from .language_model_prior import LanguageModelPrior -------------------------------------------------------------------------------- /dso/dso/language_model/language_model_prior.py: -------------------------------------------------------------------------------- 1 | """Language model to get prior""" 2 | 3 | import pickle 4 | 5 | import numpy as np 6 | import tensorflow as tf 7 | 8 | from .model.model_dyn_rnn import LanguageModel 9 | 10 | class LanguageModelPrior(object): 11 | """ 12 | Language model to get prior for DSO, given token. 13 | 14 | History of tokens of a sequence is holded as a state of language model. 15 | Usage: LanguageModelPrior.get_lm_prior(token) 16 | 17 | Parameters 18 | ---------- 19 | dso_library: dso.library.Library 20 | Library used in main DSO model 21 | 22 | model_path: str 23 | Path to separately trained mathematical language model to use as prior 24 | 25 | lib_path: str 26 | Path to token library of mathematical language model 27 | 28 | embedding_size: int 29 | num_layers: int 30 | num_hidden: int 31 | Model architecture of loaded mathematical language model 32 | 33 | prob_sharing: bool 34 | Share probabilities among terminal tokens? 35 | """ 36 | 37 | def __init__(self, dso_library, 38 | model_path="./language_model/model/saved_model", 39 | lib_path="./language_model/model/saved_model/word_dict.pkl", 40 | embedding_size=32, num_layers=1, num_hidden=256, 41 | prob_sharing=True 42 | ): 43 | 44 | self.dso_n_input_var = len(dso_library.input_tokens) 45 | self.prob_sharing = prob_sharing 46 | 47 | with open(lib_path, 'rb') as f: 48 | self.lm_token2idx = pickle.load(f) 49 | self.dso2lm, self.lm2dso = self.set_lib_to_lib(dso_library) 50 | 51 | self.language_model = LanguageModel(len(self.lm_token2idx), embedding_size, num_layers, num_hidden, mode='predict') 52 | self.lsess = self.load_model(model_path) 53 | self.next_state = None 54 | self._zero_state = np.zeros(num_hidden, dtype=np.float32) 55 | 56 | def load_model(self, saved_language_model_path): 57 | sess = tf.compat.v1.Session() 58 | saver = tf.train.Saver() 59 | saver.restore(sess,tf.train.latest_checkpoint(saved_language_model_path)) 60 | return sess 61 | 62 | def set_lib_to_lib(self, dso_library): 63 | """match token libraries of DSO and lm (LanguageModel)""" 64 | 65 | # dso token -> lm token 66 | dso2lm = [self.lm_token2idx['TERMINAL']] * self.dso_n_input_var 67 | dso2lm += [self.lm_token2idx[t.name.lower()] for t in dso_library.tokens if t.input_var is None] # ex) [1,1,1,2,3,4,5,6,7,8,9], len(dso2lm) = len(library of dso) 68 | 69 | # lm token -> DSO token 70 | lm2dso = {lm_idx: i for i, lm_idx in enumerate(dso2lm)} 71 | 72 | # TODO: if DSO token missing in lm token library 73 | 74 | return dso2lm, lm2dso 75 | 76 | def get_lm_prior(self, next_input): 77 | """return language model prior based on given current token""" 78 | 79 | # set feed_dict 80 | next_input = np.array(self.dso2lm)[next_input] # match library with DSO 81 | next_input = np.array([next_input]) 82 | 83 | if self.next_state is None: # first input of a sequence 84 | # For dynamic_rnn, not passing language_model.initial_state == passing zero_state. 85 | # Here, explicitly passing zero_state 86 | self.next_state = np.atleast_2d(self._zero_state) # initialize the cell 87 | 88 | feed_dict = {self.language_model.x: next_input, self.language_model.keep_prob: 1.0, self.language_model.initial_state: self.next_state} 89 | 90 | # get language model prior 91 | self.next_state, lm_logit = self.lsess.run([self.language_model.last_state, self.language_model.logits], feed_dict=feed_dict) 92 | 93 | if self.prob_sharing is True: 94 | # sharing probability among tokens in same group (e.g., TERMINAL to multiple variables) 95 | lm_logit[:, :, self.lm_token2idx['TERMINAL']] = lm_logit[:, :, self.lm_token2idx['TERMINAL']] - np.log(self.dso_n_input_var) 96 | lm_prior = lm_logit[0, :, self.dso2lm] 97 | lm_prior = np.transpose(lm_prior) # make its shape to (batch size, dso size) 98 | 99 | return lm_prior 100 | -------------------------------------------------------------------------------- /dso/dso/language_model/model/model_dyn_rnn.py: -------------------------------------------------------------------------------- 1 | """Model architecture of default (saved) LanguageModel""" 2 | 3 | import tensorflow as tf 4 | from tensorflow.contrib import rnn 5 | 6 | class LanguageModel(object): 7 | def __init__(self, vocabulary_size, embedding_size, num_layers, num_hidden, mode='train'): 8 | self.embedding_size = embedding_size 9 | self.num_layers = num_layers 10 | self.num_hidden = num_hidden 11 | 12 | self.x = tf.compat.v1.placeholder(tf.int32, [None, None], name="x") # whole seq + seq len 13 | self.keep_prob = tf.compat.v1.placeholder(tf.float32, [], name="keep_prob") 14 | self.batch_size = tf.compat.v1.shape(self.x)[0] 15 | 16 | if mode == 'train': 17 | self.lm_input = self.x[:, :-2] 18 | self.seq_len = self.x[:, -1] 19 | elif mode == 'predict': 20 | self.lm_input = self.x[:,:] 21 | self.seq_len = tf.reduce_sum(tf.sign(self.lm_input), 1) 22 | 23 | self.logits=tf.Variable(2.0, name="logits") 24 | 25 | # embedding, one-hot encoding 26 | # if embedding: 27 | with tf.name_scope("embedding"): 28 | init_embeddings = tf.random.uniform([vocabulary_size, self.embedding_size]) 29 | embeddings = tf.compat.v1.get_variable("embeddings", initializer=init_embeddings) 30 | lm_input_emb = tf.nn.embedding_lookup(embeddings, self.lm_input) 31 | 32 | with tf.compat.v1.variable_scope("rnn"): 33 | def make_cell(): 34 | cell = rnn.BasicRNNCell(self.num_hidden) 35 | cell = rnn.DropoutWrapper(cell, output_keep_prob=self.keep_prob) 36 | return cell 37 | 38 | cell = rnn.MultiRNNCell([make_cell() for _ in range(self.num_layers)]) 39 | 40 | self.initial_state = cell.zero_state(self.batch_size, dtype=tf.float32) 41 | 42 | # rnn_outputs: [batch_size, max_len, num_hidden(cell output)] 43 | rnn_outputs, self.last_state = tf.nn.dynamic_rnn( 44 | cell=cell, 45 | initial_state=self.initial_state, 46 | inputs=lm_input_emb, 47 | sequence_length=self.seq_len, 48 | dtype=tf.float32) 49 | 50 | # with tf.name_scope("output"): 51 | self.logits = tf.layers.dense(rnn_outputs, vocabulary_size) 52 | 53 | 54 | with tf.name_scope("loss"): 55 | if mode == "train": 56 | target = self.x[:, 1:-1] 57 | elif mode == "predict": 58 | target = self.x[:, :] 59 | 60 | self.loss = tf.contrib.seq2seq.sequence_loss( 61 | logits=self.logits, 62 | targets=target, 63 | weights=tf.sequence_mask(self.seq_len, tf.shape(self.x)[1] - 2, dtype=tf.float32), 64 | average_across_timesteps=True, 65 | average_across_batch=True 66 | ) 67 | 68 | -------------------------------------------------------------------------------- /dso/dso/language_model/model/saved_model/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "dynrnn" 2 | all_model_checkpoint_paths: "dynrnn" 3 | -------------------------------------------------------------------------------- /dso/dso/language_model/model/saved_model/dynrnn.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/language_model/model/saved_model/dynrnn.data-00000-of-00001 -------------------------------------------------------------------------------- /dso/dso/language_model/model/saved_model/dynrnn.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/language_model/model/saved_model/dynrnn.index -------------------------------------------------------------------------------- /dso/dso/language_model/model/saved_model/dynrnn.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/language_model/model/saved_model/dynrnn.meta -------------------------------------------------------------------------------- /dso/dso/language_model/model/saved_model/word_dict.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/language_model/model/saved_model/word_dict.pkl -------------------------------------------------------------------------------- /dso/dso/policy/__init__.py: -------------------------------------------------------------------------------- 1 | from dso.policy.policy import make_policy, Policy -------------------------------------------------------------------------------- /dso/dso/policy/policy.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | from typing import Tuple, TypeVar 4 | 5 | import tensorflow as tf 6 | import dso 7 | from dso.prior import LengthConstraint 8 | from dso.program import Program 9 | from dso.utils import import_custom_source 10 | from dso.prior import JointPrior 11 | from dso.tf_state_manager import StateManager 12 | from dso.memory import Batch 13 | 14 | # Used for function annotations using the type system 15 | actions = tf.TensorArray 16 | obs = tf.TensorArray 17 | priors = tf.TensorArray 18 | neglogp = tf.TensorArray 19 | entropy = tf.TensorArray 20 | 21 | def make_policy(sess, prior, state_manager, policy_type, **config_policy): 22 | """Factory function for Policy object.""" 23 | 24 | if policy_type == "rnn": 25 | from dso.policy.rnn_policy import RNNPolicy 26 | policy_class = RNNPolicy 27 | else: 28 | # Custom policy import 29 | policy_class = import_custom_source(policy_type) 30 | assert issubclass(policy_class, Policy), \ 31 | "Custom policy {} must subclass dso.policy.Policy.".format(policy_class) 32 | 33 | policy = policy_class(sess, 34 | prior, 35 | state_manager, 36 | **config_policy) 37 | 38 | return policy 39 | 40 | class Policy(ABC): 41 | """Abstract class for a policy. A policy is a parametrized probability 42 | distribution over discrete objects. DSO algorithms optimize the parameters 43 | of this distribution to generate discrete objects with high rewards. 44 | """ 45 | 46 | def __init__(self, 47 | sess : tf.Session, 48 | prior : JointPrior, 49 | state_manager : StateManager, 50 | debug : int = 0, 51 | max_length : int = 30) -> None: 52 | '''Parameters 53 | ---------- 54 | sess : tf.Session 55 | TenorFlow Session object. 56 | 57 | prior : dso.prior.JointPrior 58 | JointPrior object used to adjust probabilities during sampling. 59 | 60 | state_manager: dso.tf_state_manager.StateManager 61 | Object that handles the state features to be used 62 | 63 | debug : int 64 | Debug level, also used in learn(). 0: No debug. 1: Print shapes and 65 | number of parameters for each variable. 66 | 67 | max_length : int or None 68 | Maximum sequence length. This will be overridden if a LengthConstraint 69 | with a maximum length is part of the prior. 70 | ''' 71 | self.sess = sess 72 | self.prior = prior 73 | self.state_manager = state_manager 74 | self.debug = debug 75 | 76 | # Set self.max_length depending on the Prior 77 | self._set_max_length(max_length) 78 | 79 | # Samples produced during attempt to get novel samples. 80 | # Will be combined with checkpoint-loaded samples for next training step 81 | self.extended_batch = None 82 | self.valid_extended_batch = False 83 | 84 | def _set_max_length(self, max_length : int) -> None: 85 | """Set the max legnth depending on the Prior 86 | """ 87 | # Find max_length from the LengthConstraint prior, if it exists 88 | # For binding task, max_length is # of allowed mutations or master-seq length 89 | # Both priors will never happen in the same experiment 90 | prior_max_length = None 91 | for single_prior in self.prior.priors: 92 | if isinstance(single_prior, LengthConstraint): 93 | if single_prior.max is not None: 94 | prior_max_length = single_prior.max 95 | self.max_length = prior_max_length 96 | break 97 | 98 | if prior_max_length is None: 99 | assert max_length is not None, "max_length must be specified if "\ 100 | "there is no LengthConstraint." 101 | self.max_length = max_length 102 | print("WARNING: Maximum length not constrained. Sequences will " 103 | "stop at {} and complete by repeating the first input " 104 | "variable.".format(self.max_length)) 105 | elif max_length is not None and max_length != self.max_length: 106 | print("WARNING: max_length ({}) will be overridden by value from " 107 | "LengthConstraint ({}).".format(max_length, self.max_length)) 108 | 109 | @abstractmethod 110 | def _setup_tf_model(self, **kwargs) -> None: 111 | """"Setup the TensorFlow graph(s). 112 | 113 | Returns 114 | ------- 115 | None 116 | """ 117 | raise NotImplementedError 118 | 119 | @abstractmethod 120 | def make_neglogp_and_entropy(self, 121 | B : Batch, 122 | entropy_gamma : float 123 | ) -> Tuple[neglogp, entropy]: 124 | """Computes the negative log-probabilities for a given 125 | batch of actions, observations and priors 126 | under the current policy. 127 | 128 | Returns 129 | ------- 130 | neglogp, entropy : 131 | Tensorflow tensors 132 | """ 133 | raise NotImplementedError 134 | 135 | @abstractmethod 136 | def sample(self, n : int) -> Tuple[actions, obs, priors]: 137 | """Sample batch of n expressions. 138 | 139 | Returns 140 | ------- 141 | actions, obs, priors : 142 | Or a batch 143 | """ 144 | raise NotImplementedError 145 | 146 | @abstractmethod 147 | def compute_probs(self, memory_batch, log=False): 148 | """Compute the probabilities of a Batch. 149 | 150 | Returns 151 | ------- 152 | probs : 153 | Or a batch 154 | """ 155 | raise NotImplementedError 156 | 157 | 158 | -------------------------------------------------------------------------------- /dso/dso/policy_optimizer/__init__.py: -------------------------------------------------------------------------------- 1 | from dso.policy_optimizer.policy_optimizer import make_policy_optimizer, PolicyOptimizer -------------------------------------------------------------------------------- /dso/dso/policy_optimizer/pg_policy_optimizer.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | from dso.policy_optimizer import PolicyOptimizer 4 | from dso.policy import Policy 5 | 6 | class PGPolicyOptimizer(PolicyOptimizer): 7 | """Vanilla policy gradient policy optimizer. 8 | 9 | Parameters 10 | ---------- 11 | cell : str 12 | Recurrent cell to use. Supports 'lstm' and 'gru'. 13 | 14 | num_layers : int 15 | Number of RNN layers. 16 | 17 | num_units : int or list of ints 18 | Number of RNN cell units in each of the RNN's layers. If int, the value 19 | is repeated for each layer. 20 | 21 | initiailizer : str 22 | Initializer for the recurrent cell. Supports 'zeros' and 'var_scale'. 23 | 24 | """ 25 | def __init__(self, 26 | sess : tf.Session, 27 | policy : Policy, 28 | debug : int = 0, 29 | summary : bool = False, 30 | # Optimizer hyperparameters 31 | optimizer : str = 'adam', 32 | learning_rate : float = 0.001, 33 | # Loss hyperparameters 34 | entropy_weight : float = 0.005, 35 | entropy_gamma : float = 1.0) -> None: 36 | super()._setup_policy_optimizer(sess, policy, debug, summary, optimizer, learning_rate, entropy_weight, entropy_gamma) 37 | 38 | 39 | def _set_loss(self): 40 | with tf.name_scope("losses"): 41 | # Retrieve rewards from batch 42 | r = self.sampled_batch_ph.rewards 43 | # Baseline is the worst of the current samples r 44 | self.pg_loss = tf.reduce_mean((r - self.baseline) * self.neglogp, name="pg_loss") 45 | # Loss already is set to entropy loss 46 | self.loss += self.pg_loss 47 | 48 | 49 | def _preppend_to_summary(self): 50 | with tf.name_scope("summary"): 51 | tf.summary.scalar("pg_loss", self.pg_loss) 52 | 53 | 54 | def train_step(self, baseline, sampled_batch): 55 | """Computes loss, trains model, and returns summaries.""" 56 | feed_dict = { 57 | self.baseline : baseline, 58 | self.sampled_batch_ph : sampled_batch 59 | } 60 | 61 | summaries, _ = self.sess.run([self.summaries, self.train_op], feed_dict=feed_dict) 62 | 63 | return summaries -------------------------------------------------------------------------------- /dso/dso/policy_optimizer/ppo_policy_optimizer.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | import numpy as np 4 | from dso.policy_optimizer import PolicyOptimizer 5 | from dso.policy import Policy 6 | from dso.memory import Batch 7 | 8 | class PPOPolicyOptimizer(PolicyOptimizer): 9 | """Proximal policy optimization policy optimizer. 10 | 11 | Parameters 12 | ---------- 13 | 14 | ppo_clip_ratio : float 15 | Clip ratio to use for PPO. 16 | 17 | ppo_n_iters : int 18 | Number of optimization iterations for PPO. 19 | 20 | ppo_n_mb : int 21 | Number of minibatches per optimization iteration for PPO. 22 | 23 | """ 24 | def __init__(self, 25 | sess : tf.Session, 26 | policy : Policy, 27 | debug : int = 0, 28 | summary : bool = False, 29 | # Optimizer hyperparameters 30 | optimizer : str = 'adam', 31 | learning_rate : float = 0.001, 32 | # Loss hyperparameters 33 | entropy_weight : float = 0.005, 34 | entropy_gamma : float = 1.0, 35 | # PPO hyperparameters 36 | ppo_clip_ratio : float = 0.2, 37 | ppo_n_iters : int = 10, 38 | ppo_n_mb : int = 4) -> None: 39 | self.ppo_clip_ratio = ppo_clip_ratio 40 | self.ppo_n_iters = ppo_n_iters 41 | self.ppo_n_mb = ppo_n_mb 42 | self.rng = np.random.RandomState(0) # Used for PPO minibatch sampling 43 | super()._setup_policy_optimizer(sess, policy, debug, summary, optimizer, learning_rate, entropy_weight, entropy_gamma) 44 | 45 | 46 | def _set_loss(self): 47 | with tf.name_scope("losses"): 48 | # Retrieve rewards from batch 49 | r = self.sampled_batch_ph.rewards 50 | 51 | self.old_neglogp_ph = tf.placeholder(dtype=tf.float32, 52 | shape=(None,), name="old_neglogp") 53 | ratio = tf.exp(self.old_neglogp_ph - self.neglogp) 54 | clipped_ratio = tf.clip_by_value(ratio, 1. - self.ppo_clip_ratio, 55 | 1. + self.ppo_clip_ratio) 56 | ppo_loss = -tf.reduce_mean((r - self.baseline) * 57 | tf.minimum(ratio, clipped_ratio)) 58 | # Loss already is set to entropy loss 59 | self.loss += ppo_loss 60 | 61 | # Define PPO diagnostics 62 | clipped = tf.logical_or(ratio < (1. - self.ppo_clip_ratio), 63 | ratio > 1. + self.ppo_clip_ratio) 64 | self.clip_fraction = tf.reduce_mean(tf.cast(clipped, tf.float32)) 65 | self.sample_kl = tf.reduce_mean(self.neglogp - self.old_neglogp_ph) 66 | 67 | 68 | def _preppend_to_summary(self): 69 | with tf.name_scope("summary"): 70 | tf.summary.scalar("ppo_loss", self.ppo_loss) 71 | 72 | 73 | def train_step(self, baseline, sampled_batch): 74 | feed_dict = { 75 | self.baseline : baseline, 76 | self.sampled_batch_ph : sampled_batch 77 | } 78 | n_samples = sampled_batch.rewards.shape[0] 79 | 80 | 81 | # Compute old_neglogp to be used for training 82 | old_neglogp = self.sess.run(self.neglogp, feed_dict=feed_dict) 83 | 84 | # Perform multiple steps of minibatch training 85 | # feed_dict[self.old_neglogp_ph] = old_neglogp 86 | indices = np.arange(n_samples) 87 | for ppo_iter in range(self.ppo_n_iters): 88 | self.rng.shuffle(indices) # in-place 89 | # list of [ppo_n_mb] arrays 90 | minibatches = np.array_split(indices, self.ppo_n_mb) 91 | for i, mb in enumerate(minibatches): 92 | sampled_batch_mb = Batch( 93 | **{name: array[mb] for name, array 94 | in sampled_batch._asdict().items()}) 95 | mb_feed_dict = { 96 | self.baseline: baseline, 97 | self.batch_size: len(mb), 98 | self.old_neglogp_ph: old_neglogp[mb], 99 | self.sampled_batch_ph: sampled_batch_mb 100 | } 101 | 102 | summaries, _ = self.sess.run([self.summaries, self.train_op], 103 | feed_dict=mb_feed_dict) 104 | 105 | # Diagnostics 106 | # kl, cf, _ = self.sess.run( 107 | # [self.sample_kl, self.clip_fraction, self.train_op], 108 | # feed_dict=mb_feed_dict) 109 | # print("ppo_iter", ppo_iter, "i", i, "KL", kl, "CF", cf) 110 | 111 | return summaries 112 | 113 | 114 | 115 | 116 | -------------------------------------------------------------------------------- /dso/dso/policy_optimizer/pqt_policy_optimizer.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | from dso.policy_optimizer import PolicyOptimizer 4 | from dso.policy import Policy 5 | from dso.utils import make_batch_ph 6 | 7 | class PQTPolicyOptimizer(PolicyOptimizer): 8 | """Priority Queue Ttraining policy gradient policy optimizer. 9 | 10 | Parameters 11 | ---------- 12 | pqt_k : int 13 | Size of priority queue. 14 | 15 | pqt_batch_size : int 16 | Size of batch to sample (with replacement) from priority queue. 17 | 18 | pqt_weight : float 19 | Coefficient for PQT loss function. 20 | 21 | pqt_use_pg : bool 22 | Use policy gradient loss when using PQT? 23 | 24 | """ 25 | def __init__(self, 26 | sess : tf.Session, 27 | policy : Policy, 28 | debug : int = 0, 29 | summary : bool = False, 30 | # Optimizer hyperparameters 31 | optimizer : str = 'adam', 32 | learning_rate : float = 0.001, 33 | # Loss hyperparameters 34 | entropy_weight : float = 0.005, 35 | entropy_gamma : float = 1.0, 36 | # PQT hyperparameters 37 | pqt_k : int = 10, 38 | pqt_batch_size : int = 1, 39 | pqt_weight : float = 200.0, 40 | pqt_use_pg: bool = False) -> None: 41 | self.pqt_k = pqt_k 42 | self.pqt_batch_size = pqt_batch_size 43 | self.pqt_weight = pqt_weight 44 | self. pqt_use_pg = pqt_use_pg 45 | super()._setup_policy_optimizer(sess, policy, debug, summary, optimizer, learning_rate, entropy_weight, entropy_gamma) 46 | 47 | 48 | def _set_loss(self): 49 | with tf.name_scope("losses"): 50 | # Create placeholder for PQT batch 51 | self.pqt_batch_ph = make_batch_ph("pqt_batch", self.n_choices) #self.n_choices is defined in parent class 52 | 53 | pqt_neglogp, _ = self.policy.make_neglogp_and_entropy(self.pqt_batch_ph, self.entropy_gamma) 54 | self.pqt_loss = self.pqt_weight * tf.reduce_mean(pqt_neglogp, name="pqt_loss") 55 | 56 | # Loss already is set to entropy loss 57 | self.loss += self.pqt_loss 58 | 59 | 60 | def _preppend_to_summary(self): 61 | with tf.name_scope("summary"): 62 | tf.summary.scalar("pqt_loss", self.pqt_loss) 63 | 64 | 65 | def train_step(self, baseline, sampled_batch, pqt_batch): 66 | feed_dict = { 67 | self.baseline : baseline, 68 | self.sampled_batch_ph : sampled_batch 69 | } 70 | feed_dict.update({ 71 | self.pqt_batch_ph : pqt_batch 72 | }) 73 | 74 | summaries, _ = self.sess.run([self.summaries, self.train_op], feed_dict=feed_dict) 75 | 76 | return summaries -------------------------------------------------------------------------------- /dso/dso/scripts/search_space.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools for Monte Carlo estimates of search space reduction. Search space 3 | reduction "from A to B" is estimated by generating samples from configuration 4 | A, then computing the fraction of samples that are constrained under 5 | configuration B. 6 | """ 7 | 8 | from copy import deepcopy 9 | 10 | import tensorflow as tf 11 | import numpy as np 12 | import click 13 | 14 | from dso import DeepSymbolicOptimizer 15 | from dso.program import Program, from_tokens 16 | 17 | 18 | def create_model(config, prior_override=None): 19 | model = DeepSymbolicOptimizer(config) 20 | if prior_override is not None: 21 | model.config["prior"] = prior_override 22 | model.config_prior = prior_override 23 | model.setup() 24 | model.sess.run(tf.global_variables_initializer()) 25 | 26 | return model 27 | 28 | 29 | def count_violations(actions, obs, model): 30 | """ 31 | Given candidate actions, count the number of sequences that are constrained 32 | under model. 33 | """ 34 | 35 | # Compute sequence lengths 36 | programs = [from_tokens(a) for a in actions] 37 | lengths = np.array([min(len(p.traversal), model.policy.max_length) for p in programs], dtype=np.int32) 38 | 39 | # Compute priors under model 40 | priors = model.prior.at_once(actions, obs[:, 1, :], obs[:, 2, :]) 41 | 42 | # Count constraint violations under model 43 | count = 0 44 | T = actions.shape[1] 45 | for i in range(actions.shape[0]): 46 | count += (priors[i, np.arange(T), actions[i]][:lengths[i]] == -np.inf).any() 47 | 48 | return count 49 | 50 | 51 | @click.command() 52 | @click.argument("config1", required=False, default=None, type=str) 53 | @click.argument("config2", required=False, default=None, type=str) 54 | @click.option("--n", type=int, default=1000, help="Number of samples to generate.") 55 | @click.option("--mode", type=click.Choice(["marginal", "single", "all", None]), default=None) 56 | def main(config1, config2, n, mode): 57 | 58 | """ 59 | For each prior P, estimate search space reduction from all-but-P to all 60 | priors. 61 | """ 62 | if mode == "marginal": 63 | assert config2 is None, "'Marginal' mode works with a single config." 64 | 65 | # Create the original model 66 | model1 = create_model(config1) 67 | 68 | # For each Prior, compute search space reduction with that prior "off" 69 | counts = {} 70 | for k, v in model1.config["prior"].items(): 71 | 72 | # Skip non-dict values and disabled priors 73 | if not isinstance(v, dict) or not v.get("on", False): 74 | continue 75 | 76 | # Disable the prior 77 | config2 = deepcopy(model1.config) 78 | config2["prior"][k]["on"] = False 79 | 80 | # Sample from config2 81 | model2 = create_model(config2) 82 | actions, obs, _ = model2.policy.sample(n) 83 | 84 | # Count violations under model1 85 | counts[k] = count_violations(actions, obs, model1) 86 | 87 | for k, v in counts.items(): 88 | print("Full prior constrained {}/{} samples from all-but-{}.".format(v, n, k)) 89 | 90 | """ 91 | For each prior P, estimate search space reduction from no priors to P. 92 | """ 93 | if mode == "single": 94 | assert config2 is None, "'Single' mode works with a single config." 95 | 96 | # Get the config with all priors 97 | config_all = create_model(config1).config 98 | 99 | # Sample from the no-prior model 100 | model_none = create_model(config1, prior_override={}) 101 | actions, obs, _ = model_none.policy.sample(n) 102 | 103 | # For each Prior, compute search space reduction with that prior "on" 104 | counts = {} 105 | for k, v in config_all["prior"].items(): 106 | 107 | # Skip non-dict values and disabled priors 108 | if not isinstance(v, dict) or not v.get("on", False): 109 | continue 110 | 111 | # Create a model with a single prior 112 | config2 = deepcopy(config_all) 113 | model2 = create_model(config2, prior_override={k : v}) 114 | 115 | # Count constraints 116 | counts[k] = count_violations(actions, obs, model2) 117 | 118 | for k, v in counts.items(): 119 | print("Prior '{}' alone constrained {}/{} samples from no-prior config.".format(k, v, n)) 120 | 121 | """ 122 | Estimate search space reduction from no priors to config1. 123 | """ 124 | if mode == "all": 125 | assert config2 is None, "'All' mode works with a single config." 126 | 127 | # Sample from the no-prior model 128 | model_none = create_model(config1, prior_override={}) 129 | actions, obs, _ = model_none.policy.sample(n) 130 | 131 | # Compare to full prior 132 | model2 = create_model(config1) 133 | count = count_violations(actions, obs, model2) 134 | 135 | print("The config constrained {}/{} samples from no-prior config.".format(count, n)) 136 | 137 | """ 138 | Estimate search space reduction from config1 to config2. 139 | """ 140 | if mode is None: 141 | 142 | # Sample from config1 143 | model1 = create_model(config1) 144 | actions, obs, _ = model1.policy.sample(n) 145 | tokens1 = set(Program.library.names) 146 | 147 | # Count violations under model2 148 | model2 = create_model(config2) 149 | tokens2 = set(Program.library.names) 150 | assert tokens1 == tokens2, "Tokens must be the same between config1 and config2." 151 | count = count_violations(actions, obs, model2) 152 | 153 | print("The new config constrained {}/{} samples from the old config.".format(count, n)) 154 | 155 | 156 | if __name__ == "__main__": 157 | main() 158 | -------------------------------------------------------------------------------- /dso/dso/task/__init__.py: -------------------------------------------------------------------------------- 1 | from dso.task.task import make_task, set_task, Task, HierarchicalTask, SequentialTask 2 | -------------------------------------------------------------------------------- /dso/dso/task/control/__init__.py: -------------------------------------------------------------------------------- 1 | from gym import register 2 | import pybullet_envs 3 | # the above import is here to register the pybullet environments to Gym. Don't delete! 4 | # without the import you won't be able to use pybullet environments. 5 | 6 | # Gym Pendulum-v0 with added dt parameter 7 | register( 8 | id='CustomPendulum-v0', 9 | entry_point='dso.task.control.envs.pendulum:CustomPendulumEnv', 10 | max_episode_steps=200, 11 | ) 12 | 13 | # PyBullet CartPoleContinuousBulletEnv-v0 with added dt parameter 14 | register( 15 | id='CustomCartPoleContinuousBulletEnv-v0', 16 | entry_point='dso.task.control.envs.cartpole_bullet:CustomCartPoleContinuousBulletEnv', 17 | max_episode_steps=200, 18 | reward_threshold=190.0, 19 | ) 20 | 21 | # Gym-modified ContinuousCartPole-v0 with added dt parameter 22 | register( 23 | id='CustomCartPoleContinuous-v0', 24 | entry_point='dso.task.control.envs.continuous_cartpole:CustomCartPoleContinuousEnv', 25 | max_episode_steps=1000, 26 | reward_threshold=995.0, 27 | ) 28 | 29 | # Modified discrete LunarLander to turn off reward shaping. 30 | register( 31 | id='LunarLanderNoRewardShaping-v0', 32 | entry_point='dso.task.control.envs.lander:CustomLunarLander', 33 | kwargs=dict(reward_shaping_coef=0, continuous=False), 34 | max_episode_steps=1000, 35 | reward_threshold=200, 36 | ) 37 | 38 | # Modified continuous LunarLander to turn off reward shaping. 39 | register( 40 | id='LunarLanderContinuousNoRewardShaping-v0', 41 | entry_point='dso.task.control.envs.lander:CustomLunarLander', 42 | kwargs=dict(reward_shaping_coef=0, continuous=True), 43 | max_episode_steps=1000, 44 | reward_threshold=200, 45 | ) 46 | 47 | # Modified LunarLander with no kwargs provided, it is up to the user to declare kwargs. 48 | register( 49 | id='LunarLanderCustom-v0', 50 | entry_point='dso.task.control.envs.lander:CustomLunarLander', 51 | max_episode_steps=1000, 52 | reward_threshold=200, 53 | ) 54 | 55 | # Modified LunarLander with a MultiDiscrete action space instead of Box. 56 | register( 57 | id='LunarLanderMultiDiscrete-v0', 58 | entry_point='dso.task.control.envs.lander:LunarLanderMultiDiscrete', 59 | max_episode_steps=1000, 60 | reward_threshold=200, 61 | ) -------------------------------------------------------------------------------- /dso/dso/task/control/data/BipedalWalker-v2/model-td3.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/task/control/data/BipedalWalker-v2/model-td3.zip -------------------------------------------------------------------------------- /dso/dso/task/control/data/CustomCartPoleContinuous-v0/model-td3.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/task/control/data/CustomCartPoleContinuous-v0/model-td3.zip -------------------------------------------------------------------------------- /dso/dso/task/control/data/HopperBulletEnv-v0/model-td3.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/task/control/data/HopperBulletEnv-v0/model-td3.pkl -------------------------------------------------------------------------------- /dso/dso/task/control/data/InvertedDoublePendulumBulletEnv-v0/model-td3.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/task/control/data/InvertedDoublePendulumBulletEnv-v0/model-td3.pkl -------------------------------------------------------------------------------- /dso/dso/task/control/data/InvertedPendulumSwingupBulletEnv-v0/model-sac.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/task/control/data/InvertedPendulumSwingupBulletEnv-v0/model-sac.pkl -------------------------------------------------------------------------------- /dso/dso/task/control/data/LunarLanderContinuous-v2/model-sac.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/task/control/data/LunarLanderContinuous-v2/model-sac.pkl -------------------------------------------------------------------------------- /dso/dso/task/control/data/MountainCarContinuous-v0/model-td3.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/task/control/data/MountainCarContinuous-v0/model-td3.pkl -------------------------------------------------------------------------------- /dso/dso/task/control/data/Pendulum-v0/model-td3.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/task/control/data/Pendulum-v0/model-td3.pkl -------------------------------------------------------------------------------- /dso/dso/task/control/data/ReacherBulletEnv-v0/model-td3.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/task/control/data/ReacherBulletEnv-v0/model-td3.pkl -------------------------------------------------------------------------------- /dso/dso/task/control/envs/pendulum.py: -------------------------------------------------------------------------------- 1 | import gym 2 | from gym import spaces 3 | from gym.utils import seeding 4 | import numpy as np 5 | from os import path 6 | 7 | class CustomPendulumEnv(gym.Env): 8 | metadata = { 9 | 'render.modes' : ['human', 'rgb_array'], 10 | 'video.frames_per_second' : 30 11 | } 12 | 13 | def __init__(self, dt=0.05, g=10.0): 14 | self.max_speed=8 15 | self.max_torque=2. 16 | self.dt = dt 17 | self.g = g 18 | self.m = 1. 19 | self.l = 1. 20 | self.viewer = None 21 | 22 | high = np.array([1., 1., self.max_speed], dtype=np.float32) 23 | self.action_space = spaces.Box(low=-self.max_torque, high=self.max_torque, shape=(1,), dtype=np.float32) 24 | self.observation_space = spaces.Box(low=-high, high=high, dtype=np.float32) 25 | 26 | self.seed() 27 | 28 | def seed(self, seed=None): 29 | self.np_random, seed = seeding.np_random(seed) 30 | return [seed] 31 | 32 | def step(self, u): 33 | th, thdot = self.state # th := theta 34 | 35 | g = self.g 36 | m = self.m 37 | l = self.l 38 | dt = self.dt 39 | 40 | u = np.clip(u, -self.max_torque, self.max_torque)[0] 41 | self.last_u = u # for rendering 42 | costs = angle_normalize(th)**2 + .1*thdot**2 + .001*(u**2) 43 | 44 | newthdot = thdot + (-3*g/(2*l) * np.sin(th + np.pi) + 3./(m*l**2)*u) * dt 45 | newth = th + newthdot*dt 46 | newthdot = np.clip(newthdot, -self.max_speed, self.max_speed) #pylint: disable=E1111 47 | 48 | self.state = np.array([newth, newthdot]) 49 | return self._get_obs(), -costs, False, {} 50 | 51 | def reset(self): 52 | high = np.array([np.pi, 1]) 53 | self.state = self.np_random.uniform(low=-high, high=high) 54 | self.last_u = None 55 | return self._get_obs() 56 | 57 | def _get_obs(self): 58 | theta, thetadot = self.state 59 | return np.array([np.cos(theta), np.sin(theta), thetadot]) 60 | 61 | def render(self, mode='human'): 62 | 63 | if self.viewer is None: 64 | from gym.envs.classic_control import rendering 65 | self.viewer = rendering.Viewer(500,500) 66 | self.viewer.set_bounds(-2.2,2.2,-2.2,2.2) 67 | rod = rendering.make_capsule(1, .2) 68 | rod.set_color(.8, .3, .3) 69 | self.pole_transform = rendering.Transform() 70 | rod.add_attr(self.pole_transform) 71 | self.viewer.add_geom(rod) 72 | axle = rendering.make_circle(.05) 73 | axle.set_color(0,0,0) 74 | self.viewer.add_geom(axle) 75 | fname = path.join(path.dirname(__file__), "assets/clockwise.png") 76 | self.img = rendering.Image(fname, 1., 1.) 77 | self.imgtrans = rendering.Transform() 78 | self.img.add_attr(self.imgtrans) 79 | 80 | self.viewer.add_onetime(self.img) 81 | self.pole_transform.set_rotation(self.state[0] + np.pi/2) 82 | if self.last_u: 83 | self.imgtrans.scale = (-self.last_u/2, np.abs(self.last_u)/2) 84 | 85 | return self.viewer.render(return_rgb_array = mode=='rgb_array') 86 | 87 | def close(self): 88 | if self.viewer: 89 | self.viewer.close() 90 | self.viewer = None 91 | 92 | def angle_normalize(x): 93 | return (((x+np.pi) % (2*np.pi)) - np.pi) 94 | -------------------------------------------------------------------------------- /dso/dso/task/control/envs/test/test_envs.py: -------------------------------------------------------------------------------- 1 | """Test that given the same seed, changing dt does not alter the starting state and does alter the next state""" 2 | 3 | import gym 4 | import dso.task.control.envs # Needed to register the environments so they're seen by gym.make() 5 | import numpy as np 6 | 7 | env_ids = ["CustomPendulum-v0", "CustomCartPoleContinuous-v0", "CustomCartPoleContinuousBulletEnv-v0"] 8 | dts = [0.01, 0.02] 9 | 10 | data = {} 11 | 12 | for env_id in env_ids: 13 | 14 | data[env_id] = {} 15 | 16 | for dt in dts: 17 | 18 | data[env_id][dt] = {} 19 | 20 | env = gym.make(env_id, dt=dt) 21 | env.seed(0) 22 | s0 = env.reset() 23 | a = env.action_space.sample() 24 | s1 = env.step(a)[0] 25 | 26 | data[env_id][dt]["s0"] = s0 27 | data[env_id][dt]["s1"] = s1 28 | 29 | # Assert starting states are identical 30 | s0 = data[env_id][dts[0]]["s0"] 31 | for dt in dts[1:]: 32 | assert np.array_equal(s0, data[env_id][dt]["s0"]), "Starting states for {} are not identical.".format(env_id) 33 | 34 | # Assert next states are not identical 35 | s1 = data[env_id][dts[0]]["s1"] 36 | for dt in dts[1:]: 37 | assert not np.array_equal(s1, data[env_id][dt]["s1"]), "Next states for {} are identical.".format(env_id) 38 | 39 | print("Test passed for {}.".format(env_id)) 40 | 41 | print("All tests passed.") -------------------------------------------------------------------------------- /dso/dso/task/control/envs/test/test_lander.py: -------------------------------------------------------------------------------- 1 | import gym 2 | import dso.task.control 3 | import numpy as np 4 | 5 | import click 6 | 7 | @click.command() 8 | @click.option('--seed', default=0, help='Value for seeding environments.') 9 | @click.option('--rollout_length', default=10, help='How long of episodes to do rollouts for comparing environments.') 10 | def setup_envs_run_rollouts(seed: int, rollout_length: int): 11 | continuous_envs = {"LunarLanderContinuous-v2": {}, "LunarLanderCustom-v0": dict(reward_shaping_coef=1, continuous=True)} 12 | discrete_envs = {"LunarLander-v2": {}, "LunarLanderCustom-v0": dict(reward_shaping_coef=1, continuous=False)} 13 | multi_discrete_envs = {"LunarLanderMultiDiscrete-v0": {}} 14 | all_envs = [discrete_envs, continuous_envs, multi_discrete_envs] 15 | 16 | results = {} 17 | for i, envs in enumerate(all_envs): 18 | for env_name, kwargs in envs.items(): 19 | env = gym.make(env_name, **kwargs) 20 | env.seed(seed) 21 | env.action_space.seed(seed) 22 | env.observation_space.seed(seed) 23 | 24 | results[env_name] = {} 25 | 26 | obs = env.reset() 27 | r, done = 0, False 28 | for i in range(rollout_length): 29 | action = env.action_space.sample() 30 | 31 | results[env_name]["state"] = obs 32 | results[env_name]["reward"] = r 33 | results[env_name]["done"] = done 34 | 35 | obs, r, done, _ = env.step(action) 36 | 37 | results[env_name]["state"] = obs 38 | results[env_name]["reward"] = r 39 | results[env_name]["done"] = done 40 | 41 | names = list(envs.keys()) 42 | print(f"Comparing: {names}...") 43 | assert np.array_equal(results[names[0]]['state'], results[names[1]]['state']), "States do not match between above environments." 44 | assert np.array_equal(results[names[0]]['reward'], results[names[1]]['reward']), "Rewards do not match between above environments." 45 | assert np.array_equal(results[names[0]]['done'], results[names[1]]['done']), "Episode dones do not match between above environments." 46 | print(f"Tests passed.") 47 | 48 | return 0 49 | 50 | if __name__ == '__main__': 51 | setup_envs_run_rollouts() -------------------------------------------------------------------------------- /dso/dso/task/control/scripts/README.md: -------------------------------------------------------------------------------- 1 | # Scripts 2 | 3 | ## benchmark_zoo.py 4 | 5 | Generates benchmark results for given algotrithm/environment pairs and writes them to `benchmark_zoo_results.csv`. 6 | 7 | ## policy_eval.py 8 | 9 | Also generates benchmarks but allows more control over the output during episodes. This can be used to debug policies and environments. Results are saved to `policy_eval_results.csv`. -------------------------------------------------------------------------------- /dso/dso/task/control/scripts/sample_zoo.py: -------------------------------------------------------------------------------- 1 | """Sampling obs, and action data from a Zoo policy on a Gym environment. Do, python sample_zoo.py --env ENV_NAME """ 2 | import os 3 | import sys 4 | 5 | import numpy as np 6 | import click 7 | import gym 8 | 9 | import dso.task.control.utils as U 10 | 11 | DSP_DATA_ROOT = "." #../data" 12 | DSO_DATA_ROOT = "./dso/task/regression/data" #"../../regression/data" 13 | REGRESSION_SEED_SHIFT = int(2e6) 14 | 15 | 16 | @click.command() 17 | @click.option("--env", type=str, default="LunarLanderContinuous-v2", help="Name of environment to sample") 18 | @click.option("--n_episodes", type=int, default=1000, help="Number of episodes to sample.") 19 | @click.option("--n_samples", type=int, default=10000, help="Number of transitions to save.") 20 | def main(env, n_episodes, n_samples): 21 | 22 | env_name = env 23 | 24 | # Make gym environment 25 | env = gym.make(env_name) 26 | if "Bullet" in env_name: 27 | env = U.TimeFeatureWrapper(env) 28 | 29 | #Load model 30 | U.load_default_model(env_name) 31 | 32 | for i in range(n_episodes): 33 | env.seed(i + REGRESSION_SEED_SHIFT) 34 | obs=env.reset() 35 | done = False 36 | obs_list = [] 37 | action_list = [] 38 | while not done: 39 | obs_list.append(obs) 40 | action, _states = U.model.predict(obs) 41 | obs, rewards, done, info = env.step(action) 42 | action = np.array(action) 43 | action_list.append(action) 44 | 45 | # Convert to array 46 | # Columns correspond to [s1, s2, ..., sn, a1, a2, ..., an] for use by DSO 47 | obs_array = np.array(obs_list) 48 | action_array = np.array(action_list) 49 | data = np.hstack([obs_array, action_array]) 50 | 51 | # Save all data as numpy file 52 | path = os.path.join(DSP_DATA_ROOT, env_name + ".npz") 53 | np.savez(path, data) 54 | 55 | # Save randomly subset as CSV 56 | np.random.seed(0) # For reproducibility 57 | rows_to_keep = np.random.choice(data.shape[0], n_samples, replace=False) 58 | data = data[rows_to_keep, :] 59 | path = os.path.join(DSO_DATA_ROOT, env_name + ".csv") 60 | np.savetxt(path, data, delimiter=",") 61 | 62 | 63 | if __name__ == "__main__": 64 | main() 65 | 66 | 67 | -------------------------------------------------------------------------------- /dso/dso/task/regression/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/task/regression/__init__.py -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-1.csv: -------------------------------------------------------------------------------- 1 | 0.11871958285046258,0.24687325606028884 2 | 0.1834206614934224,0.41873140299638295 3 | -0.7185623204618259,-1.442163710851701 4 | -0.7436491559462435,-1.5454379012719912 5 | -0.4883952613840026,-0.7585845616999927 6 | 0.09264477143591132,0.18579941035123673 7 | -0.5643544277452686,-0.9386736662166705 8 | -0.44448461244947524,-0.6700353493431952 9 | 0.4430668428455389,1.4996867113003767 10 | 0.31524228429941314,0.8780140832710299 11 | -0.12282387338272138,-0.1929260811452264 12 | 0.9340910699876552,6.275351441454106 13 | 0.8531404270227412,5.166673384806487 14 | -0.9426777153953942,-2.633857718783823 15 | 0.8596123022332922,5.249963511236734 16 | 0.9443762044722155,6.426892671545376 17 | 0.7631484553320866,4.099782842742867 18 | 0.5110944114601714,1.9161172094845647 19 | -0.8225509469786829,-1.91640576226322 20 | 0.785830287238031,4.353015982072755 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-1_test.csv: -------------------------------------------------------------------------------- 1 | 0.31996478768553027,0.8976244022895927 2 | -0.21257967841271386,-0.31515481262088474 3 | 0.8203831549867104,4.758861356656606 4 | 0.9127308598575128,5.968458959910604 5 | 0.963248806307291,6.711429392689262 6 | -0.30853830764495616,-0.44695255584096627 7 | -0.09356432968140305,-0.1507621358752572 8 | -0.8640289365909228,-2.141970594396725 9 | -0.3411046055818452,-0.4950426271807473 10 | -0.7560757969089531,-1.5991115614218006 11 | 0.8453489035755657,5.067599328355612 12 | 0.33934628603140404,0.980640397625079 13 | -0.0542002366183969,-0.09078833317134172 14 | 0.3931254541788556,1.2333691574443373 15 | 0.8237726526809206,4.800005211331611 16 | -0.28443576693616857,-0.412790175970997 17 | -0.9994878034792221,-3.046053076248751 18 | 0.05621376337688022,0.10736185353822494 19 | 0.5059242722326449,1.8821707026175336 20 | -0.6251875437818604,-1.112593751659523 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-2.csv: -------------------------------------------------------------------------------- 1 | 0.85302312595144,-0.3125476168797983 2 | -0.5192040880755251,-0.5187773454014789 3 | -0.3474524919824893,-0.6367663150176254 4 | -0.24464917745462356,-0.6919637337246688 5 | -0.32774285061859487,-0.6484976528498856 6 | 0.30485236524547754,-0.6614777030485579 7 | 0.32322183691506834,-0.6511176391116538 8 | -0.9881878683813008,-0.2941278284421446 9 | 0.35503024318511067,-0.6321269933112722 10 | 0.3600939990133216,-0.6289884919253449 11 | -0.02892110182386376,-0.7491639197500828 12 | 0.6826069333366456,-0.4013946592895088 13 | 0.7798358722637526,-0.3437570206907898 14 | -0.3640108735834957,-0.6265402323605163 15 | -0.2623187002838796,-0.6835952715977307 16 | 0.8259336262040122,-0.3226186500346784 17 | -0.3522262469316655,-0.6338517788225435 18 | -0.9060849905888937,-0.2985894993079571 19 | 0.8357079612172764,-0.31877092297227516 20 | -0.4983353645145161,-0.5341001221175112 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-2_test.csv: -------------------------------------------------------------------------------- 1 | 0.5087115635488217,-0.5264964222300395 2 | -0.3562797927337269,-0.6313553457897202 3 | -0.20623770127778296,-0.7083799310673098 4 | 0.7779705731853299,-0.34470884778807537 5 | 0.6758268534584182,-0.40591565094520615 6 | -0.5263400600523134,-0.5135145170587679 7 | 0.25104660173755744,-0.688991639188815 8 | -0.9694643351979817,-0.2931773722788741 9 | -0.2111704464318409,-0.706412068547282 10 | -0.14940513480519768,-0.727928609107673 11 | -0.666999217230819,-0.41187663320974977 12 | -0.7004309841582765,-0.38976906924925236 13 | -0.8437682010291312,-0.3157778576862624 14 | 0.05698828521737287,-0.7467576132834849 15 | -0.5153070278932821,-0.5216473270260304 16 | 0.9627538311519459,-0.2931277157605629 17 | 0.9091362111119801,-0.29803685002103547 18 | -0.2575840533949676,-0.6858865079199243 19 | -0.24549807599169604,-0.6915731723308585 20 | 0.9952320944357536,-0.2947988331162131 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-3.csv: -------------------------------------------------------------------------------- 1 | 0.17227716026273798,0.02643609089664467,0.2555268932701041 2 | 0.3566509134971355,0.800579212094254,0.46951845495538597 3 | 0.1302556109682198,0.09437471313331636,0.1939265670703596 4 | 0.3202582331482283,0.41967334227103326,0.4519860328282529 5 | 0.5610897275698851,0.7806331798048574,0.689645556194825 6 | 0.17913037952096467,0.4568480225647106,0.25857821043530016 7 | 0.9507059090186074,0.7438698428650716,0.921884642451076 8 | 0.7030732578867109,0.7817285448219432,0.8041145000186881 9 | 0.04841875802389706,0.5274523789075003,0.07005541741944252 10 | 0.6840580605240315,0.9491814309174783,0.7607525217564559 11 | 0.24500673448447596,0.7644999452799437,0.3333619261558622 12 | 0.7856913416485816,0.12864125470689503,0.9221369349087634 13 | 0.2957493650529648,0.4035115784596721,0.42050942172965483 14 | 0.39576340767024365,0.14254734981768424,0.5579659696407079 15 | 0.2676918247804134,0.5719021247488286,0.37496394608407396 16 | 0.5015707924239793,0.6278884362210335,0.6499601985720599 17 | 0.816413716229895,0.9162397787004117,0.843679790827311 18 | 0.12938762632033285,0.20558134313449516,0.191847298622783 19 | 0.3195506594760398,0.4503109851500654,0.4495407374834547 20 | 0.8871674118280858,0.5224996100202376,0.9383680771741569 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-3_test.csv: -------------------------------------------------------------------------------- 1 | 0.3188634787288198,0.6157405129257801,0.43862515669189134 2 | 0.20225609690342872,0.9261037495675445,0.26729101624480445 3 | 0.3691710670811109,0.38632700103986817,0.516105627362292 4 | 0.5090078797196218,0.10854459967005414,0.6904445892916952 5 | 0.5785161489295412,0.757500377805743,0.708823748523179 6 | 0.07836418485334296,0.7446116766849875,0.1092413311845956 7 | 0.11373920943124172,0.2891737436876287,0.16801076423611858 8 | 0.516782902505783,0.7785123431706307,0.6474866684017614 9 | 0.27014702295068405,0.8315467752822294,0.3606354429174288 10 | 0.6432687636932711,0.4832228377062805,0.7981178643616832 11 | 0.43595672776635896,0.552215122687233,0.5852737697421015 12 | 0.8604227083441006,0.7191060295245346,0.8995579843593858 13 | 0.13677537668173612,0.835309013713542,0.18621502737268492 14 | 0.6447308985031706,0.6889002984700272,0.7748846840226388 15 | 0.8962170744233188,0.11242053296838506,0.9729258755358503 16 | 0.3321730632703018,0.9355979982941892,0.4265535612617032 17 | 0.4742537169243566,0.38872525477574416,0.6405870269143612 18 | 0.5619559935938065,0.9080386558954572,0.6709615418802273 19 | 0.4130621130536729,0.1381080416597994,0.579320020772753 20 | 0.036342895428388755,0.09980564952804372,0.05441951545604222 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-4.csv: -------------------------------------------------------------------------------- 1 | 0.8013494772494848,0.34639757369268653,2.5006212863283914 2 | 0.26155627498611755,0.348119861888576,1.6928037888578709 3 | 0.3766585134618361,0.6872144946441879,1.3802219748422353 4 | 0.5376200325077618,0.10221691093660856,2.534041928587247 5 | 0.3773759777899848,0.382500959776534,1.8598568092224785 6 | 0.19221928762069196,0.8864921038850134,0.6258286304899868 7 | 0.91905433922056,0.5392764844242849,2.5798504417134023 8 | 0.1316959602394575,0.09351602036016793,2.2337248913791563 9 | 0.6308206358589296,0.033610465128141254,2.6585112846473455 10 | 0.05202012851930893,0.17193321475405932,1.6241681486557196 11 | 0.7457146493410285,0.5211533736667815,2.3171523214035434 12 | 0.9165493364076949,0.21428718979753536,2.6500510918552598 13 | 0.2121193813118909,0.6463112407382539,0.9911173431820702 14 | 0.18052715584963475,0.06555747169508275,2.413373683988311 15 | 0.07800902427469436,0.4896390158950862,0.7743096231467986 16 | 0.44775940769008493,0.9475619220646179,1.2609766009712624 17 | 0.660537545113137,0.3130023920266224,2.3713259470099692 18 | 0.18069905543919795,0.9012309483874249,0.5777093671243244 19 | 0.3119062564185605,0.24691692020133116,2.025022018850477 20 | 0.050012158171824916,0.10841162276895933,1.9513197419333015 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-4_test.csv: -------------------------------------------------------------------------------- 1 | 0.9724086881002747,0.02692098995858283,2.6979670579014323 2 | 0.7433045528549092,0.3104570346321426,2.4624452815000675 3 | 0.8004839152003531,0.6106132498260741,2.356945753412071 4 | 0.7254610383446598,0.0438813722729523,2.662240704296289 5 | 0.5466265971907477,0.4787171334407674,2.022049748736397 6 | 0.08400693968883999,0.7182962165037851,0.4557255904442856 7 | 0.11654937829522938,0.29228509965014293,1.4405174884432792 8 | 0.4530097695731119,0.2789326397555024,2.164915452619227 9 | 0.2716871086444135,0.14543141327872477,2.233878887759959 10 | 0.4448611184510918,0.26846221434086537,2.172327604157895 11 | 0.8128212793415077,0.9928875067244604,2.1978547533427415 12 | 0.18038617539542623,0.33014801301026697,1.5339130605150044 13 | 0.0751808473110146,0.6819651912472063,0.462283089221884 14 | 0.3630415716128804,0.05004752903599097,2.566496446231056 15 | 0.42973917246835036,0.7201789652786995,1.46962211848448 16 | 0.09154529695198332,0.04559352597226396,2.421146617761551 17 | 0.9291776526556865,0.8565095360180149,2.5353623956963274 18 | 0.2729701563096014,0.6453580834548013,1.1680357353906736 19 | 0.8496414627162954,0.956743301298686,2.310258049925713 20 | 0.4244685993539462,0.5492268536428543,1.6864231549668365 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-5.csv: -------------------------------------------------------------------------------- 1 | 1.8752486146662828,1.51873493277778 2 | 2.1620863831533526,1.6307563433200631 3 | 0.16191375322005852,0.44626664278284567 4 | 1.7882436266066368,1.4830845089630473 5 | 1.231306974495869,1.2306533137443374 6 | 2.3736204152489293,1.7086699829856504 7 | 1.4724147690191156,1.345760069958056 8 | 3.6521206524019174,2.119459459969536 9 | 1.114429758929016,1.1707897349578573 10 | 3.1067967741229827,1.9548299241036977 11 | 3.121230907369868,1.9593657177936274 12 | 3.52191436496954,2.081334828640633 13 | 0.7589428368037594,0.9661778766193232 14 | 2.0285301906728024,1.5795860642989819 15 | 2.2920602407449944,1.6790575023257373 16 | 2.2485288986929035,1.6630365436129992 17 | 0.5609552435048388,0.830647307532476 18 | 3.4265543376869685,2.052964158322052 19 | 1.991698048056468,1.5651800532556808 20 | 3.2081475471218166,1.986459534689754 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-5_test.csv: -------------------------------------------------------------------------------- 1 | 3.3624232792716264,2.0336618778705815 2 | 2.2410393100993398,1.6602645426022287 3 | 2.376302030651409,1.7096349018726873 4 | 0.3886215540399185,0.6913787033667581 5 | 0.16847856604172895,0.45522372107714976 6 | 2.442845242804561,1.733406948367754 7 | 3.025407563840029,1.9290545102519099 8 | 1.6532469219427428,1.4260062110627616 9 | 0.270481357646716,0.576794651418909 10 | 1.2645833343865358,1.2471718010344202 11 | 0.3276262317268315,0.6348072660453744 12 | 2.005522489636268,1.5706026430171984 13 | 2.5572473934414366,1.7735315880843416 14 | 2.829261832038427,1.865473680706127 15 | 3.3346087169929253,2.025233004348215 16 | 3.189351918891453,1.9806319345694916 17 | 0.0714061571329001,0.29636054608106516 18 | 0.6272197624203217,0.8783395173718394 19 | 2.042938696027822,1.585185981553654 20 | 1.1160306823202926,1.1716303765496863 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-6.csv: -------------------------------------------------------------------------------- 1 | 1.4667603049611322,1.177250807734646 2 | 0.905912071342085,0.9587793150766859 3 | 0.8401938425212632,0.9285080313505853 4 | 1.7198776239389826,1.259859034588008 5 | 0.1911484777304393,0.4941558936465681 6 | 0.871734832698849,0.9431999179946786 7 | 3.725899706010337,1.7512351850837296 8 | 3.056988701815853,1.6096637847621014 9 | 0.6097681979723522,0.8099909977432309 10 | 1.989414901063396,1.340469713964137 11 | 0.16179046200173808,0.4602713491207515 12 | 3.505904285212168,1.7064156422903551 13 | 2.488814311704347,1.4746630656857886 14 | 1.7156315971913467,1.258533091938467 15 | 2.3842226287196846,1.447937243808323 16 | 1.6340889326776171,1.2326944576786194 17 | 0.7646123083281249,0.8919612496798034 18 | 1.3151877215231584,1.1237993580073107 19 | 0.3971008272962555,0.67473266764142 20 | 2.623991975446022,1.5082662951808654 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-6_test.csv: -------------------------------------------------------------------------------- 1 | 3.72258750136051,1.7505718225849765 2 | 3.4047690060952074,1.6852694190126147 3 | 3.438010830481035,1.6922592024015333 4 | 2.4521301601895535,1.4653640715150509 5 | 3.09031516909255,1.617116027814489 6 | 0.9875694351514435,0.9946855633747081 7 | 1.9056138657384913,1.316118122291083 8 | 3.7799703233790067,1.7620168378719516 9 | 0.297955686460881,0.5970196354985718 10 | 1.192524821486018,1.0778916907157354 11 | 2.820144928522089,1.555305214642467 12 | 2.9248596227558115,1.5796495262350754 13 | 3.4051784356508636,1.685355747769776 14 | 1.8472449702740055,1.2987914886275607 15 | 3.75457294975819,1.7569637169755958 16 | 1.0246475172957044,1.0104265137127062 17 | 3.0004762300602144,1.5969194893405627 18 | 0.6435574020818473,0.8288161127736796 19 | 2.4417868402412624,1.4627277511806838 20 | 0.5910875659376966,0.7993255163316177 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-7.csv: -------------------------------------------------------------------------------- 1 | 0.6195363224154055,0.5057071436714135,1.261693232613405 2 | 0.23754154250279957,0.6714274384428591,0.4758979866028217 3 | 0.0045216554116890295,0.7175391988763881,0.008857454915193876 4 | 0.7152056193232839,0.6223901068045844,1.3023861084090114 5 | 0.8391491852942714,0.21569085208954875,1.732972373155241 6 | 0.999338454933063,0.8509539051939774,1.2701788781009058 7 | 0.8361283273949456,0.38283693884226666,1.6422754802962536 8 | 0.23637272143261479,0.31478364554856475,0.5752181292065692 9 | 0.7025885137972812,0.5829833279549572,1.3216341794863296 10 | 0.35424321666870284,0.879333890150607,0.566769603247402 11 | 0.09431595801687376,0.4332823874707199,0.22200396911442233 12 | 0.5051095513855706,0.7312815856622572,0.9087555183958025 13 | 0.27938555740220694,0.06508020108843116,0.7090326936813415 14 | 0.7382324113378991,0.3890073938391796,1.5156567450877567 15 | 0.8014976803960596,0.9768712728923539,0.9663355225704232 16 | 0.4492742291602704,0.3272630451132146,1.044293376512564 17 | 0.29904467994587824,0.12180873472441034,0.7524619279370507 18 | 0.04825496901880577,0.02316396299312584,0.12534701174631804 19 | 0.7302888018658177,0.6220254682963786,1.3215346740654748 20 | 0.6557577189456483,0.0017005583017118964,1.5058341520342806 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-7_test.csv: -------------------------------------------------------------------------------- 1 | 0.7523000074422268,0.852468091145251,1.0916694369058249 2 | 0.10817926806574418,0.18540045903500102,0.2755355604727166 3 | 0.5802591207750668,0.44818565583337167,1.2343412174873432 4 | 0.92691466089456,0.08112542711956283,1.8615280934591334 5 | 0.5601150679036051,0.7426194666462409,0.9805320688310134 6 | 0.45406143683315303,0.7459154207958674,0.8175991577326143 7 | 0.8872090189401445,0.9021790765517254,1.1333487430949185 8 | 0.6482830815964608,0.04350618792777339,1.4915628915930073 9 | 0.8191094434393327,0.41838938892600497,1.5985274697371834 10 | 0.9763606312095716,0.18673238451429786,1.8765687353921012 11 | 0.32190791135715635,0.09787785748894562,0.808854724546072 12 | 0.9475451286398117,0.34481537447976063,1.775158799119685 13 | 0.9667887693590269,0.6825600523146658,1.4760547409855793 14 | 0.8926415090434078,0.5776965602476144,1.5363438862387877 15 | 0.339930935961245,0.7405053408376399,0.6313456736129831 16 | 0.7772496089408704,0.08263602679008375,1.6883340555775874 17 | 0.03074740479004734,0.5241399764843211,0.06919281598370453 18 | 0.09321451965168848,0.9730200985401906,0.13606674945756364 19 | 0.7929962540487898,0.6104456943851079,1.4056835554423226 20 | 0.9957234369641742,0.9064854012370696,1.1862474265368084 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-8.csv: -------------------------------------------------------------------------------- 1 | 1.456381312032055,2.2795020542120836 2 | 1.371736482801502,2.1768773935187156 3 | 0.68871994251986,1.3099772052275118 4 | 0.8361586205480174,1.4974872041526321 5 | 0.5425859181723942,1.1305171958970817 6 | 0.6422665524641396,1.2520182304566436 7 | 1.9702403423589268,2.8601456177378584 8 | 1.5864228025519416,2.4334722339806576 9 | 0.12588126664339172,0.6970519985950037 10 | 0.38646315401626796,0.9513555544144539 11 | 0.5408775709977967,1.1284758196290916 12 | 1.1416531225357907,1.8896221760246852 13 | 1.5037190604035164,2.336081776903181 14 | 0.07271252643340076,0.653528981863025 15 | 1.3593326812667585,2.1616880067278785 16 | 0.2956201090580304,0.8554737043596841 17 | 0.5217486671035079,1.1057252322114466 18 | 1.2503796526199513,2.0267298334751036 19 | 1.9006193763636328,2.785863400131068 20 | 0.9918049066011314,1.6978351232769635 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Constant-8_test.csv: -------------------------------------------------------------------------------- 1 | 0.22968413375168217,0.7905293198867664 2 | 0.012067636210907784,0.6075313185348663 3 | 0.005503731793912614,0.6027833315113436 4 | 0.6835457966547689,1.303487249131239 5 | 1.561442471169111,2.404254624995175 6 | 1.515231874574492,2.3497513542516053 7 | 1.6566394986598516,2.514662775338651 8 | 0.3044654456781384,0.8644952800995092 9 | 1.707598820825581,2.5727119498944075 10 | 1.7967093285981688,2.6724385960892914 11 | 0.8742616326019734,1.5464628092921984 12 | 1.2906182308759986,2.0768825400444313 13 | 1.0095479892889885,1.720655928348942 14 | 1.9722148599751637,2.862232440811976 15 | 1.5191382053430986,2.3543813081332754 16 | 0.12596656280031926,0.6971242218553662 17 | 1.808096867710682,2.685019146027221 18 | 1.3014512012222907,2.0903243199721757 19 | 1.0702731532435144,1.7985720458025058 20 | 0.5272923323961984,1.1122978799983634 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-10.csv: -------------------------------------------------------------------------------- 1 | 0.4907031557689563,0.2849656576941816,2.713448679874401 2 | 0.48834490496942584,0.46247297031305423,2.5192800761861363 3 | 0.026129752288513886,0.8085026570994222,0.10825612569543006 4 | 0.794298233883321,0.3506693190248297,4.01974985828669 5 | 0.931520417534097,0.8698060139348422,3.1056619271156114 6 | 0.593772659709018,0.19717128153219954,3.2919093173917595 7 | 0.33198340046854635,0.7065541585639896,1.4873697166390127 8 | 0.5572216872729236,0.8631873019385359,2.0624971112570005 9 | 0.7756511152989407,0.5964860234257952,3.475620501085106 10 | 0.8603357780819464,0.7446685925171019,3.3444732101675365 11 | 0.9794741298129286,0.39901650038942504,4.589918529672075 12 | 0.1683428007008828,0.31988928708295594,0.9542945768974314 13 | 0.7202756932320975,0.8560474292717368,2.593889226547279 14 | 0.7174408544614789,0.37803011290646793,3.6662261499243676 15 | 0.5830349020036034,0.20164916113317155,3.236425971625287 16 | 0.5479768097649167,0.5390711335077653,2.6824899858535134 17 | 0.5800737612171324,0.6400502066819955,2.6376043674929837 18 | 0.08032515592979628,0.7643955321595783,0.347498625181586 19 | 0.009311524560681295,0.5445858040544551,0.04778653456307742 20 | 0.4792792292731801,0.8871062628696619,1.7476957006848102 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-10_test.csv: -------------------------------------------------------------------------------- 1 | 0.058259635087205885,0.7114524195819453,0.26461034974065895 2 | 0.3152778608568141,0.11788139668631259,1.8475721882337282 3 | 0.46261407385902786,0.9117217064220117,1.639805360611521 4 | 0.7722635493995025,0.6982881323605318,3.206663031607589 5 | 0.7524062917410893,0.368285884379154,3.8254367460227177 6 | 0.9657643372476383,0.10989665349812938,4.905133305066547 7 | 0.08786568647421744,0.5721400626522904,0.4426652945328947 8 | 0.23695440118791444,0.9507981826842778,0.818362269409466 9 | 0.34074823089657147,0.17152468696119083,1.9757300868843055 10 | 0.015507651941464418,0.5699137218557582,0.078336635618643 11 | 0.8773681471071524,0.8205066942694854,3.1463016081655044 12 | 0.46464716631413483,0.7056721216298143,2.0465311263421926 13 | 0.9817703397496174,0.6792928878932063,3.8814444190780892 14 | 0.4130750331024762,0.9543125620645716,1.39255991335925 15 | 0.07376732443915657,0.8440346960922612,0.2938228689519786 16 | 0.1711334602841138,0.9880385781067427,0.5623236584616685 17 | 0.17081739193392032,0.304019391650972,0.9731545211297361 18 | 0.9666400003894146,0.985335967740936,2.7285946746255227 19 | 0.6258393254654664,0.8223127347818452,2.3918304162604724 20 | 0.714732681246489,0.4938159875259598,3.4626803958193593 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-11.csv: -------------------------------------------------------------------------------- 1 | -0.3577939241342365,-0.41813196696575283,-0.02884512477374494 2 | 0.5651125634574108,0.29759287977731264,0.03278328422334883 3 | -0.44174167649352936,-0.22751788434139053,-0.015092919208170633 4 | 0.9891126879013872,0.47978399637510427,0.15331752871289261 5 | -0.9201898482073441,0.6151502563634847,-1.050415406387059 6 | 0.6731505297550924,0.1412702448061709,0.011103935643682878 7 | 0.5977973988084413,-0.7749898392986636,-1.2113087918233323 8 | -0.36515469286944047,-0.41857654468978,-0.029808244038205388 9 | -0.6793547158674433,-0.2736716894310032,-0.03627003725221148 10 | -0.30103313705591495,-0.2683205087143332,-0.011459198438500686 11 | -0.8546839679717526,0.025825989059343968,-0.0005878207575367685 12 | -0.15109241034687781,-0.3232720199678072,-0.005029321702097209 13 | -0.8878106535247121,0.4761576295506855,-0.4341210944628945 14 | 0.58047989668275,-0.9052827806122923,-0.8502037384116021 15 | -0.457508650262898,0.1416214936204634,-0.013290000330298082 16 | 0.5345761877696511,-0.6104801241241511,-1.4031302188129156 17 | -0.9383381798700234,-0.7819612765249098,-0.3129573941428644 18 | 0.5373121519972526,0.7709161858926081,0.13115477340909015 19 | 0.5825955109502559,0.6767592265918216,0.12343961499395478 20 | 0.39466572199843664,-0.9821465386190007,-0.2557512159469327 21 | -0.5396326763405188,0.47969326965390824,-1.117921451303354 22 | -0.6107568654498616,-0.19270770493522482,-0.017241225916439274 23 | 0.8303722575627515,-0.19767403251247018,0.04258418009906371 24 | -0.36767959930081173,-0.7109047865434823,-0.06334433996049149 25 | 0.5660366828271399,-0.11061817644430727,0.008608579000480935 26 | -0.969022494307417,0.1909125680819277,-0.04398410922154301 27 | 0.34805482506174457,-0.2131048935317914,0.04076706952440148 28 | 0.7937727981787261,0.6747809726274427,0.19535666246302963 29 | -0.15967824060705804,0.8625695011500458,0.02698929438955198 30 | -0.5911404044501325,-0.7879515219204791,-0.15732113471349848 31 | -0.2752365117072879,-0.8371660041550693,-0.047728006575834904 32 | 0.6624755852404502,0.5503142602206863,0.10959121365896696 33 | 0.5839537581640022,0.21538358509844668,0.019790283519401947 34 | -0.8290742786991352,-0.736921820681087,-0.23836310846584421 35 | -0.8923950025241751,0.9524872252785617,12.02304049645575 36 | -0.8502880265447246,-0.54735507037294,-0.1549794567450839 37 | 0.42286948876823516,-0.11911846368356871,0.00835320454656352 38 | 0.1298831796726332,-0.5114750663394687,-0.01156526601604377 39 | -0.9902244196953489,0.23576958221526967,-0.07224529431290626 40 | 0.43518023513247606,-0.8477106978098796,-0.3298964398534275 41 | -0.548429307458169,0.5121462127595457,-2.1743272996977665 42 | 0.012128887246011022,-0.02023322075389844,-7.431126469303634e-06 43 | 0.4250165463081852,0.8709075130886272,0.10572463565975379 44 | -0.3634263798104851,-0.8014840398135279,-0.0728333227027959 45 | 0.898552506330778,-0.39510782831125746,0.2503608599088086 46 | 0.17707406659848757,0.529042992841156,0.012428397974893703 47 | 0.7994356319108897,-0.11084264230799312,0.011402974316643756 48 | 0.6121654872627884,0.5527183312746695,0.09827947027446157 49 | -0.16566779221781847,0.3107611092918767,0.018267624917760288 50 | -0.4057717009917381,0.6180171488621871,0.29629626629585254 51 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-11_test.csv: -------------------------------------------------------------------------------- 1 | 0.6305653604727217,-0.2264901011508884,0.050477334105602845 2 | 0.20014302431581155,0.49401900023948286,0.014083395312916525 3 | -0.5330352383234362,-0.34507669646308825,-0.03852948684788095 4 | 0.03369019378455573,0.8271896498953539,0.0009021415009377283 5 | -0.8908517010071901,0.39247278886485537,-0.2452845943127634 6 | -0.01278452402387309,0.9107321885383102,0.00015097306593311215 7 | -0.45869152083626474,-0.6350211354660635,-0.07757370721951244 8 | -0.10552757698495796,0.09124850430625187,-0.006493568671859808 9 | 0.13479555341561245,-0.590060759430256,-0.013895690730710052 10 | 0.577738340054009,-0.13507874268365594,0.01375835505959327 11 | -0.41613910334766335,-0.19079550489898733,-0.010386552701684702 12 | 0.17290381289720846,0.8948209793302939,0.02241930417756262 13 | -0.7917158709521839,-0.5762652862160962,-0.15216105272001243 14 | 0.866390695163342,-0.19472295186963828,0.04237477020201873 15 | -0.03416327247156703,0.6831776542041992,0.0008393283921374287 16 | 0.19582857127345532,-0.08228640994017344,0.0022869211120783287 17 | 0.4380260392512203,0.8662226355152032,0.11038206378806212 18 | -0.49743014948928055,-0.2695580900399217,-0.023441221858824093 19 | -0.8186447806248796,-0.9399919566728197,-0.33671551515925613 20 | 0.5840597436604509,-0.4892420125007453,0.8611374483753492 21 | -0.8035347863736415,-0.21754626362259,-0.02992625737500322 22 | -0.6646858136267337,-0.8600232559500218,-0.21432173985104183 23 | -0.49539351361961814,0.8901310477593292,0.4926064646595938 24 | 0.9402565272120555,0.8133271508845794,0.33350070792368447 25 | -0.29536373549606587,0.7471231494853845,0.10779323823670349 26 | 0.7385585076907815,0.2347317378277627,0.03087955746724816 27 | -0.709336417847731,-0.3864970583767646,-0.06858866298555047 28 | 0.7061851278099898,0.396968740664827,0.07123831869563384 29 | -0.9440348956307492,0.3681778903607986,-0.20978616864870925 30 | -0.07217111779579821,0.2273236740039799,0.0017348313487384273 31 | 0.5999187982864618,0.3403887968235424,0.04434712936869378 32 | 0.0022179397056927286,-0.848683554369263,-4.185831326382377e-06 33 | -0.6294150878932121,0.8167018652835403,1.410893642984635 34 | 0.6044252944923569,0.3821455320310665,0.054077245871805686 35 | -0.0812117142710167,0.2077929318901548,0.002249726006484479 36 | 0.675440970273582,-0.24230084960999143,0.06183811762154039 37 | -0.6232998299612567,-0.6865317345088611,-0.1397976268308446 38 | 0.7432200131646125,-0.12630773102989656,0.01428470387716684 39 | 0.7308974960850836,-0.35643403483080593,0.18124324722313873 40 | 0.1788939857271683,0.506786814561945,0.011987264975090804 41 | 0.673615027581292,0.38362251126559066,0.06316247325660536 42 | -0.5354252227461618,0.2143181200825819,-0.0410077104530105 43 | -0.013408495234983775,0.522456964870825,9.640554055403465e-05 44 | 0.5728172350610379,-0.11612001254409776,0.009687638647133463 45 | -0.09943600208382519,-0.4237212509861652,-0.003393247598545874 46 | -0.2637241900234626,-0.972440922242622,-0.053204688498072465 47 | 0.058167681062007315,-0.8258902852972636,-0.0030061032161989893 48 | 0.984268254155449,-0.1832603898463998,0.040618827976068674 49 | -0.21434618460884836,-0.7964229285140616,-0.02883148830883323 50 | -0.254498338689084,-0.23799963106600375,-0.007449343943671097 51 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-12.csv: -------------------------------------------------------------------------------- 1 | -0.2272121046457456,-0.1726046772264107,0.11776050046225263 2 | -0.9830674019835415,0.5124731549255561,-6.821864913223302 3 | 0.843755524682505,0.5383315499188199,2.741155342310687 4 | 0.7404814399207875,0.5737934546218062,1.1784308890320674 5 | -0.832341712950404,-0.19884148899212395,50.8144559055624 6 | -0.5535055321128923,-0.9751867725370862,0.05602042815761372 7 | -0.27253088874919884,-0.45900252073978565,0.015546529336116539 8 | 0.587571847303981,0.27843867381692133,3.2442642824153536 9 | 0.2653757770280736,0.15698146488328568,0.3402211136064928 10 | -0.4346458418189527,0.5325175259952264,-0.1027249476972148 11 | -0.8300314024327273,0.1086720755119599,-306.9861140210641 12 | 0.9592848958213871,0.14049126184110872,292.94770003809595 13 | -0.12834583666949628,-0.40742746472834446,0.0005149416528588606 14 | 0.7662895178413447,-0.7603022106868595,-0.6011815784895431 15 | -0.8095318091404233,-0.5211148819264018,2.456796381299461 16 | -0.28819016037282186,0.3763926322281852,-0.03727969945668696 17 | 0.9545591630748171,0.7543534743386755,1.8462460101361517 18 | -0.5431413753584324,0.30582312500047504,-1.652545679909189 19 | -0.14031417068302598,-0.35297105137419704,0.0012367741603778212 20 | 0.9966451990573879,-0.19910974241791268,-124.57341314652733 21 | -0.06933024428341517,-0.8196181539231591,2.909235944311782e-06 22 | -0.3418527916742822,-0.7424013275262509,0.011409836858251159 23 | 0.225373886066758,0.15003191100685354,0.17217379128608 24 | 0.9277206061704135,0.877072357563706,1.0185438817079697 25 | 0.8163940961246698,-0.5664767149782874,-1.9950468018554979 26 | 0.2743399675659506,-0.05049587355239815,-12.069179481792741 27 | 0.28989590496887363,-0.2959759362204144,-0.07896619175631643 28 | -0.36308306787303946,-0.203089823914127,0.7532957074531484 29 | 0.5620688420675617,-0.6588610625970568,-0.19613993567993393 30 | -0.5268004554605854,-0.7336488276267592,0.1027462606800378 31 | -0.9904303181828382,-0.33606744636830066,25.109628809002903 32 | -0.0916628504363477,-0.29163781551159773,0.0002608766940213088 33 | -0.7449406840712227,0.9850411549985822,-0.24001884414107036 34 | -0.7838676264334665,0.6580736695791003,-1.0384607810465554 35 | 0.028709144562956812,0.7078348431385091,5.499275153408611e-08 36 | 0.7233417649732881,0.347568310623501,4.716254022479607 37 | -0.06539417379172918,-0.47546424654767194,1.1126067517338188e-05 38 | -0.8431332348146354,0.671165540753393,-1.4092631363525576 39 | -0.9078620040087326,-0.5364886944385465,3.9940866122563174 40 | -0.8991708391557356,-0.642477718851233,2.2163424103824814 41 | -0.9820617995816461,-0.6057065924317393,4.110618656927211 42 | -0.3485173919662432,0.029750839907196402,-195.265080295785 43 | -0.01572607863911024,0.7730597307601612,-2.081915323842032e-09 44 | -0.3173358472134544,-0.9826139257409781,0.00339193035305668 45 | 0.06855105499625869,-0.36532235068010976,-3.104859727915133e-05 46 | -0.17295793644961877,0.8925030930467821,-0.00021770744745751059 47 | 0.8099685477573069,-0.4935721217765767,-2.899271655327376 48 | 0.3687753851466127,-0.4363296848739322,-0.08210419456660337 49 | 0.2708264124007973,0.4541709693989102,0.01555238403172548 50 | -0.49910735417275287,-0.4367286275193105,0.37182181201794784 51 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-12_test.csv: -------------------------------------------------------------------------------- 1 | -0.718525227149871,0.22123159386049474,-17.6875898335431 2 | 0.5225233389406758,-0.13409298510500367,-16.155123782140585 3 | -0.7256579434169097,0.9647807940674178,-0.22406489529380852 4 | -0.5630735051658848,-0.7007079103292804,0.16451837146533793 5 | -0.18502794542240286,-0.9243394218170866,0.0002745947346645074 6 | 0.7644627533766186,-0.9729686054496711,-0.2834556838354374 7 | 0.1205465641615957,-0.7362523937626362,-6.378139641413625e-05 8 | -0.9495107786847892,-0.2060912211698831,88.17003594790074 9 | -0.7477762432206807,0.23901458345976767,-17.123207404586314 10 | 0.31287199049027703,-0.10489569319214298,-2.5975292932355702 11 | 0.5950291894525581,-0.8671805084903563,-0.11438335499495211 12 | -0.6154774309068238,-0.344008971369697,2.1694585439531235 13 | -0.9512256045911376,0.7064100012833252,-2.209261587132607 14 | -0.4623428121719686,0.6293884791761621,-0.08473526349264986 15 | 0.7370055306918617,0.46859259003787956,2.113330809607058 16 | -0.6867344073444295,0.00539200750425306,-974299.708870876 17 | -0.18288260486301433,0.10174349912869318,-0.1942414831697233 18 | 0.3929797598394382,0.290057897999088,0.3840580109462378 19 | -0.6768302436956983,-0.6367003951465593,0.5502922859818147 20 | 0.6383868790635345,0.8559576209930915,0.16906860617113012 21 | 0.48148798481096033,0.339116494649925,0.663560502286645 22 | 0.8412547855421975,-0.9433096537857948,-0.5019666934568703 23 | 0.942492086627152,0.6028644485126047,3.3941383491800576 24 | -0.1051103509047957,-0.5268010689181206,8.775815562119515e-05 25 | -0.042802605202555855,-0.6791288570571166,4.586637981774792e-07 26 | -0.19253039884665535,0.6956239490680478,-0.0007859102915045813 27 | -0.9370007281997421,-0.053795244962248745,4639.465757166524 28 | 0.7130309333307538,0.32563298583817324,5.33773098031119 29 | 0.1793284955598895,-0.9769248057327997,-0.00019891297541328257 30 | -0.4258219161042103,-0.8672989656419638,0.021460190058509996 31 | -0.14809118334645843,-0.14050304987588014,0.02567963004088903 32 | 0.12950881240593692,0.5939562109785674,0.0001738734695502236 33 | -0.6049209969629179,-0.3852466114962947,1.4166964763840852 34 | -0.8605668330656071,0.24492964331284428,-32.12173315319229 35 | 0.8245583026742342,0.9316096353989329,0.47141666400931787 36 | -0.9083442312723484,-0.2897766088037115,25.413374421684125 37 | -0.5163191850953774,-0.14867905610869503,11.164568101338599 38 | -0.7594339701207808,-0.2818638496464687,11.280593784989517 39 | -0.548475735868748,0.4779978073597422,-0.4544743572586436 40 | 0.7867892585795615,0.4915629862800115,2.538368029483676 41 | -0.7702535089483433,0.768641829941265,-0.5970303118554554 42 | -0.07441914801766902,0.9238152198355114,-2.8951304007404404e-06 43 | -0.2890786955178881,-0.741736529946396,0.004946880583913652 44 | -0.2312259282138973,0.7840222292024213,-0.0013715054213778795 45 | 0.41459376104451495,-0.4794890155970395,-0.11111631346373549 46 | -0.9499612089558758,0.437246726307829,-9.254429639536786 47 | -0.12127018051929572,0.907985301014417,-3.503754482117566e-05 48 | -0.7455245982434799,-0.9025134669615922,0.3132916455628487 49 | -0.6720207940477636,-0.04718997251822832,1304.260626878089 50 | -0.4079217811867206,-0.815509342245927,0.020825606342530622 51 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-13.csv: -------------------------------------------------------------------------------- 1 | 3.7775490152277795,1.5574114719397003 2 | 1.7478972745361552,1.2045882845976803 3 | 2.516278012427513,1.3601481262441275 4 | 2.6218087613139924,1.3789028528994567 5 | 3.7277888320379775,1.5505428320053274 6 | 3.8120466586205155,1.562138019937875 7 | 1.8789863309080475,1.2339792974758486 8 | 2.6665228815200352,1.3866976244901723 9 | 1.747640353307891,1.2045292613985963 10 | 2.4203533211890353,1.3426400245924823 11 | 1.4534017861560526,1.1327356347271618 12 | 3.6443138683229517,1.5388817806271287 13 | 2.1735188262478196,1.2953520062452024 14 | 0.9343286012323504,0.9776120540499345 15 | 0.9455275403910552,0.9815024643364195 16 | 2.4465545328080456,1.3474675032857617 17 | 0.9989988025888903,0.9996661560902765 18 | 3.0961664544025678,1.4574984457697668 19 | 0.20530022324938368,0.589924555372177 20 | 3.8652412542757104,1.569370682945455 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-13_test.csv: -------------------------------------------------------------------------------- 1 | 0.1363125161784806,0.5146499222355517 2 | 3.9889559082035473,1.5859387550849258 3 | 1.4463596709948163,1.1309032043152591 4 | 0.11090204359040667,0.4804481399104467 5 | 0.36766751016444044,0.7163936890641314 6 | 3.3388867006255656,1.4946306848096627 7 | 0.5792328785639351,0.8335872598839538 8 | 0.029258878274879496,0.3081431780583595 9 | 1.2546670833588949,1.078556336784995 10 | 0.8338988436640977,0.9412488469863324 11 | 3.365768330767462,1.4986310964142258 12 | 2.845397822596367,1.417036373852008 13 | 0.8667326728832312,0.9534437058469283 14 | 1.9807202251864755,1.2558594717284954 15 | 3.438354204094476,1.5093276839902383 16 | 1.3344583874696703,1.1009518998438592 17 | 3.4482200666177656,1.5107699049730694 18 | 0.7164877758850006,0.8948211936983741 19 | 1.894363971214006,1.2373364547933234 20 | 0.7295630439374965,0.9002316456970363 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-14.csv: -------------------------------------------------------------------------------- 1 | -0.5946315300185212,-0.21148844772368425,-0.966789764776284 2 | -0.5932390728951964,-0.008680877747963,-1.0090608626581743 3 | 0.6004795936401996,0.7141289748459263,2.230771905520921 4 | 0.4243759786147796,0.03309739017891222,1.0937465603637049 5 | -0.7684660581989782,0.7697064868677719,-0.7683756123632121 6 | -0.6464983141733747,0.9375227845229273,-0.33107500117517363 7 | 0.4045843400395521,0.621435381943177,1.4047896081802893 8 | 0.9643153140749026,-0.5460538054806272,3.906374878496156 9 | 0.02764594720451785,0.7963210290713063,0.6485484079697362 10 | -0.46524082474717066,-0.2675814662951497,-0.7265920851984993 11 | 0.7253340350209037,-0.19803799239887399,2.3356430204436593 12 | 0.15262904805617605,0.11407289381582886,0.3445296612159601 13 | 0.08015424309409958,-0.7609416610477435,0.7143765171148498 14 | -0.7574565365107297,0.08100412705006832,-1.2988135548800428 15 | 0.6135768505587718,-0.43745643244957333,1.9870484741817684 16 | -0.6207282889348009,0.1637450614447995,-1.029412042403921 17 | 0.539591002510603,0.23150755245788934,1.5552111719140944 18 | -0.09741318426431689,-0.2752809900548434,-0.1104003183162856 19 | 0.6312154619553103,-0.7188510095107103,2.3653262165476603 20 | 0.7075687373482811,0.08093211906692455,2.2190070635401393 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-14_test.csv: -------------------------------------------------------------------------------- 1 | 0.5543541507548109,-0.8650594689726707,2.238829053731447 2 | -0.49262539754795953,-0.8390068917278699,-0.19521608837556237 3 | -0.870773791732045,0.8737561806625367,-0.8461981514165573 4 | 0.44838667971686985,-0.9189558698188196,1.920723769071834 5 | -0.07678176539575055,0.545178514568295,0.14481753034681544 6 | 0.3155364224717987,0.6825179801004053,1.2060071656357207 7 | -0.30264267887199425,-0.727877808385436,-0.03144769747240472 8 | 0.8274033095944564,0.08115276997402643,2.821198904602063 9 | 0.5840981687480122,0.2941987041840406,1.762438144724092 10 | -0.33343483439892263,0.45037450073943575,-0.38516837243731006 11 | 0.7144268579673054,-0.47904106976403193,2.472136216181784 12 | -0.004742452641406603,-0.3447986112409447,0.10914372283818695 13 | -0.9793678428831656,-0.49014204012514306,-1.5517883709615767 14 | 0.055955321979740846,-0.8704219352691485,0.8023924506115726 15 | -0.7931056121158493,0.38409657214701554,-1.2285057503760477 16 | 0.07810009482054658,0.36481296975786925,0.295392784318824 17 | 0.21037800762638148,0.6398215339636235,0.8708105285958054 18 | 0.6018404142437062,0.678005890493931,2.1918786151848386 19 | -0.5329046731160554,-0.09664432658785538,-0.8989528775593848 20 | -0.36912866959584845,-0.20432839849435802,-0.602233644586819 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-15.csv: -------------------------------------------------------------------------------- 1 | 1.1848042236568141,1.034497198126093 2 | 3.4744411888527584,1.2828532919383233 3 | 2.760779526173306,1.2251981859673144 4 | 3.3835026730031097,1.2760664920732674 5 | 3.9520522402493157,1.31632926068402 6 | 2.3665921780462473,1.1880225522833632 7 | 2.5327821020391657,1.204258071152953 8 | 3.166468097698175,1.2592588837288294 9 | 1.6310607954633491,1.1027930737912581 10 | 3.384772028045306,1.2761622235430603 11 | 1.8466669399113034,1.1305186073642073 12 | 2.5783147071183956,1.2085571308660752 13 | 2.3073150461443976,1.1820106228974923 14 | 2.3833183245321057,1.1896971204814784 15 | 1.065194194218591,1.0127115383133876 16 | 0.5117779951525692,0.8746137923449151 17 | 0.20410730958323287,0.7277324011377402 18 | 1.4359921236396458,1.0750543270958974 19 | 3.590763146091848,1.291330330686923 20 | 2.7468600181635248,1.2239602273261658 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-15_test.csv: -------------------------------------------------------------------------------- 1 | 1.7339321469702393,1.1163655335272842 2 | 0.9447257496670338,0.9886922970277009 3 | 2.511015748229698,1.202181076015206 4 | 2.231446364218319,1.1741329893332983 5 | 0.8090746408451506,0.9585123607805218 6 | 1.659929100778371,1.1066694155814816 7 | 2.95914514076875,1.2423193605621086 8 | 3.7530657206889315,1.3027984521201883 9 | 3.1099492270063984,1.2547310917053958 10 | 0.29677950909518813,0.7843082484180269 11 | 3.570606703897207,1.2898773099497038 12 | 0.9012520266377604,0.9794206375232091 13 | 0.39175287429795835,0.8290914534489203 14 | 2.533023086331213,1.2042809863681452 15 | 0.7128808079885949,0.9345518135121142 16 | 1.3896646512620823,1.0680264348779578 17 | 1.1823894152303738,1.0340751618763335 18 | 1.1566561214109257,1.0295343757610762 19 | 3.1642366734536793,1.2590813526043878 20 | 1.6156047217598637,1.1006950737338057 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-16.csv: -------------------------------------------------------------------------------- 1 | 0.8695615974483859,0.9110318656762952 2 | 1.7966792007326529,1.4779067300368571 3 | 2.7835492100054355,1.9787880046970117 4 | 3.1790383526448007,2.16204059899995 5 | 3.3813126004843035,2.252804726342324 6 | 3.9823391680102707,2.5124195463956847 7 | 0.01864231790918014,0.07030722224981939 8 | 0.3336175264091179,0.4810230695166974 9 | 1.1926997526172727,1.1246587145553602 10 | 2.4657434194002437,1.825150108157579 11 | 2.8282931994402913,1.9999368665071187 12 | 2.1302218676194853,1.655578936663323 13 | 0.36036966238987445,0.5064059680701051 14 | 2.270435460743189,1.7274524725598117 15 | 0.2394308845727129,0.3855849846246991 16 | 0.32465676724645576,0.47237072359865995 17 | 0.9260809028514165,0.9500926368117452 18 | 1.4285259543162594,1.2684073702121446 19 | 0.7764052170777114,0.8447450001909089 20 | 2.4847021894795134,1.8344937160890398 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-16_test.csv: -------------------------------------------------------------------------------- 1 | 2.040462135922412,1.6087393817756193 2 | 1.7331736717931507,1.442872829744703 3 | 2.60662975274483,1.8940272893171908 4 | 3.4567798208163922,2.2862013597885307 5 | 1.5155954267545368,1.3194376043145613 6 | 2.4380145860569247,1.811441026741514 7 | 1.5570416772141442,1.3433839593237307 8 | 1.892424780873451,1.5299565042761745 9 | 0.6497555624625413,0.7501820541461495 10 | 3.2328371220858982,2.18636441184901 11 | 0.0602829609505271,0.15374336551926193 12 | 2.0526866687721435,1.615158353370422 13 | 0.6163400572766693,0.7242362571319287 14 | 0.19685529217996356,0.33840081199162064 15 | 2.8460207506478516,2.008285140258509 16 | 1.458525101833963,1.2861035547526345 17 | 3.29304848038966,2.2134280856702655 18 | 1.6226460698235727,1.380858575836619 19 | 2.9184571174777925,2.0422185228401557 20 | 2.132563685289046,1.6567920666352496 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-17.csv: -------------------------------------------------------------------------------- 1 | 0.7364173626933004,0.34689713849529824,2.52651960058309 2 | 0.2373419237661648,0.2701970707677236,0.9063574107322203 3 | 0.8260974629772231,0.4189410413995347,2.6868165798375134 4 | 0.7919054460083834,0.2603551980597666,2.7508322745613145 5 | 0.23336689360883978,0.07769872502055986,0.9222270209831305 6 | 0.944960002628049,0.13269503889086876,3.213394283500909 7 | 0.8548569244794918,0.7761205115591816,2.1536894965557978 8 | 0.4714133288573795,0.9584868647280859,1.0440975765586158 9 | 0.7174479891250225,0.9204609883279328,1.5922548537115693 10 | 0.9541782800282111,0.4536634652110313,2.9332590479222906 11 | 0.48774246919716047,0.2965258420888043,1.7927217836293303 12 | 0.9904468961492111,0.6142593687024497,2.733604774091078 13 | 0.9386824795411747,0.3342771453799406,3.0484928266805036 14 | 0.11995365976424788,0.6107696289343255,0.3921254978323425 15 | 0.9506471345338259,0.43082376306911907,2.9577165176795064 16 | 0.8878305906828513,0.7159826342831943,2.340916737128587 17 | 0.7240990365433758,0.6040878254826547,2.1808754591581643 18 | 0.002204412445070414,0.1684497479724535,0.00869283647073717 19 | 0.692376566744537,0.7997068143059998,1.779558878338557 20 | 0.9428840891985358,0.9174483179131414,1.9676189197954452 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-17_test.csv: -------------------------------------------------------------------------------- 1 | 0.1755490241734422,0.0802207030761718,0.6963483496423101 2 | 0.4103626252469573,0.9512608423982643,0.9265936260810145 3 | 0.3274668759586825,0.4316379811868213,1.168579068358342 4 | 0.07164695182009884,0.6084293141325109,0.2349575706976814 5 | 0.8186974141806449,0.720480930632591,2.195117691510551 6 | 0.9145741663355549,0.38317934701125256,2.93938226201344 7 | 0.5755211156961115,0.5972197306876472,1.800239142955474 8 | 0.7646087548614445,0.17653231291489968,2.7259843892185445 9 | 0.2071358684042477,0.7636703280651604,0.5941887408707733 10 | 0.5149687070791713,0.6033040307275427,1.6222521511388472 11 | 0.1276613302115851,0.9739536266860531,0.2862210215803023 12 | 0.3592491358643355,0.25887085693620193,1.3594276717437663 13 | 0.4026963489498123,0.39841939929067527,1.4448198602542395 14 | 0.3065768628461034,0.3737376031501972,1.1238545004576483 15 | 0.8951089498097633,0.7734674404079331,2.2331301743653227 16 | 0.680466858231589,0.4032027532053436,2.314813455092776 17 | 0.7339982962654695,0.8329365288488892,1.8024312936839466 18 | 0.6553479132967788,0.4278759262148518,2.217976129104273 19 | 0.645821945177187,0.9631887614913903,1.374407915397772 20 | 0.5479894210454719,0.19611127716446486,2.0439438985672496 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-18.csv: -------------------------------------------------------------------------------- 1 | -0.9516852024426672,-4.543375947934035 2 | 0.09043620788894513,-4.991854805951268 3 | 0.9016371625183137,-4.549445707385229 4 | -0.09866639775210762,-4.990312442221663 5 | -0.23887569819512255,-4.944588768649234 6 | 0.37413110299281493,-4.8701335961602314 7 | -0.8883927196279324,-4.552345235733787 8 | -0.9335073016825375,-4.544659466300207 9 | 0.329133877017987,-4.897686097119882 10 | 0.17798697487550919,-4.968826319333211 11 | 0.9891238008331533,-4.54420704013424 12 | -0.651903651040195,-4.67224938475077 13 | 0.510581114938256,-4.775123025250942 14 | -0.11791949072118602,-4.986192001309165 15 | -0.9824920819203016,-4.543711168380598 16 | -0.9330318941020239,-4.54470740746687 17 | -0.1393500705672075,-4.980770999404259 18 | 0.2711695768658413,-4.929217908903557 19 | 0.8035872933159305,-4.582273511825515 20 | -0.41897624090347274,-4.840464470790624 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-18_test.csv: -------------------------------------------------------------------------------- 1 | 0.1025504779483073,-4.989538843303909 2 | -0.5839910574828304,-4.72095993594357 3 | 0.3328555699784759,-4.895502330148758 4 | 0.84332948994465,-4.565936503352916 5 | 0.443071015051119,-4.823781061056141 6 | -0.9257231341375023,-4.545535246107697 7 | -0.3588867567164815,-4.879739481972372 8 | -0.8531742208403514,-4.562496744602593 9 | -0.1784999973583119,-4.968649311234296 10 | -0.8666017787823261,-4.558220465577352 11 | -0.7583274489755314,-4.6051460038456105 12 | 0.3088717237204164,-4.909250745463196 13 | -0.5944085692912442,-4.713332646991817 14 | -0.1676115171649346,-4.972303725624444 15 | -0.7679104307385736,-4.59996151855808 16 | 0.7994366477796866,-4.584192642652418 17 | 0.5985920478520173,-4.710280951506084 18 | -0.6646081534395589,-4.663505051511271 19 | -0.9008706755514448,-4.549599282491743 20 | 0.1501839394987725,-4.977700566314001 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-19.csv: -------------------------------------------------------------------------------- 1 | 0.9065294770112042,1.0488437566706954 2 | 1.181798325371983,1.9882948172567465 3 | 1.1386175845468673,1.8511538485662289 4 | 1.3368835414230658,2.454558776223089 5 | 1.4748075089843853,2.838543220958931 6 | 1.205593452025144,2.06243795415724 7 | 1.2113538946850262,2.0802384135832215 8 | 0.6540966831724175,0.010419382702119381 9 | 1.535931665250232,3.0005043173367545 10 | 0.512627683733395,-0.6892412980402713 11 | 1.153710170070433,1.8994755735938729 12 | 0.7019149809264271,0.22444436311360935 13 | 1.8940478287231475,3.863405808599695 14 | 1.8014184449970134,3.65304199420555 15 | 1.1250462807718025,1.8073385484181546 16 | 0.5087954582949628,-0.7098900318634235 17 | 0.757102946346583,0.4602691731265385 18 | 1.6993354711031512,3.4113084517869567 19 | 1.6452692174333754,3.278779762051851 20 | 0.3852572201347033,-1.4434124798029133 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-19_test.csv: -------------------------------------------------------------------------------- 1 | 0.008556521231624314,-9.513529806143028 2 | 1.168003464724972,1.9448516563185105 3 | 0.5402481801894654,-0.543420437290598 4 | 1.4344945319891445,2.729051624031083 5 | 0.8253832348667165,0.7375170259511361 6 | 0.5552810831805444,-0.46614845705742686 7 | 1.2865037517697009,2.307359712222851 8 | 1.3749134775996412,2.5631330415700178 9 | 0.11149629092500879,-4.269466003120874 10 | 1.6997305581029958,3.4122651079169644 11 | 0.007640551770572213,-9.740901027349288 12 | 0.08542692693557719,-4.830943197690219 13 | 1.9034859437070106,3.884385700367453 14 | 1.4562969608824294,2.788538334462774 15 | 1.3092828933003267,2.3744014394740534 16 | 0.7500757529621505,0.4308567537001744 17 | 1.9733254602544565,4.0371340483517955 18 | 1.3402879914794714,2.4643658418581422 19 | 1.6460098969731065,3.2806173125442344 20 | 1.7658716497262932,3.570085779759414 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-2.csv: -------------------------------------------------------------------------------- 1 | -0.40801111627347764,-1.8478972270877856 2 | 0.2574608929888693,-1.9359456357690248 3 | -0.5902160560583076,-1.7163977185816077 4 | 0.7907238768556002,-1.5883436933803792 5 | -0.06849042982596631,-1.9953200763397791 6 | -0.32890057474831025,-1.8978223883377898 7 | 0.9645525419635892,-1.5431261147411166 8 | -0.6545045047293272,-1.6704471290930907 9 | 0.5775750480097739,-1.7256743309029394 10 | -0.987429289111365,-1.5440658562284206 11 | -0.19532433044286868,-1.9625829457075519 12 | 0.31192232481015325,-1.907545451788376 13 | 0.9379255591933195,-1.5442486858725275 14 | -0.09290441339411992,-1.9914060989592521 15 | -0.7702753746940905,-1.598708899300079 16 | 0.5725192323917483,-1.7293966034340893 17 | 0.8788235041917636,-1.554759272132305 18 | -0.834787549050285,-1.5691231509985342 19 | -0.5703754977274373,-1.7309765661346035 20 | 0.18800212969105656,-1.965285218079758 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-20.csv: -------------------------------------------------------------------------------- 1 | -0.49031865796019014,-0.22044765053704313 2 | -0.9216435649165677,-0.01568049962953244 3 | -0.49458762721969296,-0.2197281246805337 4 | 0.2220700685585908,0.2743572346091614 5 | 0.4477229152905984,0.7063518955420467 6 | 0.9453981395031099,3.393234662088241 7 | -0.24692365509708658,-0.1831528035578957 8 | 0.01538083634167453,0.015617463294463731 9 | 0.1499780931605479,0.17305335797557797 10 | -0.940106473095643,-0.009523293197838623 11 | 0.5900264006583107,1.1308612807980258 12 | -0.8206014645426625,-0.06586648285680163 13 | 0.30586090378315745,0.4108403898498888 14 | -0.0799918220772986,-0.07355546234937396 15 | 0.7994142161403159,2.173361007048066 16 | 0.5032672526960302,0.8529793365822433 17 | -0.14716878878529194,-0.12511007563950083 18 | 0.256962988380673,0.32847327150839933 19 | -0.47891187425305093,-0.22214374070668486 20 | 0.910251641483752,3.0502135735584 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-20_test.csv: -------------------------------------------------------------------------------- 1 | 0.03672238190251642,0.03807280055845911 2 | 0.40249200752063863,0.6012987403267053 3 | 0.713480413080779,1.6665596533646279 4 | 0.5644383566533224,1.0418195202970684 5 | 0.3636306283582458,0.5196996365711809 6 | 0.9266500833534372,3.2059111999564207 7 | -0.6445639331501631,-0.16774972515711284 8 | 0.4127976441709118,0.6242226700760484 9 | 0.4799916203101049,0.7889422018020948 10 | -0.7224310195751542,-0.12491840504873808 11 | 0.3523188795666752,0.4972838767742387 12 | 0.7325422542966178,1.7680616190950722 13 | -0.6597141873703387,-0.16003481625422722 14 | -0.07324423379931222,-0.06785284380102419 15 | 0.7331202424143166,1.7712301357236049 16 | 0.1309520114568048,0.14843301826729466 17 | -0.38789040623932736,-0.22357456270819612 18 | -0.32195174675100313,-0.21101391700340805 19 | -0.7395882842437447,-0.11468253736235079 20 | -0.8879572241171343,-0.02983423349185743 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-21.csv: -------------------------------------------------------------------------------- 1 | 0.0023050933559765063,0.9999946865587366 2 | 0.016394610608542637,0.9997312528619777 3 | 0.2450854533804463,0.9417015509881652 4 | 0.4461729397845893,0.8194922873958207 5 | 0.8146401733320021,0.514974130300454 6 | -0.1677904704979103,0.9722389785940507 7 | -0.026777723429732392,0.9992832105443078 8 | -0.1187872361736999,0.9859886777300988 9 | 0.5824393909839052,0.7123145765339198 10 | -0.3819479683042146,0.8642577348030587 11 | -0.770415470435702,0.5523684369648084 12 | -0.2755520406137293,0.9268820804284592 13 | 0.5202352144813442,0.7628875186762716 14 | -0.2783810005116738,0.9254307432826903 15 | 0.017947040365506872,0.9996779556095496 16 | -0.7330384497677374,0.5842972891954366 17 | -0.8624351165646673,0.47530701789350766 18 | 0.7756395895993775,0.5479250491820042 19 | -0.8980275962163193,0.4464385279239223 20 | -0.30718681346556465,0.909951717685658 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-21_test.csv: -------------------------------------------------------------------------------- 1 | -0.6743537411546188,0.6346047161259949 2 | -0.5103325140151669,0.7707129566064493 3 | -0.8889168261231966,0.4537662307635882 4 | -0.8489102491157205,0.4864366454316388 5 | -0.625949785137911,0.6758303922120636 6 | -0.6110131127992551,0.688433175230511 7 | -0.48670344716925373,0.7890857789432582 8 | -0.2089521399819274,0.9572784229182902 9 | -0.47349579927117613,0.7991566683272368 10 | 0.2992034394527361,0.9143675104101036 11 | 0.28845772042928086,0.9201598661548573 12 | -0.9127234090115104,0.43471526677115324 13 | 0.28054005304564233,0.9243146626093666 14 | -0.5907135343432848,0.7054337058921771 15 | -0.10679829380642869,0.9886589248362695 16 | -0.933138691905687,0.4186383667155339 17 | 0.9837324486387744,0.37994471240999633 18 | 0.6820509443042742,0.6280135786922022 19 | -0.905475529486262,0.44048189542890287 20 | 0.14615172623943962,0.9788661890176747 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-22.csv: -------------------------------------------------------------------------------- 1 | -0.2741341237629098,-0.21514640845194166 2 | -0.7235933233456122,-0.3882655161995995 3 | -0.5132456015742379,-0.33753561987193115 4 | -0.2370775857563603,-0.19164135094478713 5 | 0.8526946080638818,4.1708245560704915 6 | 0.7682211754217028,2.912387020334341 7 | 0.7729987070088336,2.9711704565132284 8 | -0.7105283138721634,-0.38840155277836325 9 | -0.4134222172703914,-0.2922477087485862 10 | 0.41536840312350964,0.7098493481704512 11 | -0.10767088529570379,-0.09720476071023293 12 | -0.5407307678355873,-0.34839224513213823 13 | 0.9096765259127906,5.348707146925397 14 | 0.7555747639169554,2.76287004404211 15 | -0.3186386740794118,-0.24161646430315042 16 | -0.6675032853182019,-0.38452441143771177 17 | 0.45668979972780943,0.838978624339525 18 | -0.3950021935804433,-0.28298743797592907 19 | -0.02640074975737261,-0.02572167816869499 20 | -0.842878563974333,-0.34085389840065816 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-22_test.csv: -------------------------------------------------------------------------------- 1 | 0.7438779813633962,2.6320760027394936 2 | 0.3581376759898771,0.5578154915896761 3 | -0.49725306856412455,-0.33086886281694616 4 | 0.15144014370960068,0.17846720027032023 5 | -0.26961469099402247,-0.21235353110841781 6 | -0.6194792514144116,-0.3742216218497687 7 | 0.9180995582183582,5.551180010804005 8 | -0.14526916593206018,-0.12684279070995416 9 | 0.6750444325503242,1.987772832110684 10 | -0.0752289466315772,-0.06996551459117403 11 | 0.65407346024833,1.8274505443587103 12 | 0.2418331209985165,0.3189671016199865 13 | 0.7718678743461653,2.957139534915516 14 | -0.3023101269590922,-0.232117550815289 15 | 0.31725714134718364,0.46463258598384566 16 | 0.664376761695675,1.9043903112625362 17 | 0.678308387162389,2.0140730569138965 18 | -0.7887868320116249,-0.37488064115387437 19 | -0.7669787899440501,-0.3820843358551881 20 | -0.17407593068440796,-0.14826620609549124 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-23.csv: -------------------------------------------------------------------------------- 1 | 0.5147514401843278,0.8759164909235203 2 | 0.4072397703878179,0.920422831291293 3 | -0.025944156723555212,0.9996635069924271 4 | -0.2006887088958058,0.9800634358638471 5 | -0.0017551185351722953,0.9999984597806499 6 | -0.312595024570439,0.9523165159669114 7 | -0.1460205936584047,0.9893956202350414 8 | -0.9898995193985187,0.6126567020196159 9 | -0.5529048939277905,0.8582568109099395 10 | -0.5109594323360025,0.8776215885888772 11 | 0.7712585091675088,0.74273211528389 12 | 0.812532083849064,0.7188488300520787 13 | 0.2958242637777091,0.9571874851015977 14 | -0.9643516013310758,0.6281432675474983 15 | -0.3163080023807159,0.9512052862330928 16 | 0.42373103542241486,0.9141377614577657 17 | 0.25658406004762635,0.9676182010281094 18 | -0.4091099254901651,0.9197204943689953 19 | 0.9302145827590684,0.6487877969602102 20 | -0.026423655749504116,0.999650956138403 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-23_test.csv: -------------------------------------------------------------------------------- 1 | -0.924474139713023,0.652250741912772 2 | 0.36542379793000457,0.9354128689883063 3 | -0.7233910571007207,0.7697824698412259 4 | -0.32270695577801,0.9492625208955607 5 | -0.5478047872880065,0.8606692088696054 6 | -0.6935199130711045,0.786246490230614 7 | -0.6765458777243747,0.7954421444911511 8 | -0.44625758900700885,0.9052239404574197 9 | 0.6569374464888782,0.8059098395768503 10 | 0.45890928291397515,0.9000554873273635 11 | 0.46288932986612363,0.8984059381794495 12 | -0.9854087846623021,0.6153800488396407 13 | 0.7648907238642337,0.7463736672651863 14 | 0.4441096753780047,0.9060899440419695 15 | -0.3315898837798541,0.9465079343725669 16 | -0.09410528077507596,0.9955818867697569 17 | -0.29747063545213503,0.9567201165700597 18 | -0.8241678362840477,0.7120363530759894 19 | 0.5259990600108053,0.8708047406965376 20 | -0.4124876010147003,0.9184452258125336 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-2_test.csv: -------------------------------------------------------------------------------- 1 | -0.3587555570078156,-1.879820994467437 2 | 0.16466631649273644,-1.9732550633425816 3 | -0.13395649286405353,-1.982217371483762 4 | -0.03392125150607228,-1.998850010885762 5 | -0.7124326615931573,-1.6321726292252847 6 | -0.08249957057463808,-1.9932170221276546 7 | -0.6488017932921559,-1.6744065605716447 8 | -0.6398033482412957,-1.6807098654621428 9 | -0.5072657033173029,-1.777557911889196 10 | 0.7085026928492799,-1.6346383388557908 11 | 0.8842624073071399,-1.5533548225604454 12 | 0.9018771016548821,-1.5493979955791313 13 | -0.7096820641754107,-1.6338961168771766 14 | 0.5678046356890469,-1.7328724801406907 15 | 0.25549671778123084,-1.9368853552500742 16 | -0.6281135683752097,-1.6889908266936293 17 | 0.5138404139577268,-1.772726537235057 18 | 0.5677236984784144,-1.7329321873272925 19 | -0.7517500752267601,-1.6088024375853731 20 | 0.6836493490389408,-1.6507041928831205 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-3.csv: -------------------------------------------------------------------------------- 1 | -0.8852714098568202,-1.452929249614521 2 | 0.7152082456404398,-0.6880517127439556 3 | -0.7607638260594021,-1.35679913225476 4 | -0.06426900072363106,-1.000265461126085 5 | 0.26278291673761833,-0.9818978047757951 6 | -0.21503015893371535,-1.009931767790311 7 | 0.729843466030754,-0.6734681896130594 8 | 0.16425942925934311,-0.9955697176239853 9 | 0.8021521136183931,-0.605160732754341 10 | 0.7328491237269947,-0.6704867570411995 11 | -0.4253405097725371,-1.075619743261083 12 | -0.544699465145821,-1.1538778628501065 13 | -0.319888486921728,-1.0325567097535566 14 | -0.10064350919591258,-1.001019377301235 15 | 0.7893345870653716,-0.616518275027367 16 | -0.9795393136392674,-1.4634376305556298 17 | -0.6408207859697828,-1.2385004375421795 18 | -0.5864694922056992,-1.188614786077181 19 | 0.9566789955912338,-0.5318731642784442 20 | -0.8317228999982738,-1.4190489470311793 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-3_test.csv: -------------------------------------------------------------------------------- 1 | 0.14627336890490894,-0.9968710712859764 2 | 0.3919309055869975,-0.9405403887239895 3 | 0.34279003420085497,-0.9600090289516965 4 | -0.37430111392394205,-1.0519024699769335 5 | -0.8411432031761288,-1.426052633663363 6 | -0.588993219060987,-1.1908225267695585 7 | 0.6099319604738724,-0.7904253588068907 8 | 0.23455514782920717,-0.9871155683418593 9 | 0.9407293334515332,-0.5315845605183533 10 | 0.5223785567786019,-0.8631924579309695 11 | -0.6342874036841575,-1.232271191296134 12 | -0.33499888277685974,-1.03734970650396 13 | 0.7208490073608822,-0.682420526324558 14 | -0.5523490953450665,-1.1599742041896433 15 | -0.6063895384301432,-1.2063495769431096 16 | 0.8300106925462216,-0.5822643911178695 17 | 0.22540420234550362,-0.9885629042525003 18 | -0.2128059665372508,-1.0096271821354341 19 | -0.22782554937629085,-1.0118089662975032 20 | 0.2975889127211422,-0.9737520911563324 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-4.csv: -------------------------------------------------------------------------------- 1 | 0.34438417476801453,-0.657983097576365 2 | 1.3498903218337546,2.1919100993725986 3 | 1.4944310610972937,2.489317015820959 4 | 0.5676381027670434,0.1626057469948895 5 | 1.7313147333725132,2.939320996400152 6 | 1.306853755748102,2.0996682646489084 7 | 1.9171936508047736,3.263848784947939 8 | 1.9312712739016913,3.287499452795412 9 | 1.059905346301841,1.5338579097878602 10 | 0.8047422820876755,0.8725104406067372 11 | 0.13552720882146718,-1.8532848344238784 12 | 0.009484340978767936,-4.648583562702584 13 | 1.437426460222301,2.3742344108368876 14 | 1.671255451783876,2.8293059319682428 15 | 0.3578040364924082,-0.6014339220654423 16 | 0.0015345391272285624,-6.477989468139973 17 | 1.7023273745996126,2.886552441183494 18 | 1.5500984076062352,2.5990504803648524 19 | 1.7356499928521942,2.9471607716219266 20 | 0.8505924234215447,0.9980450547908153 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-4_test.csv: -------------------------------------------------------------------------------- 1 | 0.5811527218441161,0.2063929467265896 2 | 0.5201465550442255,0.004613874521752437 3 | 0.11684880658489494,-2.0228020045695385 4 | 1.739160969316085,2.9535000758734684 5 | 1.9742606412488615,3.3589561978631277 6 | 0.13868715096993656,-1.8266071298876843 7 | 0.11001738196543198,-2.090710074356827 8 | 0.24992358148749916,-1.1029290022490013 9 | 1.4890546273901977,2.478581837945686 10 | 0.05126997271048306,-2.918025932973134 11 | 0.8198791260692988,0.9143129178286395 12 | 0.2002956938117506,-1.386058078147033 13 | 0.6860842620903791,0.5314003833627078 14 | 1.6316370947860108,2.7552648607790053 15 | 1.569739112431068,2.6371611556817003 16 | 0.2417518205485656,-1.1465206218107309 17 | 1.5607599739765485,2.6197767114215895 18 | 1.1934368749473265,1.84784628515034 19 | 0.3649738543931156,-0.57174357744775 20 | 1.8668940900799722,3.1783093242582545 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-5.csv: -------------------------------------------------------------------------------- 1 | 0.09576080145857058,0.6702668420125966,-0.22180325030997183 2 | 0.5544024752498204,0.3335424091509436,-0.29822271869504524 3 | 0.3683812193414322,0.32961040472143033,-0.2525426852824941 4 | 0.6216466196279428,0.7666976335354886,-0.2697649348064347 5 | 0.6925741450139052,0.48524156651626815,-0.35190886849633873 6 | 0.44039301030211364,0.6453995986538427,-0.276656368756314 7 | 0.6326736934500593,0.7283625761428106,-0.2908737525606568 8 | 0.875897167480403,0.8358249783358535,-0.22061678490308312 9 | 0.10295076385469926,0.900303026608891,-0.0907363117693234 10 | 0.1741031630077743,0.22343449535855298,-0.1778701093177692 11 | 0.5996028094447202,0.6359248920905191,-0.3178385738442193 12 | 0.6823073232344006,0.1440177327801323,-0.22418966826766065 13 | 0.9302755385685533,0.24753488183161276,-0.2423945863035443 14 | 0.7469995063823333,0.05585281829287414,-0.15819195659503038 15 | 0.024996218318800056,0.8107362291385523,-0.15345822342247495 16 | 0.4473070681459217,0.9514825254724544,-0.09562889132700902 17 | 0.8530613170915465,0.3643294016588866,-0.32281072074609907 18 | 0.5934474734436764,0.5459178946768313,-0.33286113642263987 19 | 0.3810927653323878,0.44391428822695056,-0.28110889603527545 20 | 0.3602622357969162,0.6491594866718282,-0.2576643261076608 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-5_test.csv: -------------------------------------------------------------------------------- 1 | 0.9010983816485689,0.3301819908445126,-0.29352542092320716 2 | 0.9265704497087447,0.7430898691715152,-0.2493198707808605 3 | 0.3797808832702313,0.9661319069635448,-0.06669487057756696 4 | 0.45442557357370905,0.12992254702879924,-0.16423941752448667 5 | 0.4552358485782757,0.0189432159914672,-0.06997901708834606 6 | 0.8017678388394509,0.8786954532329921,-0.20875895779856757 7 | 0.4284520719135829,0.7006923772103635,-0.2546756438905896 8 | 0.4609313181769298,0.7758922174791794,-0.2266736170454 9 | 0.4954350330028212,0.5688711788517631,-0.3066156136690119 10 | 0.8869610991557864,0.47023024940262437,-0.32798917454786536 11 | 0.6274253516635911,0.37185681679401206,-0.3256029816924373 12 | 0.5130706228761659,0.3794099426400884,-0.3012234315975986 13 | 0.36934442805078627,0.7671260837760803,-0.21041875287868494 14 | 0.8614147493717038,0.24905476473174082,-0.27561021056768353 15 | 0.4527546809208167,0.30154053317451357,-0.2614029851269586 16 | 0.06144004052187868,0.41676997737940413,-0.2432904423191656 17 | 0.6878440116224989,0.34283886959816545,-0.32688817850967256 18 | 0.8203181694875368,0.08598386875177322,-0.17777682120766522 19 | 0.9022625752739623,0.09986709235103952,-0.1616829569188245 20 | 0.0750366225866691,0.8648119278682849,-0.1173030481221019 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-6.csv: -------------------------------------------------------------------------------- 1 | -0.013486097645286854,-0.01312957401721331 2 | 0.07679927166206801,0.09009359541252453 3 | -0.335127704164349,-0.17296704273069696 4 | -0.028830139345354278,-0.027236910896164025 5 | 0.7113113293252751,3.8269298055970093 6 | 0.1453259895420358,0.19855713249280765 7 | 0.2551088022028969,0.4520195164883151 8 | 0.003945504664918165,0.003976823907463714 9 | 0.4102393459088256,1.0672519121824608 10 | -0.7455161757009154,0.3586410041542494 11 | 0.8126397523399123,5.487792580213331 12 | -0.04154738495762844,-0.03829825114529985 13 | 0.4130284739061949,1.0819994511967064 14 | 0.4752938206237973,1.4533457452010077 15 | -0.5326308149609393,-0.09662193696230664 16 | -0.8273698269413294,0.7169911433144496 17 | -0.5631765263274202,-0.06232365305264076 18 | -0.8764802621889765,1.0006056470643403 19 | 0.6661583450515272,3.2282657693628862 20 | -0.04310858295890241,-0.03961840174737235 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-6_test.csv: -------------------------------------------------------------------------------- 1 | -0.9809565941860179,1.8156417606013775 2 | 0.7199607587749719,3.9509285850629965 3 | 0.6066896666674373,2.554660471099628 4 | 0.29140996786506546,0.5643343773181174 5 | -0.5968537560921972,-0.014632665336960948 6 | 0.35444427168881476,0.8024253784942443 7 | -0.16271635590914757,-0.11988362061356858 8 | -0.6823138625115472,0.16278780749153177 9 | 2.6672024018070672e-05,2.6673446868726322e-05 10 | -0.11311798150386165,-0.09121397226319343 11 | 0.2517055608263665,0.4423136509795567 12 | 0.5439222419817102,1.9684977477807257 13 | -0.949744742273466,1.5387579680688142 14 | 0.6352608156213975,2.8628947051280305 15 | -0.9527674873303673,1.5642504259900627 16 | 0.8133439786412249,5.501030765719842 17 | -0.5976587070433892,-0.013354890142917686 18 | 0.31142777979112957,0.6336420028652984 19 | 0.5219605381827785,1.7903596464558789 20 | -0.2381866747475554,-0.15238546324421826 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-7.csv: -------------------------------------------------------------------------------- 1 | -0.45438604441950625,-0.47018418458669137 2 | 0.731032067640556,0.7979057263838989 3 | 0.7871295330568444,0.8709656526016187 4 | 0.579658850752083,0.6126699104729479 5 | -0.7201809493170719,-0.7840704030630896 6 | 0.4566510258433061,0.4726882466213322 7 | -0.5721261530851118,-0.6038531675447691 8 | 0.010375892486716554,0.01037607866433754 9 | -0.9298388808783982,-1.0697417834902658 10 | -0.5906962624112628,-0.6256517201390632 11 | 0.691558603698307,0.7480152244291316 12 | 0.32595986979707137,0.3317628093934281 13 | 0.7774577102886,0.8581802399383902 14 | 0.8303523410009257,0.9291156187547627 15 | -0.23047293458608342,-0.23251872862785528 16 | 0.017999185177820065,0.018000157061567768 17 | -0.48182062206070775,-0.5006807484399793 18 | 0.6483288435527297,0.6947117038182677 19 | 0.2669164902536265,0.2700971842394389 20 | 0.119226945431119,0.11950961605230292 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-7_test.csv: -------------------------------------------------------------------------------- 1 | 0.35816533817040286,0.36587232475181175 2 | 0.5742383404084295,0.6063219260612425 3 | -0.9335369934543172,-1.0751644754451828 4 | 0.7084901383791307,0.7692677288621326 5 | 0.2984710481065129,0.3029223764616028 6 | 0.9347834613528887,1.076995531274213 7 | -0.37199852632047636,-0.3806377925572052 8 | 0.37579338323578937,0.38470101360589154 9 | -0.9625890909032142,-1.1182821652276982 10 | 0.5062738029411658,0.5281801103467871 11 | 0.949312527043928,1.0984628379399946 12 | -0.5566065518144183,-0.5857955138003745 13 | -0.49354575947234935,-0.5138281394877553 14 | 0.8265284455043085,0.9239027383486408 15 | -0.8605097193978382,-0.9707095531181986 16 | -0.7179363753037369,-0.7812201179489241 17 | 0.18420919103305544,0.18525275590613455 18 | 0.9157857554119408,1.0492679179743172 19 | 0.7054037464032477,0.765377425377725 20 | -0.45360716245420685,-0.4693236455725742 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-8.csv: -------------------------------------------------------------------------------- 1 | 0.27179958752150335,1.037165465173442 2 | -0.4038107249124083,1.0826454892205297 3 | 0.7716424764906691,1.3127848691707678 4 | -0.5590400816331844,1.160375222548281 5 | -0.4139146808001406,1.0868927031875368 6 | 0.8525556301854638,1.3859788914978524 7 | -0.9149828954077095,1.4486280113852006 8 | 0.9742741024175701,1.513354844705996 9 | -0.8734338901250103,1.4063183574479279 10 | -0.22102851501627252,1.0245264090855204 11 | 0.628859541930721,1.2043350049141308 12 | -0.5229456516047006,1.1398807479984985 13 | -0.1142989003760786,1.0065392338416148 14 | -0.6352939950509471,1.2086783554971712 15 | -0.32673633215794196,1.0538548825586762 16 | -0.6319503116392797,1.2064150857880156 17 | 0.668615695116132,1.2319756804736566 18 | 0.5134153430313948,1.1347183189201866 19 | -0.5450779409781028,1.1522696984721663 20 | 0.8345035207808724,1.3688799922992714 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-8_test.csv: -------------------------------------------------------------------------------- 1 | -0.5952902056820892,1.182479847848382 2 | -0.9822650964717365,1.5224801389071625 3 | 0.9200963326374803,1.4540063903422407 4 | 0.49610276449750224,1.125603692393829 5 | 0.8014165963367581,1.3386943763524328 6 | 0.3764547439321191,1.0716998853300255 7 | -0.0201973809169238,1.0002039740318087 8 | 0.7579335093004995,1.3012479548192561 9 | 0.9268554773129265,1.4611741098580766 10 | 0.4668347851724821,1.1109607715584202 11 | -0.6180723491986164,1.1971652679782507 12 | 0.7997387797714712,1.3372030006851925 13 | -0.3224586031876526,1.0524418291345135 14 | 0.39612893255347714,1.0794904179433233 15 | -0.7290803333716005,1.2777627460608816 16 | 0.41030042547515255,1.0853607211043075 17 | -0.6356465499986381,1.2089177796939246 18 | 0.40104143473543874,1.0815007299137334 19 | -0.38107943793987253,1.0734937543269036 20 | -0.7758528846718913,1.3163776014694155 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-9.csv: -------------------------------------------------------------------------------- 1 | 0.13508364212151647,0.1561811600253573 2 | -0.6606650020634521,-0.40737121018547645 3 | -0.9074663796300231,-0.6742846065109798 4 | 0.6708180199770308,1.9817787725104652 5 | -0.8576730196472431,-0.5776361151242996 6 | -0.6383482478744777,-0.39648673145316343 7 | -0.44790135577230017,-0.30956964671270293 8 | 0.5141003941848821,1.055384107658906 9 | -0.015308938968490837,-0.015078109116269622 10 | -0.4855793426669426,-0.3273525136413119 11 | 0.9089605929083222,5.755415502732426 12 | 0.4646995740194071,0.8672325992417916 13 | -0.0410007154693135,-0.03938586675307193 14 | -0.5950313482614,-0.37654149822176963 15 | 0.47033880891309776,0.8869991259680438 16 | -0.6442074760676646,-0.39929113140211303 17 | 0.4627774687348851,0.8605873982244909 18 | 0.06996971787015349,0.0752338060510293 19 | 0.9091065578164823,5.759457280939048 20 | -0.47488617228173013,-0.32237706583783265 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Custom-9_test.csv: -------------------------------------------------------------------------------- 1 | -0.491225117514289,-0.3299590453113824 2 | 0.1954172193932584,0.24288009001681657 3 | -0.32724353948723994,-0.24656938478725807 4 | 0.325421826430059,0.4823881357683011 5 | -0.944854537516653,-0.7774058665793467 6 | 0.22793450323552022,0.2952264097545864 7 | -0.5683836769301629,-0.3646446074672375 8 | 0.6523530538534865,1.8363311936879223 9 | 0.016328802685002364,0.016599858499032016 10 | 0.3906074026630202,0.6408425759758141 11 | -0.21487728032805942,-0.17687176610802038 12 | 0.8886399801874059,5.222296705449116 13 | -0.044139405528321074,-0.042273479283175965 14 | -0.7709156148130227,-0.4771872313980404 15 | -0.30909640309652,-0.23612039789436712 16 | 0.4960757184858886,0.9826340367615787 17 | -0.7948037661130471,-0.49888687382422703 18 | -0.7910714248477226,-0.4952600765160381 19 | 0.985459842039816,8.370428418038706 20 | -0.25024709348507734,-0.20015887893786416 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-1.csv: -------------------------------------------------------------------------------- 1 | -0.7403801071368961,-0.5980661649010526 2 | 0.5206056048792103,0.9327356397523248 3 | 0.6525207575940659,1.3561365636798772 4 | 0.5356322049996658,0.9762079394047138 5 | -0.586133136442661,-0.443948325535601 6 | -0.3769022485680784,-0.28838790745876036 7 | -0.6201560629882985,-0.4740705376705796 8 | -0.8404642346449558,-0.7277713400189059 9 | 0.31264230226858225,0.4409467986663268 10 | -0.20966006871096332,-0.17491882414882493 11 | 0.48421935624763335,0.8322218716406187 12 | -0.5506969824140331,-0.41443812939783053 13 | -0.6350467381160174,-0.4878667956161994 14 | 0.5237341566285307,0.941690575897929 15 | 0.4441307242480925,0.7289885425998979 16 | 0.8378658262994001,2.12808281840724 17 | 0.4609140775860472,0.771273274760623 18 | 0.27611115397676866,0.37339851135463886 19 | -0.8594323772124557,-0.7556057558522368 20 | -0.10877247441601634,-0.09822795944736544 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-10.csv: -------------------------------------------------------------------------------- 1 | 0.0228132421457784,0.7286851234041879,0.034036713854823375 2 | 0.530507349451884,0.9873358031684089,0.55749430125988 3 | 0.24331232012318327,0.827287958499968,0.3261434844277611 4 | 0.9135949085667239,0.3341692334141876,1.4958206770472613 5 | 0.3162726006933466,0.05596186925905966,0.6210786121446784 6 | 0.4688051872054112,0.8409155094701379,0.6025307591157697 7 | 0.32195313305861517,0.44203548849301566,0.5720130838558007 8 | 0.00615841955188301,0.9682140891322488,0.006980792958391796 9 | 0.47449303125388287,0.5592392014755403,0.7745696708783979 10 | 0.4160672057791913,0.393573569918341,0.7465311821318746 11 | 0.7983464334539391,0.6881769604316976,1.1063978854528767 12 | 0.8956609496694983,0.7294206783654651,1.1640025455643779 13 | 0.29670765090250684,0.8362264887171637,0.39193693874046714 14 | 0.6241898591730041,0.228276967758137,1.1385568058165207 15 | 0.8411895503863956,0.39296964833881876,1.3772327688047012 16 | 0.4783408175881989,0.6404008463536714,0.7381999110484314 17 | 0.3515281171021475,0.07646737407365445,0.6866533333548202 18 | 0.7232222057867876,0.21679085933359166,1.2926254281630476 19 | 0.6044366075250462,0.14250546950735699,1.1250758074776122 20 | 0.3533217170399622,0.9563160460232419,0.3989802206568933 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-10_test.csv: -------------------------------------------------------------------------------- 1 | 0.9980273895374367,0.8163407522925702,1.1511714561584796 2 | 0.9017372534752636,0.9986149305169777,0.8494599608797637 3 | 0.20137644687162592,0.9505646212451497,0.2325105904861465 4 | 0.9884399933989239,0.36026762111001664,1.5631065033464489 5 | 0.203710819355445,0.044751109573220704,0.4042045287450107 6 | 0.19682641803805223,0.3283423244454744,0.37022190632404844 7 | 0.7615890879666736,0.32630280686214896,1.3073199810979679 8 | 0.6796637546607204,0.3830964029466509,1.1659406594954822 9 | 0.4529342134134964,0.631726793513821,0.7063035526966084 10 | 0.5555004627999165,0.3439116052167913,0.9929748977856105 11 | 0.4502480806857929,0.6361116209165397,0.7001421889107501 12 | 0.39608452030330277,0.1041747245462401,0.7674347838508891 13 | 0.8669068975909605,0.8037488635954642,1.058134242056377 14 | 0.19665312388467115,0.8389868799578961,0.26112320968505476 15 | 0.4544746698842206,0.3137097280678691,0.8351311832552073 16 | 0.5913443809270517,0.6335050845997221,0.8986065909618182 17 | 0.05018102379512368,0.579151057122629,0.08396051888543743 18 | 0.43325283250071556,0.9622378513280131,0.48001592995664727 19 | 0.4141426701056061,0.3149107583015194,0.7652331117501066 20 | 0.28478007972114294,0.4062589989559169,0.5161578641127561 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-11.csv: -------------------------------------------------------------------------------- 1 | 0.009650119087422748,0.9754678757794766,0.01081375106076273 2 | 0.7542717041611646,0.8873869234583852,0.7786096558849593 3 | 0.5712927685247489,0.5785827192892782,0.7233069880671271 4 | 0.8811673844743888,0.46554404190664944,0.9428058571327314 5 | 0.5033434391322769,0.0450333974998971,0.9695583290897475 6 | 0.3010551522520114,0.23886740013016705,0.750698487383017 7 | 0.4271415891455096,0.05973432734490913,0.9504569670453555 8 | 0.5075855174161593,0.08205847688730294,0.945876716970352 9 | 0.6795624939661843,0.06995981455635603,0.9733360301827569 10 | 0.7825899776142577,0.581055619924391,0.8672363962173748 11 | 0.5015737328270045,0.21148068721355284,0.8642245404056014 12 | 0.9677280775898672,0.6882752264946584,0.977674703014148 13 | 0.9850343561587092,0.20077624353882195,0.9969771215223752 14 | 0.6700429414533945,0.08021746140750508,0.9683902204543579 15 | 0.36498072144512117,0.37729275198075873,0.6836717196239555 16 | 0.8411866628892927,0.183007689173595,0.9688459490149101 17 | 0.1037735305541786,0.9290346850890403,0.12187379009095527 18 | 0.3584673725052546,0.40140116389359326,0.6624537516175066 19 | 0.8322379431573198,0.22924595429703343,0.958775802498225 20 | 0.6664874585819388,0.35925954287117234,0.8643618450643513 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-11_test.csv: -------------------------------------------------------------------------------- 1 | 0.9330964684434145,0.24250947449143034,0.9833472382753231 2 | 0.9393885932152682,0.6003184466319978,0.9631601863378697 3 | 0.9823387149514972,0.5126529076724765,0.9909065807329187 4 | 0.19345863548739384,0.7836469053896788,0.2760177352776936 5 | 0.13108611073012544,0.7685858871066561,0.20978117653952616 6 | 0.35579107784241226,0.07985523500690928,0.9207899605385508 7 | 0.6021616938118552,0.22101557306485753,0.8939498791504068 8 | 0.8374149354743102,0.8575514294776241,0.8588507300515819 9 | 0.6494106932929331,0.3241221181594962,0.8694275590808833 10 | 0.4046609592118948,0.7155710595502166,0.5234146932998274 11 | 0.9819037727317954,0.5566903675981135,0.989885240890773 12 | 0.8505183251366303,0.25523824566572106,0.9595168033601454 13 | 0.9356183696289436,0.7644829574948679,0.9503979246700507 14 | 0.15899659291424306,0.34774105089317686,0.5275817457361087 15 | 0.15085716258327397,0.9068693910131009,0.17991467543267664 16 | 0.897697392846848,0.36597539221364817,0.961272946923653 17 | 0.34916763529467854,0.6921417231643349,0.4827416301664693 18 | 0.5386488575843715,0.2426645637656235,0.8605922393067684 19 | 0.6330851831276092,0.38397242469260184,0.8390096193118747 20 | 0.6356313521324786,0.04232120905637449,0.9810054289450404 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-12.csv: -------------------------------------------------------------------------------- 1 | 0.571529988212336,0.2086097294428253,-0.26684105192363916 2 | 0.5831134362712086,0.25461957369993216,-0.30486051622030813 3 | 0.5083018008714596,0.6179518856363723,-0.4915944924642439 4 | 0.23664520611648598,0.15937300223089534,-0.1567893757113804 5 | 0.1788389410352389,0.6798352991666836,-0.4534442178710625 6 | 0.11401635766993878,0.9024214887985181,-0.49655240594490385 7 | 0.03766301311214426,0.6373546822771162,-0.4342955996977231 8 | 0.8712838274891328,0.837020551073922,-0.5718546210720648 9 | 0.9640600259796086,0.22501552060152052,-0.23190205775040304 10 | 0.039085476468475266,0.13155320524765401,-0.12295745843747838 11 | 0.8473442950289002,0.19225796174261,-0.2666501042625061 12 | 0.4122794511854264,0.27412751430208004,-0.27774021903029494 13 | 0.9515880236354662,0.25933184243175444,-0.2674210574319746 14 | 0.08665544886811449,0.12499282055501426,-0.11777554059484234 15 | 0.37315446589660495,0.005192758436682321,-0.037749929159407405 16 | 0.5954126176550698,0.8017482683519765,-0.5657498097043985 17 | 0.7474428457445278,0.9411986788230424,-0.6037326292961982 18 | 0.3759854347894287,0.07496650261475468,-0.10532363652082886 19 | 0.5020183307491553,0.3456109114912851,-0.3488920348690695 20 | 0.7998737827494171,0.1488335811120166,-0.2401739965508571 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-12_test.csv: -------------------------------------------------------------------------------- 1 | 0.9011456712736252,0.5390860021600181,-0.46611950801172675 2 | 0.05646339496533603,0.9063115836612818,-0.4957810880397063 3 | 0.9304186693981396,0.10649027023247115,-0.15686403347218786 4 | 0.4684093342789124,0.22380857713794622,-0.253396301704723 5 | 0.5399395011488534,0.1949724804444618,-0.24838396707444 6 | 0.012497902714201814,0.2714959213121543,-0.23464283141196718 7 | 0.3194537791611215,0.926122481603283,-0.5194571945116537 8 | 0.8837187191954148,0.7476232565933104,-0.5484042700646763 9 | 0.5571900724835851,0.6450681758833615,-0.5136114689071276 10 | 0.19738914993062606,0.6532172982300527,-0.44604357441831655 11 | 0.4570383019573264,0.3479576891905871,-0.339255876038556 12 | 0.7563575150128997,0.601323940027959,-0.5259514638061414 13 | 0.07912842299471923,0.6176613262426129,-0.42736481267985765 14 | 0.47393551647785137,0.7700034085095716,-0.5295519086416973 15 | 0.07410054952297718,0.09077883880272763,-0.08703516819978618 16 | 0.48239132491485914,0.7064450084398278,-0.5150159283947497 17 | 0.018209914318982112,0.3836586690224434,-0.31006761033062225 18 | 0.13365096385246922,0.3105763510342461,-0.2644157954065896 19 | 0.6350503261404045,0.24279470106231815,-0.3067868753711106 20 | 0.5592008838741044,0.3332234651847834,-0.3547849800622398 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-1_test.csv: -------------------------------------------------------------------------------- 1 | -0.11630564721763292,-0.10435191054859261 2 | 0.18652302418157118,0.22780315515153893 3 | -0.42211751752765747,-0.31914856838368183 4 | -0.907420735656995,-0.8311898222009845 5 | -0.7526127868564954,-0.6124862355476299 6 | -0.14202367497162816,-0.1247176711043533 7 | -0.7209470449885367,-0.5759051858178993 8 | 0.9453456843279184,2.683859225263316 9 | -0.9834628755738535,-0.9674681816016304 10 | 0.06018216457229597,0.06402203086143628 11 | 0.10885993568553376,0.12200046438297399 12 | -0.14100188005558678,-0.12392368301002721 13 | 0.2923371301491917,0.40278155060979837 14 | 0.26064767772808084,0.34629256614749193 15 | 0.7577582383840484,1.7670587085577871 16 | -0.29153340886053547,-0.23131960870795243 17 | -0.6482426087966402,-0.50042765258583 18 | -0.4850382627291796,-0.3638872745994818 19 | -0.3461860529744316,-0.2678298621916234 20 | -0.3089884641633469,-0.24301491794102403 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-2.csv: -------------------------------------------------------------------------------- 1 | -0.2789157327227092,-0.21676780910640264 2 | -0.6155652159480249,-0.3263142099605031 3 | -0.964870900776263,-0.06545051468545005 4 | -0.3605853701882056,-0.2605418523290945 5 | -0.12677496698803492,-0.11248228262260684 6 | 0.8090322384509245,2.421517492461333 7 | 0.037100271503096804,0.038529662140752964 8 | 0.8283530701610371,2.553739552693992 9 | -0.7005783650849302,-0.3127247179046436 10 | 0.7363783106066262,1.9719730207473138 11 | -0.6309956596471198,-0.325546723619473 12 | -0.1106929782457815,-0.0996462223014604 13 | 0.26579712660786114,0.3602144602287284 14 | 0.02752774308022632,0.02830695382538511 15 | -0.5330340434118443,-0.3196300236719729 16 | 0.3094396724926862,0.44399095274621003 17 | 0.6072125567661473,1.3357481376760187 18 | -0.20762052637618167,-0.1716058409326518 19 | 0.8027611702209694,2.3797906526866015 20 | -0.5385609520738912,-0.32059374188921175 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-2_test.csv: -------------------------------------------------------------------------------- 1 | -0.5472392297334601,-0.32196786635655783 2 | 0.38890369021866333,0.6218453060265308 3 | 0.34687084012370883,0.5234022573194569 4 | -0.8604504554438437,-0.20897634300909174 5 | 0.14977557070505143,0.17607139295967775 6 | 0.1378817081524819,0.15987582515313542 7 | -0.8165468635070887,-0.24967577247174144 8 | 0.21299967985860424,0.27009043069121086 9 | -0.5546425004908602,-0.3230027555098659 10 | 0.27116371665813865,0.37003870701742303 11 | -0.7279571139338363,-0.30297886397368323 12 | -0.24947070907682134,-0.198887768079884 13 | 0.8374208984166802,2.6177408505966255 14 | -0.23930562873899253,-0.1924632738960526 15 | 0.6348808661786001,1.456326788359588 16 | -0.34741430054622446,-0.254081670139717 17 | 0.5334863683363693,1.0509299504448069 18 | -0.14852461841779463,-0.1292548199233567 19 | -0.28708794777858704,-0.22153712789869012 20 | -0.874193576171304,-0.19402683204341087 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-3.csv: -------------------------------------------------------------------------------- 1 | -0.2742410761563563,-0.21555298984435148 2 | -0.4000719670413029,-0.2886797251856613 3 | 0.5858603898654235,1.317006850623232 4 | 0.2848047139027472,0.3974733011165669 5 | -0.9865643021084112,-0.9607591664062742 6 | 0.2967323734634284,0.42096312367092664 7 | 0.18483989171830695,0.22670394207032996 8 | 0.9196991925776321,3.91693146177716 9 | -0.3420146701349107,-0.25604430763443126 10 | 0.4261019135230011,0.7320405705368198 11 | 0.9307271475090158,4.052032022277849 12 | 0.45711411683980296,0.825202713876745 13 | -0.2870593661121752,-0.22346980461208035 14 | -0.5418099652934234,-0.36781946549768096 15 | 0.7869656106826004,2.5790522780510168 16 | 0.31159272467674093,0.4512989901301369 17 | 0.6919974575049623,1.8902156450500738 18 | 0.7619465758722248,2.3787360622769427 19 | 0.32883831547384745,0.48807000445235493 20 | 0.1890193090463268,0.23301874332445205 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-3_test.csv: -------------------------------------------------------------------------------- 1 | 0.12330602006862224,0.1406448636014227 2 | -0.8341482726994274,-0.6384522518461678 3 | -0.4164143199352468,-0.2976728693089716 4 | -0.39119288373453753,-0.2837684868226354 5 | -0.28577338425424004,-0.22268158157920692 6 | 0.9764679991160452,4.657901456234402 7 | 0.43249605135489166,0.750569762736142 8 | -0.44199098916594415,-0.31168471674153886 9 | 0.5220122157356137,1.049771829672018 10 | 0.09599930524622513,0.10619297427483858 11 | 0.3834719176142225,0.6168285222751912 12 | -0.4335991633810119,-0.3070905171985884 13 | -0.3373620249758591,-0.25336168451454166 14 | -0.6169593490785408,-0.41566200150572 15 | -0.7683048836186812,-0.5508039881847488 16 | -0.2786488694359457,-0.21829055777030593 17 | 0.789023431904786,2.5961811382676463 18 | 0.4463050013961438,0.7917753078283539 19 | -0.4861753646450431,-0.3360175137764131 20 | -0.5311395294961603,-0.3615551352011317 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-4.csv: -------------------------------------------------------------------------------- 1 | 0.577201329610098,1.3147074703149828 2 | -0.5537074910525015,-0.34610766172171636 3 | 0.8047324726161706,3.0019200410687663 4 | -0.2598260108204391,-0.20617614334781337 5 | -0.21524322761845438,-0.17710184937386503 6 | 0.7015682365555389,2.070536012802263 7 | 0.5592376986930361,1.2299841635234094 8 | 0.8882761025028312,4.045018482442029 9 | 0.01686621688709411,0.017155566390265 10 | 0.24727312796392487,0.32842802937297627 11 | 0.05544924017913666,0.05870435018019409 12 | -0.1511171785013936,-0.13127714679025645 13 | -0.3617287072067332,-0.2650442338487058 14 | -0.006720736867676491,-0.00667587010136183 15 | -0.9900341669795012,-0.029016397801101435 16 | 0.15775649843693573,0.18730220706912443 17 | 0.34982018264255155,0.5370500447001955 18 | 0.9998240193286838,5.996305489633515 19 | 0.13810231720361843,0.16022941222507822 20 | 0.34668480430122695,0.5297333964211179 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-4_test.csv: -------------------------------------------------------------------------------- 1 | -0.7895605646919419,-0.3343099912166266 2 | 0.5353633224954755,1.1250906821681923 3 | 0.5746887672392376,1.3025425396784742 4 | 0.35505408323224574,0.549414702306875 5 | -0.44657068177233383,-0.3062614408384454 6 | 0.16809023803736678,0.20204889321130032 7 | -0.7581721426314456,-0.3493219231598674 8 | -0.04236700540533911,-0.040644998297742686 9 | -0.007852216260799239,-0.007791039335042253 10 | 0.947275599327726,4.9851123902624135 11 | -0.23143165873068372,-0.18790819391774727 12 | 0.20114692823401947,0.251777970936226 13 | 0.041988557151692474,0.043828867843952214 14 | -0.4037666250402405,-0.28638458985204446 15 | -0.07883096175984705,-0.07307070860586212 16 | 0.2669050603526706,0.36394817712277516 17 | 0.15664597895478938,0.18573891943998821 18 | 0.9820670578247823,5.6344641905455415 19 | 0.26359160721575803,0.35782209409358984 20 | 0.7768184851096915,2.7158036496221074 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-5.csv: -------------------------------------------------------------------------------- 1 | -0.4349478698587148,-0.8294560158408567 2 | -0.6789389560180512,-0.65383409564023 3 | 0.8631063896896904,-0.5592866026547315 4 | 0.9244707419231957,-0.545694117765421 5 | 0.5705122592040333,-0.7308757439115042 6 | -0.3693485850591103,-0.8731755308713461 7 | -0.7780542064116212,-0.5946660210709676 8 | -0.28731158801138346,-0.920925636119759 9 | -0.29019228653076734,-0.9194047541060653 10 | 0.9051894569628551,-0.5487570777798731 11 | -0.8190372974056328,-0.5754733728363357 12 | 0.10793618643830061,-0.9884178396214068 13 | -0.8072879594107076,-0.58059490130306 14 | 0.6006538046345222,-0.7087796484056164 15 | 0.46455125792189267,-0.8085569917450148 16 | 0.009538392515657135,-0.9999090232070763 17 | 0.2201270826789623,-0.9527318257628095 18 | -0.3926415063595725,-0.8581280959246165 19 | 0.17103608107058998,-0.9711776071375837 20 | 0.7011622553345034,-0.6393006509286788 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-5_test.csv: -------------------------------------------------------------------------------- 1 | -0.27078103597181635,-0.9294126188343877 2 | -0.5216652082837137,-0.7669632675082511 3 | -0.16403747726520423,-0.9734561266055237 4 | 0.10152132402690706,-0.9897466694835366 5 | 0.8469465208676263,-0.5646433564254277 6 | -0.833604625744051,-0.5695789475019584 7 | 0.6737842559314944,-0.6572876140070592 8 | 0.7358931685346111,-0.6179283737235202 9 | 0.06539489258021702,-0.9957326619650867 10 | -0.07610218324880869,-0.9942252528761033 11 | -0.2257472471034241,-0.9503527249855077 12 | 0.9595150523716058,-0.5431580089777353 13 | -0.8490119556720586,-0.563920158701124 14 | -0.6565373114771327,-0.6690427342730567 15 | 0.5551940879364445,-0.7421861034435271 16 | 0.23714005201311528,-0.9453672158891828 17 | 0.8359148155431357,-0.5686920550495316 18 | -0.9217236647257525,-0.5460599098414161 19 | -0.3917342793018872,-0.8587247864344467 20 | -0.48436782176950777,-0.7942748449592015 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-6.csv: -------------------------------------------------------------------------------- 1 | -0.8734439429536089,-0.8768597906089837 2 | -0.10156933056691053,-0.1925211922609868 3 | 0.30900969522488086,0.6976715007985357 4 | -0.4839204931903258,-0.7124065135167501 5 | 0.2018208424631982,0.44063473194384073 6 | 0.17293664663931008,0.37353149844688727 7 | 0.5209559041342855,1.2097154813846882 8 | 0.7147120672245526,1.5963837670771632 9 | 0.21527069119452902,0.4722500734136445 10 | 0.44684862792336943,1.0345399654299894 11 | -0.17394606563632942,-0.31626510123678486 12 | 0.4747633386316681,1.1014707541981035 13 | -0.3021009460096211,-0.5068040893274725 14 | 0.7608347179666672,1.6629430695042198 15 | 0.9758133646643441,1.7650275135502962 16 | -0.17168957124566364,-0.3125807114547759 17 | -0.2506032317891631,-0.4346876533349173 18 | -0.06253579069047377,-0.1210865285572473 19 | -0.5599489153360604,-0.7750631368536841 20 | -0.5647486202454308,-0.7785432308187508 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-6_test.csv: -------------------------------------------------------------------------------- 1 | 0.8379728658130738,1.742819647477015 2 | -0.7695011150523388,-0.8722176078553304 3 | -0.3421671444267138,-0.5587223058775501 4 | -0.64064886974132,-0.8259054468631615 5 | 0.6632363010179283,1.5082879397136804 6 | 0.5438622420859585,1.2618532082941376 7 | 0.720884003822003,1.6060140210944391 8 | 0.7826864820388859,1.6898240516404288 9 | -0.9385027242873822,-0.8643574441235508 10 | 0.08961952943204854,0.18699567916647436 11 | 0.9069444019411992,1.775058856230849 12 | 0.9229637458259201,1.776651706500103 13 | 0.32576384443833084,0.7386168532895526 14 | 0.4473797486901194,1.0358215950888467 15 | 0.685114187735959,1.54735255846332 16 | 0.34198992356873004,0.7783667862370781 17 | -0.08857676129423964,-0.16910423368621252 18 | 0.8299769925885963,1.736566355146619 19 | 0.3149844423220556,0.7122590589581266 20 | -0.7340839155155925,-0.8638746661747178 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-7.csv: -------------------------------------------------------------------------------- 1 | 0.605917598286303,0.7864134456820748 2 | 1.2652366195650215,1.7735074251657648 3 | 1.3557549346534747,1.899985994907216 4 | 1.3098613940969044,1.836250844380595 5 | 1.9979696623329108,2.7057484278660313 6 | 1.6322149316173509,2.2664153921371333 7 | 1.1534635096160086,1.613150906618281 8 | 1.493719163435045,2.0866278740462456 9 | 0.4097370905838347,0.4985972047417509 10 | 1.4341814440339147,2.0070041524146154 11 | 0.7317274666947928,0.9779267173424435 12 | 1.682893813702461,2.330317192029723 13 | 0.9200674520274958,1.2656653324484644 14 | 0.219347081712848,0.24530707680136993 15 | 1.6837673644761237,2.3314098897068516 16 | 0.3893606558155618,0.46999745507063917 17 | 0.11259268294138591,0.11929047247901335 18 | 1.851571099029555,2.5358893764955734 19 | 0.3008842487402652,0.34970926181347417 20 | 1.3727163932621331,1.9233351303245287 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-7_test.csv: -------------------------------------------------------------------------------- 1 | 0.1528127056831623,0.16528803037304635 2 | 0.32977955728060837,0.3882505462167066 3 | 0.7216023275396501,0.9624327208173018 4 | 0.4671216504861371,0.5806789439454347 5 | 0.031510012862542736,0.03201614890165631 6 | 0.6630361996390657,0.8730220750463573 7 | 1.0795612979275175,1.5047863478783337 8 | 0.5381932884871417,0.6849809651742536 9 | 1.7089644128961698,2.3628007650561127 10 | 1.7150407018587497,2.370333955565539 11 | 0.7754378039328833,1.044865207394016 12 | 0.4271429834942715,0.5232640435772772 13 | 0.1846887635555321,0.20302118809059083 14 | 0.6178328548887495,0.804414754428093 15 | 0.6036654071841054,0.7830153994448583 16 | 0.43693817584429095,0.5372364767249402 17 | 1.478131297519765,2.0659169085832096 18 | 0.8636826764901531,1.1798520042369485 19 | 1.7305172521124375,2.389457030050565 20 | 1.6912140860847071,2.340712737797519 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-8.csv: -------------------------------------------------------------------------------- 1 | 0.4439140598156808,0.6662687594474777 2 | 3.5777730471526197,1.8915002107196868 3 | 1.0466773330204768,1.0230724964637046 4 | 1.6694231732779343,1.2920615980973718 5 | 3.0704934856227504,1.7522823646954706 6 | 0.45397423890315913,0.6737761044317016 7 | 0.22181861957280047,0.47097624098546675 8 | 2.831831649485261,1.6828046973684323 9 | 3.9901252725788634,1.9975297926636446 10 | 0.42738411234255347,0.6537462140177589 11 | 0.2323588425519505,0.4820361423710368 12 | 2.326902068858495,1.525418653635288 13 | 0.5207992232823195,0.7216642039635328 14 | 1.28014231796383,1.1314337443985971 15 | 0.8870307283896941,0.9418230876282945 16 | 1.8589725772243075,1.3634414462030657 17 | 0.37696405495299556,0.6139739855669746 18 | 2.1509845926842086,1.4666235347505536 19 | 2.87004851642217,1.6941217537184776 20 | 2.4482719636111043,1.5646954859048787 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-8_test.csv: -------------------------------------------------------------------------------- 1 | 1.9777714965525988,1.406332640790435 2 | 1.2805067118634428,1.131594764862158 3 | 3.243101479387957,1.8008613159785396 4 | 0.11838333693050895,0.34406879679870556 5 | 2.3334164393538024,1.52755243424041 6 | 0.5605706676571969,0.7487126736320128 7 | 0.04752788270320174,0.21800890510069018 8 | 3.4256972244179797,1.8508639129925193 9 | 0.09056942054034378,0.3009475378539319 10 | 3.145354663146166,1.773514776692364 11 | 0.18863769182822754,0.4343244085107669 12 | 3.8173079319939087,1.9537932162831124 13 | 2.367980686684381,1.5388244495992325 14 | 0.13952949053484964,0.37353646479942176 15 | 2.645612511645075,1.6265338950188142 16 | 1.0517761267292376,1.0255613715079355 17 | 1.4046163663712155,1.185165121985631 18 | 2.4363339586180954,1.5608760228211898 19 | 0.19412965679911665,0.4406014716261359 20 | 1.7865699972991491,1.3366263491713566 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-9.csv: -------------------------------------------------------------------------------- 1 | 0.1907237370646968,0.43778758645344096,0.3800563273710924 2 | 0.511780233869418,0.9110790491530697,1.2277054211091303 3 | 0.8858198354295761,0.9831237709939898,1.5973544165353837 4 | 0.4191935407445384,0.715828260124807,0.8973031783409327 5 | 0.7255901482471144,0.8817616601796886,1.3650795759234557 6 | 0.07879841618396055,0.7788615221602095,0.6488150088067735 7 | 0.14147498867211494,0.16722011801524495,0.16896244369623037 8 | 0.9209427422077568,0.7349852941729912,1.3104828227166232 9 | 0.44383273867988626,0.60944363731408,0.7923444181168122 10 | 0.824940720150802,0.3830269618006097,0.880691486120089 11 | 0.6300409543269153,0.13117175420790128,0.6063830298186593 12 | 0.21288215293833168,0.8179895579627416,0.8315635864271486 13 | 0.8031071101218406,0.29089148945157783,0.8040342840649523 14 | 0.055934964589704594,0.34951138403910587,0.17776041561667874 15 | 0.8502982564202176,0.6267076050163439,1.1342191665086163 16 | 0.6839084685223353,0.06541661405742705,0.6361066523549319 17 | 0.6318571163130142,0.5282030714236922,0.8660373405673901 18 | 0.5745968824299336,0.6726768974483016,0.9807065348410606 19 | 0.9189499483043391,0.07225657556150356,0.8001860280439798 20 | 0.5637282843395133,0.32270742147289166,0.6382932528110492 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Nguyen-9_test.csv: -------------------------------------------------------------------------------- 1 | 0.7502932187066848,0.047314504091698684,0.684091936008948 2 | 0.8075119483251983,0.4058736134183446,0.8865587628228955 3 | 0.27917336732952724,0.23666144174411963,0.3315404741324787 4 | 0.1225202019894216,0.3556025112319542,0.2483303120785762 5 | 0.11912237182198515,0.10800720183702617,0.1305061355291077 6 | 0.14141795269416413,0.4345943329175056,0.32869835765416444 7 | 0.30599333744221036,0.463084472394905,0.5140478698912032 8 | 0.914680984981933,0.273612145904338,0.8671617105867683 9 | 0.12317403622814427,0.36214716413062176,0.2536377259867269 10 | 0.7444080683816883,0.25998893280087954,0.7450793540564267 11 | 0.8398852897487666,0.8649176050509522,1.424801017859552 12 | 0.6112972093905025,0.9029676376085652,1.3018962210897291 13 | 0.6883180758408367,0.8683203575462749,1.3197847578932302 14 | 0.5744133336064882,0.5174020923423437,0.8078611810115688 15 | 0.8512388129482399,0.5197807844640945,1.018994687035828 16 | 0.0972372314533061,0.9790117175507246,0.9153937158111217 17 | 0.3082606471759375,0.7138784041170811,0.7912493744339566 18 | 0.792856187257642,0.5125744784110978,0.9720810283417941 19 | 0.649259724624558,0.17081750686088293,0.6337713991531597 20 | 0.25452629606056987,0.7708740954095349,0.811671887358697 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/PiecewiseFunction-1.csv: -------------------------------------------------------------------------------- 1 | 0.91209114396153,0.9550346297184883 2 | 1.3327788333544879,1.154460407876549 3 | -0.7618102704952134,0.5803548882319902 4 | 1.737040403025226,1.3179682860468327 5 | -0.7636456308920145,0.5831546495804629 6 | -0.6829882228343624,0.4664729125304407 7 | -1.4567511139811962,2.122123808085456 8 | -0.6818747151482625,0.4649531271585241 9 | -1.7555571408847905,3.0819808749115802 10 | 1.3773968747702794,1.1736255257833648 11 | 0.4226314974103107,0.6501011439847731 12 | 1.5954331153318715,1.2631045543943984 13 | -1.9702929885019183,3.88205446053982 14 | -0.32571226024828714,0.10608847647604794 15 | 0.4934480468065301,0.7024585730180322 16 | -0.5147391885975097,0.2649564322780226 17 | -1.0652601335978793,1.1347791522329715 18 | 1.9376874440002059,1.392008420951614 19 | 0.2704763129384631,0.5200733726489591 20 | 1.2566940488827782,1.1210236611609847 21 | 1.8800815008079592,1.3711606400447611 22 | -1.7543294824847235,3.077671933115118 23 | 0.7614573780045353,0.8726152519894065 24 | -0.9970381616585033,0.9940850958033678 25 | 1.7679937054354515,1.329659244105591 26 | 1.7655573232522253,1.3287427603762232 27 | 0.530411521078586,0.7282935679233932 28 | 1.126580405073657,1.0614049204114597 29 | -0.7565453630835841,0.5723608864032721 30 | 0.9800213770864481,0.9899602906614224 31 | -1.1859956911956928,1.4065857795347492 32 | -0.6129542969481183,0.375712970147162 33 | 1.8807131536202357,1.3713909557891344 34 | 1.6330876036745843,1.2779231603169983 35 | -1.6972747003526392,2.880741408457141 36 | 1.5585133781674072,1.2484043328054446 37 | 0.4927250505249581,0.7019437659278399 38 | -0.23593797368882896,0.9722955129642071 39 | 1.7202157580439583,1.3115699592640715 40 | 1.248754981437303,1.117477060810334 -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Test-1.csv: -------------------------------------------------------------------------------- 1 | -0.7403801071368961,-0.5980661649010526 2 | 0.5206056048792103,0.9327356397523248 3 | 0.6525207575940659,1.3561365636798772 4 | 0.5356322049996658,0.9762079394047138 5 | -0.586133136442661,-0.443948325535601 6 | -0.3769022485680784,-0.28838790745876036 7 | -0.6201560629882985,-0.4740705376705796 8 | -0.8404642346449558,-0.7277713400189059 9 | 0.31264230226858225,0.4409467986663268 10 | -0.20966006871096332,-0.17491882414882493 11 | 0.48421935624763335,0.8322218716406187 12 | -0.5506969824140331,-0.41443812939783053 13 | -0.6350467381160174,-0.4878667956161994 14 | 0.5237341566285307,0.941690575897929 15 | 0.4441307242480925,0.7289885425998979 16 | 0.8378658262994001,2.12808281840724 17 | 0.4609140775860472,0.771273274760623 18 | 0.27611115397676866,0.37339851135463886 19 | -0.8594323772124557,-0.7556057558522368 20 | -0.10877247441601634,-0.09822795944736544 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/data/Test-1_test.csv: -------------------------------------------------------------------------------- 1 | -0.11630564721763292,-0.10435191054859261 2 | 0.18652302418157118,0.22780315515153893 3 | -0.42211751752765747,-0.31914856838368183 4 | -0.907420735656995,-0.8311898222009845 5 | -0.7526127868564954,-0.6124862355476299 6 | -0.14202367497162816,-0.1247176711043533 7 | -0.7209470449885367,-0.5759051858178993 8 | 0.9453456843279184,2.683859225263316 9 | -0.9834628755738535,-0.9674681816016304 10 | 0.06018216457229597,0.06402203086143628 11 | 0.10885993568553376,0.12200046438297399 12 | -0.14100188005558678,-0.12392368301002721 13 | 0.2923371301491917,0.40278155060979837 14 | 0.26064767772808084,0.34629256614749193 15 | 0.7577582383840484,1.7670587085577871 16 | -0.29153340886053547,-0.23131960870795243 17 | -0.6482426087966402,-0.50042765258583 18 | -0.4850382627291796,-0.3638872745994818 19 | -0.3461860529744316,-0.2678298621916234 20 | -0.3089884641633469,-0.24301491794102403 21 | -------------------------------------------------------------------------------- /dso/dso/task/regression/function_sets.csv: -------------------------------------------------------------------------------- 1 | name,function_set 2 | Koza,"add,sub,mul,div,sin,cos,exp,log" 3 | CKoza,"add,sub,mul,div,sin,cos,exp,log,const" 4 | KozaPlus1,"add,sub,mul,div,sin,cos,exp,log,1.0" 5 | Korns,"add,sub,mul,div,sin,cos,exp,log,n2,n3,sqrt,tan,tanh,const" 6 | Keijzer,"add,mul,inv,neg,sqrt,const" 7 | KeijzerPlus1,"add,mul,inv,neg,sqrt,1.0,const" 8 | Vladislavleva-A,"add,sub,mul,div,n2" 9 | Vladislavleva-B,"add,sub,mul,div,n2,exp,expneg" 10 | Vladislavleva-C,"add,sub,mul,div,n2,exp,expneg,sin,cos" 11 | Real,"add,sub,mul,div,sin,cos,exp,log" 12 | None,"add,sub,mul,div,sin,cos,exp,log" 13 | Jin,"add,sub,mul,div,sin,cos,exp,n2,n3,const" 14 | GrammarVAE,"add,mul,div,sin,exp,1.0,2.0,3.0" 15 | KozaUserConst,"add,sub,mul,div,sin,cos,exp,log,3.14159265358979323846,2.71828182845904523536" 16 | PKoza,"add,sub,mul,div,sin,cos,exp,log,poly" 17 | CPKoza,"add,sub,mul,div,sin,cos,exp,log,const,poly" 18 | PKozaPlusSqrt,"add,sub,mul,div,sin,cos,exp,log,sqrt,poly" 19 | Base,"add,sub,mul,div,sqrt,n2,log,exp,sin,cos,1" 20 | BaseC,"add,sub,mul,div,sqrt,n2,log,exp,sin,cos,1,const" 21 | BaseP,"add,sub,mul,div,sqrt,n2,log,exp,sin,cos,1,poly" 22 | BaseCP,"add,sub,mul,div,sqrt,n2,log,exp,sin,cos,1,const,poly" -------------------------------------------------------------------------------- /dso/dso/task/regression/mat_mult_benchmark.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy 3 | from scipy import linalg 4 | import time 5 | 6 | import sys 7 | args = sys.argv 8 | if len(args) == 4: 9 | nparams = int(args[1]) 10 | nsamples = int(args[2]) 11 | ntries = int(args[3]) 12 | else: 13 | nparams = 100 14 | nsamples = 1000 15 | ntries = 100 16 | 17 | def mat_mult_benchmark(nparams, nsamples): 18 | # create data 19 | X = 100.0 * np.random.rand(nsamples, nparams) - 50.0 20 | X[:,0] = 1.0 21 | X_pinv = scipy.linalg.pinv(X) 22 | X_pinv_32 = X_pinv.astype(np.float32, copy=True) 23 | y = 100.0 * np.random.rand(nsamples) - 50.0 24 | # do least squares 25 | tls = time.time() 26 | beta = scipy.linalg.lstsq(X, y) 27 | tls = time.time() - tls 28 | # use pinv with dot 29 | tdot = time.time() 30 | beta = np.dot(X_pinv, y) 31 | tdot = time.time() - tdot 32 | # use pinv with matmul 33 | tmatmul = time.time() 34 | beta = np.matmul(X_pinv, y) 35 | tmatmul = time.time() - tmatmul 36 | # use pinv with matmul and float32 37 | tmatmul32 = time.time() 38 | y_32 = y.astype(np.float32, copy=True) 39 | beta = np.matmul(X_pinv_32, y_32) 40 | tmatmul32 = time.time() - tmatmul32 41 | # print results 42 | print("pinv-dot: ", tls/tdot, "x; pinv-matmul: ", tls/tmatmul, 43 | "x; pinv-matmul32:", tls/tmatmul32, "x") 44 | 45 | for i in range(ntries): 46 | mat_mult_benchmark(nparams, nsamples) 47 | -------------------------------------------------------------------------------- /dso/dso/task/regression/sklearn.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | 3 | from sklearn.base import BaseEstimator, RegressorMixin 4 | from sklearn.utils.validation import check_is_fitted 5 | 6 | from dso import DeepSymbolicOptimizer 7 | 8 | 9 | class DeepSymbolicRegressor(DeepSymbolicOptimizer, 10 | BaseEstimator, RegressorMixin): 11 | """ 12 | Sklearn interface for deep symbolic regression. 13 | """ 14 | 15 | def __init__(self, config=None): 16 | if config is None: 17 | config = { 18 | "task" : {"task_type" : "regression"} 19 | } 20 | DeepSymbolicOptimizer.__init__(self, config) 21 | 22 | def fit(self, X, y): 23 | 24 | # Update the Task 25 | config = deepcopy(self.config) 26 | config["task"]["dataset"] = (X, y) 27 | 28 | # Turn off file saving 29 | config["experiment"]["logdir"] = None 30 | 31 | # TBD: Add support for gp-meld and sklearn interface. Currently, gp-meld 32 | # relies on BenchmarkDataset objects, not (X, y) data. 33 | if config["gp_meld"].get("run_gp_meld"): 34 | print("WARNING: GP-meld not yet supported for sklearn interface.") 35 | config["gp_meld"]["run_gp_meld"] = False 36 | 37 | self.set_config(config) 38 | 39 | train_result = self.train() 40 | self.program_ = train_result["program"] 41 | 42 | return self 43 | 44 | def predict(self, X): 45 | 46 | check_is_fitted(self, "program_") 47 | 48 | return self.program_.execute(X) 49 | -------------------------------------------------------------------------------- /dso/dso/task/regression/test_sklearn.py: -------------------------------------------------------------------------------- 1 | """Tests for sklearn interface.""" 2 | 3 | import pytest 4 | import numpy as np 5 | 6 | from dso import DeepSymbolicRegressor 7 | from dso.test.generate_test_data import CONFIG_TRAINING_OVERRIDE 8 | 9 | 10 | @pytest.fixture 11 | def model(): 12 | return DeepSymbolicRegressor() 13 | 14 | 15 | def test_task(model): 16 | """Test regression for various configs.""" 17 | 18 | # Generate some data 19 | np.random.seed(0) 20 | X = np.random.random(size=(10, 3)) 21 | y = np.random.random(size=(10,)) 22 | 23 | model.config_training.update(CONFIG_TRAINING_OVERRIDE) 24 | model.fit(X, y) 25 | -------------------------------------------------------------------------------- /dso/dso/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/test/__init__.py -------------------------------------------------------------------------------- /dso/dso/test/custom_tasks/custom_task_prior.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file implements an "empty" task and prior for test purposes. 3 | """ 4 | 5 | from dso.task import HierarchicalTask 6 | from dso.library import Library 7 | from dso.functions import create_tokens 8 | from dso.prior import Prior 9 | 10 | 11 | class CustomTask(HierarchicalTask): 12 | 13 | task_type = "dummy" 14 | 15 | def __init__(self, param): 16 | 17 | super(HierarchicalTask).__init__() 18 | 19 | self.param = param 20 | 21 | # Create a Library 22 | tokens = create_tokens(n_input_var=1, 23 | function_set=["add", "sin"], 24 | protected=False, 25 | decision_tree_threshold_set=None) 26 | self.library = Library(tokens) 27 | 28 | self.stochastic = False 29 | self.name = "dummy" 30 | 31 | def reward_function(self, p): 32 | r = 0. 33 | return r 34 | 35 | def evaluate(self, p): 36 | info = {} 37 | return info 38 | 39 | 40 | class CustomPrior(Prior): 41 | 42 | def __init__(self, library, param): 43 | Prior.__init__(self, library) 44 | #Just check whether the prior parameter got lost for some reason 45 | assert param == "test" 46 | 47 | def __call__(self, actions, parent, sibling, dangling): 48 | # Initialize the prior 49 | prior = self.init_zeros(actions) 50 | return prior 51 | 52 | def validate(self): 53 | return None -------------------------------------------------------------------------------- /dso/dso/test/custom_tasks/test_custom_task_prior.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from dso.test.generate_test_data import CONFIG_TRAINING_OVERRIDE 3 | from dso.test.test_core import model 4 | 5 | 6 | def test_custom_task(model): 7 | model.config_task = { 8 | "task_type": "dso.test.custom_tasks.custom_task_prior:CustomTask", 9 | "param" : "test" 10 | } 11 | model.config_training.update(CONFIG_TRAINING_OVERRIDE) 12 | model.train() 13 | 14 | 15 | def test_custom_prior(model): 16 | model.config_prior["dso.test.custom_tasks.custom_task_prior:CustomPrior"] = { 17 | "param" : "test", 18 | "on" : True 19 | } 20 | model.config_training.update(CONFIG_TRAINING_OVERRIDE) 21 | model.train() 22 | -------------------------------------------------------------------------------- /dso/dso/test/data/test_model.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/test/data/test_model.meta -------------------------------------------------------------------------------- /dso/dso/test/data/test_model_strong.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/test/data/test_model_strong.data-00000-of-00001 -------------------------------------------------------------------------------- /dso/dso/test/data/test_model_strong.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/test/data/test_model_strong.index -------------------------------------------------------------------------------- /dso/dso/test/data/test_model_weak.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/test/data/test_model_weak.data-00000-of-00001 -------------------------------------------------------------------------------- /dso/dso/test/data/test_model_weak.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/dso/dso/test/data/test_model_weak.index -------------------------------------------------------------------------------- /dso/dso/test/generate_test_data.py: -------------------------------------------------------------------------------- 1 | """Generate model parity test case data for DeepSymbolicOptimizer.""" 2 | 3 | from pkg_resources import resource_filename 4 | 5 | import tensorflow as tf 6 | import click 7 | 8 | from dso import DeepSymbolicOptimizer 9 | from dso.config import load_config 10 | 11 | 12 | # Shorter config run for parity test 13 | CONFIG_TRAINING_OVERRIDE = { 14 | "n_samples" : 1000, 15 | "batch_size" : 100 16 | } 17 | 18 | @click.command() 19 | @click.option('--stringency', '--t', default="strong", type=str, help="stringency of the test data to generate") 20 | def main(stringency): 21 | # Load config 22 | config = load_config() 23 | 24 | # Train the model 25 | model = DeepSymbolicOptimizer(config) 26 | 27 | if stringency == "strong": 28 | n_samples = 1000 29 | suffix = "_" + stringency 30 | elif stringency == "weak": 31 | n_samples = 100 32 | suffix = "_" + stringency 33 | 34 | model.config_training.update({"n_samples" : n_samples, 35 | "batch_size" : 100}) 36 | 37 | # Turn on GP meld 38 | model.config_gp_meld.update({"run_gp_meld" : True, 39 | "generations" : 3, 40 | "population_size" : 10}) 41 | 42 | model.train() 43 | 44 | # Save the TF model 45 | tf_save_path = resource_filename("dso.test", "data/test_model" + suffix) 46 | saver = tf.train.Saver() 47 | saver.save(model.sess, tf_save_path) 48 | 49 | 50 | if __name__ == "__main__": 51 | main() 52 | -------------------------------------------------------------------------------- /dso/dso/test/test_checkpoint.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | from dso import DeepSymbolicOptimizer 6 | from dso.config import load_config 7 | from datetime import datetime 8 | 9 | 10 | N_STEPS = 3 11 | 12 | 13 | def make_config(logdir=None, seed=None, timestamp=None): 14 | config = load_config() 15 | 16 | if timestamp is None: 17 | timestamp = datetime.now().strftime("%Y-%m-%d-%H%M%S") 18 | config["experiment"]["timestamp"] = timestamp 19 | config["experiment"]["logdir"] = str(logdir) 20 | 21 | # IMPORTANT: CHANGE THE SEED EACH CHECKPOINT 22 | # If the seed does not change each checkpoint, almost the same sequences 23 | # will be sampled again, even as neural network weights change. 24 | config["experiment"]["seed"] = seed 25 | 26 | # Turn on debugging to see neural network weights changing 27 | config["training"]["verbose"] = False 28 | config["training"]["debug"] = True 29 | 30 | return config 31 | 32 | 33 | @pytest.mark.parametrize("pqt", [False, True]) 34 | def test_checkpoint_manual(tmp_path, pqt): 35 | """ 36 | Run a model N_STEPS iterations, manually saving to/loading from 37 | checkpointing each iteration with a new model instance. 38 | """ 39 | 40 | timestamp = None 41 | for i in range(N_STEPS): 42 | load_path = os.path.join(tmp_path, "checkpoint_{}".format(i - 1)) if i > 0 else None 43 | save_path = os.path.join(tmp_path, "checkpoint_{}".format(i)) 44 | 45 | config = make_config(logdir=tmp_path, seed=i, timestamp=timestamp) 46 | if pqt: 47 | config["policy_optimizer"] = { 48 | "policy_optimizer_type" : "pqt", 49 | "pqt_k" : 10, 50 | "pqt_batch_size" : 3 51 | } 52 | timestamp = config["experiment"]["timestamp"] # Reuse for next checkpoint 53 | 54 | # Load the model from checkpoint, train it one step, then save the checkpoint 55 | model = DeepSymbolicOptimizer(config) 56 | model.setup() 57 | if load_path is not None: 58 | model.load(load_path) 59 | if pqt: 60 | assert len(model.trainer.priority_queue) == i 61 | model.train_one_step() 62 | model.save(save_path) 63 | 64 | 65 | def test_checkpoint_config(tmp_path): 66 | """ 67 | Run a model N_STEPS iterations, checkpointing via config. 68 | """ 69 | 70 | config = make_config(logdir=tmp_path, seed=0, timestamp=None) 71 | config["checkpoint"] = { 72 | "save_freq" : 1, 73 | "units" : "iterations" 74 | } 75 | 76 | model = DeepSymbolicOptimizer(config) 77 | for i in range(N_STEPS): 78 | model.train_one_step() 79 | -------------------------------------------------------------------------------- /dso/dso/test/test_constant.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import numpy as np 4 | 5 | from dso.library import HardCodedConstant 6 | from dso.config import load_config 7 | from dso.core import DeepSymbolicOptimizer 8 | from dso.test.test_core import model 9 | 10 | 11 | def test_constant(): 12 | valid_cases = np.arange(0, 25, 0.1) 13 | for number in valid_cases: 14 | const = HardCodedConstant(value=number) 15 | assert const() == number, "Value returned from Constant.function() ({}) does not match input value ({}).".format(const(), number) 16 | 17 | def test_regression_with_hard_coded_constants(model): 18 | model.config["task"]["function_set"] = ["add", "sin", 0, -1.0, 25, "1.23"] 19 | model.config_training.update( 20 | { 21 | "n_samples" : 10, 22 | "batch_size" : 4 23 | } 24 | ) 25 | model.train() -------------------------------------------------------------------------------- /dso/dso/test/test_core.py: -------------------------------------------------------------------------------- 1 | """Test cases for DeepSymbolicOptimizer on each Task.""" 2 | 3 | from pkg_resources import resource_filename 4 | 5 | import pytest 6 | import tensorflow as tf 7 | import numpy as np 8 | 9 | from dso import DeepSymbolicOptimizer 10 | from dso.config import load_config 11 | from dso.test.generate_test_data import CONFIG_TRAINING_OVERRIDE 12 | 13 | 14 | @pytest.fixture 15 | def model(): 16 | config = load_config() 17 | config["experiment"]["logdir"] = None # Turn off saving results 18 | return DeepSymbolicOptimizer(config) 19 | 20 | 21 | @pytest.fixture(params=("strong", "weak")) 22 | def cached_results(model, request): 23 | if request.param == "strong": 24 | model_data = "data/test_model_strong" 25 | elif request.param == "weak": 26 | model_data = "data/test_model_weak" 27 | tf_load_path = resource_filename("dso.test", model_data) 28 | model.setup() 29 | saver = tf.train.Saver() 30 | saver.restore(model.sess, tf_load_path) 31 | results = model.sess.run(tf.trainable_variables()) 32 | 33 | return [request.param, results] 34 | 35 | 36 | @pytest.mark.parametrize("config", ["config/config_regression.json", 37 | "config/config_control.json"]) 38 | def test_task(model, config): 39 | """Test that Tasks do not crash for various configs.""" 40 | config = load_config(config) 41 | config["experiment"]["logdir"] = None # Turn off saving results 42 | model.set_config(config) 43 | model.config_training.update({"n_samples" : 10, 44 | "batch_size" : 5 45 | }) 46 | model.train() 47 | 48 | 49 | @pytest.mark.parametrize("config", ["config/config_regression.json"]) 50 | def test_model_parity(model, cached_results, config): 51 | """Compare results with gp meld on to previous set""" 52 | config = load_config(config) 53 | config["experiment"]["logdir"] = None # Turn off saving results 54 | model.set_config(config) 55 | 56 | [stringency, cached_results]= cached_results 57 | 58 | if stringency == "strong": 59 | n_samples = 1000 60 | elif stringency == "weak": 61 | n_samples = 100 62 | 63 | model.config_training.update({"n_samples" : n_samples, 64 | "batch_size" : 100}) 65 | 66 | # Turn on GP meld 67 | model.config_gp_meld.update({"run_gp_meld" : True, 68 | "generations" : 3, 69 | "population_size" : 10, 70 | "crossover_operator" : "cxOnePoint", 71 | "mutation_operator" : "multi_mutate" 72 | }) 73 | 74 | model.train() 75 | results = model.sess.run(tf.trainable_variables()) 76 | results = np.concatenate([a.flatten() for a in results]) 77 | cached_results = np.concatenate([a.flatten() for a in cached_results]) 78 | assert np.linalg.norm(cached_results, ord=1) > 0 79 | 80 | if stringency == "weak": 81 | results = np.where(results, 1, 0) 82 | cached_results = np.where(cached_results, 1, 0) 83 | 84 | np.testing.assert_array_almost_equal(results, cached_results) 85 | -------------------------------------------------------------------------------- /dso/dso/test/test_polynomial.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import numpy as np 4 | 5 | from dso.config import load_config 6 | from dso.program import Program, from_str_tokens 7 | from dso.task import make_task 8 | from dso.library import Polynomial 9 | from dso.task.regression.polyfit import regressors, PolyOptimizer, PolyGenerator 10 | 11 | np.random.seed(0) 12 | 13 | options = { 14 | "linear_regression": {"fit_intercept": False}, 15 | "lasso": {"alpha": 1e-6, "fit_intercept": False, "max_iter": 200000, "tol": 1e-9}, 16 | "ridge": {"alpha": 1e-6, "fit_intercept": False}, 17 | "dso_least_squares" : {"cutoff_p_value": 0.05, "n_max_terms": 10, "coef_tol": 1e-12}, 18 | "dso_lasso" : {"gamma": 1E-6, "comp_tol": 1E-4, "rtrn_constrnd_ls": True} 19 | } 20 | 21 | coef_tol = 1e-6 22 | rel_tol = 1e-4 23 | 24 | n_input_var = 3 25 | degree = 5 26 | n_pts = 500 27 | X_range = 10.0 28 | n_tests = 10 29 | 30 | def check_error(poly, X, y, regressor): 31 | diff = poly(X) 32 | diff -= y 33 | rel_err = np.linalg.norm(diff) / np.linalg.norm(y) 34 | assert rel_err < rel_tol, "\nregressor: {}\npoly(X) = {}".format(regressor, repr(poly)) 35 | 36 | 37 | def test_polyfit(): 38 | X = np.random.uniform(-X_range / 2, X_range / 2, n_input_var * n_pts) 39 | X = X.reshape((n_pts, n_input_var)) 40 | poly_generator = PolyGenerator(degree, n_input_var) 41 | 42 | for test in range(1, n_tests+1): 43 | target_poly = poly_generator.generate() 44 | y = target_poly(X) 45 | 46 | print("\ntest_polyfit #{}: \ny = {}".format(test, target_poly)) 47 | for regressor in regressors: 48 | poly_optimizer = PolyOptimizer(degree, coef_tol, regressor, options[regressor]) 49 | poly = poly_optimizer.fit(X, y) 50 | check_error(poly, X, y, regressor) 51 | 52 | 53 | def test_poly_optimize(): 54 | config = load_config() 55 | config["task"]["dataset"] = "Poly-4" 56 | task = make_task(**config["task"]) 57 | Program.set_task(task) 58 | Program.set_execute(protected=False) 59 | 60 | target_poly = Polynomial([(1, 1, 0, 0, 0, 0, 0, 0, 0, 0), (0, 0, 0, 0, 0, 0, 0, 0, 0, 0), 61 | (0, 0, 1, 0, 0, 1, 0, 0, 0, 1)], np.array([12.0, 1.3, -0.05])) 62 | y = target_poly(task.X_train) 63 | 64 | print("target_poly = {}".format(repr(target_poly))) 65 | for regressor in regressors: 66 | task.poly_optimizer = PolyOptimizer(degree, coef_tol, regressor, options[regressor]) 67 | my_program = from_str_tokens(['div', 'sin', 'x4', 'mul', 'sqrt', 'poly', 'exp', 'x7']) 68 | my_program.r 69 | my_poly = my_program.traversal[my_program.poly_pos] 70 | check_error(my_poly, task.X_train, y, regressor) 71 | 72 | 73 | def test_poly_to_traversal(): 74 | config = load_config() 75 | config["task"]["dataset"] = "Korns-3" 76 | task = make_task(**config["task"]) 77 | Program.set_task(task) 78 | Program.set_execute(protected=False) 79 | 80 | X = np.random.uniform(-X_range / 2, X_range / 2, n_input_var * n_pts) 81 | X = X.reshape((n_pts, n_input_var)) 82 | poly_generator = PolyGenerator(degree, n_input_var) 83 | for test in range(n_tests): 84 | poly = poly_generator.generate() 85 | equivalent_program = from_str_tokens(poly.to_str_tokens()) 86 | 87 | y = equivalent_program.execute(X) 88 | diff = poly(X) 89 | diff -= y 90 | rel_err = np.linalg.norm(diff) / np.linalg.norm(y) 91 | assert rel_err < 1e-8, \ 92 | "The converted traversal for {} is incorrect!".format(poly) 93 | -------------------------------------------------------------------------------- /dso/dso/variance.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from dso.program import from_tokens 4 | from dso.utils import weighted_quantile 5 | 6 | 7 | def quantile_variance(memory_queue, policy, batch_size, epsilon, step, 8 | n_experiments=1000, estimate_bias=True, 9 | n_samples_bias=1e6): 10 | 11 | print("Running quantile variance/bias experiments...") 12 | empirical_quantiles = [] 13 | memory_augmented_quantiles = [] 14 | 15 | if len(memory_queue) < memory_queue.capacity: 16 | print("WARNING: Memory queue not yet at capacity.") 17 | 18 | memory_r = memory_queue.get_rewards() 19 | memory_w = memory_queue.compute_probs() 20 | for exp in range(n_experiments): 21 | actions, obs, priors = policy.sample(batch_size) 22 | programs = [from_tokens(a) for a in actions] 23 | r = np.array([p.r for p in programs]) 24 | quantile = np.quantile(r, 1 - epsilon, interpolation="higher") 25 | empirical_quantiles.append(quantile) 26 | unique_programs = [p for p in programs if p.str not in memory_queue.unique_items] 27 | N = len(unique_programs) 28 | sample_r = [p.r for p in unique_programs] 29 | combined_r = np.concatenate([memory_r, sample_r]) 30 | if N == 0: 31 | print("WARNING: Found no unique samples in batch!") 32 | combined_w = memory_w / memory_w.sum() # Renormalize 33 | else: 34 | sample_w = np.repeat((1 - memory_w.sum()) / N, N) 35 | combined_w = np.concatenate([memory_w, sample_w]) 36 | 37 | # Compute the weighted quantile 38 | quantile = weighted_quantile(values=combined_r, weights=combined_w, q=1 - epsilon) 39 | memory_augmented_quantiles.append(quantile) 40 | 41 | empirical_quantiles = np.array(empirical_quantiles) 42 | memory_augmented_quantiles = np.array(memory_augmented_quantiles) 43 | print("Train step:", step) 44 | print("Memory weight:", memory_w.sum()) 45 | print("Mean(empirical quantile):", np.mean(empirical_quantiles)) 46 | print("Var(empirical quantile):", np.var(empirical_quantiles)) 47 | print("Mean(Memory augmented quantile):", np.mean(memory_augmented_quantiles)) 48 | print("Var(Memory augmented quantile):", np.var(memory_augmented_quantiles)) 49 | if estimate_bias: 50 | actions, obs, priors = policy.sample(int(n_samples_bias)) 51 | programs = [from_tokens(a) for a in actions] 52 | r = np.array([p.r for p in programs]) 53 | true_quantile = np.quantile(r, 1 - epsilon, interpolation="higher") 54 | print("'True' empirical quantile:", true_quantile) 55 | print("Empirical quantile bias:", np.mean(np.abs(empirical_quantiles - true_quantile))) 56 | print("Memory-augmented quantile bias:", np.mean(np.abs(memory_augmented_quantiles - true_quantile))) 57 | exit() 58 | -------------------------------------------------------------------------------- /dso/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | import os 3 | from setuptools import dist 4 | 5 | dist.Distribution().fetch_build_eggs(['Cython', 'numpy']) 6 | 7 | import numpy 8 | from Cython.Build import cythonize 9 | 10 | required = [ 11 | "pytest", 12 | "cython", 13 | "numpy<=1.19", 14 | "tensorflow==1.14", 15 | "numba==0.53.1", 16 | "sympy", 17 | "pandas", 18 | "scikit-learn", 19 | "click", 20 | "deap", 21 | "pathos", 22 | "seaborn", 23 | "progress", 24 | "tqdm", 25 | "commentjson", 26 | "PyYAML", 27 | "prettytable" 28 | ] 29 | 30 | extras = { 31 | "control": [ 32 | "mpi4py", 33 | "gym[box2d]==0.15.4", 34 | "pybullet", 35 | "stable-baselines[mpi]==2.10.0" 36 | ], 37 | "regression": [] 38 | } 39 | extras['all'] = list(set([item for group in extras.values() for item in group])) 40 | 41 | setup( name='dso', 42 | version='1.0dev', 43 | description='Deep symbolic optimization.', 44 | author='LLNL', 45 | packages=['dso'], 46 | setup_requires=["numpy", "Cython"], 47 | ext_modules=cythonize([os.path.join('dso','cyfunc.pyx')]), 48 | include_dirs=[numpy.get_include()], 49 | install_requires=required, 50 | extras_require=extras 51 | ) 52 | -------------------------------------------------------------------------------- /images/banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/images/banner.png -------------------------------------------------------------------------------- /images/code_map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/images/code_map.png -------------------------------------------------------------------------------- /images/srbench_accuracy-solution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/images/srbench_accuracy-solution.png -------------------------------------------------------------------------------- /images/srbench_symbolic-solution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dso-org/deep-symbolic-optimization/d652666ea2ea2679ce3d68e554bf2092147ddec1/images/srbench_symbolic-solution.png --------------------------------------------------------------------------------