├── .gitignore ├── LICENSE ├── README.md ├── compare_EAs.py ├── compare_GAs.py ├── evoalgo ├── __init__.py ├── const.py ├── evolutionary_algorithms │ ├── __init__.py │ ├── evolution_strategy.py │ └── genetic_algorithm.py ├── optimization_problems │ ├── __init__.py │ ├── policy_nn.py │ ├── rastrigin.py │ └── string_opt.py └── utils │ ├── __init__.py │ ├── evolution_process.py │ ├── fitness_shaping.py │ ├── genetic_operators.py │ ├── optimizers.py │ └── utils.py ├── images ├── EvoAlgo_bear.jpeg ├── Rastrigin_function.png └── results │ ├── ea_comparison │ ├── CartPole407D-Pop500-Avg.png │ ├── CartPole407D-Pop500-Max.png │ ├── Legend.png │ ├── Rastrigin100D-Pop100-Avg.png │ └── Rastrigin100D-Pop100-Max.png │ └── ga_comparison │ ├── Legend.png │ ├── Rastrigin100D-Pop100-Avg-Sigma0.5.png │ ├── Rastrigin100D-Pop100-Avg-Sigma0.5_Min0.01_Decay0.9.png │ ├── Rastrigin100D-Pop100-Avg-Sigma1.png │ ├── Rastrigin100D-Pop100-Max-Sigma0.5.png │ ├── Rastrigin100D-Pop100-Max-Sigma0.5_Min0.01_Decay0.9.png │ ├── Rastrigin100D-Pop100-Max-Sigma1.png │ ├── String12D-Pop100-Avg.png │ ├── String12D-Pop100-Max.png │ ├── String12D-Pop1000-Avg.png │ └── String12D-Pop1000-Max.png ├── requirements.txt ├── results ├── CartPole407D-Pop500-Avg.png ├── CartPole407D-Pop500-Max.png ├── Rastrigin100D-Pop100-Avg-Sigma0.5.png ├── Rastrigin100D-Pop100-Avg-Sigma0.5_Min0.01_Decay0.9.png ├── Rastrigin100D-Pop100-Avg-Sigma1.png ├── Rastrigin100D-Pop100-Avg.png ├── Rastrigin100D-Pop100-Max-Sigma0.5.png ├── Rastrigin100D-Pop100-Max-Sigma0.5_Min0.01_Decay0.9.png ├── Rastrigin100D-Pop100-Max-Sigma1.png ├── Rastrigin100D-Pop100-Max.png ├── String12D-Pop100-Avg.png ├── String12D-Pop100-Max.png ├── String12D-Pop1000-Avg.png └── String12D-Pop1000-Max.png └── tests ├── __init__.py ├── results ├── CartPole407D-Pop10-Avg.png ├── CartPole407D-Pop10-Max.png ├── CartPole407D-Pop20-Avg.png ├── CartPole407D-Pop20-Max.png ├── Rastrigin100D-Pop10-Avg.png ├── Rastrigin100D-Pop10-Max.png ├── Rastrigin100D-Pop20-Avg.png ├── Rastrigin100D-Pop20-Max.png ├── String12D-Pop20-Avg.png └── String12D-Pop20-Max.png ├── test_es.py └── test_ga.py /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /ignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Elior Ben-Yosef 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Evolutionary Algorithms implementations 2 | Evolutionary Algorithms implementations, for various (discrete & continuous) optimization problems. 3 | 4 | ### TL;DR 5 | 6 | This repository contains implementations of the following: 7 | 8 | **Evolutionary Algorithms (EAs):** 9 | * [Genetic Algorithm (GA)](../master/evoalgo/evolutionary_algorithms/genetic_algorithm.py) - 10 | with multiple choices of Selection, Crossover & Mutation strategies. 11 | * [Evolution Strategy (ES)](../master/evoalgo/evolutionary_algorithms/evolution_strategy.py) - 12 | PEPG, OpenAI-ES, CMA-ES. 13 | 14 | **Optimization problems (of increasing difficulty):** 15 | 16 | Optimization problem | Type | Parameters number 17 | --- | --- | --- 18 | **[String](../master/evoalgo/optimization_problems/string_opt.py)** | Discrete-number vector | 12 19 | **[Rastrigin function](../master/evoalgo/optimization_problems/rastrigin.py)** | Real-number vector | 100 20 | **[Policy Neural-Network](../master/evoalgo/optimization_problems/policy_nn.py) for autonomous agent control** | Real-number vector | 407 21 | 22 | Note that the higher the dimensionality (parameters number) of the optimization problem, 23 | the harder it is, and the slower the optimization process is. 24 | 25 | ### Table of contents: 26 | 27 | * [Intro](https://github.com/EliorBenYosef/evolutionary-algorithms#intro) 28 | * [Evolutionary Algorithms](https://github.com/EliorBenYosef/evolutionary-algorithms#evolutionary-algorithms) 29 | * [Optimization Problems](https://github.com/EliorBenYosef/evolutionary-algorithms#optimization-problems) 30 | * [Results](https://github.com/EliorBenYosef/evolutionary-algorithms#results) 31 | * [Evolutionary Algorithms Comparison](https://github.com/EliorBenYosef/evolutionary-algorithms#evolutionary-algorithms-comparison) 32 | * [Genetic Algorithms Comparison](https://github.com/EliorBenYosef/evolutionary-algorithms#genetic-algorithms-comparison) 33 | * [How to use](https://github.com/EliorBenYosef/evolutionary-algorithms#how-to-use) 34 | 35 | ## Intro 36 | 37 | 38 | 39 | ### Evolutionary Algorithms 40 | A family of problem-independent, gradient-free, population-based, metaheuristic or stochastic, 41 | global optimization algorithms. 42 | 43 | Inspired by the biological concept of **evolution by natural selection** - 44 | a population of candidate solution models follows an evolutionary process towards optimality 45 | (optimization via evolution). 46 | 47 | #### Genetic Algorithm 48 | Employs a more **biologically-accurate** form of evolution - utilizes a loop of 49 | parents **selection**, reproduction via **crossover** (AKA genetic recombination) and **mutation**. 50 | 51 | #### Evolution Strategy 52 | Employs a more **mathematical** (and less biologically-accurate) form of evolution - 53 | a single parent produces multiple variants of itself via **cloning** and **mutation** 54 | (a small amount of random noise). 55 | After the new population is evaluated, a single parent is created using a weighted-sum of 56 | all the individuals (AKA population averaging). 57 | 58 | ### Optimization Problems 59 | 60 | #### String 61 | Optimizing a fixed-length String (discrete-valued vector) towards a chosen target String 62 | of 12 characters. 63 | 64 | For generality purposes, the String (individual & target) is converted into a discrete-number 65 | vector (and vice-versa), which is optimized. 66 | 67 | 68 | 69 | #### Rastrigin function 70 | Optimizing the Rastrigin function's input parameters (x). 71 | 72 | The Rastrigin function is a **non-convex** function (with multiple local minima), 73 | and is used as a performance-test problem for optimization algorithms. 74 | It is a typical example of non-linear multimodal function. 75 | 76 | #### Policy Neural-Network for autonomous agent control 77 | Optimizing the Policy Neural-Network's parameters (layers' weights and biases). 78 | 79 | A Policy network receives the **environment state** as an input, 80 | and outputs a **probability distribution over all possible actions**. 81 | 82 | Any AI Gym environment can be chosen, as long as the relevant variables parameters 83 | (`env_name`, `input_dims`, `n_actions`, `optimal_fit`) are changed accordingly. 84 | Here, AI Gym's **CartPole** environment is chosen as an example. 85 | 86 | ## Results 87 | optimization process results: 88 | 89 | ## Evolutionary Algorithms Comparison 90 | 91 | Legend: 92 |

93 | 94 |

95 | 96 | #### Rastrigin function (100 input parameters) 97 | 98 |

99 | 100 | 101 |

102 | 103 | #### Policy Neural-Network (407 parameters) 104 | Environment: CartPole. 105 | 106 | Neural-Network layers' units number: (Input) 4,25,10,2 (Output). 107 | 108 |

109 | 110 | 111 |

112 | 113 | ## Genetic Algorithms Comparison 114 | 115 | * **Selection** options: 116 | * fitness-proportionate selection (FPS) 117 | * stochastic top-sampling (STS) selection 118 | * tournament (Tour) selection 119 | * **Crossover** options: 120 | * Single-point crossover(1PtCross) 121 | * Two-point crossover(2PtCross) 122 | * Uniform crossover(UniCross) 123 | * **Mutation** options: 124 | * Deterministic mutation (DetMut) 125 | * Stochastic uniform mutation (StoUniMut) 126 | * Gaussian-noise mutation (GaussMut) 127 | 128 | Legend: 129 |

130 | 131 |

132 | 133 | #### String (12 characters) 134 | Notice how the optimization is better with a larger population size. 135 | 136 |

137 | 138 | 139 |

140 | 141 |

142 | 143 | 144 |

145 | 146 | #### Rastrigin function (100 input parameters) 147 | Notice how the optimization is better with a higher (and constant) mutation sigma. 148 | 149 | **Sigma = 1** 150 |

151 | 152 | 153 |

154 | 155 | **Sigma = 0.5** 156 |

157 | 158 | 159 |

160 | 161 | **Sigma: Init = 0.5, Decay = 0.9, Min = 0.01** 162 |

163 | 164 | 165 |

166 | 167 | ## How to use 168 | 169 | Examples of how to use: 170 | * [compare_EAs.py](../master/compare_EAs.py) 171 | * [compare_GAs.py](../master/compare_GAs.py) 172 | 173 | -------------------------------------------------------------------------------- /compare_EAs.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import evoalgo.evolutionary_algorithms.genetic_algorithm as GA 4 | import evoalgo.evolutionary_algorithms.evolution_strategy as ES 5 | import evoalgo.utils.genetic_operators as GenOp 6 | from evoalgo.utils.evolution_process import Evolution 7 | from evoalgo.utils.utils import plot_fit_history_comparison 8 | from evoalgo.optimization_problems.policy_nn import max_gen_num, pop_size, params_num, task_name, \ 9 | fitness_function, optimal_fit # , discrete_values_num # import only when optimizing the string 10 | 11 | discrete_values_num = None 12 | 13 | max_fit_history_dict = {} 14 | avg_fit_history_dict = {} 15 | 16 | 17 | def compare_evolutionary_algorithms(): 18 | """ 19 | Compares ES algorithms (CMA-ES, OpenAI-ES, PEPG) 20 | and the best GA (Tournament selection, Uniform crossover, Gaussian mutation). 21 | """ 22 | if discrete_values_num is not None: 23 | print('Evolution Strategy Algorithms are implemented only for real-number vector optimization') 24 | return 25 | 26 | start_time = datetime.datetime.now() 27 | cma_es = ES.CMA_ES(params_num, pop_size, sigma_init=0.5) 28 | Evolution.test_solver(cma_es, max_gen_num, task_name, fitness_function, 29 | print_progress=True, plot=False) 30 | print(f"CMA-ES ~~~ Runtime: {str(datetime.datetime.now() - start_time).split('.')[0]}") 31 | max_fit_history_dict['CMA-ES'] = cma_es.pop_max_fit_history 32 | avg_fit_history_dict['CMA-ES'] = cma_es.pop_avg_fit_history 33 | 34 | start_time = datetime.datetime.now() 35 | open_ai_es = ES.OpenAI_ES(params_num, pop_size, sigma_init=0.5, sigma_decay=1.0, # sigma_decay=0.999 36 | alpha_init=0.1, alpha_decay=1.0, antithetic_sampling=False, rank_fitness=False) 37 | Evolution.test_solver(open_ai_es, max_gen_num, task_name, fitness_function, 38 | print_progress=True, plot=False) 39 | print(f"OpenAI-ES ~~~ Runtime: {str(datetime.datetime.now() - start_time).split('.')[0]}") 40 | max_fit_history_dict['OpenAI-ES'] = open_ai_es.pop_max_fit_history 41 | avg_fit_history_dict['OpenAI-ES'] = open_ai_es.pop_avg_fit_history 42 | 43 | start_time = datetime.datetime.now() 44 | pepg = ES.PEPG(params_num, pop_size, sigma_init=0.5, sigma_decay=1.0, # sigma_decay=0.999 45 | alpha_init=0.1, alpha_decay=1.0, avg_fit_baseline=False, rank_fitness=False) 46 | Evolution.test_solver(pepg, max_gen_num, task_name, fitness_function, 47 | print_progress=True, plot=False) 48 | print(f"PEPG ~~~ Runtime: {str(datetime.datetime.now() - start_time).split('.')[0]}") 49 | max_fit_history_dict['PEPG'] = pepg.pop_max_fit_history 50 | avg_fit_history_dict['PEPG'] = pepg.pop_avg_fit_history 51 | 52 | start_time = datetime.datetime.now() 53 | ga = GA.SimpleGA(params_num, pop_size, mutation_var=1) # sigma 54 | Evolution.test_solver(ga, max_gen_num, task_name, fitness_function, 55 | GenOp.Selection.tournament, GenOp.Crossover.uniform, GenOp.Mutation.gaussian_noise, 56 | print_progress=True, plot=False) 57 | print(f"GA Tour UniCross GaussMut ~~~ Runtime: {str(datetime.datetime.now() - start_time).split('.')[0]}") 58 | max_fit_history_dict['GA Tour UniCross GaussMut'] = ga.pop_max_fit_history 59 | avg_fit_history_dict['GA Tour UniCross GaussMut'] = ga.pop_avg_fit_history 60 | 61 | plot_fit_history_comparison(max_fit_history_dict, 'Max', max_gen_num, task_name, pop_size, optimal_fit, 62 | colors=['red', 'orange', 'green', 'blue']) 63 | plot_fit_history_comparison(avg_fit_history_dict, 'Avg', max_gen_num, task_name, pop_size, optimal_fit, 64 | colors=['red', 'orange', 'green', 'blue']) 65 | 66 | 67 | if __name__ == '__main__': 68 | compare_evolutionary_algorithms() 69 | -------------------------------------------------------------------------------- /compare_GAs.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import evoalgo.evolutionary_algorithms.genetic_algorithm as GA 4 | import evoalgo.utils.genetic_operators as GenOp 5 | from evoalgo.utils.evolution_process import Evolution 6 | from evoalgo.utils.utils import colors_28, plot_fit_history_comparison 7 | from evoalgo.optimization_problems.policy_nn import max_gen_num, pop_size, params_num, task_name, \ 8 | fitness_function, optimal_fit # , discrete_values_num # import only when optimizing the string 9 | 10 | discrete_values_num = None 11 | 12 | max_fit_history_dict = {} 13 | avg_fit_history_dict = {} 14 | 15 | 16 | def run_algo(selection_f, crossover_f, mutation_f, key): 17 | if mutation_f.__name__ == GenOp.Mutation.gaussian_noise.__name__: 18 | ga = GA.SimpleGA(params_num, pop_size, discrete_values_num, 19 | # mutation_var=0.5, sigma_decay=0.999, sigma_min=0.01) 20 | mutation_var=0.5) 21 | else: 22 | ga = GA.SimpleGA(params_num, pop_size, discrete_values_num) 23 | 24 | Evolution.test_solver(ga, max_gen_num, task_name, fitness_function, 25 | selection_f, crossover_f, mutation_f, 26 | print_progress=False, plot=False) 27 | max_fit_history_dict[key] = ga.pop_max_fit_history 28 | avg_fit_history_dict[key] = ga.pop_avg_fit_history 29 | 30 | 31 | def compare_genetic_algorithms(): 32 | algo_type = 'GA' 33 | 34 | selection_types = [('FPS', GenOp.Selection.fitness_proportionate), 35 | ('STS', GenOp.Selection.stochastic_top_sampling), 36 | ('Tour', GenOp.Selection.tournament)] 37 | crossover_types = [('1PtCross', GenOp.Crossover.single_pt), 38 | ('2PtCross', GenOp.Crossover.two_pt), 39 | ('UniCross', GenOp.Crossover.uniform)] 40 | mutation_types = [('DetMut', GenOp.Mutation.deterministic), 41 | ('StoUniMut', GenOp.Mutation.stochastic_uniform), 42 | ('GaussMut', GenOp.Mutation.gaussian_noise)] 43 | 44 | for selection_key, selection_f in selection_types: 45 | for crossover_key, crossover_f in crossover_types: 46 | for mutation_key, mutation_f in mutation_types: 47 | start_time = datetime.datetime.now() 48 | description = f'{selection_key} {crossover_key} {mutation_key}' 49 | run_algo(selection_f, crossover_f, mutation_f, description) 50 | print(f"{description} ~~~ Runtime: {str(datetime.datetime.now() - start_time).split('.')[0]}") 51 | 52 | plot_fit_history_comparison(max_fit_history_dict, 'Max', max_gen_num, task_name, pop_size, optimal_fit, algo_type, 53 | colors=colors_28) 54 | plot_fit_history_comparison(avg_fit_history_dict, 'Avg', max_gen_num, task_name, pop_size, optimal_fit, algo_type, 55 | colors=colors_28) 56 | 57 | 58 | if __name__ == '__main__': 59 | compare_genetic_algorithms() 60 | -------------------------------------------------------------------------------- /evoalgo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/evoalgo/__init__.py -------------------------------------------------------------------------------- /evoalgo/const.py: -------------------------------------------------------------------------------- 1 | 2 | KEY_PARAMS_VEC = 'params_vec' 3 | KEY_FITNESS = 'fitness' 4 | -------------------------------------------------------------------------------- /evoalgo/evolutionary_algorithms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/evoalgo/evolutionary_algorithms/__init__.py -------------------------------------------------------------------------------- /evoalgo/evolutionary_algorithms/evolution_strategy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Evolution Strategy Algorithms. 3 | Implemented only for real-number vector optimization. 4 | """ 5 | 6 | import numpy as np 7 | from cma import CMAEvolutionStrategy 8 | 9 | from evoalgo.const import KEY_PARAMS_VEC, KEY_FITNESS 10 | import evoalgo.utils.optimizers as optimizers 11 | from evoalgo.utils.fitness_shaping import compute_zero_centered_ranks 12 | import evoalgo.utils.genetic_operators as GenOp 13 | 14 | 15 | class CMA_ES: 16 | """ 17 | CMA-ES (Covariance Matrix Adaptation) algorithm (cma's CMAEvolutionStrategy) wrapper. 18 | """ 19 | 20 | def __init__(self, params_num, pop_size, sigma_init=0.1): 21 | """ 22 | :param params_num: number of model parameters 23 | :param pop_size: population size 24 | :param sigma_init: mut_rate / sigma = sigma_init: initial STD 25 | """ 26 | self.params_num = params_num 27 | self.pop_size = pop_size 28 | 29 | self.sigma_init = sigma_init 30 | self.sigma = sigma_init 31 | 32 | self.cma_es = CMAEvolutionStrategy(self.params_num * [0], self.sigma_init, {'popsize': self.pop_size}) 33 | 34 | self.population = None 35 | self.fitness_scores = None 36 | self.top_individual = None 37 | self.pop_avg_fit_history = [] 38 | self.pop_max_fit_history = [] 39 | 40 | def evolve(self, fitness_f): 41 | self.population = np.array(self.cma_es.ask()) 42 | self.eval_pop(fitness_f) 43 | 44 | def eval_pop(self, fitness_function): 45 | self.fitness_scores = np.zeros(self.pop_size) 46 | for i, params in enumerate(self.population): 47 | self.fitness_scores[i] = fitness_function(params) 48 | 49 | avg_fit = np.sum(self.fitness_scores) / self.pop_size 50 | self.pop_avg_fit_history.append(avg_fit) 51 | 52 | self.cma_es.tell(self.population, (-self.fitness_scores).tolist()) # sign flip since the optimizer is a minimizer 53 | 54 | result = self.cma_es.result 55 | top_individual_params = result[0] # best_mu 56 | top_individual_fitness = -result[1] 57 | # top_individual_index = result[2] - 1 58 | self.sigma = result[6] 59 | 60 | self.pop_max_fit_history.append(top_individual_fitness) 61 | 62 | self.top_individual = { 63 | KEY_PARAMS_VEC: top_individual_params, 64 | KEY_FITNESS: top_individual_fitness 65 | } 66 | 67 | 68 | class OpenAI_ES: 69 | """ 70 | OpenAI's ES algorithm (basic version). 71 | """ 72 | 73 | def __init__(self, params_num, pop_size, 74 | sigma_init=0.1, sigma_decay=0.999, sigma_min=0.01, 75 | alpha_init=0.01, alpha_decay=0.9999, alpha_min=0.001, 76 | antithetic_sampling=False, rank_fitness=True): 77 | """ 78 | :param params_num: number of model parameters 79 | :param pop_size: population size 80 | :param sigma_init: initial STD 81 | :param sigma_decay: anneal STD. sigma_decay=1 --> don't anneal the STD 82 | :param sigma_min: stop annealing if less than this 83 | :param alpha_init: learning rate for STD 84 | :param alpha_decay: annealing the learning rate. alpha_decay=1.0 --> don't anneal the learning rate 85 | :param alpha_min: stop annealing learning rate 86 | :param antithetic_sampling: antithetic sampling of epsilon (gaussian-noise vector). 87 | half of the population have some params, and the other half has the opposite params (sign change). 88 | :param rank_fitness: rank-based fitness-shaping - use rank rather than fitness numbers. 89 | """ 90 | self.params_num = params_num 91 | self.pop_size = pop_size 92 | self.rank_fitness = rank_fitness 93 | 94 | #################################### 95 | 96 | # new population's params variables: 97 | self.mu = np.zeros(self.params_num) 98 | 99 | self.sigma = sigma_init 100 | self.sigma_init = sigma_init 101 | self.sigma_decay = sigma_decay 102 | self.sigma_min = sigma_min 103 | 104 | self.epsilon = None 105 | self.antithetic_sampling = antithetic_sampling 106 | if self.antithetic_sampling: 107 | if self.pop_size % 2 != 0: # Antithetic sampling -> requires even pop_size 108 | self.pop_size += 1 109 | 110 | #################################### 111 | 112 | self.alpha = alpha_init 113 | self.alpha_init = alpha_init 114 | self.alpha_decay = alpha_decay 115 | self.alpha_min = alpha_min 116 | self.optimizer = optimizers.Adam(pi=self, step_size=alpha_init) # updates self.mu 117 | 118 | #################################### 119 | 120 | self.population = None 121 | self.fitness_scores = None 122 | self.top_individual = None 123 | self.pop_avg_fit_history = [] 124 | self.pop_max_fit_history = [] 125 | 126 | def evolve(self, fitness_f): 127 | self.update_pop() 128 | self.eval_pop(fitness_f) 129 | self.decay_variables() 130 | 131 | def update_pop(self): 132 | self.epsilon = GenOp.Mutation.sample_epsilon(self.pop_size, self.params_num, self.antithetic_sampling) 133 | self.population = self.mu[None, :] + self.epsilon * self.sigma 134 | 135 | def eval_pop(self, fitness_function): 136 | self.fitness_scores = np.zeros(self.pop_size) 137 | for i, params in enumerate(self.population): 138 | self.fitness_scores[i] = fitness_function(params) 139 | 140 | avg_fit = np.sum(self.fitness_scores) / self.pop_size 141 | self.pop_avg_fit_history.append(avg_fit) 142 | 143 | fit_arr = compute_zero_centered_ranks(self.fitness_scores) if self.rank_fitness else self.fitness_scores 144 | 145 | indices_sort = np.argsort(fit_arr)[::-1] # sorted by fitness score \ fitness rank 146 | top_individual_params = self.population[indices_sort[0]] 147 | top_individual_fitness = self.fitness_scores[indices_sort[0]] 148 | self.pop_max_fit_history.append(top_individual_fitness) 149 | 150 | if self.top_individual is None or top_individual_fitness > self.top_individual[KEY_FITNESS]: 151 | self.top_individual = { 152 | KEY_PARAMS_VEC: top_individual_params, 153 | KEY_FITNESS: top_individual_fitness 154 | } 155 | 156 | self.update_mu(fit_arr) 157 | 158 | def update_mu(self, fit_arr): 159 | standardized_fit = (fit_arr - np.mean(fit_arr)) / np.std(fit_arr) 160 | # formula: ∆μ = α ∙ 1/(Nσ) ∙ ∑(Fi∙ei), i=1,...,N 161 | delta_mu = 1. / (self.pop_size * self.sigma) * np.dot(standardized_fit, self.epsilon) # Population averaging 162 | 163 | if self.optimizer is not None: 164 | # the optimizer updates self.mu within itself 165 | self.optimizer.step_size = self.alpha 166 | update_ratio = self.optimizer.update(-delta_mu) # sign flip since the optimizer is a minimizer 167 | else: 168 | self.mu += self.alpha * delta_mu # normal SGD method? 169 | 170 | def decay_variables(self): 171 | if self.sigma_decay < 1 and self.sigma > self.sigma_min: 172 | self.sigma *= self.sigma_decay 173 | 174 | if self.alpha_decay < 1 and self.alpha > self.alpha_min: 175 | self.alpha *= self.alpha_decay 176 | 177 | 178 | class PEPG: 179 | """ 180 | PEPG (NES) (extension). 181 | """ 182 | 183 | def __init__(self, params_num, pop_size, 184 | sigma_init=0.1, sigma_decay=0.999, sigma_min=0.01, 185 | sigma_alpha=0.2, sigma_max_change=0.2, 186 | alpha_init=0.01, alpha_decay=0.9999, alpha_min=0.001, 187 | elite_ratio=0, 188 | avg_fit_baseline=True, rank_fitness=True): 189 | """ 190 | :param params_num: number of model parameters 191 | :param pop_size: population size 192 | :param sigma_init: initial STD 193 | :param sigma_decay: anneal STD. sigma_decay=1 --> don't anneal the STD 194 | :param sigma_min: stop annealing if less than this 195 | :param sigma_alpha: learning rate for STD 196 | :param sigma_max_change: clips adaptive sigma to 20%. 197 | restricts sigma from moving more than 20% of the original value. increases stability. 198 | :param alpha_init: learning rate for STD 199 | :param alpha_decay: annealing the learning rate. alpha_decay=1.0 --> don't anneal the learning rate 200 | :param alpha_min: stop annealing learning rate 201 | :param elite_ratio: (0,1) 202 | :param avg_fit_baseline: set baseline to average of batch 203 | :param rank_fitness: rank-based fitness-shaping - use rank rather than fitness numbers. 204 | """ 205 | self.params_num = params_num 206 | self.pop_size = pop_size 207 | if self.pop_size % 2 == 0: # (Antithetic sampling) + (self.population[-1] == self.mu) -> requires odd pop_size 208 | self.pop_size += 1 209 | self.rank_fitness = rank_fitness 210 | self.elite_ratio = elite_ratio 211 | self.avg_fit_baseline = avg_fit_baseline 212 | 213 | #################################### 214 | 215 | # new population's params variables: 216 | self.mu = np.zeros(self.params_num) 217 | 218 | self.sigma = np.ones(self.params_num) * sigma_init # sigma_arr 219 | self.sigma_init = sigma_init 220 | self.sigma_decay = sigma_decay 221 | self.sigma_min = sigma_min 222 | # adaptive sigma params: 223 | self.sigma_alpha = sigma_alpha 224 | self.sigma_max_change = sigma_max_change 225 | 226 | self.epsilon_half = None 227 | self.epsilon = None 228 | 229 | #################################### 230 | 231 | self.alpha = alpha_init 232 | self.alpha_init = alpha_init 233 | self.alpha_decay = alpha_decay 234 | self.alpha_min = alpha_min 235 | self.optimizer = optimizers.Adam(pi=self, step_size=alpha_init) # updates self.mu 236 | 237 | #################################### 238 | 239 | self.population = None 240 | self.fitness_scores = None 241 | self.top_individual = None 242 | self.pop_avg_fit_history = [] 243 | self.pop_max_fit_history = [] 244 | 245 | def evolve(self, fitness_f): 246 | self.update_pop() 247 | self.eval_pop(fitness_f) 248 | self.decay_variables() 249 | 250 | def update_pop(self): 251 | # sampling epsilon (gaussian-noise vector), which constitutes the mutation: 252 | # antithetic sampling (#antithetic_sampling): 253 | self.epsilon_half = np.random.randn(int(self.pop_size / 2), self.params_num) * self.sigma[None, :] 254 | self.epsilon = np.concatenate([self.epsilon_half, - self.epsilon_half, np.zeros((1, self.params_num))]) 255 | self.population = self.mu[None, :] + self.epsilon # self.population[-1] == self.mu 256 | 257 | def eval_pop(self, fitness_function): 258 | self.fitness_scores = np.zeros(self.pop_size) 259 | for i, params in enumerate(self.population): 260 | self.fitness_scores[i] = fitness_function(params) 261 | 262 | avg_fit = np.sum(self.fitness_scores) / self.pop_size 263 | self.pop_avg_fit_history.append(avg_fit) 264 | 265 | fit_arr = compute_zero_centered_ranks(self.fitness_scores) if self.rank_fitness else self.fitness_scores 266 | 267 | indices_sort = np.argsort(fit_arr)[::-1] # sorted by fitness score \ fitness rank 268 | top_individual_params = self.population[indices_sort[0]] 269 | top_individual_fitness = self.fitness_scores[indices_sort[0]] 270 | self.pop_max_fit_history.append(top_individual_fitness) 271 | 272 | if self.top_individual is None or top_individual_fitness > self.top_individual[KEY_FITNESS]: 273 | self.top_individual = { 274 | KEY_PARAMS_VEC: top_individual_params, 275 | KEY_FITNESS: top_individual_fitness 276 | } 277 | 278 | self.update_mu(fit_arr, indices_sort) 279 | 280 | def update_mu(self, fit_arr, indices_sort): 281 | if 0 < self.elite_ratio < 1: 282 | # Greedy-ES method (ignoring learning rate): 283 | self.mu = self.population[indices_sort[0:int(self.pop_size * self.elite_ratio)]].mean(axis=0) 284 | else: 285 | # using drift param (utilizing learning rate) 286 | fit_T = fit_arr[:int(self.pop_size / 2)] - fit_arr[int(self.pop_size / 2):-1] 287 | delta_mu = np.dot(fit_T, self.epsilon_half) 288 | 289 | if self.optimizer is not None: 290 | # the optimizer updates self.mu within itself 291 | self.optimizer.step_size = self.alpha 292 | update_ratio = self.optimizer.update(-delta_mu) # sign flip since the optimizer is a minimizer 293 | else: 294 | self.mu += self.alpha * delta_mu # normal SGD method? 295 | 296 | if self.sigma_alpha > 0: 297 | self.adaptive_sigma(fit_arr) 298 | 299 | def adaptive_sigma(self, fit_arr): 300 | fit_mean = (fit_arr[:int(self.pop_size / 2)] + fit_arr[int(self.pop_size / 2):-1]) / 2.0 301 | fit_std = 1.0 if self.rank_fitness else fit_arr.std() 302 | 303 | fit_baseline = np.mean(fit_arr) if self.avg_fit_baseline else fit_arr[-1] # current mu's fitness 304 | fit_S = fit_mean - fit_baseline 305 | S = ((self.epsilon_half * self.epsilon_half - (self.sigma * self.sigma)[None, :]) / self.sigma[None, :]) 306 | delta_sigma = (np.dot(fit_S, S)) / ((self.pop_size - 1) * fit_std) # adaptive sigma calculation 307 | 308 | d_sigma = self.sigma_alpha * delta_sigma 309 | d_sigma = np.minimum(d_sigma, self.sigma_max_change * self.sigma) # clip d_sigma max 310 | d_sigma = np.maximum(d_sigma, - self.sigma_max_change * self.sigma) # clip d_sigma min 311 | self.sigma += d_sigma 312 | 313 | def decay_variables(self): 314 | if self.sigma_decay < 1: 315 | self.sigma[self.sigma > self.sigma_min] *= self.sigma_decay 316 | 317 | if self.alpha_decay < 1 and self.alpha > self.alpha_min: 318 | self.alpha *= self.alpha_decay 319 | -------------------------------------------------------------------------------- /evoalgo/evolutionary_algorithms/genetic_algorithm.py: -------------------------------------------------------------------------------- 1 | """ 2 | Genetic Algorithms 3 | """ 4 | 5 | import numpy as np 6 | 7 | from evoalgo.const import KEY_PARAMS_VEC, KEY_FITNESS 8 | from evoalgo.utils.genetic_operators import Selection 9 | 10 | 11 | class SimpleGA: 12 | """ 13 | Simple Genetic Algorithm. 14 | """ 15 | 16 | def __init__(self, params_num, pop_size, discrete_values_num=None, enable_elitism_selection=True, 17 | selection_var=None, mutation_var=None, **kwargs): 18 | """ 19 | :param params_num: number of model parameters 20 | :param pop_size: population size 21 | :param selection_var: top_size / tournament_size / truncation_size / elite_size 22 | :param mutation_var: mut_rate / sigma = sigma_init: initial STD 23 | :param kwargs: sigma_decay: anneal STD. sigma_decay=1 --> don't anneal the STD 24 | """ 25 | self.params_num = params_num 26 | self.pop_size = pop_size 27 | self.enable_elitism_selection = enable_elitism_selection 28 | self.selection_var = selection_var 29 | self.mutation_var = mutation_var 30 | self.set_mutation_var(**kwargs) 31 | 32 | self.discrete_values_num = discrete_values_num 33 | 34 | self.population = None 35 | self.top_individual = None 36 | self.pop_avg_fit_history = [] 37 | self.pop_max_fit_history = [] 38 | 39 | def set_mutation_var(self, **kwargs): 40 | if kwargs.get('sigma_min'): 41 | self.sigma_min = kwargs.get('sigma_min') 42 | if kwargs.get('sigma_decay'): 43 | self.sigma_decay = kwargs.get('sigma_decay') 44 | 45 | def evolve(self, i, fitness_f, selection_f, crossover_f, mutation_f): 46 | self.init_pop() if i == 0 else self.update_pop(selection_f, crossover_f, mutation_f) 47 | self.eval_pop(fitness_f) 48 | 49 | if i != 0 and hasattr(self, 'sigma_decay') and hasattr(self, 'sigma_min'): 50 | if self.mutation_var > self.sigma_min: # stop annealing if less than sigma_min 51 | self.mutation_var *= self.sigma_decay # decay sigma. 52 | 53 | def init_pop(self): 54 | """ 55 | initialize population (create the first generation). 56 | creates a population of agents (individuals), 57 | each is a dict that stores its NN parameters vector & fitness score 58 | """ 59 | self.population = [] # solution models 60 | for i in range(self.pop_size): 61 | 62 | if self.discrete_values_num is not None: 63 | param = np.random.randint(self.discrete_values_num, size=self.params_num) 64 | else: 65 | # sample a random number from a standard normal distribution (mean 0, variance 1) 66 | param = np.random.randn(self.params_num) 67 | # the division gives a number which is close to 0 -> good for the NN weights. 68 | param /= 2.0 # TODO: test with 2.0, 10.0, and without 69 | 70 | self.population.append({KEY_PARAMS_VEC: param, KEY_FITNESS: None}) 71 | 72 | def update_pop(self, selection_f, crossover_f, mutation_f): # was called: next_gen 73 | """ 74 | construct new population (create the next generation). 75 | :param selection_f: selection function 76 | :param crossover_f: crossover function 77 | :param mutation_f: mutation function 78 | """ 79 | if self.enable_elitism_selection or \ 80 | selection_f.__name__ == Selection.stochastic_universal_sampling.__name__ or \ 81 | selection_f.__name__ == Selection.stochastic_top_sampling.__name__: 82 | # sort population by descending fitness-score: 83 | self.population.sort(key=lambda individual: individual[KEY_FITNESS], reverse=True) 84 | 85 | new_pop = [] 86 | 87 | if self.enable_elitism_selection: 88 | new_pop.extend(Selection.elitism(self.population)) 89 | 90 | while len(new_pop) < self.pop_size: 91 | p1_params, p2_params = \ 92 | selection_f(self.population) if self.selection_var is None else \ 93 | selection_f(self.population, self.selection_var) 94 | offspring = crossover_f(p1_params, p2_params, self.params_num) 95 | for o in offspring: 96 | mut_offspring = \ 97 | mutation_f(o, self.params_num, self.discrete_values_num) if self.mutation_var is None else \ 98 | mutation_f(o, self.params_num, self.discrete_values_num, self.mutation_var) 99 | new_pop.append({KEY_PARAMS_VEC: mut_offspring, KEY_FITNESS: None}) 100 | 101 | self.population = new_pop 102 | 103 | def eval_pop(self, fitness_function): 104 | """ 105 | evaluates all individuals and updates their fitness score 106 | :return: update population (with fitness scores), average population fitness 107 | """ 108 | fit_sum = 0 109 | 110 | top_individual = None 111 | 112 | for individual in self.population: 113 | fit = fitness_function(individual[KEY_PARAMS_VEC]) 114 | individual[KEY_FITNESS] = fit 115 | 116 | fit_sum += fit 117 | 118 | if top_individual is None or fit > top_individual[KEY_FITNESS]: 119 | top_individual = { 120 | KEY_PARAMS_VEC: np.copy(individual[KEY_PARAMS_VEC]), 121 | KEY_FITNESS: individual[KEY_FITNESS] 122 | } 123 | 124 | avg_fit = fit_sum / self.pop_size 125 | self.pop_avg_fit_history.append(avg_fit) 126 | 127 | self.pop_max_fit_history.append(top_individual[KEY_FITNESS]) 128 | 129 | if self.top_individual is None or top_individual[KEY_FITNESS] > self.top_individual[KEY_FITNESS]: 130 | self.top_individual = { 131 | KEY_PARAMS_VEC: np.copy(top_individual[KEY_PARAMS_VEC]), 132 | KEY_FITNESS: top_individual[KEY_FITNESS] 133 | } 134 | -------------------------------------------------------------------------------- /evoalgo/optimization_problems/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/evoalgo/optimization_problems/__init__.py -------------------------------------------------------------------------------- /evoalgo/optimization_problems/policy_nn.py: -------------------------------------------------------------------------------- 1 | """ 2 | Optimizing: Policy Neural-Network's parameters (layers' weights and biases), for autonomous agent control. 3 | Policy Neural Network I/O: environment state --> probability distribution over actions. 4 | 5 | Any AI Gym environment can be chosen, as long as the relevant variables parameters 6 | (`env_name`, `input_dims`, `n_actions`, `optimal_fit`) are changed accordingly. 7 | Here, AI Gym's CartPole environment is chosen as an example. 8 | """ 9 | 10 | import numpy as np 11 | import torch 12 | import torch.distributions 13 | import torch.nn.functional 14 | import gym 15 | 16 | max_gen_num = 25 17 | pop_size = 500 18 | 19 | env_name = 'CartPole-v0' 20 | input_dims = 4 # input layer 21 | n_actions = 2 # output layer 22 | optimal_fit = 200 # game automatically terminates after 200 time-steps 23 | 24 | hidden_layers_units = [25, 10] # The individual NN hidden layers 25 | 26 | ######################################### 27 | 28 | # NN-model specific 29 | 30 | layers_weights_shapes = [] # the parameter vector's layers (each layer's weight matrix dimension) 31 | params_num = 0 # the total number parameters 32 | for i in range(len(hidden_layers_units) + 1): 33 | v_curr = n_actions if i == len(hidden_layers_units) else hidden_layers_units[i] 34 | v_prev = input_dims if i == 0 else hidden_layers_units[i - 1] 35 | layers_weights_shapes.append((v_curr, v_prev)) 36 | params_num += (v_curr * (v_prev + 1)) 37 | task_name = 'CartPole' + str(params_num) + 'D' 38 | 39 | 40 | def split_model_params_vec(params_vec): 41 | """ 42 | :param params_vec: NN parameters vector 43 | :return: params_by_layer: a list of layer-wise tuples: (layer's weight matrix, layer's bias vector) 44 | """ 45 | params_by_layer = [] 46 | end_pt = 0 47 | for i, layer in enumerate(layers_weights_shapes): 48 | start_pt, end_pt = end_pt, end_pt + np.prod(layer) 49 | weights = params_vec[start_pt:end_pt].view(layer) 50 | start_pt, end_pt = end_pt, end_pt + layer[0] 51 | bias = params_vec[start_pt:end_pt] 52 | params_by_layer.append((weights, bias)) 53 | return params_by_layer 54 | 55 | 56 | def construct_model_and_pass_state(s, params_by_layer): 57 | """ 58 | constructs the NN model, and passes the input state into it 59 | :param s: input state 60 | :param params_by_layer: a list of layer-wise tuples: (layer's weight matrix, layer's bias vector) 61 | :return: probabilities over actions 62 | """ 63 | for i, layer_params in enumerate(params_by_layer): 64 | w, b = layer_params 65 | x = torch.nn.functional.linear(s if i == 0 else x, w, b) # logits 66 | x = torch.relu(x) if i != len(params_by_layer) - 1 else torch.softmax(x, dim=0) # probs 67 | return x 68 | 69 | 70 | ######################################### 71 | 72 | # RL specific 73 | 74 | env = gym.make(env_name) 75 | 76 | 77 | def fitness_function(individual_params): 78 | """ 79 | assessing the individual's fitness by testing its model in the CartPole environment 80 | (until it loses the game and returns the number of time steps it lasted as its fitness score). 81 | :return: individual's fitness score 82 | """ 83 | if isinstance(individual_params, np.ndarray): 84 | individual_params = torch.as_tensor(individual_params, dtype=torch.float32) 85 | 86 | params_by_layer = split_model_params_vec(individual_params) 87 | 88 | done = False 89 | fitness_score = 0 90 | s = torch.from_numpy(env.reset()).float() 91 | while not done: 92 | probs = construct_model_and_pass_state(s, params_by_layer) 93 | a = torch.distributions.Categorical(probs=probs).sample().item() 94 | s_, r, done, info = env.step(a) 95 | s = torch.from_numpy(s_).float() 96 | fitness_score += r 97 | return fitness_score 98 | -------------------------------------------------------------------------------- /evoalgo/optimization_problems/rastrigin.py: -------------------------------------------------------------------------------- 1 | """ 2 | Optimizing: Rastrigin function's input parameters (x). 3 | """ 4 | 5 | import numpy as np 6 | 7 | max_gen_num = 1000 # number of generations (iterations) to run each solver 8 | pop_size = 100 9 | 10 | params_num = 100 # number of model parameters (expresses the problem's dimensionality) 11 | 12 | task_name = 'Rastrigin' + str(params_num) + 'D' 13 | optimal_fit = 0 # global maximum point 14 | 15 | 16 | def rastrigin_function(x): 17 | """ 18 | taken from: https://github.com/CMA-ES/pycma/blob/master/cma/fitness_functions.py 19 | """ 20 | if not np.isscalar(x[0]): 21 | N = len(x[0]) 22 | return np.array([10 * N + sum(xi ** 2 - 10 * np.cos(2 * np.pi * xi)) for xi in x]) 23 | N = len(x) 24 | return 10 * N + sum(x ** 2 - 10 * np.cos(2 * np.pi * x)) 25 | 26 | 27 | def fitness_function(individual_params): 28 | """ 29 | modifies the Rastrigin function: 30 | * -10. units shift - to move the optimum point away from origin. 31 | * sign flip - to have a global maximum (instead of a global minimum). 32 | :return: individual's fitness score 33 | """ 34 | individual_params = np.copy(individual_params) 35 | individual_params -= 10.0 # -10. units shift 36 | return -rastrigin_function(individual_params) # sign flip 37 | 38 | 39 | def test_fitness_function(): 40 | x = np.zeros(params_num) 41 | print(f"F(zeros(params_num)) = {fitness_function(x)}") 42 | x = np.ones(params_num) * 10. 43 | print(f"F(ones(params_num) * 10) = {fitness_function(x)}") 44 | 45 | 46 | if __name__ == '__main__': 47 | test_fitness_function() 48 | -------------------------------------------------------------------------------- /evoalgo/optimization_problems/string_opt.py: -------------------------------------------------------------------------------- 1 | """ 2 | Optimizing: fixed-length String (discrete-valued vector) towards a chosen target String of 12 characters. 3 | For generality purposes, the String (individual & target) is converted into a discrete-number vector (and vice-versa), 4 | which is optimized.. 5 | """ 6 | 7 | from difflib import SequenceMatcher 8 | 9 | alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ,.! " # possible char values 10 | discrete_values_num = len(alphabet) # 26+26+4=56 characters 11 | 12 | target = "FullyEvolved" # 12-character string 13 | params_num = len(target) # target string length 14 | 15 | max_gen_num = 100 16 | pop_size = 1000 17 | 18 | task_name = 'String' + str(params_num) + 'D' 19 | optimal_fit = 1 # max similarity of SequenceMatcher 20 | 21 | 22 | def convert_indices_to_string(individual_params): 23 | individual_string = '' 24 | for i in individual_params: 25 | individual_string += alphabet[i] 26 | return individual_string 27 | 28 | 29 | def fitness_function(individual_params): 30 | """ 31 | computes the strings' similarity 32 | :param individual_params: discrete-numbers vector 33 | :return: individual's fitness score 34 | """ 35 | individual_string = convert_indices_to_string(individual_params) 36 | return SequenceMatcher(None, individual_string, target).ratio() 37 | -------------------------------------------------------------------------------- /evoalgo/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/evoalgo/utils/__init__.py -------------------------------------------------------------------------------- /evoalgo/utils/evolution_process.py: -------------------------------------------------------------------------------- 1 | from evoalgo.const import KEY_FITNESS 2 | from evoalgo.utils.utils import plot_fit_history 3 | 4 | 5 | class Evolution: 6 | 7 | @staticmethod 8 | def test_solver(solver, max_gen_num, task_name, fitness_f, 9 | selection_f=None, crossover_f=None, mutation_f=None, 10 | print_progress=True, plot=True, show=False, save=True): 11 | """ 12 | uses solver (evolutionary algorithm) to solve fit_func. 13 | :param solver: an optimization method 14 | """ 15 | for i in range(max_gen_num): 16 | 17 | if selection_f is None and crossover_f is None and mutation_f is None: 18 | solver.evolve(fitness_f) 19 | else: 20 | solver.evolve(i, fitness_f, selection_f, crossover_f, mutation_f) 21 | 22 | if print_progress and ((i + 1) % 100 == 0 or i + 1 == max_gen_num): 23 | print(f"top fitness score at iteration {(i + 1)} : {solver.top_individual[KEY_FITNESS]}") 24 | 25 | # TODO: if implemented, change max_gen_num to i + 1 when plotting 26 | # if solver.top_individual[KEY_FITNESS] > REQUIRED_FITNESS_VALUE: # stop the algorithm when target is reached. 27 | # print(f"Required fitness ({str(REQUIRED_FITNESS_VALUE)}) reached at Gen {str(i + 1)}") 28 | # break 29 | 30 | if plot: 31 | plot_fit_history(solver.pop_max_fit_history, 'Max', max_gen_num, task_name, show, save) 32 | plot_fit_history(solver.pop_avg_fit_history, 'Average', max_gen_num, task_name, show, save) 33 | -------------------------------------------------------------------------------- /evoalgo/utils/fitness_shaping.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def compute_ranks(x): 5 | """ 6 | Returns int ranks in range [0, len(x) - 1]. 7 | Note: scipy.stats.rankdata returns ranks in range [1, len(x)]. 8 | taken from: https://github.com/openai/evolution-strategies-starter/blob/master/es_distributed/es.py 9 | """ 10 | assert x.ndim == 1 11 | ranks = np.empty(len(x), dtype=int) 12 | ranks[x.argsort()] = np.arange(len(x)) 13 | return ranks 14 | 15 | 16 | def compute_zero_centered_ranks(x): 17 | """ 18 | Returns float ranks in range [-0.5, 0.5]. 19 | taken from: https://github.com/openai/evolution-strategies-starter/blob/master/es_distributed/es.py 20 | """ 21 | y = compute_ranks(x.ravel()).reshape(x.shape).astype(np.float64) 22 | y /= (x.size - 1) # Normalization - scaling to [0, 1]. 23 | y -= .5 # shifting to [-0.5, 0.5]. 24 | return y -------------------------------------------------------------------------------- /evoalgo/utils/genetic_operators.py: -------------------------------------------------------------------------------- 1 | """ 2 | Genetic Operators. 3 | """ 4 | 5 | import random 6 | import numpy as np 7 | 8 | from evoalgo.const import KEY_PARAMS_VEC, KEY_FITNESS 9 | 10 | 11 | class Selection: 12 | """ 13 | p - parent 14 | o - offspring 15 | """ 16 | 17 | @staticmethod 18 | def fitness_proportionate(population): 19 | """ 20 | Fitness-Proportionate Selection (FPS) of parents. Stochastic selection method. 21 | """ 22 | # np approach - works only with positive weights, problematic with negative weights. 23 | # p = np.array([individual[KEY_FITNESS] for individual in population]) 24 | # p /= p.sum() # normalize. problem with negative weights -> sum = 0 25 | # parents = np.random.choice(population, size=2, p=p) 26 | 27 | # a single pointer, on a wheel that is spun 2 times: 28 | parents = random.choices(population, k=2, weights=[individual[KEY_FITNESS] for individual in population]) 29 | 30 | p1_params = parents[0][KEY_PARAMS_VEC] 31 | p2_params = parents[1][KEY_PARAMS_VEC] 32 | return p1_params, p2_params 33 | 34 | @staticmethod 35 | def stochastic_top_sampling(population_sort, top_ratio=0.1): 36 | """ 37 | Stochastic Top Sampling (STS) Selection of parents. Stochastic selection method. 38 | selecting from the top T (% of the) individuals. 39 | :param population_sort: sorted population by descending fitness-score. 40 | :param top_ratio: default: 0.1 -> top 10% of the population. 41 | """ 42 | pop_size = len(population_sort) 43 | rand_elite_pair_indices = np.random.default_rng().choice(int(top_ratio * pop_size), size=2, replace=False) 44 | 45 | p1_params = population_sort[rand_elite_pair_indices[0]][KEY_PARAMS_VEC] 46 | p2_params = population_sort[rand_elite_pair_indices[1]][KEY_PARAMS_VEC] 47 | return p1_params, p2_params 48 | 49 | @staticmethod 50 | def tournament(population, tournament_ratio=0.2): 51 | """ 52 | Tournament Selection of parents. Semi-deterministic selection method. 53 | the batch is the tournament batch (a subset of the population). 54 | :param tournament_ratio: default: 0.2 -> random 20% of the population. 55 | """ 56 | pop_size = len(population) 57 | rand_indices = np.random.default_rng().choice(pop_size, size=(int(tournament_ratio * pop_size)), replace=False) 58 | batch = np.array([[i, individual[KEY_FITNESS]] for (i, individual) in enumerate(population) 59 | if i in rand_indices]) 60 | batch_sort = batch[batch[:, 1].argsort()] # sort by ascending fitness-score 61 | 62 | p1_params = population[int(batch_sort[-1, 0])][KEY_PARAMS_VEC] # top_0_parent params 63 | p2_params = population[int(batch_sort[-2, 0])][KEY_PARAMS_VEC] # top_1_parent params 64 | return p1_params, p2_params 65 | 66 | ###################################### 67 | 68 | # stochastic_universal_sampling is not operational yet... (there's a TODO) 69 | @staticmethod 70 | def stochastic_universal_sampling(population_sort): 71 | """ 72 | Stochastic Universal Sampling (SUS) of parents. Stochastic selection method. 73 | provides no bias and minimal spread. 74 | :param population_sort: sorted population by descending fitness-score. 75 | :return: a list of parameters of N parents # TODO: to be randomly? paired (non-identical individuals), recombined and mutated. 76 | """ 77 | # adjust negative weights (crucial for the sum later) 78 | min_fit = np.array([individual[KEY_FITNESS] for individual in population_sort]).min() 79 | if min_fit < 0: 80 | for individual in population_sort: 81 | individual[KEY_FITNESS] -= min_fit 82 | 83 | total_fit = sum([individual[KEY_FITNESS] for individual in population_sort]) 84 | if total_fit == 0: # meaning: each individual's fittness is 0 85 | for individual in population_sort: 86 | individual[KEY_FITNESS] += 1 87 | 88 | wheel = [] 89 | fit_limit = 0.0 90 | for i, individual in enumerate(population_sort): 91 | fit_limit += individual[KEY_FITNESS] 92 | wheel.append((fit_limit, i)) 93 | 94 | # N equally-spaced pointers, on a wheel that is spun once (single sampling): 95 | pop_size = len(population_sort) 96 | pointer_size = 0 97 | while pointer_size == 0: 98 | pointer_size = random.uniform(0, total_fit/pop_size) 99 | 100 | individuals_indices = [] 101 | curr_i = 0 102 | current_pointer = pointer_size 103 | for fit_limit, i in wheel: 104 | while current_pointer < fit_limit and curr_i < pop_size: 105 | individuals_indices.append(i) 106 | curr_i += 1 107 | current_pointer = (curr_i + 1) * pointer_size 108 | 109 | p_params_list = [population_sort[i][KEY_PARAMS_VEC] for i in individuals_indices] 110 | 111 | return p_params_list 112 | 113 | # truncation is not operational yet... (there's a TODO) 114 | @staticmethod 115 | def truncation(population_sort, truncation_ratio=0.1): 116 | """ 117 | Truncation Selection of parents. Deterministic selection method. 118 | selecting the top T (% of the) individuals. 119 | :param population_sort: sorted population by descending fitness-score. 120 | :param truncation_ratio: default: 0.1 -> top 10% of the population. 121 | :return: a list of parameters of (N * truncation_ratio) parents # TODO: to be randomly? paired (non-identical individuals), recombined and mutated. 122 | """ 123 | pop_size = len(population_sort) 124 | p_params_list = [individual[KEY_PARAMS_VEC] for individual in population_sort[:int(truncation_ratio * pop_size)]] 125 | 126 | return p_params_list 127 | 128 | @staticmethod 129 | def elitism(population_sort, elite_ratio=0.1): 130 | """ 131 | Elitism Selection of individuals. Deterministic selection method. 132 | :param population_sort: sorted population by descending fitness-score. 133 | :param elite_ratio: default: 0.1 -> top 10% of the population. 134 | :return: a list of individuals, to be directly copied to the next generation's (new) population. 135 | """ 136 | pop_size = len(population_sort) 137 | individuals_list = [] 138 | for individual in population_sort[:int(elite_ratio * pop_size)]: 139 | individuals_list.append({KEY_PARAMS_VEC: individual[KEY_PARAMS_VEC], KEY_FITNESS: None}) 140 | return individuals_list 141 | 142 | 143 | class Crossover: 144 | """ 145 | Parents crossover (AKA genetic recombination) 146 | 2 parents (p) --> 2 offspring (o) 147 | """ 148 | 149 | @staticmethod 150 | def single_pt(p1, p2, params_num): 151 | """ 152 | Positional method. 153 | :param p1: p1_params 154 | :param p2: p2_params 155 | :param params_num: vector's length 156 | :return: o1_params, o2_params 157 | """ 158 | cross_pt = np.random.randint(low=1, high=params_num) 159 | 160 | o1 = np.copy(p1) 161 | o1[cross_pt:] = p2[cross_pt:] 162 | 163 | o2 = np.copy(p2) 164 | o2[cross_pt:] = p1[cross_pt:] 165 | 166 | return o1, o2 167 | 168 | @staticmethod 169 | def two_pt(p1, p2, params_num): 170 | """ 171 | Positional method. 172 | :param p1: p1_params 173 | :param p2: p2_params 174 | :param params_num: vector's length 175 | :return: o1_params, o2_params 176 | """ 177 | cross_pts = np.random.default_rng().choice(params_num - 1, size=2, replace=False) + 1 178 | cross_pt_low, cross_pt_high = min(cross_pts), max(cross_pts) 179 | 180 | o1 = np.copy(p1) 181 | o1[cross_pt_low:cross_pt_high] = p2[cross_pt_low:cross_pt_high] 182 | 183 | o2 = np.copy(p2) 184 | o2[cross_pt_low:cross_pt_high] = p1[cross_pt_low:cross_pt_high] 185 | 186 | return o1, o2 187 | 188 | @staticmethod 189 | def uniform(p1, p2, params_num): 190 | """ 191 | Non-positional method. 192 | :param p1: p1_params 193 | :param p2: p2_params 194 | :param params_num: vector's length 195 | :return: o1_params, o2_params 196 | """ 197 | indices = np.where(np.random.rand(params_num) > 0.5) 198 | 199 | o1 = np.copy(p1) 200 | o1[indices] = p2[indices] 201 | 202 | o2 = np.copy(p2) 203 | o2[indices] = p1[indices] 204 | 205 | return o1, o2 206 | 207 | 208 | class Mutation: 209 | """ 210 | Mutating an individual's parameter vector 211 | """ 212 | 213 | @staticmethod 214 | def deterministic(individual, params_num, discrete_values_num, mut_rate=1e-2): 215 | """ 216 | mutation by randomly changing a proportional number elements of the parameter vector. 217 | recommended for a high number of params (long vectors - NN params). 218 | :param individual: individual_params 219 | :param params_num: vector's length 220 | :param mut_rate: the % of elements to mutate in each individual. determines the number of changed elements. 221 | :return: individual's mutated params 222 | """ 223 | mut_num = int(mut_rate * params_num) # number of elements to mutate 224 | 225 | if mut_num > 0: 226 | mut_indices = np.random.default_rng().choice(params_num, size=mut_num, replace=False) 227 | if discrete_values_num is not None: 228 | # randomly flipping values (replacing them with random values) 229 | individual[mut_indices] = np.random.default_rng().choice( 230 | discrete_values_num, size=mut_num, replace=False) 231 | else: 232 | # sample a random number from a standard normal distribution (mean 0, variance 1) 233 | # the division gives a number which is close to 0 -> good for the NN weights. 234 | individual[mut_indices] = np.random.randn(mut_num) / 10.0 # TODO: test with 2.0, 10.0, and without 235 | 236 | return individual 237 | 238 | @staticmethod 239 | def stochastic_uniform(individual, params_num, discrete_values_num, mut_rate=1e-2): 240 | """ 241 | mutation by randomly changing a random number elements of the parameter vector. 242 | recommended for a small number of params (short vectors). 243 | :param individual: individual_params 244 | :param params_num: vector's length 245 | :param mut_rate: the chance of changing each single element. 246 | :return: individual's mutated params 247 | """ 248 | for i in range(params_num): 249 | if random.random() < mut_rate: 250 | if discrete_values_num is not None: 251 | # randomly flipping values (replacing them with random values) 252 | individual[i] = np.random.randint(discrete_values_num) 253 | else: 254 | # sample a random number from a standard normal distribution (mean 0, variance 1) 255 | # the division gives a number which is close to 0 -> good for the NN weights. 256 | individual[i] = np.random.randn() / 10.0 # TODO: test with 2.0, 10.0, and without 257 | 258 | return individual 259 | 260 | @staticmethod 261 | def gaussian_noise(individual, params_num, discrete_values_num, sigma=0.5): 262 | """ 263 | Mutation by adding a random noise vector (epsilon), 264 | which is sampled from a "standard normal" distribution. 265 | recommended for when values distance (from each other) is meaningful. 266 | :param individual: individual_params 267 | :param params_num: vector's length 268 | :param sigma: 269 | :return: individual's mutated params 270 | """ 271 | # sample a random number from a standard normal distribution (mean 0, variance 1) 272 | epsilon = np.random.randn(params_num) * sigma 273 | if discrete_values_num is not None: 274 | epsilon_disctere = np.zeros(params_num, dtype=np.int32) 275 | for i in range(params_num): 276 | epsilon_disctere[i] = int(round(epsilon[i])) 277 | epsilon = epsilon_disctere 278 | 279 | individual += epsilon 280 | 281 | if discrete_values_num is not None: 282 | for i in range(params_num): 283 | if individual[i] < 0: 284 | individual[i] = 0 285 | elif individual[i] > discrete_values_num - 1: 286 | individual[i] = discrete_values_num - 1 287 | 288 | return individual 289 | 290 | @staticmethod 291 | def sample_epsilon(pop_size, params_num, antithetic_sampling=False): 292 | """ 293 | sampling epsilon (gaussian-noise vector), which constitutes the mutation. 294 | :param params_num: vector's length 295 | :param antithetic_sampling: sample noise vector only for half of the population, 296 | the other half gets the opposite-sign (-) of the noise. 297 | antithetic sampling reduces the variance in the gradient estimate. 298 | :return: epsilon (gaussian-noise vector) 299 | """ 300 | if antithetic_sampling: 301 | assert (pop_size % 2 == 0), "Antithetic sampling requires even population size" 302 | epsilon_half = np.random.randn(int(pop_size / 2), params_num) 303 | epsilon = np.concatenate([epsilon_half, -epsilon_half]) 304 | else: 305 | epsilon = np.random.randn(pop_size, params_num) 306 | 307 | return epsilon 308 | -------------------------------------------------------------------------------- /evoalgo/utils/optimizers.py: -------------------------------------------------------------------------------- 1 | """ 2 | taken from: https://github.com/openai/evolution-strategies-starter/blob/master/es_distributed/optimizers.py 3 | """ 4 | 5 | import numpy as np 6 | 7 | 8 | class Optimizer(object): 9 | def __init__(self, pi, epsilon=1e-08): 10 | self.pi = pi 11 | self.dim = pi.params_num 12 | self.epsilon = epsilon 13 | self.t = 0 14 | 15 | def update(self, global_g): 16 | self.t += 1 17 | step = self._compute_step(global_g) 18 | theta = self.pi.mu 19 | ratio = np.linalg.norm(step) / (np.linalg.norm(theta) + self.epsilon) 20 | self.pi.mu = theta + step 21 | return ratio 22 | 23 | def _compute_step(self, globalg): 24 | raise NotImplementedError 25 | 26 | 27 | class BasicSGD(Optimizer): 28 | def __init__(self, pi, step_size): 29 | Optimizer.__init__(self, pi) 30 | self.step_size = step_size 31 | 32 | def _compute_step(self, global_g): 33 | step = -self.step_size * global_g 34 | return step 35 | 36 | 37 | class SGD(Optimizer): 38 | def __init__(self, pi, step_size, momentum=0.9): 39 | Optimizer.__init__(self, pi) 40 | self.v = np.zeros(self.dim, dtype=np.float32) 41 | self.step_size, self.momentum = step_size, momentum 42 | 43 | def _compute_step(self, global_g): 44 | self.v = self.momentum * self.v + (1. - self.momentum) * global_g 45 | step = -self.step_size * self.v 46 | return step 47 | 48 | 49 | class Adam(Optimizer): 50 | def __init__(self, pi, step_size, beta1=0.99, beta2=0.999): 51 | Optimizer.__init__(self, pi) 52 | self.step_size = step_size 53 | self.beta1 = beta1 54 | self.beta2 = beta2 55 | self.m = np.zeros(self.dim, dtype=np.float32) 56 | self.v = np.zeros(self.dim, dtype=np.float32) 57 | 58 | def _compute_step(self, global_g): 59 | a = self.step_size * np.sqrt(1 - self.beta2 ** self.t) / (1 - self.beta1 ** self.t) 60 | self.m = self.beta1 * self.m + (1 - self.beta1) * global_g 61 | self.v = self.beta2 * self.v + (1 - self.beta2) * (global_g * global_g) 62 | step = -a * self.m / (np.sqrt(self.v) + self.epsilon) 63 | return step 64 | -------------------------------------------------------------------------------- /evoalgo/utils/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | from matplotlib import pyplot as plt 3 | 4 | from evoalgo.const import KEY_PARAMS_VEC, KEY_FITNESS 5 | 6 | 7 | # Plotter: 8 | 9 | colors_28 = ['#f00082', '#f000bf', '#ff75e3', # pink 10 | '#FF0000', '#fa3c3c', '#fa7a7a', '#ff2b0a', # red 11 | '#FF7F00', '#ff6400', '#ffa538', # orange 12 | '#FFFF00', '#e6dc32', '#fff76b', # yellow 13 | '#00FF00', '#a0e632', '#00dc00', '#17A858', '#00d28c', # green 14 | '#0000FF', '#00c8c8', '#0DB0DD', '#00a0ff', '#1e3cff', # blue 15 | '#4B0082', '#a000c8', '#6e00dc', '#8B00FF', '#9400D3'] # purple 16 | 17 | 18 | def plot_fit_history(fit_history, fit_history_type, max_gen_num, task_name, show=False, save=True): 19 | plt.title(f"{task_name} - Population {fit_history_type} Fitness") 20 | plt.ylabel(f"{fit_history_type} Fitness") 21 | plt.xlabel('Generation') 22 | 23 | plt.xlim(0, max_gen_num) 24 | 25 | # x = [i + 1 for i in range(curr_gen + 1)] 26 | x = [i + 1 for i in range(max_gen_num)] 27 | plt.plot(x, fit_history) 28 | 29 | if save: 30 | path_dir = 'results' 31 | if not os.path.isdir(path_dir): 32 | os.mkdir(path_dir) 33 | plt.savefig(f'{path_dir}/pop_{fit_history_type}_fit_{task_name}.png') 34 | 35 | if show: 36 | plt.show() 37 | 38 | plt.close() 39 | 40 | 41 | def plot_fit_history_comparison(fit_history_dict, fit_history_type, max_gen_num, task_name, pop_size, 42 | optimal_fit=None, algo_type='Evolutionary Algorithms', 43 | colors=None, show=False, save=True): 44 | 45 | plt.figure(figsize=(16, 8), dpi=150) 46 | 47 | plt.title(f"{task_name} - {algo_type} Comparison - Population: {str(pop_size)} - {fit_history_type} Fitness") 48 | plt.ylabel(f"{fit_history_type} Fitness") 49 | plt.xlabel('Generation') 50 | 51 | plt.xlim(0, max_gen_num) 52 | 53 | x = [i + 1 for i in range(max_gen_num)] 54 | 55 | handles = [] 56 | if optimal_fit is not None: 57 | line_optimum, = plt.plot(x, [optimal_fit] * max_gen_num, label='Global Optimum', 58 | linewidth=0.5, linestyle="-.", color="black") 59 | handles.append(line_optimum) 60 | for i, (method_name, fit_history) in enumerate(fit_history_dict.items()): 61 | if colors is not None: 62 | line, = plt.plot(x, fit_history, label=method_name, linewidth=1.0, linestyle="-", color=colors[i]) 63 | else: 64 | line, = plt.plot(x, fit_history, label=method_name, linewidth=1.0, linestyle="-") # auto colors 65 | 66 | handles.append(line) 67 | 68 | plt.legend(handles=handles, bbox_to_anchor=(1.01, 0), loc="lower left") 69 | 70 | if save: 71 | path_dir = 'results' 72 | if not os.path.isdir(path_dir): 73 | os.mkdir(path_dir) 74 | plt.savefig(f'{path_dir}/{task_name}-Pop{str(pop_size)}-{fit_history_type}.png', bbox_inches='tight') 75 | 76 | if show: 77 | plt.show() 78 | 79 | plt.close() 80 | 81 | 82 | # Printer: 83 | 84 | def print_top_individual(pop): 85 | pop.sort(key=lambda individual: individual[KEY_FITNESS], reverse=True) # descending fitness score 86 | print(f"Top individual's fitness: {str(pop[0][KEY_FITNESS])}") 87 | print(f"Top individual's params: {str(pop[0][KEY_PARAMS_VEC])}") 88 | -------------------------------------------------------------------------------- /images/EvoAlgo_bear.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/EvoAlgo_bear.jpeg -------------------------------------------------------------------------------- /images/Rastrigin_function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/Rastrigin_function.png -------------------------------------------------------------------------------- /images/results/ea_comparison/CartPole407D-Pop500-Avg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ea_comparison/CartPole407D-Pop500-Avg.png -------------------------------------------------------------------------------- /images/results/ea_comparison/CartPole407D-Pop500-Max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ea_comparison/CartPole407D-Pop500-Max.png -------------------------------------------------------------------------------- /images/results/ea_comparison/Legend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ea_comparison/Legend.png -------------------------------------------------------------------------------- /images/results/ea_comparison/Rastrigin100D-Pop100-Avg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ea_comparison/Rastrigin100D-Pop100-Avg.png -------------------------------------------------------------------------------- /images/results/ea_comparison/Rastrigin100D-Pop100-Max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ea_comparison/Rastrigin100D-Pop100-Max.png -------------------------------------------------------------------------------- /images/results/ga_comparison/Legend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ga_comparison/Legend.png -------------------------------------------------------------------------------- /images/results/ga_comparison/Rastrigin100D-Pop100-Avg-Sigma0.5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ga_comparison/Rastrigin100D-Pop100-Avg-Sigma0.5.png -------------------------------------------------------------------------------- /images/results/ga_comparison/Rastrigin100D-Pop100-Avg-Sigma0.5_Min0.01_Decay0.9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ga_comparison/Rastrigin100D-Pop100-Avg-Sigma0.5_Min0.01_Decay0.9.png -------------------------------------------------------------------------------- /images/results/ga_comparison/Rastrigin100D-Pop100-Avg-Sigma1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ga_comparison/Rastrigin100D-Pop100-Avg-Sigma1.png -------------------------------------------------------------------------------- /images/results/ga_comparison/Rastrigin100D-Pop100-Max-Sigma0.5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ga_comparison/Rastrigin100D-Pop100-Max-Sigma0.5.png -------------------------------------------------------------------------------- /images/results/ga_comparison/Rastrigin100D-Pop100-Max-Sigma0.5_Min0.01_Decay0.9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ga_comparison/Rastrigin100D-Pop100-Max-Sigma0.5_Min0.01_Decay0.9.png -------------------------------------------------------------------------------- /images/results/ga_comparison/Rastrigin100D-Pop100-Max-Sigma1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ga_comparison/Rastrigin100D-Pop100-Max-Sigma1.png -------------------------------------------------------------------------------- /images/results/ga_comparison/String12D-Pop100-Avg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ga_comparison/String12D-Pop100-Avg.png -------------------------------------------------------------------------------- /images/results/ga_comparison/String12D-Pop100-Max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ga_comparison/String12D-Pop100-Max.png -------------------------------------------------------------------------------- /images/results/ga_comparison/String12D-Pop1000-Avg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ga_comparison/String12D-Pop1000-Avg.png -------------------------------------------------------------------------------- /images/results/ga_comparison/String12D-Pop1000-Max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/images/results/ga_comparison/String12D-Pop1000-Max.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy~=2.0.2 2 | matplotlib~=3.9.2 3 | gym~=0.26.2 4 | torch~=2.4.1 5 | cma~=4.0.0 -------------------------------------------------------------------------------- /results/CartPole407D-Pop500-Avg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/CartPole407D-Pop500-Avg.png -------------------------------------------------------------------------------- /results/CartPole407D-Pop500-Max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/CartPole407D-Pop500-Max.png -------------------------------------------------------------------------------- /results/Rastrigin100D-Pop100-Avg-Sigma0.5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/Rastrigin100D-Pop100-Avg-Sigma0.5.png -------------------------------------------------------------------------------- /results/Rastrigin100D-Pop100-Avg-Sigma0.5_Min0.01_Decay0.9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/Rastrigin100D-Pop100-Avg-Sigma0.5_Min0.01_Decay0.9.png -------------------------------------------------------------------------------- /results/Rastrigin100D-Pop100-Avg-Sigma1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/Rastrigin100D-Pop100-Avg-Sigma1.png -------------------------------------------------------------------------------- /results/Rastrigin100D-Pop100-Avg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/Rastrigin100D-Pop100-Avg.png -------------------------------------------------------------------------------- /results/Rastrigin100D-Pop100-Max-Sigma0.5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/Rastrigin100D-Pop100-Max-Sigma0.5.png -------------------------------------------------------------------------------- /results/Rastrigin100D-Pop100-Max-Sigma0.5_Min0.01_Decay0.9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/Rastrigin100D-Pop100-Max-Sigma0.5_Min0.01_Decay0.9.png -------------------------------------------------------------------------------- /results/Rastrigin100D-Pop100-Max-Sigma1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/Rastrigin100D-Pop100-Max-Sigma1.png -------------------------------------------------------------------------------- /results/Rastrigin100D-Pop100-Max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/Rastrigin100D-Pop100-Max.png -------------------------------------------------------------------------------- /results/String12D-Pop100-Avg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/String12D-Pop100-Avg.png -------------------------------------------------------------------------------- /results/String12D-Pop100-Max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/String12D-Pop100-Max.png -------------------------------------------------------------------------------- /results/String12D-Pop1000-Avg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/String12D-Pop1000-Avg.png -------------------------------------------------------------------------------- /results/String12D-Pop1000-Max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/results/String12D-Pop1000-Max.png -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/tests/__init__.py -------------------------------------------------------------------------------- /tests/results/CartPole407D-Pop10-Avg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/tests/results/CartPole407D-Pop10-Avg.png -------------------------------------------------------------------------------- /tests/results/CartPole407D-Pop10-Max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/tests/results/CartPole407D-Pop10-Max.png -------------------------------------------------------------------------------- /tests/results/CartPole407D-Pop20-Avg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/tests/results/CartPole407D-Pop20-Avg.png -------------------------------------------------------------------------------- /tests/results/CartPole407D-Pop20-Max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/tests/results/CartPole407D-Pop20-Max.png -------------------------------------------------------------------------------- /tests/results/Rastrigin100D-Pop10-Avg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/tests/results/Rastrigin100D-Pop10-Avg.png -------------------------------------------------------------------------------- /tests/results/Rastrigin100D-Pop10-Max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/tests/results/Rastrigin100D-Pop10-Max.png -------------------------------------------------------------------------------- /tests/results/Rastrigin100D-Pop20-Avg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/tests/results/Rastrigin100D-Pop20-Avg.png -------------------------------------------------------------------------------- /tests/results/Rastrigin100D-Pop20-Max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/tests/results/Rastrigin100D-Pop20-Max.png -------------------------------------------------------------------------------- /tests/results/String12D-Pop20-Avg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/tests/results/String12D-Pop20-Avg.png -------------------------------------------------------------------------------- /tests/results/String12D-Pop20-Max.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliorBenYosef/evolutionary-algorithms/c69606bb51be03fd56c8116fb2a226ca090eecf3/tests/results/String12D-Pop20-Max.png -------------------------------------------------------------------------------- /tests/test_es.py: -------------------------------------------------------------------------------- 1 | import evoalgo.evolutionary_algorithms.evolution_strategy as ES 2 | from evoalgo.utils.evolution_process import Evolution 3 | from evoalgo.utils.utils import plot_fit_history_comparison 4 | 5 | from evoalgo.optimization_problems.rastrigin import params_num as params_num_rst, task_name as task_name_rst, \ 6 | fitness_function as fitness_function_rst, optimal_fit as optimal_fit_rst 7 | from evoalgo.optimization_problems.policy_nn import params_num as params_num_nn, task_name as task_name_nn, \ 8 | fitness_function as fitness_function_nn, optimal_fit as optimal_fit_nn 9 | 10 | 11 | def run_oes(max_gen_num, pop_size, params_num, task_name, fitness_function, 12 | antithetic_sampling=False, rank_fitness=False): 13 | open_ai_es = ES.OpenAI_ES(params_num, pop_size, sigma_init=0.5, sigma_decay=1.0, # sigma_decay=0.999 14 | alpha_init=0.1, alpha_decay=1.0, 15 | antithetic_sampling=antithetic_sampling, rank_fitness=rank_fitness) 16 | Evolution.test_solver(open_ai_es, max_gen_num, task_name, fitness_function, 17 | print_progress=True, plot=True, save=False) 18 | return open_ai_es 19 | 20 | 21 | def run_pepg(max_gen_num, pop_size, params_num, task_name, fitness_function, 22 | avg_fit_baseline=False, rank_fitness=False): 23 | pepg = ES.PEPG(params_num, pop_size, sigma_init=0.5, sigma_decay=1.0, # sigma_decay=0.999 24 | alpha_init=0.1, alpha_decay=1.0, 25 | avg_fit_baseline=avg_fit_baseline, rank_fitness=rank_fitness) 26 | Evolution.test_solver(pepg, max_gen_num, task_name, fitness_function, 27 | print_progress=True, plot=True, save=False) 28 | return pepg 29 | 30 | 31 | def run_evolution_strategy_algorithms(max_gen_num, pop_size, params_num, task_name, 32 | fitness_function, optimal_fit, discrete_values_num=None): 33 | 34 | if discrete_values_num is not None: 35 | print('Evolution Strategy Algorithms are implemented only for real-number vector optimization') 36 | return 37 | 38 | max_fit_history_dict = {} 39 | avg_fit_history_dict = {} 40 | 41 | ############################## 42 | 43 | # CMA-ES 44 | cma_es = ES.CMA_ES(params_num, pop_size, sigma_init=0.5) 45 | Evolution.test_solver(cma_es, max_gen_num, task_name, fitness_function, 46 | print_progress=True, plot=True, save=False) 47 | max_fit_history_dict['CMA-ES'] = cma_es.pop_max_fit_history 48 | avg_fit_history_dict['CMA-ES'] = cma_es.pop_avg_fit_history 49 | 50 | ############################## 51 | 52 | # OpenAI-ES 53 | open_ai_es = run_oes(max_gen_num, pop_size, params_num, task_name, fitness_function, 54 | antithetic_sampling=False, rank_fitness=False) 55 | max_fit_history_dict['OpenAI-ES'] = open_ai_es.pop_max_fit_history 56 | avg_fit_history_dict['OpenAI-ES'] = open_ai_es.pop_avg_fit_history 57 | 58 | open_ai_es_r = run_oes(max_gen_num, pop_size, params_num, task_name, fitness_function, 59 | antithetic_sampling=False, rank_fitness=True) 60 | max_fit_history_dict['OpenAI-ES rank'] = open_ai_es_r.pop_max_fit_history 61 | avg_fit_history_dict['OpenAI-ES rank'] = open_ai_es_r.pop_avg_fit_history 62 | 63 | open_ai_es_a = run_oes(max_gen_num, pop_size, params_num, task_name, fitness_function, 64 | antithetic_sampling=True, rank_fitness=False) 65 | max_fit_history_dict['OpenAI-ES antithetic'] = open_ai_es_a.pop_max_fit_history 66 | avg_fit_history_dict['OpenAI-ES antithetic'] = open_ai_es_a.pop_avg_fit_history 67 | 68 | open_ai_es_r_a = run_oes(max_gen_num, pop_size, params_num, task_name, fitness_function, 69 | antithetic_sampling=True, rank_fitness=True) 70 | max_fit_history_dict['OpenAI-ES rank antithetic'] = open_ai_es_r_a.pop_max_fit_history 71 | avg_fit_history_dict['OpenAI-ES rank antithetic'] = open_ai_es_r_a.pop_avg_fit_history 72 | 73 | ############################## 74 | 75 | # PEPG 76 | pepg = run_pepg(max_gen_num, pop_size, params_num, task_name, fitness_function, 77 | avg_fit_baseline=False, rank_fitness=False) 78 | max_fit_history_dict['PEPG'] = pepg.pop_max_fit_history 79 | avg_fit_history_dict['PEPG'] = pepg.pop_avg_fit_history 80 | 81 | pepg_r = run_pepg(max_gen_num, pop_size, params_num, task_name, fitness_function, 82 | avg_fit_baseline=False, rank_fitness=True) 83 | max_fit_history_dict['PEPG rank'] = pepg_r.pop_max_fit_history 84 | avg_fit_history_dict['PEPG rank'] = pepg_r.pop_avg_fit_history 85 | 86 | pepg_a = run_pepg(max_gen_num, pop_size, params_num, task_name, fitness_function, 87 | avg_fit_baseline=True, rank_fitness=False) 88 | max_fit_history_dict['PEPG avg_base'] = pepg_a.pop_max_fit_history 89 | avg_fit_history_dict['PEPG avg_base'] = pepg_a.pop_avg_fit_history 90 | 91 | pepg_r_a = run_pepg(max_gen_num, pop_size, params_num, task_name, fitness_function, 92 | avg_fit_baseline=True, rank_fitness=True) 93 | max_fit_history_dict['PEPG rank avg_base'] = pepg_r_a.pop_max_fit_history 94 | avg_fit_history_dict['PEPG rank avg_base'] = pepg_r_a.pop_avg_fit_history 95 | 96 | ############################## 97 | 98 | plot_fit_history_comparison(max_fit_history_dict, 'Max', max_gen_num, task_name, pop_size, optimal_fit) 99 | plot_fit_history_comparison(avg_fit_history_dict, 'Avg', max_gen_num, task_name, pop_size, optimal_fit) 100 | 101 | 102 | def test_optimization_problems(): 103 | """ 104 | Runs the algorithms with a population of 10, for 10 generation. 105 | """ 106 | run_evolution_strategy_algorithms(10, 10, params_num_rst, task_name_rst, fitness_function_rst, optimal_fit_rst) 107 | run_evolution_strategy_algorithms(10, 10, params_num_nn, task_name_nn, fitness_function_nn, optimal_fit_nn) 108 | -------------------------------------------------------------------------------- /tests/test_ga.py: -------------------------------------------------------------------------------- 1 | import evoalgo.evolutionary_algorithms.genetic_algorithm as GA 2 | import evoalgo.utils.genetic_operators as GenOp 3 | from evoalgo.utils.evolution_process import Evolution 4 | from evoalgo.utils.utils import colors_28, plot_fit_history_comparison 5 | 6 | from evoalgo.optimization_problems.string_opt import params_num as params_num_str, task_name as task_name_str, \ 7 | fitness_function as fitness_function_str, optimal_fit as optimal_fit_str, \ 8 | discrete_values_num as discrete_values_num_str 9 | from evoalgo.optimization_problems.rastrigin import params_num as params_num_rst, task_name as task_name_rst, \ 10 | fitness_function as fitness_function_rst, optimal_fit as optimal_fit_rst 11 | from evoalgo.optimization_problems.policy_nn import params_num as params_num_nn, task_name as task_name_nn, \ 12 | fitness_function as fitness_function_nn, optimal_fit as optimal_fit_nn 13 | 14 | 15 | def run_genetic_algorithms(max_gen_num, pop_size, params_num, task_name, 16 | fitness_function, optimal_fit, discrete_values_num=None): 17 | algo_type = 'GA' 18 | 19 | selection_types = [('FPS', GenOp.Selection.fitness_proportionate), 20 | ('STS', GenOp.Selection.stochastic_top_sampling), 21 | ('Tour', GenOp.Selection.tournament)] 22 | crossover_types = [('1PtCross', GenOp.Crossover.single_pt), 23 | ('2PtCross', GenOp.Crossover.two_pt), 24 | ('UniCross', GenOp.Crossover.uniform)] 25 | mutation_types = [('DetMut', GenOp.Mutation.deterministic), 26 | ('StoUniMut', GenOp.Mutation.stochastic_uniform), 27 | ('GaussMut', GenOp.Mutation.gaussian_noise)] 28 | 29 | max_fit_history_dict = {} 30 | avg_fit_history_dict = {} 31 | 32 | for selection_key, selection_f in selection_types: 33 | for crossover_key, crossover_f in crossover_types: 34 | for mutation_key, mutation_f in mutation_types: 35 | 36 | if mutation_f.__name__ == GenOp.Mutation.gaussian_noise.__name__: 37 | ga = GA.SimpleGA(params_num, pop_size, discrete_values_num, 38 | # mutation_var=0.5, sigma_decay=0.999, sigma_min=0.01) 39 | mutation_var=0.5) 40 | else: 41 | ga = GA.SimpleGA(params_num, pop_size, discrete_values_num) 42 | 43 | Evolution.test_solver(ga, max_gen_num, task_name, fitness_function, 44 | selection_f, crossover_f, mutation_f, 45 | print_progress=True, plot=True, save=False) 46 | 47 | description = f'{selection_key} {crossover_key} {mutation_key}' 48 | max_fit_history_dict[description] = ga.pop_max_fit_history 49 | avg_fit_history_dict[description] = ga.pop_avg_fit_history 50 | 51 | plot_fit_history_comparison(max_fit_history_dict, 'Max', max_gen_num, task_name, pop_size, optimal_fit, algo_type) 52 | plot_fit_history_comparison(avg_fit_history_dict, 'Avg', max_gen_num, task_name, pop_size, optimal_fit, algo_type) 53 | 54 | 55 | def test_string(): 56 | run_genetic_algorithms(10, 20, params_num_str, task_name_str, fitness_function_str, optimal_fit_str, 57 | discrete_values_num_str) 58 | 59 | 60 | def test_rastrigin(): 61 | run_genetic_algorithms(10, 20, params_num_rst, task_name_rst, fitness_function_rst, optimal_fit_rst) 62 | 63 | 64 | def test_nn(): 65 | run_genetic_algorithms(10, 20, params_num_nn, task_name_nn, fitness_function_nn, optimal_fit_nn) 66 | --------------------------------------------------------------------------------