├── .gitignore ├── LICENSE ├── README.md ├── dataset ├── __init__.py ├── pima-indians-diabetes.csv └── synthetic.py ├── envs ├── __init__.py ├── blackjack.py ├── cliff_walking.py ├── gridworld.py └── windy_gridworld.py ├── reinforcement ├── double_q_learning.ipynb ├── double_q_learning.py ├── mc_control_epsilon_greedy.ipynb ├── mc_control_epsilon_greedy.py ├── mc_control_importance_sampling.ipynb ├── mc_control_importance_sampling.py ├── mc_prediction.ipynb ├── mc_prediction.py ├── policy_evaluation.ipynb ├── policy_evaluation.py ├── policy_iteration.ipynb ├── policy_iteration.py ├── q_learning.ipynb ├── q_learning.py ├── q_learning_fa.ipynb ├── q_learning_fa.py ├── reinforce_baseline.ipynb ├── reinforce_baseline.py ├── sarsa.ipynb ├── sarsa.py ├── td_actor_critic_baseline.ipynb ├── td_actor_critic_baseline.py ├── value_iteration.ipynb └── value_iteration.py ├── supervised ├── __init__.py ├── adaboost.ipynb ├── adaboost.py ├── base_regression.py ├── linear_regression.ipynb ├── linear_regression.py ├── logistic_regression.ipynb ├── logistic_regression.py ├── naive_bayes.ipynb └── naive_bayes.py ├── unsupervised ├── __init__.py ├── kmeans.ipynb ├── kmeans.py ├── pca.ipynb └── pca.py └── utils ├── __init__.py ├── base_estimator.py ├── metrics.py ├── plotting.py └── policy.py /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints/ 2 | tmp/ 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Chen Hung-Tu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ml-playground 2 | 3 | This project reuses code and is highly inspired by (selected) great resources: 4 | 5 | 1. [MLAlgorithms](https://github.com/rushter/MLAlgorithms) 6 | 2. [reinforcement-learning](https://github.com/dennybritz/reinforcement-learning). 7 | 8 | It's good to learn machine learning by implementing algorithms, and better if we can write them in an elegant and readable way. Then we can review later or others might easily learn from them. 9 | 10 | So I manage to use succinct codes with demo example to illustrate the essence of ml algorithms. 11 | 12 | ## Supervised 13 | 14 | - Linear Regression [[code]](supervised/linear_regression.py) [[demo]](supervised/linear_regression.ipynb) 15 | 16 | - Logistic Regression [[code]](supervised/logistic_regression.py) [[demo]](supervised/logistic_regression.ipynb) 17 | 18 | - Naive Bayes [[code]](supervised/naive_bayes.py) [[demo]](supervised/naive_bayes.ipynb) 19 | 20 | - AdaBoost [[code]](supervised/adaboost.py) [[demo]](supervised/adaboost.ipynb) 21 | 22 | ## Unsupervised 23 | 24 | - Kmeans [[code]](unsupervised/kmeans.py) [[demo]](unsupervised/kmeans.ipynb) 25 | 26 | - PCA [[code]](unsupervised/pca.py) [[demo]](unsupervised/pca.ipynb) 27 | 28 | ## Reinforcement 29 | 30 | - Dynamic Programming 31 | 32 | - Policy Evaluation [[code]](reinforcement/policy_evaluation.py) [[demo]](reinforcement/policy_evaluation.ipynb) 33 | 34 | - Policy Iteration [[code]](reinforcement/policy_iteration.py) [[demo]](reinforcement/policy_iteration.ipynb) 35 | 36 | - Value Iteration [[code]](reinforcement/value_iteration.py) [[demo]](reinforcement/value_iteration.ipynb) 37 | 38 | - Monte Carlo Methods 39 | 40 | - Monte Carlo Prediction [[code]](reinforcement/mc_prediction.py) [[demo]](reinforcement/mc_prediction.ipynb) 41 | 42 | - Monte Carlo Control (with Epsilon-Greedy Policy) [[code]](reinforcement/mc_control_epsilon_greedy.py) [[demo]](reinforcement/mc_control_epsilon_greedy.ipynb) 43 | 44 | - Monte Carlo Off-Policy Control (with Weighted Importance Sampling) [[code]](reinforcement/mc_control_importance_sampling.py) [[demo]](reinforcement/mc_control_importance_sampling.ipynb) 45 | 46 | - Temporal Difference Methods 47 | 48 | - SARSA [[code]](reinforcement/sarsa.py) [[demo]](reinforcement/sarsa.ipynb) 49 | 50 | - Q-Learning [[code]](reinforcement/q_learning.py) [[demo]](reinforcement/q_learning.ipynb) 51 | 52 | - Double Q-Learning [[code]](reinforcement/double_q_learning.py) [[demo]](reinforcement/double_q_learning.ipynb) 53 | 54 | - Value Function Approximation 55 | 56 | - Q-Learning (with Function Approximation) [[code]](reinforcement/q_learning_fa.py) [[demo]](reinforcement/q_learning_fa.ipynb) 57 | 58 | - [Deep Q-Networks](https://github.com/transedward/pytorch-dqn) 59 | 60 | - Policy Gradient Methods 61 | 62 | - REINFORCE with Baseline [[code]](reinforcement/reinforce_baseline.py) [[demo]](reinforcement/reinforce_baseline.ipynb) 63 | 64 | - TD Actor-Critic with Baseline [[code]](reinforcement/td_actor_critic_baseline.py) [[demo]](reinforcement/td_actor_critic_baseline.ipynb) 65 | 66 | - Hierarchical Reinforcement Learning 67 | 68 | - [Hierarchical Deep Reinforcement Learning](https://github.com/transedward/pytorch-hdqn) 69 | -------------------------------------------------------------------------------- /dataset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hungtuchen/ml-playground/f273ea923cd442f5a51b73c14eb220f3d2b11151/dataset/__init__.py -------------------------------------------------------------------------------- /dataset/pima-indians-diabetes.csv: -------------------------------------------------------------------------------- 1 | 6,148,72,35,0,33.6,0.627,50,1 2 | 1,85,66,29,0,26.6,0.351,31,0 3 | 8,183,64,0,0,23.3,0.672,32,1 4 | 1,89,66,23,94,28.1,0.167,21,0 5 | 0,137,40,35,168,43.1,2.288,33,1 6 | 5,116,74,0,0,25.6,0.201,30,0 7 | 3,78,50,32,88,31.0,0.248,26,1 8 | 10,115,0,0,0,35.3,0.134,29,0 9 | 2,197,70,45,543,30.5,0.158,53,1 10 | 8,125,96,0,0,0.0,0.232,54,1 11 | 4,110,92,0,0,37.6,0.191,30,0 12 | 10,168,74,0,0,38.0,0.537,34,1 13 | 10,139,80,0,0,27.1,1.441,57,0 14 | 1,189,60,23,846,30.1,0.398,59,1 15 | 5,166,72,19,175,25.8,0.587,51,1 16 | 7,100,0,0,0,30.0,0.484,32,1 17 | 0,118,84,47,230,45.8,0.551,31,1 18 | 7,107,74,0,0,29.6,0.254,31,1 19 | 1,103,30,38,83,43.3,0.183,33,0 20 | 1,115,70,30,96,34.6,0.529,32,1 21 | 3,126,88,41,235,39.3,0.704,27,0 22 | 8,99,84,0,0,35.4,0.388,50,0 23 | 7,196,90,0,0,39.8,0.451,41,1 24 | 9,119,80,35,0,29.0,0.263,29,1 25 | 11,143,94,33,146,36.6,0.254,51,1 26 | 10,125,70,26,115,31.1,0.205,41,1 27 | 7,147,76,0,0,39.4,0.257,43,1 28 | 1,97,66,15,140,23.2,0.487,22,0 29 | 13,145,82,19,110,22.2,0.245,57,0 30 | 5,117,92,0,0,34.1,0.337,38,0 31 | 5,109,75,26,0,36.0,0.546,60,0 32 | 3,158,76,36,245,31.6,0.851,28,1 33 | 3,88,58,11,54,24.8,0.267,22,0 34 | 6,92,92,0,0,19.9,0.188,28,0 35 | 10,122,78,31,0,27.6,0.512,45,0 36 | 4,103,60,33,192,24.0,0.966,33,0 37 | 11,138,76,0,0,33.2,0.420,35,0 38 | 9,102,76,37,0,32.9,0.665,46,1 39 | 2,90,68,42,0,38.2,0.503,27,1 40 | 4,111,72,47,207,37.1,1.390,56,1 41 | 3,180,64,25,70,34.0,0.271,26,0 42 | 7,133,84,0,0,40.2,0.696,37,0 43 | 7,106,92,18,0,22.7,0.235,48,0 44 | 9,171,110,24,240,45.4,0.721,54,1 45 | 7,159,64,0,0,27.4,0.294,40,0 46 | 0,180,66,39,0,42.0,1.893,25,1 47 | 1,146,56,0,0,29.7,0.564,29,0 48 | 2,71,70,27,0,28.0,0.586,22,0 49 | 7,103,66,32,0,39.1,0.344,31,1 50 | 7,105,0,0,0,0.0,0.305,24,0 51 | 1,103,80,11,82,19.4,0.491,22,0 52 | 1,101,50,15,36,24.2,0.526,26,0 53 | 5,88,66,21,23,24.4,0.342,30,0 54 | 8,176,90,34,300,33.7,0.467,58,1 55 | 7,150,66,42,342,34.7,0.718,42,0 56 | 1,73,50,10,0,23.0,0.248,21,0 57 | 7,187,68,39,304,37.7,0.254,41,1 58 | 0,100,88,60,110,46.8,0.962,31,0 59 | 0,146,82,0,0,40.5,1.781,44,0 60 | 0,105,64,41,142,41.5,0.173,22,0 61 | 2,84,0,0,0,0.0,0.304,21,0 62 | 8,133,72,0,0,32.9,0.270,39,1 63 | 5,44,62,0,0,25.0,0.587,36,0 64 | 2,141,58,34,128,25.4,0.699,24,0 65 | 7,114,66,0,0,32.8,0.258,42,1 66 | 5,99,74,27,0,29.0,0.203,32,0 67 | 0,109,88,30,0,32.5,0.855,38,1 68 | 2,109,92,0,0,42.7,0.845,54,0 69 | 1,95,66,13,38,19.6,0.334,25,0 70 | 4,146,85,27,100,28.9,0.189,27,0 71 | 2,100,66,20,90,32.9,0.867,28,1 72 | 5,139,64,35,140,28.6,0.411,26,0 73 | 13,126,90,0,0,43.4,0.583,42,1 74 | 4,129,86,20,270,35.1,0.231,23,0 75 | 1,79,75,30,0,32.0,0.396,22,0 76 | 1,0,48,20,0,24.7,0.140,22,0 77 | 7,62,78,0,0,32.6,0.391,41,0 78 | 5,95,72,33,0,37.7,0.370,27,0 79 | 0,131,0,0,0,43.2,0.270,26,1 80 | 2,112,66,22,0,25.0,0.307,24,0 81 | 3,113,44,13,0,22.4,0.140,22,0 82 | 2,74,0,0,0,0.0,0.102,22,0 83 | 7,83,78,26,71,29.3,0.767,36,0 84 | 0,101,65,28,0,24.6,0.237,22,0 85 | 5,137,108,0,0,48.8,0.227,37,1 86 | 2,110,74,29,125,32.4,0.698,27,0 87 | 13,106,72,54,0,36.6,0.178,45,0 88 | 2,100,68,25,71,38.5,0.324,26,0 89 | 15,136,70,32,110,37.1,0.153,43,1 90 | 1,107,68,19,0,26.5,0.165,24,0 91 | 1,80,55,0,0,19.1,0.258,21,0 92 | 4,123,80,15,176,32.0,0.443,34,0 93 | 7,81,78,40,48,46.7,0.261,42,0 94 | 4,134,72,0,0,23.8,0.277,60,1 95 | 2,142,82,18,64,24.7,0.761,21,0 96 | 6,144,72,27,228,33.9,0.255,40,0 97 | 2,92,62,28,0,31.6,0.130,24,0 98 | 1,71,48,18,76,20.4,0.323,22,0 99 | 6,93,50,30,64,28.7,0.356,23,0 100 | 1,122,90,51,220,49.7,0.325,31,1 101 | 1,163,72,0,0,39.0,1.222,33,1 102 | 1,151,60,0,0,26.1,0.179,22,0 103 | 0,125,96,0,0,22.5,0.262,21,0 104 | 1,81,72,18,40,26.6,0.283,24,0 105 | 2,85,65,0,0,39.6,0.930,27,0 106 | 1,126,56,29,152,28.7,0.801,21,0 107 | 1,96,122,0,0,22.4,0.207,27,0 108 | 4,144,58,28,140,29.5,0.287,37,0 109 | 3,83,58,31,18,34.3,0.336,25,0 110 | 0,95,85,25,36,37.4,0.247,24,1 111 | 3,171,72,33,135,33.3,0.199,24,1 112 | 8,155,62,26,495,34.0,0.543,46,1 113 | 1,89,76,34,37,31.2,0.192,23,0 114 | 4,76,62,0,0,34.0,0.391,25,0 115 | 7,160,54,32,175,30.5,0.588,39,1 116 | 4,146,92,0,0,31.2,0.539,61,1 117 | 5,124,74,0,0,34.0,0.220,38,1 118 | 5,78,48,0,0,33.7,0.654,25,0 119 | 4,97,60,23,0,28.2,0.443,22,0 120 | 4,99,76,15,51,23.2,0.223,21,0 121 | 0,162,76,56,100,53.2,0.759,25,1 122 | 6,111,64,39,0,34.2,0.260,24,0 123 | 2,107,74,30,100,33.6,0.404,23,0 124 | 5,132,80,0,0,26.8,0.186,69,0 125 | 0,113,76,0,0,33.3,0.278,23,1 126 | 1,88,30,42,99,55.0,0.496,26,1 127 | 3,120,70,30,135,42.9,0.452,30,0 128 | 1,118,58,36,94,33.3,0.261,23,0 129 | 1,117,88,24,145,34.5,0.403,40,1 130 | 0,105,84,0,0,27.9,0.741,62,1 131 | 4,173,70,14,168,29.7,0.361,33,1 132 | 9,122,56,0,0,33.3,1.114,33,1 133 | 3,170,64,37,225,34.5,0.356,30,1 134 | 8,84,74,31,0,38.3,0.457,39,0 135 | 2,96,68,13,49,21.1,0.647,26,0 136 | 2,125,60,20,140,33.8,0.088,31,0 137 | 0,100,70,26,50,30.8,0.597,21,0 138 | 0,93,60,25,92,28.7,0.532,22,0 139 | 0,129,80,0,0,31.2,0.703,29,0 140 | 5,105,72,29,325,36.9,0.159,28,0 141 | 3,128,78,0,0,21.1,0.268,55,0 142 | 5,106,82,30,0,39.5,0.286,38,0 143 | 2,108,52,26,63,32.5,0.318,22,0 144 | 10,108,66,0,0,32.4,0.272,42,1 145 | 4,154,62,31,284,32.8,0.237,23,0 146 | 0,102,75,23,0,0.0,0.572,21,0 147 | 9,57,80,37,0,32.8,0.096,41,0 148 | 2,106,64,35,119,30.5,1.400,34,0 149 | 5,147,78,0,0,33.7,0.218,65,0 150 | 2,90,70,17,0,27.3,0.085,22,0 151 | 1,136,74,50,204,37.4,0.399,24,0 152 | 4,114,65,0,0,21.9,0.432,37,0 153 | 9,156,86,28,155,34.3,1.189,42,1 154 | 1,153,82,42,485,40.6,0.687,23,0 155 | 8,188,78,0,0,47.9,0.137,43,1 156 | 7,152,88,44,0,50.0,0.337,36,1 157 | 2,99,52,15,94,24.6,0.637,21,0 158 | 1,109,56,21,135,25.2,0.833,23,0 159 | 2,88,74,19,53,29.0,0.229,22,0 160 | 17,163,72,41,114,40.9,0.817,47,1 161 | 4,151,90,38,0,29.7,0.294,36,0 162 | 7,102,74,40,105,37.2,0.204,45,0 163 | 0,114,80,34,285,44.2,0.167,27,0 164 | 2,100,64,23,0,29.7,0.368,21,0 165 | 0,131,88,0,0,31.6,0.743,32,1 166 | 6,104,74,18,156,29.9,0.722,41,1 167 | 3,148,66,25,0,32.5,0.256,22,0 168 | 4,120,68,0,0,29.6,0.709,34,0 169 | 4,110,66,0,0,31.9,0.471,29,0 170 | 3,111,90,12,78,28.4,0.495,29,0 171 | 6,102,82,0,0,30.8,0.180,36,1 172 | 6,134,70,23,130,35.4,0.542,29,1 173 | 2,87,0,23,0,28.9,0.773,25,0 174 | 1,79,60,42,48,43.5,0.678,23,0 175 | 2,75,64,24,55,29.7,0.370,33,0 176 | 8,179,72,42,130,32.7,0.719,36,1 177 | 6,85,78,0,0,31.2,0.382,42,0 178 | 0,129,110,46,130,67.1,0.319,26,1 179 | 5,143,78,0,0,45.0,0.190,47,0 180 | 5,130,82,0,0,39.1,0.956,37,1 181 | 6,87,80,0,0,23.2,0.084,32,0 182 | 0,119,64,18,92,34.9,0.725,23,0 183 | 1,0,74,20,23,27.7,0.299,21,0 184 | 5,73,60,0,0,26.8,0.268,27,0 185 | 4,141,74,0,0,27.6,0.244,40,0 186 | 7,194,68,28,0,35.9,0.745,41,1 187 | 8,181,68,36,495,30.1,0.615,60,1 188 | 1,128,98,41,58,32.0,1.321,33,1 189 | 8,109,76,39,114,27.9,0.640,31,1 190 | 5,139,80,35,160,31.6,0.361,25,1 191 | 3,111,62,0,0,22.6,0.142,21,0 192 | 9,123,70,44,94,33.1,0.374,40,0 193 | 7,159,66,0,0,30.4,0.383,36,1 194 | 11,135,0,0,0,52.3,0.578,40,1 195 | 8,85,55,20,0,24.4,0.136,42,0 196 | 5,158,84,41,210,39.4,0.395,29,1 197 | 1,105,58,0,0,24.3,0.187,21,0 198 | 3,107,62,13,48,22.9,0.678,23,1 199 | 4,109,64,44,99,34.8,0.905,26,1 200 | 4,148,60,27,318,30.9,0.150,29,1 201 | 0,113,80,16,0,31.0,0.874,21,0 202 | 1,138,82,0,0,40.1,0.236,28,0 203 | 0,108,68,20,0,27.3,0.787,32,0 204 | 2,99,70,16,44,20.4,0.235,27,0 205 | 6,103,72,32,190,37.7,0.324,55,0 206 | 5,111,72,28,0,23.9,0.407,27,0 207 | 8,196,76,29,280,37.5,0.605,57,1 208 | 5,162,104,0,0,37.7,0.151,52,1 209 | 1,96,64,27,87,33.2,0.289,21,0 210 | 7,184,84,33,0,35.5,0.355,41,1 211 | 2,81,60,22,0,27.7,0.290,25,0 212 | 0,147,85,54,0,42.8,0.375,24,0 213 | 7,179,95,31,0,34.2,0.164,60,0 214 | 0,140,65,26,130,42.6,0.431,24,1 215 | 9,112,82,32,175,34.2,0.260,36,1 216 | 12,151,70,40,271,41.8,0.742,38,1 217 | 5,109,62,41,129,35.8,0.514,25,1 218 | 6,125,68,30,120,30.0,0.464,32,0 219 | 5,85,74,22,0,29.0,1.224,32,1 220 | 5,112,66,0,0,37.8,0.261,41,1 221 | 0,177,60,29,478,34.6,1.072,21,1 222 | 2,158,90,0,0,31.6,0.805,66,1 223 | 7,119,0,0,0,25.2,0.209,37,0 224 | 7,142,60,33,190,28.8,0.687,61,0 225 | 1,100,66,15,56,23.6,0.666,26,0 226 | 1,87,78,27,32,34.6,0.101,22,0 227 | 0,101,76,0,0,35.7,0.198,26,0 228 | 3,162,52,38,0,37.2,0.652,24,1 229 | 4,197,70,39,744,36.7,2.329,31,0 230 | 0,117,80,31,53,45.2,0.089,24,0 231 | 4,142,86,0,0,44.0,0.645,22,1 232 | 6,134,80,37,370,46.2,0.238,46,1 233 | 1,79,80,25,37,25.4,0.583,22,0 234 | 4,122,68,0,0,35.0,0.394,29,0 235 | 3,74,68,28,45,29.7,0.293,23,0 236 | 4,171,72,0,0,43.6,0.479,26,1 237 | 7,181,84,21,192,35.9,0.586,51,1 238 | 0,179,90,27,0,44.1,0.686,23,1 239 | 9,164,84,21,0,30.8,0.831,32,1 240 | 0,104,76,0,0,18.4,0.582,27,0 241 | 1,91,64,24,0,29.2,0.192,21,0 242 | 4,91,70,32,88,33.1,0.446,22,0 243 | 3,139,54,0,0,25.6,0.402,22,1 244 | 6,119,50,22,176,27.1,1.318,33,1 245 | 2,146,76,35,194,38.2,0.329,29,0 246 | 9,184,85,15,0,30.0,1.213,49,1 247 | 10,122,68,0,0,31.2,0.258,41,0 248 | 0,165,90,33,680,52.3,0.427,23,0 249 | 9,124,70,33,402,35.4,0.282,34,0 250 | 1,111,86,19,0,30.1,0.143,23,0 251 | 9,106,52,0,0,31.2,0.380,42,0 252 | 2,129,84,0,0,28.0,0.284,27,0 253 | 2,90,80,14,55,24.4,0.249,24,0 254 | 0,86,68,32,0,35.8,0.238,25,0 255 | 12,92,62,7,258,27.6,0.926,44,1 256 | 1,113,64,35,0,33.6,0.543,21,1 257 | 3,111,56,39,0,30.1,0.557,30,0 258 | 2,114,68,22,0,28.7,0.092,25,0 259 | 1,193,50,16,375,25.9,0.655,24,0 260 | 11,155,76,28,150,33.3,1.353,51,1 261 | 3,191,68,15,130,30.9,0.299,34,0 262 | 3,141,0,0,0,30.0,0.761,27,1 263 | 4,95,70,32,0,32.1,0.612,24,0 264 | 3,142,80,15,0,32.4,0.200,63,0 265 | 4,123,62,0,0,32.0,0.226,35,1 266 | 5,96,74,18,67,33.6,0.997,43,0 267 | 0,138,0,0,0,36.3,0.933,25,1 268 | 2,128,64,42,0,40.0,1.101,24,0 269 | 0,102,52,0,0,25.1,0.078,21,0 270 | 2,146,0,0,0,27.5,0.240,28,1 271 | 10,101,86,37,0,45.6,1.136,38,1 272 | 2,108,62,32,56,25.2,0.128,21,0 273 | 3,122,78,0,0,23.0,0.254,40,0 274 | 1,71,78,50,45,33.2,0.422,21,0 275 | 13,106,70,0,0,34.2,0.251,52,0 276 | 2,100,70,52,57,40.5,0.677,25,0 277 | 7,106,60,24,0,26.5,0.296,29,1 278 | 0,104,64,23,116,27.8,0.454,23,0 279 | 5,114,74,0,0,24.9,0.744,57,0 280 | 2,108,62,10,278,25.3,0.881,22,0 281 | 0,146,70,0,0,37.9,0.334,28,1 282 | 10,129,76,28,122,35.9,0.280,39,0 283 | 7,133,88,15,155,32.4,0.262,37,0 284 | 7,161,86,0,0,30.4,0.165,47,1 285 | 2,108,80,0,0,27.0,0.259,52,1 286 | 7,136,74,26,135,26.0,0.647,51,0 287 | 5,155,84,44,545,38.7,0.619,34,0 288 | 1,119,86,39,220,45.6,0.808,29,1 289 | 4,96,56,17,49,20.8,0.340,26,0 290 | 5,108,72,43,75,36.1,0.263,33,0 291 | 0,78,88,29,40,36.9,0.434,21,0 292 | 0,107,62,30,74,36.6,0.757,25,1 293 | 2,128,78,37,182,43.3,1.224,31,1 294 | 1,128,48,45,194,40.5,0.613,24,1 295 | 0,161,50,0,0,21.9,0.254,65,0 296 | 6,151,62,31,120,35.5,0.692,28,0 297 | 2,146,70,38,360,28.0,0.337,29,1 298 | 0,126,84,29,215,30.7,0.520,24,0 299 | 14,100,78,25,184,36.6,0.412,46,1 300 | 8,112,72,0,0,23.6,0.840,58,0 301 | 0,167,0,0,0,32.3,0.839,30,1 302 | 2,144,58,33,135,31.6,0.422,25,1 303 | 5,77,82,41,42,35.8,0.156,35,0 304 | 5,115,98,0,0,52.9,0.209,28,1 305 | 3,150,76,0,0,21.0,0.207,37,0 306 | 2,120,76,37,105,39.7,0.215,29,0 307 | 10,161,68,23,132,25.5,0.326,47,1 308 | 0,137,68,14,148,24.8,0.143,21,0 309 | 0,128,68,19,180,30.5,1.391,25,1 310 | 2,124,68,28,205,32.9,0.875,30,1 311 | 6,80,66,30,0,26.2,0.313,41,0 312 | 0,106,70,37,148,39.4,0.605,22,0 313 | 2,155,74,17,96,26.6,0.433,27,1 314 | 3,113,50,10,85,29.5,0.626,25,0 315 | 7,109,80,31,0,35.9,1.127,43,1 316 | 2,112,68,22,94,34.1,0.315,26,0 317 | 3,99,80,11,64,19.3,0.284,30,0 318 | 3,182,74,0,0,30.5,0.345,29,1 319 | 3,115,66,39,140,38.1,0.150,28,0 320 | 6,194,78,0,0,23.5,0.129,59,1 321 | 4,129,60,12,231,27.5,0.527,31,0 322 | 3,112,74,30,0,31.6,0.197,25,1 323 | 0,124,70,20,0,27.4,0.254,36,1 324 | 13,152,90,33,29,26.8,0.731,43,1 325 | 2,112,75,32,0,35.7,0.148,21,0 326 | 1,157,72,21,168,25.6,0.123,24,0 327 | 1,122,64,32,156,35.1,0.692,30,1 328 | 10,179,70,0,0,35.1,0.200,37,0 329 | 2,102,86,36,120,45.5,0.127,23,1 330 | 6,105,70,32,68,30.8,0.122,37,0 331 | 8,118,72,19,0,23.1,1.476,46,0 332 | 2,87,58,16,52,32.7,0.166,25,0 333 | 1,180,0,0,0,43.3,0.282,41,1 334 | 12,106,80,0,0,23.6,0.137,44,0 335 | 1,95,60,18,58,23.9,0.260,22,0 336 | 0,165,76,43,255,47.9,0.259,26,0 337 | 0,117,0,0,0,33.8,0.932,44,0 338 | 5,115,76,0,0,31.2,0.343,44,1 339 | 9,152,78,34,171,34.2,0.893,33,1 340 | 7,178,84,0,0,39.9,0.331,41,1 341 | 1,130,70,13,105,25.9,0.472,22,0 342 | 1,95,74,21,73,25.9,0.673,36,0 343 | 1,0,68,35,0,32.0,0.389,22,0 344 | 5,122,86,0,0,34.7,0.290,33,0 345 | 8,95,72,0,0,36.8,0.485,57,0 346 | 8,126,88,36,108,38.5,0.349,49,0 347 | 1,139,46,19,83,28.7,0.654,22,0 348 | 3,116,0,0,0,23.5,0.187,23,0 349 | 3,99,62,19,74,21.8,0.279,26,0 350 | 5,0,80,32,0,41.0,0.346,37,1 351 | 4,92,80,0,0,42.2,0.237,29,0 352 | 4,137,84,0,0,31.2,0.252,30,0 353 | 3,61,82,28,0,34.4,0.243,46,0 354 | 1,90,62,12,43,27.2,0.580,24,0 355 | 3,90,78,0,0,42.7,0.559,21,0 356 | 9,165,88,0,0,30.4,0.302,49,1 357 | 1,125,50,40,167,33.3,0.962,28,1 358 | 13,129,0,30,0,39.9,0.569,44,1 359 | 12,88,74,40,54,35.3,0.378,48,0 360 | 1,196,76,36,249,36.5,0.875,29,1 361 | 5,189,64,33,325,31.2,0.583,29,1 362 | 5,158,70,0,0,29.8,0.207,63,0 363 | 5,103,108,37,0,39.2,0.305,65,0 364 | 4,146,78,0,0,38.5,0.520,67,1 365 | 4,147,74,25,293,34.9,0.385,30,0 366 | 5,99,54,28,83,34.0,0.499,30,0 367 | 6,124,72,0,0,27.6,0.368,29,1 368 | 0,101,64,17,0,21.0,0.252,21,0 369 | 3,81,86,16,66,27.5,0.306,22,0 370 | 1,133,102,28,140,32.8,0.234,45,1 371 | 3,173,82,48,465,38.4,2.137,25,1 372 | 0,118,64,23,89,0.0,1.731,21,0 373 | 0,84,64,22,66,35.8,0.545,21,0 374 | 2,105,58,40,94,34.9,0.225,25,0 375 | 2,122,52,43,158,36.2,0.816,28,0 376 | 12,140,82,43,325,39.2,0.528,58,1 377 | 0,98,82,15,84,25.2,0.299,22,0 378 | 1,87,60,37,75,37.2,0.509,22,0 379 | 4,156,75,0,0,48.3,0.238,32,1 380 | 0,93,100,39,72,43.4,1.021,35,0 381 | 1,107,72,30,82,30.8,0.821,24,0 382 | 0,105,68,22,0,20.0,0.236,22,0 383 | 1,109,60,8,182,25.4,0.947,21,0 384 | 1,90,62,18,59,25.1,1.268,25,0 385 | 1,125,70,24,110,24.3,0.221,25,0 386 | 1,119,54,13,50,22.3,0.205,24,0 387 | 5,116,74,29,0,32.3,0.660,35,1 388 | 8,105,100,36,0,43.3,0.239,45,1 389 | 5,144,82,26,285,32.0,0.452,58,1 390 | 3,100,68,23,81,31.6,0.949,28,0 391 | 1,100,66,29,196,32.0,0.444,42,0 392 | 5,166,76,0,0,45.7,0.340,27,1 393 | 1,131,64,14,415,23.7,0.389,21,0 394 | 4,116,72,12,87,22.1,0.463,37,0 395 | 4,158,78,0,0,32.9,0.803,31,1 396 | 2,127,58,24,275,27.7,1.600,25,0 397 | 3,96,56,34,115,24.7,0.944,39,0 398 | 0,131,66,40,0,34.3,0.196,22,1 399 | 3,82,70,0,0,21.1,0.389,25,0 400 | 3,193,70,31,0,34.9,0.241,25,1 401 | 4,95,64,0,0,32.0,0.161,31,1 402 | 6,137,61,0,0,24.2,0.151,55,0 403 | 5,136,84,41,88,35.0,0.286,35,1 404 | 9,72,78,25,0,31.6,0.280,38,0 405 | 5,168,64,0,0,32.9,0.135,41,1 406 | 2,123,48,32,165,42.1,0.520,26,0 407 | 4,115,72,0,0,28.9,0.376,46,1 408 | 0,101,62,0,0,21.9,0.336,25,0 409 | 8,197,74,0,0,25.9,1.191,39,1 410 | 1,172,68,49,579,42.4,0.702,28,1 411 | 6,102,90,39,0,35.7,0.674,28,0 412 | 1,112,72,30,176,34.4,0.528,25,0 413 | 1,143,84,23,310,42.4,1.076,22,0 414 | 1,143,74,22,61,26.2,0.256,21,0 415 | 0,138,60,35,167,34.6,0.534,21,1 416 | 3,173,84,33,474,35.7,0.258,22,1 417 | 1,97,68,21,0,27.2,1.095,22,0 418 | 4,144,82,32,0,38.5,0.554,37,1 419 | 1,83,68,0,0,18.2,0.624,27,0 420 | 3,129,64,29,115,26.4,0.219,28,1 421 | 1,119,88,41,170,45.3,0.507,26,0 422 | 2,94,68,18,76,26.0,0.561,21,0 423 | 0,102,64,46,78,40.6,0.496,21,0 424 | 2,115,64,22,0,30.8,0.421,21,0 425 | 8,151,78,32,210,42.9,0.516,36,1 426 | 4,184,78,39,277,37.0,0.264,31,1 427 | 0,94,0,0,0,0.0,0.256,25,0 428 | 1,181,64,30,180,34.1,0.328,38,1 429 | 0,135,94,46,145,40.6,0.284,26,0 430 | 1,95,82,25,180,35.0,0.233,43,1 431 | 2,99,0,0,0,22.2,0.108,23,0 432 | 3,89,74,16,85,30.4,0.551,38,0 433 | 1,80,74,11,60,30.0,0.527,22,0 434 | 2,139,75,0,0,25.6,0.167,29,0 435 | 1,90,68,8,0,24.5,1.138,36,0 436 | 0,141,0,0,0,42.4,0.205,29,1 437 | 12,140,85,33,0,37.4,0.244,41,0 438 | 5,147,75,0,0,29.9,0.434,28,0 439 | 1,97,70,15,0,18.2,0.147,21,0 440 | 6,107,88,0,0,36.8,0.727,31,0 441 | 0,189,104,25,0,34.3,0.435,41,1 442 | 2,83,66,23,50,32.2,0.497,22,0 443 | 4,117,64,27,120,33.2,0.230,24,0 444 | 8,108,70,0,0,30.5,0.955,33,1 445 | 4,117,62,12,0,29.7,0.380,30,1 446 | 0,180,78,63,14,59.4,2.420,25,1 447 | 1,100,72,12,70,25.3,0.658,28,0 448 | 0,95,80,45,92,36.5,0.330,26,0 449 | 0,104,64,37,64,33.6,0.510,22,1 450 | 0,120,74,18,63,30.5,0.285,26,0 451 | 1,82,64,13,95,21.2,0.415,23,0 452 | 2,134,70,0,0,28.9,0.542,23,1 453 | 0,91,68,32,210,39.9,0.381,25,0 454 | 2,119,0,0,0,19.6,0.832,72,0 455 | 2,100,54,28,105,37.8,0.498,24,0 456 | 14,175,62,30,0,33.6,0.212,38,1 457 | 1,135,54,0,0,26.7,0.687,62,0 458 | 5,86,68,28,71,30.2,0.364,24,0 459 | 10,148,84,48,237,37.6,1.001,51,1 460 | 9,134,74,33,60,25.9,0.460,81,0 461 | 9,120,72,22,56,20.8,0.733,48,0 462 | 1,71,62,0,0,21.8,0.416,26,0 463 | 8,74,70,40,49,35.3,0.705,39,0 464 | 5,88,78,30,0,27.6,0.258,37,0 465 | 10,115,98,0,0,24.0,1.022,34,0 466 | 0,124,56,13,105,21.8,0.452,21,0 467 | 0,74,52,10,36,27.8,0.269,22,0 468 | 0,97,64,36,100,36.8,0.600,25,0 469 | 8,120,0,0,0,30.0,0.183,38,1 470 | 6,154,78,41,140,46.1,0.571,27,0 471 | 1,144,82,40,0,41.3,0.607,28,0 472 | 0,137,70,38,0,33.2,0.170,22,0 473 | 0,119,66,27,0,38.8,0.259,22,0 474 | 7,136,90,0,0,29.9,0.210,50,0 475 | 4,114,64,0,0,28.9,0.126,24,0 476 | 0,137,84,27,0,27.3,0.231,59,0 477 | 2,105,80,45,191,33.7,0.711,29,1 478 | 7,114,76,17,110,23.8,0.466,31,0 479 | 8,126,74,38,75,25.9,0.162,39,0 480 | 4,132,86,31,0,28.0,0.419,63,0 481 | 3,158,70,30,328,35.5,0.344,35,1 482 | 0,123,88,37,0,35.2,0.197,29,0 483 | 4,85,58,22,49,27.8,0.306,28,0 484 | 0,84,82,31,125,38.2,0.233,23,0 485 | 0,145,0,0,0,44.2,0.630,31,1 486 | 0,135,68,42,250,42.3,0.365,24,1 487 | 1,139,62,41,480,40.7,0.536,21,0 488 | 0,173,78,32,265,46.5,1.159,58,0 489 | 4,99,72,17,0,25.6,0.294,28,0 490 | 8,194,80,0,0,26.1,0.551,67,0 491 | 2,83,65,28,66,36.8,0.629,24,0 492 | 2,89,90,30,0,33.5,0.292,42,0 493 | 4,99,68,38,0,32.8,0.145,33,0 494 | 4,125,70,18,122,28.9,1.144,45,1 495 | 3,80,0,0,0,0.0,0.174,22,0 496 | 6,166,74,0,0,26.6,0.304,66,0 497 | 5,110,68,0,0,26.0,0.292,30,0 498 | 2,81,72,15,76,30.1,0.547,25,0 499 | 7,195,70,33,145,25.1,0.163,55,1 500 | 6,154,74,32,193,29.3,0.839,39,0 501 | 2,117,90,19,71,25.2,0.313,21,0 502 | 3,84,72,32,0,37.2,0.267,28,0 503 | 6,0,68,41,0,39.0,0.727,41,1 504 | 7,94,64,25,79,33.3,0.738,41,0 505 | 3,96,78,39,0,37.3,0.238,40,0 506 | 10,75,82,0,0,33.3,0.263,38,0 507 | 0,180,90,26,90,36.5,0.314,35,1 508 | 1,130,60,23,170,28.6,0.692,21,0 509 | 2,84,50,23,76,30.4,0.968,21,0 510 | 8,120,78,0,0,25.0,0.409,64,0 511 | 12,84,72,31,0,29.7,0.297,46,1 512 | 0,139,62,17,210,22.1,0.207,21,0 513 | 9,91,68,0,0,24.2,0.200,58,0 514 | 2,91,62,0,0,27.3,0.525,22,0 515 | 3,99,54,19,86,25.6,0.154,24,0 516 | 3,163,70,18,105,31.6,0.268,28,1 517 | 9,145,88,34,165,30.3,0.771,53,1 518 | 7,125,86,0,0,37.6,0.304,51,0 519 | 13,76,60,0,0,32.8,0.180,41,0 520 | 6,129,90,7,326,19.6,0.582,60,0 521 | 2,68,70,32,66,25.0,0.187,25,0 522 | 3,124,80,33,130,33.2,0.305,26,0 523 | 6,114,0,0,0,0.0,0.189,26,0 524 | 9,130,70,0,0,34.2,0.652,45,1 525 | 3,125,58,0,0,31.6,0.151,24,0 526 | 3,87,60,18,0,21.8,0.444,21,0 527 | 1,97,64,19,82,18.2,0.299,21,0 528 | 3,116,74,15,105,26.3,0.107,24,0 529 | 0,117,66,31,188,30.8,0.493,22,0 530 | 0,111,65,0,0,24.6,0.660,31,0 531 | 2,122,60,18,106,29.8,0.717,22,0 532 | 0,107,76,0,0,45.3,0.686,24,0 533 | 1,86,66,52,65,41.3,0.917,29,0 534 | 6,91,0,0,0,29.8,0.501,31,0 535 | 1,77,56,30,56,33.3,1.251,24,0 536 | 4,132,0,0,0,32.9,0.302,23,1 537 | 0,105,90,0,0,29.6,0.197,46,0 538 | 0,57,60,0,0,21.7,0.735,67,0 539 | 0,127,80,37,210,36.3,0.804,23,0 540 | 3,129,92,49,155,36.4,0.968,32,1 541 | 8,100,74,40,215,39.4,0.661,43,1 542 | 3,128,72,25,190,32.4,0.549,27,1 543 | 10,90,85,32,0,34.9,0.825,56,1 544 | 4,84,90,23,56,39.5,0.159,25,0 545 | 1,88,78,29,76,32.0,0.365,29,0 546 | 8,186,90,35,225,34.5,0.423,37,1 547 | 5,187,76,27,207,43.6,1.034,53,1 548 | 4,131,68,21,166,33.1,0.160,28,0 549 | 1,164,82,43,67,32.8,0.341,50,0 550 | 4,189,110,31,0,28.5,0.680,37,0 551 | 1,116,70,28,0,27.4,0.204,21,0 552 | 3,84,68,30,106,31.9,0.591,25,0 553 | 6,114,88,0,0,27.8,0.247,66,0 554 | 1,88,62,24,44,29.9,0.422,23,0 555 | 1,84,64,23,115,36.9,0.471,28,0 556 | 7,124,70,33,215,25.5,0.161,37,0 557 | 1,97,70,40,0,38.1,0.218,30,0 558 | 8,110,76,0,0,27.8,0.237,58,0 559 | 11,103,68,40,0,46.2,0.126,42,0 560 | 11,85,74,0,0,30.1,0.300,35,0 561 | 6,125,76,0,0,33.8,0.121,54,1 562 | 0,198,66,32,274,41.3,0.502,28,1 563 | 1,87,68,34,77,37.6,0.401,24,0 564 | 6,99,60,19,54,26.9,0.497,32,0 565 | 0,91,80,0,0,32.4,0.601,27,0 566 | 2,95,54,14,88,26.1,0.748,22,0 567 | 1,99,72,30,18,38.6,0.412,21,0 568 | 6,92,62,32,126,32.0,0.085,46,0 569 | 4,154,72,29,126,31.3,0.338,37,0 570 | 0,121,66,30,165,34.3,0.203,33,1 571 | 3,78,70,0,0,32.5,0.270,39,0 572 | 2,130,96,0,0,22.6,0.268,21,0 573 | 3,111,58,31,44,29.5,0.430,22,0 574 | 2,98,60,17,120,34.7,0.198,22,0 575 | 1,143,86,30,330,30.1,0.892,23,0 576 | 1,119,44,47,63,35.5,0.280,25,0 577 | 6,108,44,20,130,24.0,0.813,35,0 578 | 2,118,80,0,0,42.9,0.693,21,1 579 | 10,133,68,0,0,27.0,0.245,36,0 580 | 2,197,70,99,0,34.7,0.575,62,1 581 | 0,151,90,46,0,42.1,0.371,21,1 582 | 6,109,60,27,0,25.0,0.206,27,0 583 | 12,121,78,17,0,26.5,0.259,62,0 584 | 8,100,76,0,0,38.7,0.190,42,0 585 | 8,124,76,24,600,28.7,0.687,52,1 586 | 1,93,56,11,0,22.5,0.417,22,0 587 | 8,143,66,0,0,34.9,0.129,41,1 588 | 6,103,66,0,0,24.3,0.249,29,0 589 | 3,176,86,27,156,33.3,1.154,52,1 590 | 0,73,0,0,0,21.1,0.342,25,0 591 | 11,111,84,40,0,46.8,0.925,45,1 592 | 2,112,78,50,140,39.4,0.175,24,0 593 | 3,132,80,0,0,34.4,0.402,44,1 594 | 2,82,52,22,115,28.5,1.699,25,0 595 | 6,123,72,45,230,33.6,0.733,34,0 596 | 0,188,82,14,185,32.0,0.682,22,1 597 | 0,67,76,0,0,45.3,0.194,46,0 598 | 1,89,24,19,25,27.8,0.559,21,0 599 | 1,173,74,0,0,36.8,0.088,38,1 600 | 1,109,38,18,120,23.1,0.407,26,0 601 | 1,108,88,19,0,27.1,0.400,24,0 602 | 6,96,0,0,0,23.7,0.190,28,0 603 | 1,124,74,36,0,27.8,0.100,30,0 604 | 7,150,78,29,126,35.2,0.692,54,1 605 | 4,183,0,0,0,28.4,0.212,36,1 606 | 1,124,60,32,0,35.8,0.514,21,0 607 | 1,181,78,42,293,40.0,1.258,22,1 608 | 1,92,62,25,41,19.5,0.482,25,0 609 | 0,152,82,39,272,41.5,0.270,27,0 610 | 1,111,62,13,182,24.0,0.138,23,0 611 | 3,106,54,21,158,30.9,0.292,24,0 612 | 3,174,58,22,194,32.9,0.593,36,1 613 | 7,168,88,42,321,38.2,0.787,40,1 614 | 6,105,80,28,0,32.5,0.878,26,0 615 | 11,138,74,26,144,36.1,0.557,50,1 616 | 3,106,72,0,0,25.8,0.207,27,0 617 | 6,117,96,0,0,28.7,0.157,30,0 618 | 2,68,62,13,15,20.1,0.257,23,0 619 | 9,112,82,24,0,28.2,1.282,50,1 620 | 0,119,0,0,0,32.4,0.141,24,1 621 | 2,112,86,42,160,38.4,0.246,28,0 622 | 2,92,76,20,0,24.2,1.698,28,0 623 | 6,183,94,0,0,40.8,1.461,45,0 624 | 0,94,70,27,115,43.5,0.347,21,0 625 | 2,108,64,0,0,30.8,0.158,21,0 626 | 4,90,88,47,54,37.7,0.362,29,0 627 | 0,125,68,0,0,24.7,0.206,21,0 628 | 0,132,78,0,0,32.4,0.393,21,0 629 | 5,128,80,0,0,34.6,0.144,45,0 630 | 4,94,65,22,0,24.7,0.148,21,0 631 | 7,114,64,0,0,27.4,0.732,34,1 632 | 0,102,78,40,90,34.5,0.238,24,0 633 | 2,111,60,0,0,26.2,0.343,23,0 634 | 1,128,82,17,183,27.5,0.115,22,0 635 | 10,92,62,0,0,25.9,0.167,31,0 636 | 13,104,72,0,0,31.2,0.465,38,1 637 | 5,104,74,0,0,28.8,0.153,48,0 638 | 2,94,76,18,66,31.6,0.649,23,0 639 | 7,97,76,32,91,40.9,0.871,32,1 640 | 1,100,74,12,46,19.5,0.149,28,0 641 | 0,102,86,17,105,29.3,0.695,27,0 642 | 4,128,70,0,0,34.3,0.303,24,0 643 | 6,147,80,0,0,29.5,0.178,50,1 644 | 4,90,0,0,0,28.0,0.610,31,0 645 | 3,103,72,30,152,27.6,0.730,27,0 646 | 2,157,74,35,440,39.4,0.134,30,0 647 | 1,167,74,17,144,23.4,0.447,33,1 648 | 0,179,50,36,159,37.8,0.455,22,1 649 | 11,136,84,35,130,28.3,0.260,42,1 650 | 0,107,60,25,0,26.4,0.133,23,0 651 | 1,91,54,25,100,25.2,0.234,23,0 652 | 1,117,60,23,106,33.8,0.466,27,0 653 | 5,123,74,40,77,34.1,0.269,28,0 654 | 2,120,54,0,0,26.8,0.455,27,0 655 | 1,106,70,28,135,34.2,0.142,22,0 656 | 2,155,52,27,540,38.7,0.240,25,1 657 | 2,101,58,35,90,21.8,0.155,22,0 658 | 1,120,80,48,200,38.9,1.162,41,0 659 | 11,127,106,0,0,39.0,0.190,51,0 660 | 3,80,82,31,70,34.2,1.292,27,1 661 | 10,162,84,0,0,27.7,0.182,54,0 662 | 1,199,76,43,0,42.9,1.394,22,1 663 | 8,167,106,46,231,37.6,0.165,43,1 664 | 9,145,80,46,130,37.9,0.637,40,1 665 | 6,115,60,39,0,33.7,0.245,40,1 666 | 1,112,80,45,132,34.8,0.217,24,0 667 | 4,145,82,18,0,32.5,0.235,70,1 668 | 10,111,70,27,0,27.5,0.141,40,1 669 | 6,98,58,33,190,34.0,0.430,43,0 670 | 9,154,78,30,100,30.9,0.164,45,0 671 | 6,165,68,26,168,33.6,0.631,49,0 672 | 1,99,58,10,0,25.4,0.551,21,0 673 | 10,68,106,23,49,35.5,0.285,47,0 674 | 3,123,100,35,240,57.3,0.880,22,0 675 | 8,91,82,0,0,35.6,0.587,68,0 676 | 6,195,70,0,0,30.9,0.328,31,1 677 | 9,156,86,0,0,24.8,0.230,53,1 678 | 0,93,60,0,0,35.3,0.263,25,0 679 | 3,121,52,0,0,36.0,0.127,25,1 680 | 2,101,58,17,265,24.2,0.614,23,0 681 | 2,56,56,28,45,24.2,0.332,22,0 682 | 0,162,76,36,0,49.6,0.364,26,1 683 | 0,95,64,39,105,44.6,0.366,22,0 684 | 4,125,80,0,0,32.3,0.536,27,1 685 | 5,136,82,0,0,0.0,0.640,69,0 686 | 2,129,74,26,205,33.2,0.591,25,0 687 | 3,130,64,0,0,23.1,0.314,22,0 688 | 1,107,50,19,0,28.3,0.181,29,0 689 | 1,140,74,26,180,24.1,0.828,23,0 690 | 1,144,82,46,180,46.1,0.335,46,1 691 | 8,107,80,0,0,24.6,0.856,34,0 692 | 13,158,114,0,0,42.3,0.257,44,1 693 | 2,121,70,32,95,39.1,0.886,23,0 694 | 7,129,68,49,125,38.5,0.439,43,1 695 | 2,90,60,0,0,23.5,0.191,25,0 696 | 7,142,90,24,480,30.4,0.128,43,1 697 | 3,169,74,19,125,29.9,0.268,31,1 698 | 0,99,0,0,0,25.0,0.253,22,0 699 | 4,127,88,11,155,34.5,0.598,28,0 700 | 4,118,70,0,0,44.5,0.904,26,0 701 | 2,122,76,27,200,35.9,0.483,26,0 702 | 6,125,78,31,0,27.6,0.565,49,1 703 | 1,168,88,29,0,35.0,0.905,52,1 704 | 2,129,0,0,0,38.5,0.304,41,0 705 | 4,110,76,20,100,28.4,0.118,27,0 706 | 6,80,80,36,0,39.8,0.177,28,0 707 | 10,115,0,0,0,0.0,0.261,30,1 708 | 2,127,46,21,335,34.4,0.176,22,0 709 | 9,164,78,0,0,32.8,0.148,45,1 710 | 2,93,64,32,160,38.0,0.674,23,1 711 | 3,158,64,13,387,31.2,0.295,24,0 712 | 5,126,78,27,22,29.6,0.439,40,0 713 | 10,129,62,36,0,41.2,0.441,38,1 714 | 0,134,58,20,291,26.4,0.352,21,0 715 | 3,102,74,0,0,29.5,0.121,32,0 716 | 7,187,50,33,392,33.9,0.826,34,1 717 | 3,173,78,39,185,33.8,0.970,31,1 718 | 10,94,72,18,0,23.1,0.595,56,0 719 | 1,108,60,46,178,35.5,0.415,24,0 720 | 5,97,76,27,0,35.6,0.378,52,1 721 | 4,83,86,19,0,29.3,0.317,34,0 722 | 1,114,66,36,200,38.1,0.289,21,0 723 | 1,149,68,29,127,29.3,0.349,42,1 724 | 5,117,86,30,105,39.1,0.251,42,0 725 | 1,111,94,0,0,32.8,0.265,45,0 726 | 4,112,78,40,0,39.4,0.236,38,0 727 | 1,116,78,29,180,36.1,0.496,25,0 728 | 0,141,84,26,0,32.4,0.433,22,0 729 | 2,175,88,0,0,22.9,0.326,22,0 730 | 2,92,52,0,0,30.1,0.141,22,0 731 | 3,130,78,23,79,28.4,0.323,34,1 732 | 8,120,86,0,0,28.4,0.259,22,1 733 | 2,174,88,37,120,44.5,0.646,24,1 734 | 2,106,56,27,165,29.0,0.426,22,0 735 | 2,105,75,0,0,23.3,0.560,53,0 736 | 4,95,60,32,0,35.4,0.284,28,0 737 | 0,126,86,27,120,27.4,0.515,21,0 738 | 8,65,72,23,0,32.0,0.600,42,0 739 | 2,99,60,17,160,36.6,0.453,21,0 740 | 1,102,74,0,0,39.5,0.293,42,1 741 | 11,120,80,37,150,42.3,0.785,48,1 742 | 3,102,44,20,94,30.8,0.400,26,0 743 | 1,109,58,18,116,28.5,0.219,22,0 744 | 9,140,94,0,0,32.7,0.734,45,1 745 | 13,153,88,37,140,40.6,1.174,39,0 746 | 12,100,84,33,105,30.0,0.488,46,0 747 | 1,147,94,41,0,49.3,0.358,27,1 748 | 1,81,74,41,57,46.3,1.096,32,0 749 | 3,187,70,22,200,36.4,0.408,36,1 750 | 6,162,62,0,0,24.3,0.178,50,1 751 | 4,136,70,0,0,31.2,1.182,22,1 752 | 1,121,78,39,74,39.0,0.261,28,0 753 | 3,108,62,24,0,26.0,0.223,25,0 754 | 0,181,88,44,510,43.3,0.222,26,1 755 | 8,154,78,32,0,32.4,0.443,45,1 756 | 1,128,88,39,110,36.5,1.057,37,1 757 | 7,137,90,41,0,32.0,0.391,39,0 758 | 0,123,72,0,0,36.3,0.258,52,1 759 | 1,106,76,0,0,37.5,0.197,26,0 760 | 6,190,92,0,0,35.5,0.278,66,1 761 | 2,88,58,26,16,28.4,0.766,22,0 762 | 9,170,74,31,0,44.0,0.403,43,1 763 | 9,89,62,0,0,22.5,0.142,33,0 764 | 10,101,76,48,180,32.9,0.171,63,0 765 | 2,122,70,27,0,36.8,0.340,27,0 766 | 5,121,72,23,112,26.2,0.245,30,0 767 | 1,126,60,0,0,30.1,0.349,47,1 768 | 1,93,70,31,0,30.4,0.315,23,0 769 | -------------------------------------------------------------------------------- /dataset/synthetic.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def make_wave(n_samples=100): 4 | rnd = np.random.RandomState(2046) 5 | x = rnd.uniform(-3, 3, size=n_samples) 6 | y_no_noise = (np.sin(4 * x) + x) 7 | y = (y_no_noise + rnd.normal(size=len(x))) / 2 8 | return x.reshape(-1, 1), y 9 | -------------------------------------------------------------------------------- /envs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hungtuchen/ml-playground/f273ea923cd442f5a51b73c14eb220f3d2b11151/envs/__init__.py -------------------------------------------------------------------------------- /envs/blackjack.py: -------------------------------------------------------------------------------- 1 | import gym 2 | from gym import spaces 3 | from gym.utils import seeding 4 | 5 | def cmp(a, b): 6 | return int((a > b)) - int((a < b)) 7 | 8 | # 1 = Ace, 2-10 = Number cards, Jack/Queen/King = 10 9 | deck = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10] 10 | 11 | 12 | def draw_card(np_random): 13 | return np_random.choice(deck) 14 | 15 | 16 | def draw_hand(np_random): 17 | return [draw_card(np_random), draw_card(np_random)] 18 | 19 | 20 | def usable_ace(hand): # Does this hand have a usable ace? 21 | return 1 in hand and sum(hand) + 10 <= 21 22 | 23 | 24 | def sum_hand(hand): # Return current hand total 25 | if usable_ace(hand): 26 | return sum(hand) + 10 27 | return sum(hand) 28 | 29 | 30 | def is_bust(hand): # Is this hand a bust? 31 | return sum_hand(hand) > 21 32 | 33 | 34 | def score(hand): # What is the score of this hand (0 if bust) 35 | return 0 if is_bust(hand) else sum_hand(hand) 36 | 37 | 38 | def is_natural(hand): # Is this hand a natural blackjack? 39 | return sorted(hand) == [1, 10] 40 | 41 | 42 | class BlackjackEnv(gym.Env): 43 | """Simple blackjack environment 44 | Blackjack is a card game where the goal is to obtain cards that sum to as 45 | near as possible to 21 without going over. They're playing against a fixed 46 | dealer. 47 | Face cards (Jack, Queen, King) have point value 10. 48 | Aces can either count as 11 or 1, and it's called 'usable' at 11. 49 | This game is placed with an infinite deck (or with replacement). 50 | The game starts with each (player and dealer) having one face up and one 51 | face down card. 52 | The player can request additional cards (hit=1) until they decide to stop 53 | (stick=0) or exceed 21 (bust). 54 | After the player sticks, the dealer reveals their facedown card, and draws 55 | until their sum is 17 or greater. If the dealer goes bust the player wins. 56 | If neither player nor dealer busts, the outcome (win, lose, draw) is 57 | decided by whose sum is closer to 21. The reward for winning is +1, 58 | drawing is 0, and losing is -1. 59 | The observation of a 3-tuple of: the players current sum, 60 | the dealer's one showing card (1-10 where 1 is ace), 61 | and whether or not the player holds a usable ace (0 or 1). 62 | This environment corresponds to the version of the blackjack problem 63 | described in Example 5.1 in Reinforcement Learning: An Introduction 64 | by Sutton and Barto (1998). 65 | https://webdocs.cs.ualberta.ca/~sutton/book/the-book.html 66 | """ 67 | def __init__(self, natural=False): 68 | self.action_space = spaces.Discrete(2) 69 | self.observation_space = spaces.Tuple(( 70 | spaces.Discrete(32), 71 | spaces.Discrete(11), 72 | spaces.Discrete(2))) 73 | self._seed() 74 | 75 | # Flag to payout 1.5 on a "natural" blackjack win, like casino rules 76 | # Ref: http://www.bicyclecards.com/how-to-play/blackjack/ 77 | self.natural = natural 78 | # Start the first game 79 | self._reset() 80 | # Number of action 81 | self.nA = 2 82 | 83 | def _seed(self, seed=None): 84 | self.np_random, seed = seeding.np_random(seed) 85 | return [seed] 86 | 87 | def _step(self, action): 88 | assert self.action_space.contains(action) 89 | if action: # hit: add a card to players hand and return 90 | self.player.append(draw_card(self.np_random)) 91 | if is_bust(self.player): 92 | done = True 93 | reward = -1 94 | else: 95 | done = False 96 | reward = 0 97 | else: # stick: play out the dealers hand, and score 98 | done = True 99 | while sum_hand(self.dealer) < 17: 100 | self.dealer.append(draw_card(self.np_random)) 101 | reward = cmp(score(self.player), score(self.dealer)) 102 | if self.natural and is_natural(self.player) and reward == 1: 103 | reward = 1.5 104 | return self._get_obs(), reward, done, {} 105 | 106 | def _get_obs(self): 107 | return (sum_hand(self.player), self.dealer[0], usable_ace(self.player)) 108 | 109 | def _reset(self): 110 | self.dealer = draw_hand(self.np_random) 111 | self.player = draw_hand(self.np_random) 112 | 113 | # Auto-draw another card if the score is less than 12 114 | while sum_hand(self.player) < 12: 115 | self.player.append(draw_card(self.np_random)) 116 | 117 | return self._get_obs() 118 | -------------------------------------------------------------------------------- /envs/cliff_walking.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sys 3 | from gym.envs.toy_text import discrete 4 | 5 | 6 | UP = 0 7 | RIGHT = 1 8 | DOWN = 2 9 | LEFT = 3 10 | 11 | class CliffWalkingEnv(discrete.DiscreteEnv): 12 | 13 | metadata = {'render.modes': ['human', 'ansi']} 14 | 15 | def _limit_coordinates(self, coord): 16 | coord[0] = min(coord[0], self.shape[0] - 1) 17 | coord[0] = max(coord[0], 0) 18 | coord[1] = min(coord[1], self.shape[1] - 1) 19 | coord[1] = max(coord[1], 0) 20 | return coord 21 | 22 | def _calculate_transition_prob(self, current, delta): 23 | new_position = np.array(current) + np.array(delta) 24 | new_position = self._limit_coordinates(new_position).astype(int) 25 | new_state = np.ravel_multi_index(tuple(new_position), self.shape) 26 | reward = -100.0 if self._cliff[tuple(new_position)] else -1.0 27 | is_done = self._cliff[tuple(new_position)] or (tuple(new_position) == (3,11)) 28 | return [(1.0, new_state, reward, is_done)] 29 | 30 | def __init__(self): 31 | self.shape = (4, 12) 32 | 33 | nS = np.prod(self.shape) 34 | nA = 4 35 | 36 | # Cliff Location 37 | self._cliff = np.zeros(self.shape, dtype=np.bool) 38 | self._cliff[3, 1:-1] = True 39 | 40 | # Calculate transition probabilities 41 | P = {} 42 | for s in range(nS): 43 | position = np.unravel_index(s, self.shape) 44 | P[s] = { a : [] for a in range(nA) } 45 | P[s][UP] = self._calculate_transition_prob(position, [-1, 0]) 46 | P[s][RIGHT] = self._calculate_transition_prob(position, [0, 1]) 47 | P[s][DOWN] = self._calculate_transition_prob(position, [1, 0]) 48 | P[s][LEFT] = self._calculate_transition_prob(position, [0, -1]) 49 | 50 | # We always start in state (3, 0) 51 | isd = np.zeros(nS) 52 | isd[np.ravel_multi_index((3,0), self.shape)] = 1.0 53 | 54 | super(CliffWalkingEnv, self).__init__(nS, nA, P, isd) 55 | 56 | def _render(self, mode='human', close=False): 57 | if close: 58 | return 59 | 60 | outfile = StringIO() if mode == 'ansi' else sys.stdout 61 | 62 | for s in range(self.nS): 63 | position = np.unravel_index(s, self.shape) 64 | # print(self.s) 65 | if self.s == s: 66 | output = " x " 67 | elif position == (3,11): 68 | output = " T " 69 | elif self._cliff[position]: 70 | output = " C " 71 | else: 72 | output = " o " 73 | 74 | if position[1] == 0: 75 | output = output.lstrip() 76 | if position[1] == self.shape[1] - 1: 77 | output = output.rstrip() 78 | output += "\n" 79 | 80 | outfile.write(output) 81 | outfile.write("\n") 82 | -------------------------------------------------------------------------------- /envs/gridworld.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sys 3 | from gym.envs.toy_text import discrete 4 | 5 | UP = 0 6 | RIGHT = 1 7 | DOWN = 2 8 | LEFT = 3 9 | 10 | class GridworldEnv(discrete.DiscreteEnv): 11 | """ 12 | Grid World environment from Sutton's Reinforcement Learning book chapter 4. 13 | You are an agent on an MxN grid and your goal is to reach the terminal 14 | state at the top left or the bottom right corner. 15 | For example, a 4x4 grid looks as follows: 16 | T o o o 17 | o x o o 18 | o o o o 19 | o o o T 20 | x is your position and T are the two terminal states. 21 | You can take actions in each direction (UP=0, RIGHT=1, DOWN=2, LEFT=3). 22 | Actions going off the edge leave you in your current state. 23 | You receive a reward of -1 at each step until you reach a terminal state. 24 | """ 25 | 26 | metadata = {'render.modes': ['human', 'ansi']} 27 | 28 | def __init__(self, shape=[4,4]): 29 | if not isinstance(shape, (list, tuple)) or not len(shape) == 2: 30 | raise ValueError('shape argument must be a list/tuple of length 2') 31 | 32 | self.shape = shape 33 | 34 | nS = np.prod(shape) 35 | nA = 4 36 | 37 | MAX_Y = shape[0] 38 | MAX_X = shape[1] 39 | 40 | P = {} 41 | grid = np.arange(nS).reshape(shape) 42 | it = np.nditer(grid, flags=['multi_index']) 43 | 44 | while not it.finished: 45 | s = it.iterindex 46 | y, x = it.multi_index 47 | 48 | P[s] = {a : [] for a in range(nA)} 49 | 50 | is_done = lambda s: s == 0 or s == (nS - 1) 51 | reward = 0.0 if is_done(s) else -1.0 52 | 53 | # We're stuck in a terminal state 54 | if is_done(s): 55 | P[s][UP] = [(1.0, s, reward, True)] 56 | P[s][RIGHT] = [(1.0, s, reward, True)] 57 | P[s][DOWN] = [(1.0, s, reward, True)] 58 | P[s][LEFT] = [(1.0, s, reward, True)] 59 | # Not a terminal state 60 | else: 61 | ns_up = s if y == 0 else s - MAX_X 62 | ns_right = s if x == (MAX_X - 1) else s + 1 63 | ns_down = s if y == (MAX_Y - 1) else s + MAX_X 64 | ns_left = s if x == 0 else s - 1 65 | P[s][UP] = [(1.0, ns_up, reward, is_done(ns_up))] 66 | P[s][RIGHT] = [(1.0, ns_right, reward, is_done(ns_right))] 67 | P[s][DOWN] = [(1.0, ns_down, reward, is_done(ns_down))] 68 | P[s][LEFT] = [(1.0, ns_left, reward, is_done(ns_left))] 69 | 70 | it.iternext() 71 | 72 | # Initial state distribution is uniform 73 | isd = np.ones(nS) / nS 74 | 75 | # We expose the model of the environment for educational purposes 76 | # This should not be used in any model-free learning algorithm 77 | self.P = P 78 | 79 | super(GridworldEnv, self).__init__(nS, nA, P, isd) 80 | 81 | def _render(self, mode='human', close=False): 82 | if close: 83 | return 84 | 85 | outfile = StringIO() if mode == 'ansi' else sys.stdout 86 | 87 | grid = np.arange(self.nS).reshape(self.shape) 88 | it = np.nditer(grid, flags=['multi_index']) 89 | while not it.finished: 90 | s = it.iterindex 91 | y, x = it.multi_index 92 | 93 | if self.s == s: 94 | output = " x " 95 | elif s == 0 or s == self.nS - 1: 96 | output = " T " 97 | else: 98 | output = " o " 99 | 100 | if x == 0: 101 | output = output.lstrip() 102 | if x == self.shape[1] - 1: 103 | output = output.rstrip() 104 | 105 | outfile.write(output) 106 | 107 | if x == self.shape[1] - 1: 108 | outfile.write("\n") 109 | 110 | it.iternext() 111 | -------------------------------------------------------------------------------- /envs/windy_gridworld.py: -------------------------------------------------------------------------------- 1 | import gym 2 | import numpy as np 3 | import sys 4 | from gym.envs.toy_text import discrete 5 | 6 | UP = 0 7 | RIGHT = 1 8 | DOWN = 2 9 | LEFT = 3 10 | 11 | class WindyGridworldEnv(discrete.DiscreteEnv): 12 | 13 | metadata = {'render.modes': ['human', 'ansi']} 14 | 15 | def _limit_coordinates(self, coord): 16 | coord[0] = min(coord[0], self.shape[0] - 1) 17 | coord[0] = max(coord[0], 0) 18 | coord[1] = min(coord[1], self.shape[1] - 1) 19 | coord[1] = max(coord[1], 0) 20 | return coord 21 | 22 | def _calculate_transition_prob(self, current, delta, winds): 23 | new_position = np.array(current) + np.array(delta) + np.array([-1, 0]) * winds[tuple(current)] 24 | new_position = self._limit_coordinates(new_position).astype(int) 25 | new_state = np.ravel_multi_index(tuple(new_position), self.shape) 26 | is_done = tuple(new_position) == (3, 7) 27 | return [(1.0, new_state, -1.0, is_done)] 28 | 29 | def __init__(self): 30 | self.shape = (7, 10) 31 | 32 | nS = np.prod(self.shape) 33 | nA = 4 34 | 35 | # Wind strength 36 | winds = np.zeros(self.shape) 37 | winds[:,[3,4,5,8]] = 1 38 | winds[:,[6,7]] = 2 39 | 40 | # Calculate transition probabilities 41 | P = {} 42 | for s in range(nS): 43 | position = np.unravel_index(s, self.shape) 44 | P[s] = { a : [] for a in range(nA) } 45 | P[s][UP] = self._calculate_transition_prob(position, [-1, 0], winds) 46 | P[s][RIGHT] = self._calculate_transition_prob(position, [0, 1], winds) 47 | P[s][DOWN] = self._calculate_transition_prob(position, [1, 0], winds) 48 | P[s][LEFT] = self._calculate_transition_prob(position, [0, -1], winds) 49 | 50 | # We always start in state (3, 0) 51 | isd = np.zeros(nS) 52 | isd[np.ravel_multi_index((3,0), self.shape)] = 1.0 53 | 54 | super(WindyGridworldEnv, self).__init__(nS, nA, P, isd) 55 | 56 | def _render(self, mode='human', close=False): 57 | if close: 58 | return 59 | 60 | outfile = StringIO() if mode == 'ansi' else sys.stdout 61 | 62 | for s in range(self.nS): 63 | position = np.unravel_index(s, self.shape) 64 | # print(self.s) 65 | if self.s == s: 66 | output = " x " 67 | elif position == (3,7): 68 | output = " T " 69 | else: 70 | output = " o " 71 | 72 | if position[1] == 0: 73 | output = output.lstrip() 74 | if position[1] == self.shape[1] - 1: 75 | output = output.rstrip() 76 | output += "\n" 77 | 78 | outfile.write(output) 79 | outfile.write("\n") 80 | -------------------------------------------------------------------------------- /reinforcement/double_q_learning.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from collections import defaultdict 3 | import itertools 4 | import random 5 | 6 | from utils import plotting 7 | 8 | def make_double_q_epsilon_greedy_policy(epsilon, nA, Q1, Q2): 9 | """ 10 | Creates an epsilon-greedy policy based on sum of given Q-functions and epsilon. 11 | 12 | Args: 13 | epsilon: The probability to select a random action . float between 0 and 1. 14 | nA: Number of actions in the environment. 15 | Q1, Q2: A dictionary that maps from state -> action-values. 16 | Each value is a numpy array of length nA (see below) 17 | 18 | Returns: 19 | A function that takes the state as an argument and returns 20 | the probabilities for each action in the form of a numpy array of length nA. 21 | 22 | """ 23 | def policy_fn(state): 24 | # 1 / epsilon for non-greedy actions 25 | probs = (epsilon / nA) * np.ones(nA) 26 | 27 | summed_Q = Q1[state] + Q2[state] 28 | 29 | greedy_action = summed_Q.argmax() 30 | # (1 / epsilon + (1 - epsilon)) for greedy action 31 | probs[greedy_action] += 1.0 - epsilon 32 | 33 | return probs 34 | 35 | return policy_fn 36 | 37 | def double_q_learning(env, num_episodes, discount_factor=1.0, alpha=0.5, epsilon=0.1): 38 | """ 39 | Double Q-Learning algorithm: Off-policy TD control that avoid maxmization bias. 40 | Finds the optimal greedy policy while following an epsilon-greedy policy. 41 | 42 | Args: 43 | env: OpenAI environment. 44 | num_episodes: Number of episodes to run for. 45 | discount_factor: Lambda time discount factor. 46 | alpha: TD learning rate. 47 | epsilon: Chance the sample a random action. Float betwen 0 and 1. 48 | 49 | Returns: 50 | A tuple (Q1, Q2, episode_lengths). 51 | Q1 + Q2 is the optimal action-value function, a dictionary mapping state -> action values. 52 | stats is an EpisodeStats object with two numpy arrays for episode_lengths and episode_rewards. 53 | """ 54 | 55 | # The final action-value functions. 56 | # A nested dictionary that maps state -> (action -> action-value). 57 | Q1 = defaultdict(lambda: np.zeros(env.action_space.n)) 58 | Q2 = defaultdict(lambda: np.zeros(env.action_space.n)) 59 | 60 | # keeps track of useful statistics 61 | stats = plotting.EpisodeStats( 62 | episode_lengths=np.zeros(num_episodes), 63 | episode_rewards=np.zeros(num_episodes)) 64 | 65 | policy = make_double_q_epsilon_greedy_policy(epsilon, env.action_space.n, Q1, Q2) 66 | 67 | for i_episode in range(num_episodes): 68 | current_state = env.reset() 69 | # keep track number of time-step per episode only for plotting 70 | for t in itertools.count(): 71 | # choose the action based on epsilon greedy policy 72 | action_probs = policy(current_state) 73 | action = np.random.choice(np.arange(len(action_probs)), p=action_probs) 74 | next_state, reward, done, _ = env.step(action) 75 | 76 | if random.random() < 0.5: 77 | # Update Q1: using Q1 to select max action yet using Q2's estimate. 78 | greedy_next_action = Q1[next_state].argmax() 79 | td_target = reward + discount_factor * Q2[next_state][greedy_next_action] 80 | td_error = td_target - Q1[current_state][action] 81 | Q1[current_state][action] += alpha * td_error 82 | else: 83 | # Update Q2: using Q2 to select max action yet using Q1's estimate. 84 | greedy_next_action = Q2[next_state].argmax() 85 | td_target = reward + discount_factor * Q1[next_state][greedy_next_action] 86 | td_error = td_target - Q2[current_state][action] 87 | Q2[current_state][action] += alpha * td_error 88 | 89 | # improve epsilon greedy policy using new evaluate Q 90 | policy = make_double_q_epsilon_greedy_policy(epsilon, env.action_space.n, Q1, Q2) 91 | 92 | # update statistics 93 | stats.episode_rewards[i_episode] += reward 94 | stats.episode_lengths[i_episode] = t 95 | 96 | if done: 97 | break 98 | else: 99 | current_state = next_state 100 | 101 | return Q1, Q2, stats 102 | -------------------------------------------------------------------------------- /reinforcement/mc_control_epsilon_greedy.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from collections import defaultdict 3 | 4 | from utils.policy import make_epsilon_greedy_policy 5 | 6 | def mc_control_epsilon_greedy(env, num_episodes, discount_factor=1.0, epsilon=0.1): 7 | """ 8 | Monte Carlo Control using Epsilon-Greedy policies. 9 | Finds an optimal epsilon-greedy policy. 10 | 11 | Args: 12 | env: OpenAI gym environment. 13 | num_episodes: Nubmer of episodes to sample. 14 | discount_factor: Lambda discount factor. 15 | epsilon: Chance the sample a random action. Float betwen 0 and 1. 16 | 17 | Returns: 18 | A tuple (Q, policy). 19 | Q is a dictionary mapping state -> action values. 20 | policy is a function that takes an state as an argument and returns 21 | action probabilities 22 | """ 23 | 24 | # Keeps track of sum and count of returns for each state 25 | # to calculate an average. We could use an array to save all 26 | # returns (like in the book) but that's memory inefficient. 27 | returns_sum = defaultdict(float) 28 | returns_count = defaultdict(float) 29 | 30 | # The final action-value function. 31 | # A nested dictionary that maps state -> [...action_value](key corresponds to action) 32 | Q = defaultdict(lambda: np.zeros(env.action_space.n)) 33 | 34 | # Initialize the epsilon-greedy policy 35 | policy = make_epsilon_greedy_policy(Q, epsilon, env.action_space.n) 36 | 37 | for episode in range(num_episodes): 38 | # keep track of visited rewards, states, actions in current episode 39 | visited_rewards = [] 40 | visited_state_actions = [] 41 | 42 | current_state = env.reset() 43 | while True: 44 | # choose the action based on policy 45 | probs = policy(current_state) 46 | action = np.random.choice(np.arange(len(probs)), p=probs) 47 | 48 | next_state, reward, done, _ = env.step(action) 49 | visited_state_actions.append((current_state, action)) 50 | visited_rewards.append(reward) 51 | 52 | if done: 53 | # we only take the first time the (state, action) is visited in the episode 54 | # into account if we use first-visit Monte Carlo methonds 55 | was_state_action_visited = {} 56 | for i, state_action in enumerate(visited_state_actions): 57 | # uncomment this part if you want to use first-visit 58 | # if state_action not in was_state_action_visited: 59 | # was_state_action_visited[state_action] = True 60 | returns_count[state_action] += 1.0 61 | # calculate the return (expected rewards from current state_action onwards) 62 | # Note: we need to take care of discount_factor 63 | return_ = 0.0 64 | for j, reward in enumerate(visited_rewards[i:]): 65 | return_ += (discount_factor ** j) * reward 66 | returns_sum[state_action] += return_ 67 | 68 | # evaluate Q after every episode (We can improve policy when we have more experience) 69 | for state_action, count in returns_count.items(): 70 | state, action = state_action 71 | Q[state][action] = returns_sum[state_action] / count 72 | 73 | # improve the policy based on new evaluated Q 74 | policy = make_epsilon_greedy_policy(Q, epsilon, env.action_space.n) 75 | break 76 | else: 77 | current_state = next_state 78 | 79 | return Q, policy 80 | -------------------------------------------------------------------------------- /reinforcement/mc_control_importance_sampling.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from collections import defaultdict 3 | 4 | from utils.policy import make_greedy_policy 5 | 6 | def mc_control_importance_sampling(env, num_episodes, behavior_policy, discount_factor=1.0): 7 | """ 8 | Monte Carlo Control Off-Policy Control using Weighted Importance Sampling. 9 | Finds an optimal greedy policy. 10 | Ch5.7 Sutton & Barto, Reinforcement Learning: An Introduction 11 | Args: 12 | env: OpenAI gym environment. 13 | num_episodes: Nubmer of episodes to sample. 14 | behavior_policy: The behavior to follow while generating episodes. 15 | A function that given an observation returns a vector of probabilities for each action. 16 | discount_factor: Lambda discount factor. 17 | 18 | Returns: 19 | A tuple (Q, policy). 20 | Q is a dictionary mapping state -> action values. 21 | policy is a function that takes an observation as an argument and returns 22 | action probabilities. This is the optimal greedy policy. 23 | """ 24 | 25 | # The final action-value function. 26 | # A dictionary that maps state -> action values 27 | Q = defaultdict(lambda: np.zeros(env.action_space.n)) 28 | # A cumulative sum of weighted importance sampling ratio 29 | C = defaultdict(lambda: np.zeros(env.action_space.n)) 30 | 31 | greedy_policy = make_greedy_policy(Q, env.action_space.n) 32 | 33 | for episode in range(num_episodes): 34 | # keep track of visited rewards, states, actions in current episode 35 | visited_rewards = [] 36 | visited_state_actions = [] 37 | 38 | current_state = env.reset() 39 | while True: 40 | # choose the action based on behavior_policy 41 | probs = behavior_policy(current_state) 42 | action = np.random.choice(np.arange(len(probs)), p=probs) 43 | 44 | next_state, reward, done, _ = env.step(action) 45 | visited_state_actions.append((current_state, action)) 46 | visited_rewards.append(reward) 47 | 48 | if done: 49 | # cumulative reward 50 | G = 0 51 | # cumulative importance sampling ratio 52 | W = 1.0 53 | # iterate episode backwards and incrementally 54 | for t, state_action in reversed(list(enumerate(visited_state_actions))): 55 | state, action = state_action 56 | G = discount_factor * G + visited_rewards[t] 57 | C[state][action] += W 58 | # We use weighted importance sampling so divide by weighted average 59 | Q[state][action] += (W / C[state][action]) * (G - Q[state][action]) 60 | 61 | # improve the policy based on new evaluated Q 62 | greedy_policy = make_greedy_policy(Q, env.action_space.n) 63 | # if target policy is not the same as behavior policy, then we can't 64 | # evaluate it (not the same trajectory), so break 65 | greedy_action = greedy_policy(state).argmax() 66 | if action != greedy_action: 67 | break 68 | W *= 1.0 / behavior_policy(state)[action] 69 | # current episode is over 70 | break 71 | else: 72 | current_state = next_state 73 | 74 | 75 | return Q, greedy_policy 76 | -------------------------------------------------------------------------------- /reinforcement/mc_prediction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from collections import defaultdict 3 | 4 | def mc_prediction(policy, env, num_episodes, discount_factor=1.0): 5 | """ 6 | Monte Carlo prediction algorithm. Calculates the value function 7 | for a given policy using sampling. 8 | 9 | Args: 10 | policy: A function that maps an state to action probabilities. 11 | env: OpenAI gym environment. 12 | num_episodes: Nubmer of episodes to sample. 13 | discount_factor: Lambda discount factor. 14 | 15 | Returns: 16 | A dictionary that maps from state -> value. 17 | The state is a tuple and the value is a float. 18 | """ 19 | 20 | # Keeps track of sum and count of returns for each state 21 | # to calculate an average. We could use an array to save all 22 | # returns (like in the book) but that's memory inefficient. 23 | returns_sum = defaultdict(float) 24 | returns_count = defaultdict(float) 25 | 26 | # The final value function 27 | V = defaultdict(float) 28 | 29 | for episode in range(num_episodes): 30 | # keep track of visited rewards and states in current episode 31 | visited_rewards = [] 32 | visited_states = [] 33 | 34 | current_state = env.reset() 35 | while True: 36 | # choose the action based on policy 37 | probs = policy(current_state) 38 | action = np.random.choice(np.arange(len(probs)), p=probs) 39 | 40 | next_state, reward, done, _ = env.step(action) 41 | visited_states.append(current_state) 42 | visited_rewards.append(reward) 43 | 44 | if done: 45 | # we only take the first time the state is visited in the episode 46 | # into account if we use first-visit Monte Carlo methonds 47 | was_state_visited = {} 48 | for i, state in enumerate(visited_states): 49 | # uncomment this part if you want to use first-visit 50 | # if state not in was_state_visited: 51 | # was_state_visited[state] = True 52 | returns_count[state] += 1.0 53 | # calculate the return (expected rewards from current state onwards) 54 | # Note: we need to take care of discount_factor 55 | return_ = 0.0 56 | for j, reward in enumerate(visited_rewards[i:]): 57 | return_ += (discount_factor ** j) * reward 58 | returns_sum[state] += return_ 59 | 60 | break 61 | else: 62 | current_state = next_state 63 | # use eperical mean to predict value function 64 | for state, count in returns_count.items(): 65 | V[state] = returns_sum[state] / count 66 | 67 | return V 68 | -------------------------------------------------------------------------------- /reinforcement/policy_evaluation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import sys\n", 13 | "if \"../\" not in sys.path:\n", 14 | " sys.path.append(\"../\")\n", 15 | "\n", 16 | "from envs.gridworld import GridworldEnv\n", 17 | "from policy_evaluation import policy_eval\n", 18 | "\n", 19 | "# for auto-reloading external modules\n", 20 | "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n", 21 | "%load_ext autoreload\n", 22 | "%autoreload 2" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 2, 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "env = GridworldEnv()" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 3, 39 | "metadata": { 40 | "collapsed": false, 41 | "scrolled": true 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "random_policy = np.ones([env.nS, env.nA]) / env.nA\n", 46 | "v = policy_eval(random_policy, env)" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 4, 52 | "metadata": { 53 | "collapsed": false 54 | }, 55 | "outputs": [], 56 | "source": [ 57 | "# Test: Make sure the evaluated policy is what we expected\n", 58 | "expected_v = np.array([0, -14, -20, -22, -14, -18, -20, -20, -20, -20, -18, -14, -22, -20, -14, 0])\n", 59 | "np.testing.assert_array_almost_equal(v, expected_v, decimal=2)" 60 | ] 61 | } 62 | ], 63 | "metadata": { 64 | "anaconda-cloud": {}, 65 | "kernelspec": { 66 | "display_name": "Python [py35]", 67 | "language": "python", 68 | "name": "Python [py35]" 69 | }, 70 | "language_info": { 71 | "codemirror_mode": { 72 | "name": "ipython", 73 | "version": 3 74 | }, 75 | "file_extension": ".py", 76 | "mimetype": "text/x-python", 77 | "name": "python", 78 | "nbconvert_exporter": "python", 79 | "pygments_lexer": "ipython3", 80 | "version": "3.5.2" 81 | } 82 | }, 83 | "nbformat": 4, 84 | "nbformat_minor": 0 85 | } 86 | -------------------------------------------------------------------------------- /reinforcement/policy_evaluation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def policy_eval(policy, env, discount_factor=1.0, theta=0.00001): 4 | """ 5 | Evaluate a policy given an environment and a full description of the environment's dynamics. 6 | 7 | Args: 8 | policy: [S, A] shaped matrix representing the policy. 9 | env: OpenAI env. env.P represents the transition probabilities of the environment. 10 | env.P[s][a] is a (prob, next_state, reward, done) tuple. 11 | theta: We stop evaluation once our value function change is less than theta for all states. 12 | discount_factor: gamma discount factor. 13 | 14 | Returns: 15 | Vector of length env.nS representing the value function. 16 | """ 17 | # Start with a random (all 0) value function 18 | V = np.zeros(env.nS) 19 | 20 | while True: 21 | V_converged = True 22 | # iterate over state 23 | for s in range(env.nS): 24 | new_v = 0 25 | # iterate over action 26 | for a in range(env.nA): 27 | # iterate over next state given transition probability 28 | for prob, next_state, reward, done in env.P[s][a]: 29 | # Bellman expectation backup 30 | new_v += policy[s][a] * prob * (reward + discount_factor * V[next_state]) 31 | # if the update between two iterration is less then theta 32 | # for every state of V, then the value function is converged 33 | # other is not 34 | if np.abs(new_v - V[s]) > theta: 35 | V_converged = False 36 | # update new value 37 | V[s] = new_v 38 | # stop if V_converged 39 | if V_converged: 40 | return np.array(V) 41 | -------------------------------------------------------------------------------- /reinforcement/policy_iteration.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import sys\n", 13 | "if \"../\" not in sys.path:\n", 14 | " sys.path.append(\"../\")\n", 15 | "\n", 16 | "from envs.gridworld import GridworldEnv\n", 17 | "from policy_iteration import policy_improvement\n", 18 | "\n", 19 | "# for auto-reloading external modules\n", 20 | "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n", 21 | "%load_ext autoreload\n", 22 | "%autoreload 2" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 2, 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "env = GridworldEnv()" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 3, 39 | "metadata": { 40 | "collapsed": false 41 | }, 42 | "outputs": [], 43 | "source": [ 44 | "policy, v = policy_improvement(env)" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 4, 50 | "metadata": { 51 | "collapsed": false 52 | }, 53 | "outputs": [ 54 | { 55 | "name": "stdout", 56 | "output_type": "stream", 57 | "text": [ 58 | "Policy Probability Distribution:\n", 59 | "[[ 1. 0. 0. 0.]\n", 60 | " [ 0. 0. 0. 1.]\n", 61 | " [ 0. 0. 0. 1.]\n", 62 | " [ 0. 0. 1. 0.]\n", 63 | " [ 1. 0. 0. 0.]\n", 64 | " [ 1. 0. 0. 0.]\n", 65 | " [ 1. 0. 0. 0.]\n", 66 | " [ 0. 0. 1. 0.]\n", 67 | " [ 1. 0. 0. 0.]\n", 68 | " [ 1. 0. 0. 0.]\n", 69 | " [ 0. 1. 0. 0.]\n", 70 | " [ 0. 0. 1. 0.]\n", 71 | " [ 1. 0. 0. 0.]\n", 72 | " [ 0. 1. 0. 0.]\n", 73 | " [ 0. 1. 0. 0.]\n", 74 | " [ 1. 0. 0. 0.]]\n", 75 | "\n", 76 | "Reshaped Grid Policy (0=up, 1=right, 2=down, 3=left):\n", 77 | "[[0 3 3 2]\n", 78 | " [0 0 0 2]\n", 79 | " [0 0 1 2]\n", 80 | " [0 1 1 0]]\n", 81 | "\n", 82 | "Value Function:\n", 83 | "[ 0. -1. -2. -3. -1. -2. -3. -2. -2. -3. -2. -1. -3. -2. -1. 0.]\n", 84 | "\n", 85 | "Reshaped Grid Value Function:\n", 86 | "[[ 0. -1. -2. -3.]\n", 87 | " [-1. -2. -3. -2.]\n", 88 | " [-2. -3. -2. -1.]\n", 89 | " [-3. -2. -1. 0.]]\n", 90 | "\n" 91 | ] 92 | } 93 | ], 94 | "source": [ 95 | "policy, v = policy_improvement(env)\n", 96 | "print(\"Policy Probability Distribution:\")\n", 97 | "print(policy)\n", 98 | "print(\"\")\n", 99 | "\n", 100 | "print(\"Reshaped Grid Policy (0=up, 1=right, 2=down, 3=left):\")\n", 101 | "print(np.reshape(np.argmax(policy, axis=1), env.shape))\n", 102 | "print(\"\")\n", 103 | "\n", 104 | "print(\"Value Function:\")\n", 105 | "print(v)\n", 106 | "print(\"\")\n", 107 | "\n", 108 | "print(\"Reshaped Grid Value Function:\")\n", 109 | "print(v.reshape(env.shape))\n", 110 | "print(\"\")" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 5, 116 | "metadata": { 117 | "collapsed": false 118 | }, 119 | "outputs": [], 120 | "source": [ 121 | "# Test the value function\n", 122 | "expected_v = np.array([ 0, -1, -2, -3, -1, -2, -3, -2, -2, -3, -2, -1, -3, -2, -1, 0])\n", 123 | "np.testing.assert_array_almost_equal(v, expected_v, decimal=2)" 124 | ] 125 | } 126 | ], 127 | "metadata": { 128 | "kernelspec": { 129 | "display_name": "Python [py35]", 130 | "language": "python", 131 | "name": "Python [py35]" 132 | }, 133 | "language_info": { 134 | "codemirror_mode": { 135 | "name": "ipython", 136 | "version": 3 137 | }, 138 | "file_extension": ".py", 139 | "mimetype": "text/x-python", 140 | "name": "python", 141 | "nbconvert_exporter": "python", 142 | "pygments_lexer": "ipython3", 143 | "version": "3.5.2" 144 | } 145 | }, 146 | "nbformat": 4, 147 | "nbformat_minor": 0 148 | } 149 | -------------------------------------------------------------------------------- /reinforcement/policy_iteration.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import math 3 | 4 | from policy_evaluation import policy_eval 5 | 6 | def policy_improvement(env, policy_eval_fn=policy_eval, discount_factor=1.0): 7 | """ 8 | Policy Improvement Algorithm. Iteratively evaluates and improves a policy 9 | until an optimal policy is found. 10 | 11 | Args: 12 | env: The OpenAI envrionment. env.P represents the transition probabilities of the environment. 13 | env.P[s][a] is a (prob, next_state, reward, done) tuple. 14 | policy_eval_fn: Policy Evaluation function that takes 3 arguments: 15 | policy, env, discount_factor. 16 | discount_factor: Lambda discount factor. 17 | 18 | Returns: 19 | A tuple (policy, V). 20 | policy is the optimal policy, a matrix of shape [S, A] where each state s 21 | contains a valid probability distribution over actions. 22 | V is the value function for the optimal policy. 23 | 24 | """ 25 | # Start with a random policy 26 | policy = np.ones([env.nS, env.nA]) / env.nA 27 | 28 | while True: 29 | # evalute how new policy performs 30 | V = policy_eval_fn(policy, env, discount_factor) 31 | # prepare for new policy, since new policy will be deterministic 32 | # we init probability for all actions as 0.0 and give only the best 1.0 33 | new_policy = np.zeros_like(policy) 34 | 35 | is_policy_optimized = True 36 | # iterate over state 37 | for s in range(env.nS): 38 | action_taken = policy[s].argmax() 39 | # value of current state given action, we will use it to choose best action 40 | action_values = np.zeros(env.nA) 41 | # iterate over action 42 | for a in range(env.nA): 43 | # iterate over next state given transition probability 44 | for prob, next_state, reward, done in env.P[s][a]: 45 | action_values[a] += prob * (reward + discount_factor * V[next_state]) 46 | # choose best action based on which action give us max value 47 | best_action = action_values.argmax() 48 | # if previous choosen action base on last policy does not equal to new action 49 | # based on max action value, then we didn't obtain optimal policy 50 | if action_taken != best_action: 51 | is_policy_optimized = False 52 | # update new policy no matter what 53 | new_policy[s][best_action] = 1.0 54 | if is_policy_optimized: 55 | return policy, V 56 | 57 | policy = new_policy 58 | -------------------------------------------------------------------------------- /reinforcement/q_learning.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from collections import defaultdict 3 | import itertools 4 | 5 | from utils import plotting 6 | from utils.policy import make_epsilon_greedy_policy 7 | 8 | def q_learning(env, num_episodes, discount_factor=1.0, alpha=0.5, epsilon=0.1): 9 | """ 10 | Q-Learning algorithm: Off-policy TD control. Finds the optimal greedy policy 11 | while following an epsilon-greedy policy 12 | 13 | Args: 14 | env: OpenAI environment. 15 | num_episodes: Number of episodes to run for. 16 | discount_factor: Lambda time discount factor. 17 | alpha: TD learning rate. 18 | epsilon: Chance the sample a random action. Float betwen 0 and 1. 19 | 20 | Returns: 21 | A tuple (Q, episode_lengths). 22 | Q is the optimal action-value function, a dictionary mapping state -> action values. 23 | stats is an EpisodeStats object with two numpy arrays for episode_lengths and episode_rewards. 24 | """ 25 | 26 | # The final action-value function. 27 | # A nested dictionary that maps state -> (action -> action-value). 28 | Q = defaultdict(lambda: np.zeros(env.action_space.n)) 29 | 30 | # keeps track of useful statistics 31 | stats = plotting.EpisodeStats( 32 | episode_lengths=np.zeros(num_episodes), 33 | episode_rewards=np.zeros(num_episodes)) 34 | 35 | policy = make_epsilon_greedy_policy(Q, epsilon, env.action_space.n) 36 | 37 | for i_episode in range(num_episodes): 38 | current_state = env.reset() 39 | # keep track number of time-step per episode only for plotting 40 | for t in itertools.count(): 41 | # choose the action based on epsilon greedy policy 42 | action_probs = policy(current_state) 43 | action = np.random.choice(np.arange(len(action_probs)), p=action_probs) 44 | next_state, reward, done, _ = env.step(action) 45 | 46 | # sse the greedy action to evaluate Q, not the one we actually follow 47 | greedy_next_action = Q[next_state].argmax() 48 | # evaluate Q using estimated action value of (next_state, greedy_next_action) 49 | td_target = reward + discount_factor * Q[next_state][greedy_next_action] 50 | td_error = td_target - Q[current_state][action] 51 | Q[current_state][action] += alpha * td_error 52 | 53 | # improve epsilon greedy policy using new evaluate Q 54 | policy = make_epsilon_greedy_policy(Q, epsilon, env.action_space.n) 55 | 56 | # update statistics 57 | stats.episode_rewards[i_episode] += reward 58 | stats.episode_lengths[i_episode] = t 59 | 60 | if done: 61 | break 62 | else: 63 | current_state = next_state 64 | 65 | return Q, stats 66 | -------------------------------------------------------------------------------- /reinforcement/q_learning_fa.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import itertools 3 | from sklearn.linear_model import SGDRegressor 4 | 5 | from utils import plotting 6 | 7 | class Estimator(): 8 | """ 9 | Value Function approximator. 10 | """ 11 | 12 | def __init__(self, env, scaler, featurizer): 13 | self.env = env 14 | self.scaler = scaler 15 | self.featurizer = featurizer 16 | # We create a separate model for each action in the environment's 17 | # action space. Alternatively we could somehow encode the action 18 | # into the features, but this way it's easier to code up. 19 | self.models = [] 20 | for _ in range(self.env.action_space.n): 21 | model = SGDRegressor(learning_rate="constant") 22 | # We need to call partial_fit once to initialize the model 23 | # or we get a NotFittedError when trying to make a prediction 24 | # This is quite hacky. 25 | initial_feature = self.featurize_state(env.reset()).reshape(1, -1) 26 | model.partial_fit(initial_feature, [0]) 27 | self.models.append(model) 28 | 29 | def featurize_state(self, state): 30 | """ 31 | Returns the featurized representation for a state. 32 | """ 33 | scaled = self.scaler.transform([state]) 34 | featurized = self.featurizer.transform(scaled) 35 | return featurized[0] 36 | 37 | def predict(self, s, a=None): 38 | """ 39 | Makes value function predictions. 40 | 41 | Args: 42 | s: state to make a prediction for 43 | a: (Optional) action to make a prediction for 44 | 45 | Returns 46 | If an action a is given this returns a single number as the prediction. 47 | If no action is given this returns a vector or predictions for all actions 48 | in the environment where pred[i] is the prediction for action i. 49 | 50 | """ 51 | models = self.models 52 | feature = self.featurize_state(s).reshape(1, -1) 53 | 54 | if a is not None: 55 | return models[a].predict(feature)[0] 56 | else: 57 | return [model.predict(feature) for model in models] 58 | 59 | def update(self, s, a, y): 60 | """ 61 | Updates the estimator parameters for a given state and action towards 62 | the target y. 63 | """ 64 | feature = self.featurize_state(s).reshape(1, -1) 65 | model = self.models[a] 66 | model.partial_fit(feature, [y]) 67 | 68 | def make_epsilon_greedy_policy(estimator, epsilon, nA): 69 | """ 70 | Creates an epsilon-greedy policy based on a given Q-function approximator and epsilon. 71 | 72 | Args: 73 | estimator: An estimator that returns q values for a given state 74 | epsilon: The probability to select a random action . float between 0 and 1. 75 | nA: Number of actions in the environment. 76 | 77 | Returns: 78 | A function that takes the observation as an argument and returns 79 | the probabilities for each action in the form of a numpy array of length nA. 80 | 81 | """ 82 | def policy_fn(observation): 83 | A = np.ones(nA, dtype=float) * epsilon / nA 84 | q_values = estimator.predict(observation) 85 | best_action = np.argmax(q_values) 86 | A[best_action] += (1.0 - epsilon) 87 | return A 88 | return policy_fn 89 | 90 | def q_learning_fa(env, estimator, num_episodes, discount_factor=1.0, epsilon=0.1, epsilon_decay=1.0): 91 | """ 92 | Q-Learning algorithm for fff-policy TD control using Function Approximation. 93 | Finds the optimal greedy policy while following an epsilon-greedy policy. 94 | 95 | Args: 96 | env: OpenAI environment. 97 | estimator: Action-Value function estimator 98 | num_episodes: Number of episodes to run for. 99 | discount_factor: Lambda time discount factor. 100 | epsilon: Chance the sample a random action. Float betwen 0 and 1. 101 | epsilon_decay: Each episode, epsilon is decayed by this factor 102 | 103 | Returns: 104 | An EpisodeStats object with two numpy arrays for episode_lengths and episode_rewards. 105 | """ 106 | 107 | # keeps track of useful statistics 108 | stats = plotting.EpisodeStats( 109 | episode_lengths=np.zeros(num_episodes), 110 | episode_rewards=np.zeros(num_episodes)) 111 | 112 | for i_episode in range(num_episodes): 113 | policy = make_epsilon_greedy_policy( 114 | estimator, epsilon * epsilon_decay**i_episode, env.action_space.n) 115 | 116 | current_state = env.reset() 117 | # keep track number of time-step per episode only for plotting 118 | for t in itertools.count(): 119 | # choose the action based on epsilon greedy policy 120 | action_probs = policy(current_state) 121 | action = np.random.choice(np.arange(len(action_probs)), p=action_probs) 122 | next_state, reward, done, _ = env.step(action) 123 | 124 | # use the greedy action to evaluate Q, not the one we actually follow 125 | greedy_next_action = np.argmax(estimator.predict(next_state)) 126 | # evaluate Q using estimated action value of (next_state, greedy_next_action) 127 | td_target = reward + discount_factor * estimator.predict(next_state, greedy_next_action) 128 | # update weights 129 | estimator.update(current_state, action, td_target) 130 | 131 | # update statistics 132 | stats.episode_rewards[i_episode] += reward 133 | stats.episode_lengths[i_episode] = t 134 | 135 | if done: 136 | break 137 | else: 138 | current_state = next_state 139 | 140 | return stats 141 | -------------------------------------------------------------------------------- /reinforcement/reinforce_baseline.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from itertools import count 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | import torch.autograd as autograd 8 | from torch.autograd import Variable 9 | 10 | from utils import plotting 11 | 12 | class PolicyEstimator(nn.Module): 13 | """ 14 | Policy Function approximator. 15 | """ 16 | 17 | def __init__(self, D_in, D_out, hidden_size = 128): 18 | super(PolicyEstimator, self).__init__() 19 | # define network structure 20 | self.W1 = nn.Linear(D_in, hidden_size) 21 | self.W2 = nn.Linear(hidden_size, D_out) 22 | 23 | def forward(self, state): 24 | h = F.relu(self.W1(state)) 25 | action_scores = self.W2(h) 26 | return F.softmax(action_scores) 27 | 28 | class ValueEstimator(nn.Module): 29 | """ 30 | Value Function approximator. 31 | """ 32 | 33 | def __init__(self, D_in, hidden_size = 128): 34 | super(ValueEstimator, self).__init__() 35 | # define network structure 36 | self.W1 = nn.Linear(D_in, hidden_size) 37 | # output a score 38 | self.W2 = nn.Linear(hidden_size, 1) 39 | 40 | def forward(self, state): 41 | h = F.relu(self.W1(state)) 42 | state_values = self.W2(h) 43 | return state_values 44 | 45 | def discount_rewards(rewards, gamma): 46 | """ 47 | take 1D float array of rewards and compute discounted reward 48 | Reference: https://gist.github.com/karpathy/a4166c7fe253700972fcbc77e4ea32c5#file-pg-pong-py-L18 49 | """ 50 | discounted_rewards = np.zeros_like(rewards) 51 | running_add = 0 52 | for t in reversed(range(len(rewards))): 53 | running_add = running_add * gamma + rewards[t] 54 | discounted_rewards[t] = running_add 55 | return discounted_rewards 56 | 57 | def reinforce_baseline(env, policy_estimator, policy_optimizer, value_estimator, value_optimizer, 58 | num_episodes, discount_factor=1.0, render=True): 59 | """ 60 | REINFORCE (Monte Carlo Policy Gradient) Algorithm with Baseline. 61 | Optimizes the policy function approximator using policy gradient. 62 | 63 | Args: 64 | env: OpenAI environment. 65 | policy_estimator: Policy Function to be optimized 66 | policy_optimizer: Optimizer for Policy Function 67 | value_estimator: Value function approximator, used as a baseline 68 | value_optimizer: Optimizer for Value Function 69 | num_episodes: Number of episodes to run for 70 | discount_factor: Time-discount factor 71 | render: Render the training process or not 72 | 73 | Returns: 74 | An EpisodeStats object with two numpy arrays for episode_lengths and episode_rewards. 75 | """ 76 | running_reward = 0 77 | # Keeps track of useful statistics 78 | stats = plotting.EpisodeStats( 79 | episode_lengths=np.zeros(num_episodes), 80 | episode_rewards=np.zeros(num_episodes)) 81 | 82 | for i_episode in range(num_episodes): 83 | episode_actions = [] 84 | episode_rewards = [] 85 | episode_baselines = [] 86 | 87 | state = env.reset() 88 | for t in count(1): 89 | state = torch.from_numpy(state).float().unsqueeze(0) 90 | # Calculate the probability distribution of actions 91 | probs = policy_estimator(Variable(state)) 92 | # Select action by distribution estimated above 93 | action = probs.multinomial() 94 | # Calculate state value as baseline 95 | baseline = value_estimator(Variable(state)) 96 | 97 | state, reward, done, _ = env.step(action.data[0, 0]) 98 | if render: 99 | env.render() 100 | # Keep track of visited action, reward and baseline for later update 101 | episode_actions.append(action) 102 | episode_rewards.append(reward) 103 | episode_baselines.append(baseline) 104 | 105 | # update statistics 106 | stats.episode_rewards[i_episode] += reward 107 | stats.episode_lengths[i_episode] = t 108 | 109 | if done: 110 | break 111 | 112 | # start updating policy and value estimator 113 | discount_rs = discount_rewards(episode_rewards, discount_factor) 114 | # standardize the rewards to be unit normal (helps control the gradient estimator variance) 115 | discount_rs -= discount_rs.mean() 116 | discount_rs /= discount_rs.std() 117 | 118 | # define creterion and calculate loss for value funcion 119 | value_target = Variable(torch.Tensor(discount_rs), requires_grad=False) 120 | value_predict = torch.cat(episode_baselines) 121 | value_loss = F.smooth_l1_loss(value_predict, value_target) 122 | 123 | # Registers a reward obtained as a result of a stochastic process. 124 | # Differentiating stochastic nodes requires providing them with reward value. 125 | for baseline, action, r in zip(episode_baselines, episode_actions, discount_rs): 126 | action.reinforce(r - baseline.data) 127 | 128 | 129 | # Remove gradient from previous steps 130 | policy_optimizer.zero_grad() 131 | value_optimizer.zero_grad() 132 | 133 | # Perform backward pass 134 | torch.cat(episode_actions).backward() 135 | value_loss.backward() 136 | 137 | # Use optimizer to update 138 | policy_optimizer.step() 139 | value_optimizer.step() 140 | 141 | # Book-keep the running reward 142 | running_reward = running_reward * 0.99 + sum(episode_rewards) * 0.01 143 | if i_episode % 10 == 0: 144 | print('Episode {}\tRunning reward: {:.2f}'.format(i_episode, running_reward)) 145 | if running_reward > 200: 146 | print("Solved! Running reward is now {} and " \ 147 | "the last episode runs to {} time steps!".format(running_reward, t)) 148 | break 149 | 150 | return stats 151 | -------------------------------------------------------------------------------- /reinforcement/sarsa.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from collections import defaultdict 3 | import itertools 4 | 5 | from utils import plotting 6 | from utils.policy import make_epsilon_greedy_policy 7 | 8 | def sarsa(env, num_episodes, discount_factor=1.0, alpha=0.5, epsilon=0.1): 9 | """ 10 | SARSA algorithm: On-policy TD control. Finds the optimal epsilon-greedy policy. 11 | 12 | Args: 13 | env: OpenAI environment. 14 | num_episodes: Number of episodes to run for. 15 | discount_factor: Lambda time discount factor. 16 | alpha: TD learning rate. 17 | epsilon: Chance the sample a random action. Float betwen 0 and 1. 18 | 19 | Returns: 20 | A tuple (Q, stats). 21 | Q is the optimal action-value function, a dictionary mapping state -> action values. 22 | stats is an EpisodeStats object with two numpy arrays for episode_lengths and episode_rewards. 23 | """ 24 | 25 | # The final action-value function. 26 | # A nested dictionary that maps state -> (action -> action-value). 27 | Q = defaultdict(lambda: np.zeros(env.action_space.n)) 28 | 29 | # Keeps track of useful statistics 30 | stats = plotting.EpisodeStats( 31 | episode_lengths=np.zeros(num_episodes), 32 | episode_rewards=np.zeros(num_episodes)) 33 | 34 | policy = make_epsilon_greedy_policy(Q, epsilon, env.action_space.n) 35 | 36 | for i_episode in range(num_episodes): 37 | current_state = env.reset() 38 | # choose the action based on epsilon greedy policy 39 | probs = policy(current_state) 40 | action = np.random.choice(np.arange(len(probs)), p=probs) 41 | # keep track number of time-step per episode only for plotting 42 | for t in itertools.count(): 43 | next_state, reward, done, _ = env.step(action) 44 | 45 | # choose next action 46 | next_probs = policy(next_state) 47 | next_action = np.random.choice(np.arange(len(next_probs)), p=next_probs) 48 | # evaluate Q using estimated action value of (next_state, next_action) 49 | td_target = reward + discount_factor * Q[next_state][next_action] 50 | Q[current_state][action] += alpha * (td_target - Q[current_state][action]) 51 | 52 | # improve policy using new evaluate Q 53 | policy = make_epsilon_greedy_policy(Q, epsilon, env.action_space.n) 54 | 55 | # Update statistics 56 | stats.episode_rewards[i_episode] += reward 57 | stats.episode_lengths[i_episode] = t 58 | 59 | if done: 60 | break 61 | else: 62 | current_state = next_state 63 | action = next_action 64 | 65 | return Q, stats 66 | -------------------------------------------------------------------------------- /reinforcement/td_actor_critic_baseline.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from itertools import count 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | import torch.autograd as autograd 8 | from torch.autograd import Variable 9 | 10 | from utils import plotting 11 | 12 | class PolicyEstimator(nn.Module): 13 | """ 14 | Policy Function approximator. 15 | """ 16 | 17 | def __init__(self, D_in, D_out, hidden_size = 128): 18 | super(PolicyEstimator, self).__init__() 19 | # define network structure 20 | self.W1 = nn.Linear(D_in, hidden_size) 21 | self.W2 = nn.Linear(hidden_size, D_out) 22 | 23 | def forward(self, state): 24 | h = F.relu(self.W1(state)) 25 | action_scores = self.W2(h) 26 | return F.softmax(action_scores) 27 | 28 | class ValueEstimator(nn.Module): 29 | """ 30 | Value Function approximator. 31 | """ 32 | 33 | def __init__(self, D_in, hidden_size = 128): 34 | super(ValueEstimator, self).__init__() 35 | # define network structure 36 | self.W1 = nn.Linear(D_in, hidden_size) 37 | # output a score 38 | self.W2 = nn.Linear(hidden_size, 1) 39 | 40 | def forward(self, state): 41 | h = F.relu(self.W1(state)) 42 | state_values = self.W2(h) 43 | return state_values 44 | 45 | def td_actor_critic_baseline(env, policy_estimator, policy_optimizer, value_estimator, value_optimizer, 46 | num_episodes, discount_factor=1.0, render=True): 47 | """ 48 | REINFORCE (Monte Carlo Policy Gradient) Algorithm with Baseline. 49 | Optimizes the policy function approximator using policy gradient. 50 | 51 | Args: 52 | env: OpenAI environment. 53 | policy_estimator: Policy Function to be optimized 54 | policy_optimizer: Optimizer for Policy Function 55 | value_estimator: Value function approximator, used as a baseline 56 | value_optimizer: Optimizer for Value Function 57 | num_episodes: Number of episodes to run for 58 | discount_factor: Time-discount factor 59 | render: Render the training process or not 60 | 61 | Returns: 62 | An EpisodeStats object with two numpy arrays for episode_lengths and episode_rewards. 63 | """ 64 | running_reward = 0 65 | # Keeps track of useful statistics 66 | stats = plotting.EpisodeStats( 67 | episode_lengths=np.zeros(num_episodes), 68 | episode_rewards=np.zeros(num_episodes)) 69 | 70 | for i_episode in range(num_episodes): 71 | episode_rewards = [] 72 | 73 | state = env.reset() 74 | state = torch.from_numpy(state).float().unsqueeze(0) 75 | for t in count(1): 76 | # Calculate the probability distribution of actions 77 | probs = policy_estimator(Variable(state)) 78 | # Select action by distribution estimated above 79 | action = probs.multinomial() 80 | 81 | next_state, reward, done, _ = env.step(action.data[0, 0]) 82 | next_state = torch.from_numpy(next_state).float().unsqueeze(0) 83 | if render: 84 | env.render() 85 | # Update statistics 86 | stats.episode_rewards[i_episode] += reward 87 | stats.episode_lengths[i_episode] = t 88 | episode_rewards.append(reward) 89 | 90 | # Calculate TD(0) target 91 | td_target = reward + discount_factor * value_estimator(Variable(next_state, requires_grad=False)) 92 | # Calculate estimated state value as baseline 93 | baseline = value_estimator(Variable(state)) 94 | # Calculate TD(0) error 95 | td_error = td_target - baseline 96 | 97 | # Registers a reward obtained as a result of a stochastic process. 98 | # Differentiating stochastic nodes requires providing them with reward value. 99 | action.reinforce(td_error.data) 100 | 101 | # Define creterion and calculate loss for value funcion 102 | value_loss = F.smooth_l1_loss(baseline, td_target) 103 | 104 | # Remove gradient from previous steps 105 | policy_optimizer.zero_grad() 106 | value_optimizer.zero_grad() 107 | 108 | # Perform backward pass 109 | action.backward() 110 | value_loss.backward() 111 | 112 | # Use optimizer to update 113 | policy_optimizer.step() 114 | value_optimizer.step() 115 | 116 | if done: 117 | break 118 | else: 119 | state = next_state 120 | 121 | # Book-keep the running reward 122 | running_reward = running_reward * 0.99 + sum(episode_rewards) * 0.01 123 | if i_episode % 10 == 0: 124 | print('Episode {}\tRunning reward: {:.2f}'.format(i_episode, running_reward)) 125 | if running_reward > 200: 126 | print("Solved! Running reward is now {} and " \ 127 | "the last episode runs to {} time steps!".format(running_reward, t)) 128 | break 129 | 130 | return stats 131 | -------------------------------------------------------------------------------- /reinforcement/value_iteration.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 4, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [ 10 | { 11 | "name": "stdout", 12 | "output_type": "stream", 13 | "text": [ 14 | "The autoreload extension is already loaded. To reload it, use:\n", 15 | " %reload_ext autoreload\n" 16 | ] 17 | } 18 | ], 19 | "source": [ 20 | "import numpy as np\n", 21 | "import sys\n", 22 | "if \"../\" not in sys.path:\n", 23 | " sys.path.append(\"../\")\n", 24 | "\n", 25 | "from envs.gridworld import GridworldEnv\n", 26 | "from value_iteration import value_iteration\n", 27 | "\n", 28 | "# for auto-reloading external modules\n", 29 | "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n", 30 | "%load_ext autoreload\n", 31 | "%autoreload 2" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 5, 37 | "metadata": { 38 | "collapsed": true 39 | }, 40 | "outputs": [], 41 | "source": [ 42 | "env = GridworldEnv()" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 12, 48 | "metadata": { 49 | "collapsed": false 50 | }, 51 | "outputs": [], 52 | "source": [ 53 | "policy, v = value_iteration(env)" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 13, 59 | "metadata": { 60 | "collapsed": false 61 | }, 62 | "outputs": [ 63 | { 64 | "name": "stdout", 65 | "output_type": "stream", 66 | "text": [ 67 | "Policy Probability Distribution:\n", 68 | "[[ 1. 0. 0. 0.]\n", 69 | " [ 0. 0. 0. 1.]\n", 70 | " [ 0. 0. 0. 1.]\n", 71 | " [ 0. 0. 1. 0.]\n", 72 | " [ 1. 0. 0. 0.]\n", 73 | " [ 1. 0. 0. 0.]\n", 74 | " [ 1. 0. 0. 0.]\n", 75 | " [ 0. 0. 1. 0.]\n", 76 | " [ 1. 0. 0. 0.]\n", 77 | " [ 1. 0. 0. 0.]\n", 78 | " [ 0. 1. 0. 0.]\n", 79 | " [ 0. 0. 1. 0.]\n", 80 | " [ 1. 0. 0. 0.]\n", 81 | " [ 0. 1. 0. 0.]\n", 82 | " [ 0. 1. 0. 0.]\n", 83 | " [ 1. 0. 0. 0.]]\n", 84 | "\n", 85 | "Reshaped Grid Policy (0=up, 1=right, 2=down, 3=left):\n", 86 | "[[0 3 3 2]\n", 87 | " [0 0 0 2]\n", 88 | " [0 0 1 2]\n", 89 | " [0 1 1 0]]\n", 90 | "\n", 91 | "Value Function:\n", 92 | "[ 0. -1. -2. -3. -1. -2. -3. -2. -2. -3. -2. -1. -3. -2. -1. 0.]\n", 93 | "\n", 94 | "Reshaped Grid Value Function:\n", 95 | "[[ 0. -1. -2. -3.]\n", 96 | " [-1. -2. -3. -2.]\n", 97 | " [-2. -3. -2. -1.]\n", 98 | " [-3. -2. -1. 0.]]\n", 99 | "\n" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "policy, v = value_iteration(env)\n", 105 | "\n", 106 | "print(\"Policy Probability Distribution:\")\n", 107 | "print(policy)\n", 108 | "print(\"\")\n", 109 | "\n", 110 | "print(\"Reshaped Grid Policy (0=up, 1=right, 2=down, 3=left):\")\n", 111 | "print(np.reshape(np.argmax(policy, axis=1), env.shape))\n", 112 | "print(\"\")\n", 113 | "\n", 114 | "print(\"Value Function:\")\n", 115 | "print(v)\n", 116 | "print(\"\")\n", 117 | "\n", 118 | "print(\"Reshaped Grid Value Function:\")\n", 119 | "print(v.reshape(env.shape))\n", 120 | "print(\"\")" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 14, 126 | "metadata": { 127 | "collapsed": false 128 | }, 129 | "outputs": [], 130 | "source": [ 131 | "# Test the value function\n", 132 | "expected_v = np.array([ 0, -1, -2, -3, -1, -2, -3, -2, -2, -3, -2, -1, -3, -2, -1, 0])\n", 133 | "np.testing.assert_array_almost_equal(v, expected_v, decimal=2)" 134 | ] 135 | } 136 | ], 137 | "metadata": { 138 | "kernelspec": { 139 | "display_name": "Python [py35]", 140 | "language": "python", 141 | "name": "Python [py35]" 142 | }, 143 | "language_info": { 144 | "codemirror_mode": { 145 | "name": "ipython", 146 | "version": 3 147 | }, 148 | "file_extension": ".py", 149 | "mimetype": "text/x-python", 150 | "name": "python", 151 | "nbconvert_exporter": "python", 152 | "pygments_lexer": "ipython3", 153 | "version": "3.5.2" 154 | } 155 | }, 156 | "nbformat": 4, 157 | "nbformat_minor": 0 158 | } 159 | -------------------------------------------------------------------------------- /reinforcement/value_iteration.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def value_iteration(env, theta=0.0001, discount_factor=1.0): 4 | """ 5 | Value Iteration Algorithm. 6 | 7 | Args: 8 | env: OpenAI environment. env.P represents the transition probabilities of the environment. 9 | theta: Stopping threshold. If the value of all states changes less than theta 10 | in one iteration we are done. 11 | discount_factor: lambda time discount factor. 12 | 13 | Returns: 14 | A tuple (policy, V) of the optimal policy and the optimal value function. 15 | """ 16 | 17 | 18 | V = np.zeros(env.nS) 19 | 20 | while True: 21 | V_converged = True 22 | # init a empty policy every time, we can choose the best policy along with 23 | # find optimal action value 24 | policy = np.zeros([env.nS, env.nA]) 25 | for s in range(env.nS): 26 | action_values = np.zeros(env.nA) 27 | # use one-step lookahead and update v to best action value 28 | for a in range(env.nA): 29 | for prob, next_state, reward, done in env.P[s][a]: 30 | action_values[a] += prob * (reward + discount_factor * V[next_state]) 31 | max_v = action_values.max() 32 | 33 | # converged only when V reach optimal (max action value doesn't change anymore) 34 | if np.abs(max_v - V[s]) > theta: 35 | V_converged = False 36 | # update v and corresponding policy 37 | V[s] = max_v 38 | policy[s][action_values.argmax()] = 1.0 39 | 40 | if V_converged: 41 | return policy, V 42 | -------------------------------------------------------------------------------- /supervised/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hungtuchen/ml-playground/f273ea923cd442f5a51b73c14eb220f3d2b11151/supervised/__init__.py -------------------------------------------------------------------------------- /supervised/adaboost.py: -------------------------------------------------------------------------------- 1 | from utils.base_estimator import BaseEstimator 2 | 3 | import numpy as np 4 | from sklearn.tree import DecisionTreeClassifier 5 | 6 | class AdaBoost(BaseEstimator): 7 | """ 8 | An AdaBoost classifier is a meta-estimator that begins by fitting a 9 | classifier on the original dataset and then fits additional copies of the 10 | classifier on the same dataset but where the weights of incorrectly 11 | classified instances are adjusted such that subsequent classifiers focus 12 | more on difficult cases. 13 | 14 | Parameters 15 | ---------- 16 | n_estimators : integer, optional (default=50) 17 | The maximum number of estimators at which boosting is terminated. 18 | """ 19 | def __init__(self, n_estimators=50): 20 | self.n_estimators = n_estimators 21 | self.estimators_ = [] 22 | self.estimator_weights_ = np.zeros(self.n_estimators) 23 | 24 | def fit(self, X, y=None): 25 | """Build a boosted classifier from the training set (X, y).""" 26 | self._setup_input(X, y) 27 | # Initialize weights to 1 / n_samples 28 | sample_weight = np.zeros(self.n_samples) 29 | sample_weight[:] = 1.0 / self.n_samples 30 | 31 | for i in range(self.n_estimators): 32 | # Boosting step 33 | sample_weight, estimator_weight = self._boost(sample_weight) 34 | self.estimator_weights_[i] = estimator_weight 35 | 36 | def _boost(self, sample_weight): 37 | """Implement a single boost. 38 | sample_weight : array-like of shape = [n_samples] 39 | The current sample weights. 40 | Returns 41 | ------- 42 | sample_weight : array-like of shape = [n_samples] or None 43 | The reweighted sample weights. 44 | estimator_weight : float 45 | The weight for the current boost. 46 | """ 47 | X, y = self.X, self.y 48 | estimator = DecisionTreeClassifier(max_depth=1) 49 | # TODO: replace with custom decision tree 50 | estimator.fit(X, y, sample_weight=sample_weight) 51 | self.estimators_.append(estimator) 52 | 53 | y_predict = estimator.predict(X) 54 | # Instances incorrectly and correctly classified 55 | incorrect = y != y_predict 56 | correct = y == y_predict 57 | 58 | # Error fraction 59 | estimator_error = np.average(incorrect, weights=sample_weight) 60 | 61 | scaling_factor = np.sqrt((1 - estimator_error) / estimator_error) 62 | # Boosting by weighting up incorrect samples and weighting down correct ones 63 | sample_weight[incorrect] *= scaling_factor 64 | sample_weight[correct] /= scaling_factor 65 | 66 | # Normalize sample_weight 67 | sample_weight /= sample_weight.sum() 68 | 69 | estimator_weight = np.log(scaling_factor) 70 | 71 | return sample_weight, estimator_weight 72 | 73 | def _predict(self, X): 74 | score = self.decision_function(X) 75 | # expect y to be 1 or 0 76 | return np.where(score >= 0.5, 1.0, 0.0) 77 | 78 | def decision_function(self, X): 79 | """Compute the decision function of ``X``.""" 80 | pred = np.sum([estimator.predict(X) * w for estimator, w in 81 | zip(self.estimators_, self.estimator_weights_)], axis=0) 82 | pred /= self.estimator_weights_.sum() 83 | return pred 84 | -------------------------------------------------------------------------------- /supervised/base_regression.py: -------------------------------------------------------------------------------- 1 | from utils.base_estimator import BaseEstimator 2 | 3 | import numpy as np 4 | 5 | class BasicRegression(BaseEstimator): 6 | def __init__(self, lr=0.001, max_iters=1000, C=0, verbose=False): 7 | """Basic class for implementing continuous regression estimators which 8 | are trained with gradient descent optimization on their particular loss function. 9 | Parameters 10 | ---------- 11 | lr : float, default 0.001 12 | Learning rate. 13 | max_iters : int, default 1000 14 | The maximum number of iterations. 15 | C : float, default 0 (no regularization) 16 | The l2-regularization coefficient. 17 | Since l1-norm is not differentiable, We don't support l1 here for simplicity. 18 | If you want to implement it, you can check https://github.com/HIPS/autograd 19 | or there are mamy algorithms out there. 20 | verbose: boolean, default False 21 | If True, print progress during optimization. 22 | """ 23 | self.lr = lr 24 | self.max_iters = max_iters 25 | self.C = C 26 | self.verbose = verbose 27 | self.loss_history = [] 28 | self.theta = [] 29 | self.n_samples, self.n_features = None, None 30 | 31 | def _loss(self, w): 32 | raise NotImplementedError() 33 | 34 | def _train(self): 35 | # default: gradient descent optimization 36 | theta = self.theta 37 | loss_history = [] 38 | for i in range(self.max_iters): 39 | # evaluate loss and gradient 40 | loss, grad = self._loss(theta) 41 | loss_history.append(loss) 42 | 43 | # perform parameter update 44 | # Update the weights using the gradient and the learning rate 45 | theta -= self.lr * grad 46 | 47 | if self.verbose and (i + 1) % 100 == 0: 48 | print('Iteration %s, loss %s' % (i + 1, loss_history[i])) 49 | 50 | return theta, loss_history 51 | 52 | @staticmethod 53 | def _add_intercept(X): 54 | b = np.ones([X.shape[0], 1]) 55 | return np.concatenate([b, X], axis=1) 56 | 57 | def init_weights(self): 58 | return np.random.normal(size=(self.n_features + 1), scale=0.5) 59 | 60 | def fit(self, X, y=None): 61 | self._setup_input(X, y) 62 | self.n_samples, self.n_features = X.shape 63 | 64 | # Initialize weights + bias term 65 | self.theta = self.init_weights() 66 | 67 | # Add an intercept column 68 | self.X = self._add_intercept(self.X) 69 | 70 | self.theta, self.loss_history = self._train() 71 | 72 | def _predict(self, X=None): 73 | X = self._add_intercept(X) 74 | return X.dot(self.theta) 75 | -------------------------------------------------------------------------------- /supervised/linear_regression.py: -------------------------------------------------------------------------------- 1 | from base_regression import BasicRegression 2 | from utils.metrics import mean_squared_error 3 | 4 | import numpy as np 5 | 6 | class LinearRegressionGD(BasicRegression): 7 | """Linear regression with gradient descent optimizer.""" 8 | 9 | def _loss(self, theta): 10 | X, y = self.X, self.y 11 | loss = (1.0 / 2) * mean_squared_error(y, X.dot(theta)) 12 | grad = (1.0 / self.n_samples) * (X.T.dot(X.dot(theta) - y)) 13 | return loss, grad 14 | 15 | class LinearRegression(BasicRegression): 16 | """ Linear regression with closed form solution.""" 17 | 18 | def _train(self): 19 | X, y = self.X, self.y 20 | pseudo_inverse = np.matrix(X.T.dot(X)).I.dot(X.T) 21 | theta = pseudo_inverse.dot(y) 22 | 23 | return np.squeeze(np.asarray(theta)), [] 24 | -------------------------------------------------------------------------------- /supervised/logistic_regression.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [ 10 | { 11 | "name": "stderr", 12 | "output_type": "stream", 13 | "text": [ 14 | "/Users/mac/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n", 15 | " \"This module will be removed in 0.20.\", DeprecationWarning)\n" 16 | ] 17 | } 18 | ], 19 | "source": [ 20 | "import numpy as np\n", 21 | "import sys\n", 22 | "if \"../\" not in sys.path:\n", 23 | " sys.path.append(\"../\")\n", 24 | "\n", 25 | "from sklearn.datasets import load_breast_cancer, load_iris\n", 26 | "from sklearn.cross_validation import train_test_split\n", 27 | "from sklearn.preprocessing import StandardScaler\n", 28 | "\n", 29 | "from supervised.logistic_regression import LogisticRegression, Softmax\n", 30 | "\n", 31 | "import matplotlib.pyplot as plt\n", 32 | "from matplotlib.colors import ListedColormap\n", 33 | "%matplotlib inline\n", 34 | "plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\n", 35 | "plt.rcParams['image.interpolation'] = 'nearest'\n", 36 | "plt.rcParams['image.cmap'] = 'gray'\n", 37 | "\n", 38 | "import seaborn as sns\n", 39 | "sns.set_context('notebook')\n", 40 | "sns.set_style('white')\n", 41 | "\n", 42 | "# for auto-reloading external modules\n", 43 | "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n", 44 | "%load_ext autoreload\n", 45 | "%autoreload 2" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 2, 51 | "metadata": { 52 | "collapsed": false 53 | }, 54 | "outputs": [], 55 | "source": [ 56 | "cancer = load_breast_cancer()\n", 57 | "X_train, X_test, y_train, y_test = train_test_split(\n", 58 | " cancer.data, cancer.target, stratify=cancer.target, random_state=2046)\n", 59 | "\n", 60 | "\n", 61 | "sc = StandardScaler()\n", 62 | "sc.fit(X_train)\n", 63 | "X_train = sc.transform(X_train)\n", 64 | "X_test = sc.transform(X_test)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 3, 70 | "metadata": { 71 | "collapsed": false 72 | }, 73 | "outputs": [ 74 | { 75 | "name": "stdout", 76 | "output_type": "stream", 77 | "text": [ 78 | "Training set score: 0.988\n", 79 | "Test set score: 0.965\n" 80 | ] 81 | } 82 | ], 83 | "source": [ 84 | "model_0 = LogisticRegression(lr=0.1, max_iters=2000)\n", 85 | "model_0.fit(X_train, y_train)\n", 86 | "\n", 87 | "print(\"Training set score: {:.3f}\".format(np.mean(model_0.predict(X_train) == y_train)))\n", 88 | "print(\"Test set score: {:.3f}\".format(np.mean(model_0.predict(X_test) == y_test)))" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 4, 94 | "metadata": { 95 | "collapsed": false 96 | }, 97 | "outputs": [ 98 | { 99 | "name": "stdout", 100 | "output_type": "stream", 101 | "text": [ 102 | "Training set score: 0.979\n", 103 | "Test set score: 0.979\n" 104 | ] 105 | } 106 | ], 107 | "source": [ 108 | "model_1 = LogisticRegression(lr=0.1, max_iters=2000, C=0.1)\n", 109 | "model_1.fit(X_train, y_train)\n", 110 | "\n", 111 | "print(\"Training set score: {:.3f}\".format(np.mean(model_1.predict(X_train) == y_train)))\n", 112 | "print(\"Test set score: {:.3f}\".format(np.mean(model_1.predict(X_test) == y_test)))" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 5, 118 | "metadata": { 119 | "collapsed": false 120 | }, 121 | "outputs": [], 122 | "source": [ 123 | "# Multi-class logistic regression\n", 124 | "iris = load_iris()\n", 125 | "X = iris.data\n", 126 | "y = iris.target\n", 127 | "\n", 128 | "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=2046)\n", 129 | "\n", 130 | "sc.fit(X_train)\n", 131 | "X_train = sc.transform(X_train)\n", 132 | "X_test = sc.transform(X_test)" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 6, 138 | "metadata": { 139 | "collapsed": false 140 | }, 141 | "outputs": [ 142 | { 143 | "name": "stdout", 144 | "output_type": "stream", 145 | "text": [ 146 | "Training set score: 0.971\n", 147 | "Test set score: 0.933\n" 148 | ] 149 | } 150 | ], 151 | "source": [ 152 | "softmax = Softmax(lr=0.05)\n", 153 | "softmax.fit(X_train, y_train)\n", 154 | "\n", 155 | "print(\"Training set score: {:.3f}\".format(np.mean(softmax.predict(X_train) == y_train)))\n", 156 | "print(\"Test set score: {:.3f}\".format(np.mean(softmax.predict(X_test) == y_test)))" 157 | ] 158 | } 159 | ], 160 | "metadata": { 161 | "anaconda-cloud": {}, 162 | "kernelspec": { 163 | "display_name": "Python [py35]", 164 | "language": "python", 165 | "name": "Python [py35]" 166 | }, 167 | "language_info": { 168 | "codemirror_mode": { 169 | "name": "ipython", 170 | "version": 3 171 | }, 172 | "file_extension": ".py", 173 | "mimetype": "text/x-python", 174 | "name": "python", 175 | "nbconvert_exporter": "python", 176 | "pygments_lexer": "ipython3", 177 | "version": "3.5.2" 178 | } 179 | }, 180 | "nbformat": 4, 181 | "nbformat_minor": 0 182 | } 183 | -------------------------------------------------------------------------------- /supervised/logistic_regression.py: -------------------------------------------------------------------------------- 1 | from base_regression import BasicRegression 2 | 3 | import numpy as np 4 | 5 | def sigmoid(s): 6 | return 1 / (1 + np.exp(-s)) 7 | 8 | def softmax(z): 9 | # Avoid numerical overflow by removing max 10 | # See: http://cs231n.github.io/linear-classify/ 11 | e = np.exp(z - np.max(z)) 12 | return e / np.sum(e) 13 | 14 | class LogisticRegression(BasicRegression): 15 | """Binary logistic regression with gradient descent optimizer.""" 16 | def _loss(self, theta): 17 | X, y, C = self.X, self.y, self.C 18 | 19 | predict = sigmoid(X.dot(theta)) 20 | # prevent overflow or underflow 21 | predict = np.clip(predict, 1e-15, 1 - 1e-15) 22 | loss = - np.mean(np.sum(y * np.log(predict) + (1 - y) * np.log(1 - predict))) 23 | grad = (1.0 / self.n_samples) * (X.T.dot(predict - y)) 24 | 25 | # Add regularization to the loss. 26 | loss += 0.5 * C * np.sum(theta * theta) 27 | grad += C * theta 28 | 29 | return loss, grad 30 | 31 | def _predict(self, X=None, threshold=0.5): 32 | """Predict class labels for samples in X.""" 33 | X = self._add_intercept(X) 34 | predict = sigmoid(X.dot(self.theta)) 35 | return np.where(predict > threshold , 1, 0) 36 | 37 | class Softmax(BasicRegression): 38 | """Multi-class logistic regression with gradient descent optimizer.""" 39 | def init_weights(self): 40 | """Mutli-class weights""" 41 | self.num_classes = np.max(self.y) + 1 # assume y takes values 0...K-1 where K is number of classes 42 | return np.random.normal(size=(self.n_features + 1, self.num_classes), scale=0.5) 43 | 44 | def _loss(self, theta): 45 | """ 46 | Naive implementation for softmax loss due to its simplicity and readibility 47 | For vectorized version: https://github.com/transedward/cs231n/blob/master/assignment1/cs231n/classifiers/softmax.py#L62 48 | """ 49 | X, y, C, n_samples, n_features = self.X, self.y, self.C, self.n_samples, self.n_features 50 | loss = 0.0 51 | grad = np.zeros_like(theta) 52 | for i in range(n_samples): 53 | scores = X[i].dot(theta) 54 | probability = softmax(scores) 55 | loss += -np.log(probability[y[i]]) 56 | # gradient for softmax loss 57 | # http://ufldl.stanford.edu/wiki/index.php/Softmax_Regression 58 | for j in range(self.num_classes): 59 | grad[:, j] += (probability[j] - (j == y[i])) * X[i, :] 60 | 61 | 62 | # Right now the loss is a sum over all training examples, but we want it 63 | # to be an average instead so we divide by n_samples. 64 | loss /= n_samples 65 | grad /= n_samples 66 | 67 | # Add regularization to the loss. 68 | loss += 0.5 * C * np.sum(theta * theta) 69 | grad += C * theta 70 | 71 | return loss, grad 72 | 73 | def _predict(self, X=None): 74 | """Predict class labels for samples in X.""" 75 | X = self._add_intercept(X) 76 | scores = X.dot(self.theta) 77 | predict = np.argmax(scores, axis=1) 78 | return predict 79 | -------------------------------------------------------------------------------- /supervised/naive_bayes.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [ 10 | { 11 | "name": "stderr", 12 | "output_type": "stream", 13 | "text": [ 14 | "/Users/mac/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n", 15 | " \"This module will be removed in 0.20.\", DeprecationWarning)\n" 16 | ] 17 | } 18 | ], 19 | "source": [ 20 | "import numpy as np\n", 21 | "import sys\n", 22 | "if \"../\" not in sys.path:\n", 23 | " sys.path.append(\"../\")\n", 24 | "\n", 25 | "import pandas as pd\n", 26 | "from sklearn.cross_validation import train_test_split\n", 27 | "\n", 28 | "from supervised.naive_bayes import GaussianNB\n", 29 | "# for auto-reloading external modules\n", 30 | "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n", 31 | "%load_ext autoreload\n", 32 | "%autoreload 2" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 2, 38 | "metadata": { 39 | "collapsed": false 40 | }, 41 | "outputs": [], 42 | "source": [ 43 | "df = pd.read_csv('../dataset/pima-indians-diabetes.csv', header=None)\n", 44 | "\n", 45 | "X, y = df[df.columns[:-1]], df[df.columns[-1]]\n", 46 | "\n", 47 | "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=2046)" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 3, 53 | "metadata": { 54 | "collapsed": false 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "model = GaussianNB()\n", 59 | "model.fit(X_train, y_train)" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": 4, 65 | "metadata": { 66 | "collapsed": false 67 | }, 68 | "outputs": [], 69 | "source": [ 70 | "y_pred = model.predict(X_test)" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 5, 76 | "metadata": { 77 | "collapsed": false 78 | }, 79 | "outputs": [ 80 | { 81 | "data": { 82 | "text/plain": [ 83 | "0.76377952755905509" 84 | ] 85 | }, 86 | "execution_count": 5, 87 | "metadata": {}, 88 | "output_type": "execute_result" 89 | } 90 | ], 91 | "source": [ 92 | "# accuracy\n", 93 | "np.mean(y_test == y_pred)\n", 94 | "\n", 95 | "# http://www.is.umk.pl/projects/datasets.html#Diabetes\n", 96 | "# other algorithms on this dataset" 97 | ] 98 | } 99 | ], 100 | "metadata": { 101 | "anaconda-cloud": {}, 102 | "kernelspec": { 103 | "display_name": "Python [py35]", 104 | "language": "python", 105 | "name": "Python [py35]" 106 | }, 107 | "language_info": { 108 | "codemirror_mode": { 109 | "name": "ipython", 110 | "version": 3 111 | }, 112 | "file_extension": ".py", 113 | "mimetype": "text/x-python", 114 | "name": "python", 115 | "nbconvert_exporter": "python", 116 | "pygments_lexer": "ipython3", 117 | "version": "3.5.2" 118 | } 119 | }, 120 | "nbformat": 4, 121 | "nbformat_minor": 0 122 | } 123 | -------------------------------------------------------------------------------- /supervised/naive_bayes.py: -------------------------------------------------------------------------------- 1 | from utils.base_estimator import BaseEstimator 2 | 3 | import numpy as np 4 | 5 | def softmax(z): 6 | # Avoid numerical overflow by removing max 7 | # See: http://cs231n.github.io/linear-classify/ 8 | e = np.exp(z - np.amax(z, axis=1, keepdims=True)) 9 | return e / np.sum(e, axis=1, keepdims=True) 10 | 11 | class GaussianNB(BaseEstimator): 12 | """Gaussian Naive Bayes.""" 13 | # Binary problem. 14 | n_classes = 2 15 | 16 | def fit(self, X, y=None): 17 | self._setup_input(X, y) 18 | # Check target labels 19 | assert list(np.unique(y)) == [0, 1] 20 | 21 | # Prepare mean, std, prior for each class 22 | self._mean = np.zeros((self.n_classes, self.n_features)) 23 | self._var = np.zeros((self.n_classes, self.n_features)) 24 | self._prior = np.zeros((self.n_classes)) 25 | 26 | for c in range(self.n_classes): 27 | # Filter X only in current class 28 | X_per_class = X[y == c] 29 | 30 | self._mean[c, :] = np.mean(X_per_class, axis=0) 31 | self._var[c, :] = np.var(X_per_class, axis=0) 32 | self._prior[c] = len(X_per_class) / float(len(X)) 33 | 34 | def _predict(self, X=None): 35 | """ 36 | Return the class that each row in X most likely belongs to. 37 | Perform classification on an array of test vectors X. 38 | """ 39 | return np.argmax(self._joint_log_likelihood(X), axis=1) 40 | 41 | def predict_proba(self, X): 42 | """Predict probability for each class for each row.""" 43 | return softmax(self._joint_log_likelihood(X)) 44 | 45 | def _joint_log_likelihood(self, X): 46 | """ 47 | The conditional probabilities for each class given an feature value are small. 48 | When they are multiplied together they result in very small values, 49 | which can lead to floating point underflow. 50 | A common fix for this is to apply log function on joint probabilities. 51 | """ 52 | joint_log_likelihood = [] 53 | for c in range(self.n_classes): 54 | prior = np.log(self._prior[c]) 55 | # log of denominator part of gaussian distribution 56 | posterior = - 0.5 * np.sum(np.log(2. * np.pi * self._var[c, :])) 57 | # log of numerator part of gaussian distribution 58 | posterior -= 0.5 * np.sum(((X - self._mean[c, :]) ** 2) / 59 | (self._var[c, :]), axis=1) 60 | joint_log_likelihood.append(prior + posterior) 61 | return np.array(joint_log_likelihood).T 62 | -------------------------------------------------------------------------------- /unsupervised/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hungtuchen/ml-playground/f273ea923cd442f5a51b73c14eb220f3d2b11151/unsupervised/__init__.py -------------------------------------------------------------------------------- /unsupervised/kmeans.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "from sklearn.datasets import make_blobs\n", 12 | "import numpy as np\n", 13 | "import sys\n", 14 | "if \"../\" not in sys.path:\n", 15 | " sys.path.append(\"../\")\n", 16 | "\n", 17 | "from unsupervised.kmeans import KMeans\n", 18 | "\n", 19 | "import matplotlib.pyplot as plt\n", 20 | "%matplotlib inline\n", 21 | "plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\n", 22 | "plt.rcParams['image.interpolation'] = 'nearest'\n", 23 | "plt.rcParams['image.cmap'] = 'gray'\n", 24 | "\n", 25 | "import seaborn as sns\n", 26 | "sns.set_context('notebook')\n", 27 | "sns.set_style('white')\n", 28 | "\n", 29 | "# for auto-reloading external modules\n", 30 | "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n", 31 | "%load_ext autoreload\n", 32 | "%autoreload 2" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 2, 38 | "metadata": { 39 | "collapsed": false 40 | }, 41 | "outputs": [ 42 | { 43 | "name": "stdout", 44 | "output_type": "stream", 45 | "text": [ 46 | "Converged on iteration 5\n" 47 | ] 48 | } 49 | ], 50 | "source": [ 51 | "X, y = make_blobs(centers=4, n_samples=500, n_features=2,\n", 52 | " shuffle=True, random_state=2046)\n", 53 | "clusters = len(np.unique(y))\n", 54 | "\n", 55 | "model = KMeans(K=clusters, max_iters=150)\n", 56 | "model.fit(X)" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": 3, 62 | "metadata": { 63 | "collapsed": false 64 | }, 65 | "outputs": [ 66 | { 67 | "data": { 68 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqkAAAHcCAYAAAD1ITMoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3XtcVHX+P/DXYWBGZeTiDZBMroOKmKS1u5W4Ze33IX5b\n1ktSkj9NU3Ddtq10c8lLKhGpXbbaRE3QEnNQcamvtu2uN9Ku6pAjyB2pBExEwFEZmOH8/hjnMJcz\nV2aGGXg//0qYOedzJsW378/7/f4wLMuyIIQQQgghxI149fYCCCGEEEIIMURBKiGEEEIIcTsUpBJC\nCCGEELdDQSohhBBCCHE7FKQSQgghhBC3Q0EqIYQQQghxOxSkEkIIIYQQt0NBKiGEEEIIcTsUpBJC\nCCGEELfj0iC1o6MD69evx/3334+HHnoIb7/9titvTwghhBBCPIS3K2+WkZGB7777Djk5OVAoFHjh\nhRcQGhqKuXPnunIZhBBCCCHEzbksk9ra2oqCggJkZGRg/Pjx+PWvf41Fixbhhx9+cNUSCCGEEEKI\nh3BZJvXs2bMYPHgwJk+ezH1tyZIlrro9IYQQQgjxIC7LpP70008IDQ3FP//5T0yfPh2PPvooPvjg\nA7As66olEEIIIYQQD+GyTOqtW7dw6dIl5OfnIysrC1evXsWaNWswaNAgLFy40OL7J0+eDKVSiREj\nRjh/sYQQQgghxGa//PILRCIRzpw50+NruSxIFQgEuHnzJt566y0EBwcDAC5fvoxPPvnEqiC1o6MD\narXayaskhBBCCCH2UqvV6OjocMi1XBakjhgxAiKRiAtQASA8PByNjY1WvX/48OEAgKNHjzplfYQQ\nQgghpGemTZvmsGu5rCb1nnvugVKpRF1dHfe16upqhIaGumoJhBBCCCHEQ7gsSA0PD8fUqVOxatUq\nlJWV4csvv8SOHTswb948Vy2BEEIIIYR4CJcO89+yZQsyMjKQkpKCgQMHYv78+UhJSXHlEgghhBBC\niAdwaZAqFouRlZWFrKwsV96WEEIIIYR4GJdt9xNCCCGEEGItClIJIYQQQojboSCVEEIIIYS4HQpS\nCSGEEEKI26EglRBCCCGEuB0KUgkhhBBCiNuhIJUQQgghhLgdClIJIYQQQojboSCVEEIIIYS4HQpS\nCSGEEEKI26EglRBCCCGEuB0KUgkhhBBCiNuhIJUQQgghhLgdClIJIYQQQojboSCVEEIIIYS4HQpS\nCSGEEEKI26EglRBCCCGEuB3v3l4AIYTokstkKNyeDW+FAiqxGElL0xAXH9/byyKEEOJilEklhLgN\nuUyGI2vTkXyrBXMEaiTfasGRtemQy2S9vTRCCCEuRkEqIcRtFG7PxqwhfmAYBgDAMAxmDfFD4Y7s\nXl4ZIYQQV6MglRDiNrwVCi5A1WIYBt4KRS+tiBBCSG+hmlRC+iBPretUicVgb7XoBaosy0IlFvfi\nqgghhPQGyqQS0sd4cl1n0tI0FDS3gWVZAJoAtaC5DUlL0np5ZYQQQlyNglRC+hhPruuMi49H4oZM\n5PsG4kCXAFLfACRuyPSILDAhhBDHou1+QvoYb4UCjMBz6zrj4uMR90F3QC2XyZCxLNXjShcIIYT0\nDGVSCeljVGIxt12u5al1nZ5cukAIIaRnKEglpI/pS3Wdnly6QAghpGdou5+QPiYuPh7YkIn8Hdsg\nUNzQbJH/ZZVHbpF7eukCIYQQ+1GQSkgfZFjX6aloJBUhhPRftN1PCHFbfal0gRBCiG0ok0oIcVt9\nqXSBEFeRyYuRXZgHhXcHxCoh0pJSEB83sbeXRYjNKEglhLi1vlK6QIgryOTFSD+yFX7JcWAYBi0s\ni/SCrcjEMgpUiceh7X5CCCGkj8guzIPfrDi9iRh+s+KQXZjXyysjxHaUSSWE9AtymQyF27PpUADS\npym8O/QaDQFNoKrw7uylFRFiPwpSCSF9HncowBA/MAIG7K0WFKxNB2w8cpUCXeLuxCohWljWaCKG\nWOXTi6sixD69tt2/dOlS/O1vf+ut2xNC+hFHHApAp18RdyeTF6OlrQVlf/8cZR/+F62VDWBZFm0F\ncqQlpfT28gixWa8EqYcPH0ZRUVFv3JoQ0g95KxS8W6C2HApAp18Rd6ZtmMKSezD2L4mIWTwNTafK\nof7HOWQmUtMU8Uwu3+5vbW3F5s2bMWHCBFffmpA+hbaereeIQwHo9CvizrIL87iOfkDzezNi4VQE\nSOsoQCUey+WZ1DfeeANJSUmIjIx09a0J6TP689azXCZDxrJUZM1PQcayVKue2RGHAqjEYu79WnT6\nFXEX1DBF+iKXBqlff/01zp49i+XLl7vytoT0OX1969lUIGpvcB4XH4/EDZnI9w3EgS4BpL4BSLSx\naYpOvyLuTKwS8v4jihqmiCdz2XZ/R0cHXn31Vaxbtw5CodBVtyWkT7Jn69lTygPMdeIXbs/WfN0g\nOJfuyEbcB9vMXrenhwLQ6VfEnaUlpSC9YCs3I1XbMLUqaVlvL40Qu7ksSH3vvfcwfvx4PPDAA666\nJSF9lq01lo4aweQK5gLR3q4LpdOviLuKj5uITCzDtvy9uCHogFjlg1VJ/adhio6C7ZtcFqQeOXIE\n165dQ/ydvxA7OzV1Ml988QXOnTvnqmUQ0ickLU1Dwdp0bsuf23r+yyre1/ckA+lqpgJRZeMVlF2+\njNljInrUAEVIXxUfNxHZ/TAwo6Ng+y6XBal79uyBSqXifr1582YAwMqVK121BEL6DFu3nns7A2kL\nU1niS1WVmBc+CvmVNZgbHWFVcE4I6fv4Jhv4zYpDtjQP2yhI9WguC1JDQkL0fu3r6wsAGDVqlKuW\nQEifYsvWsyNGMLkKX5Y4t6oWT4WPgiQwAAwY5JZWwMeLQTUjwMtbt/MG555Sg0sI6RmabNB30bGo\nhPQDtpYHmOKKwI8vS6wcMhwMurCzpAwigQAsWPw6OBgD/YeYDFA9pQaXENIzdBRs38WwhjMr3NS0\nadMAAEePHu3llRDimeQyGT7VLQ9YYluAqQ38DANdW0c52eMvc+dg2E+XkCyJ5O4trahG091heEd6\nwOj1GctSkcyTOZb6BmC1m9XgEkJ6hqtJNZhsQCdt9Q5HxmuUSSWkn+hpZ7qp5qsdm7LgFxDg1Oyq\nUCDgAlTtvZMlkdjRwf96T6rBJcQdeVK3fH+fbNCXUZBKCLGKqcDvZsVFLBk/xuptdcOSgdipD6Pk\n5HGzQe4Qb28wjNro3kN8BLz38KQaXEKcxd5As6fd8r0R4PbXyQZ9HQWphBCrmAr8mm4oUNXShuhA\nf4ujrQxrRSvq63D6nc1YGBPFBbnSVSuw0z8QwSIRF7SaundzpwoZy1KNAlxH1eAS4ql6Emj2pFue\nxkERR3LpsaiEEPdg6thRc/iOBc2vrMGCsdE4d7UJlddbAZjfVt/5+mu43tCAvPIq7CwpQ2HNJTwY\nNBw5peXYU1aJnNJy3DvABwNqqvSOPY2d+rDRvaWNTehquMx7RKojjkElxJNlF+ZxNZqATqBZmGfx\nvT3plu/JfQkxRJlUQvoZezvftV3365elIpJVoaOrCwkjQxAd6I/oAH/klJYjOtDf5La6XCaD7091\nWBgbw2U3t5z7Ad9duYpF47q/ll9Zg5t3DvvgMrNFx5Fk0PHf5K/G8oECkwcU0OlQxJM4eou8J4Fm\nT7rlaRwUcSTKpBLSzxRuz+a2wYHu4K5wh+WALi4+HuOjozF/rASLY8cgOtCfu4ZIIOjeVl+Spvc+\nuUyGN9KWwrtLjZzSclRebwXDMAgUDUBKTJTeWuZGR+Dq7XbuvdrMbFx8PF75IBurPsrD6g+2IVgk\n4v3LkJqjiKfRbpG3JI+Geo4ELcmjkX5kK2TyYruvKVYJYTi8x9pAMy0pBW0Fcr2di7YCOdKSUpx6\nX0IMUSaVkH6mp53vpupDq1VdkPoGGJ18pc3crhsToZctBYAB3gLeQPPuwWK9a/NlZqk5ivQV9tSA\nWsq8piWlIL3AeCzTqqRlFtdjb7e8TF6Ma03XULfrEiIWTrX5voQYoiCVkH6mp8Gdqaakv+7I4S0X\n4BtdNTc6Ajml5WBZzb0N1+J159fmGp6saY6iU6eIJ7B1i9ya5qSejmWytVueW9PyezGsqhHlOcfA\ntikxxnckMucvp6YpYhcKUgnpZ3ra+c53IpRh9lSXqcyt0MsLNwcOgrSxCcnBw7qH9Dc2QRkZjQNd\nApPX1gafSh8h1pfVIChoBALvukvvtXTqFPEUttaAWpt5deVYJt01+UeHwD86BCzLIkBaRwEqsRsF\nqYT0M4ZBZqNSCaHYD4ff2oJCK7ONtjQlmcrc/tDeCUlEKK4rFFhfXovwkSMhCg5CUpbpgBcwCD79\nBoIdHKE5+erOCVraALa+uBh+nUpUecGq8ViE9BZTW/MzY6chNWOl0Za+OzYnOXNNnnSwAHEsClIJ\n6Ye0Qab+Uadqm7KN1m6l82Vut9VdxuTgEUgWAszQwWCHiPUCTXNMnXy1Pm0pZr/wEirydmu+HxOu\nV/+qDVSpsYq4G76t+Zmx07C74ijvlr47nlXvrDXR3NX+jbr7CenH7O30P7RvHz75UyrvjFJDfDNL\nvUJCuS1+3fu+kbbU4uxWb4WCN2Pje0uBLza9hjgvVu+6c6MjUFTfAIAaq4j7io+biOxXNiFv1TvY\ntnozjpd8b3LeaE+6753FWWuiuav9GwWphLg5ewbvW8tUwGcu2yiXyVCQuRHPRIVbHdzGxcfj90tS\nofIVw/uGAopLl3jvGwm1xaBXJRbzjrgJFIlwl0iEwppLRtc1Nx6LEHfEt33eVtWI78qLseVwLnwb\nOsHsOA/BgUoESOuQmdi7mcX4uInITFyGwPwfHbomdyxtIK5D2/2EuDFnN//Y0+lfuD0bEQNtm1Fq\n+Bwfdip5u/o7urq4a5mqH+UrH8ivrMHU0BBEBfhh3Tdn9V5vbjwWIe7KcPu8tbIBV89WQ7I2CWqG\ngYBl0VogR2biQrfZ9nZGo5Y7ljYQ16FMKiFurCeD963Bd9SppWyjt0KBjq4u3mymbnCrmwF+I22p\n3jb81NAQSCuqjY5YTRgZwr2fYRjUy4rx0uMz8PSv7sOaWUnIWJYKAEjckIm/fSvrPkp1+DCu5lQk\n8NK7bm5VLUaEjAT0l0uIWzPcPm84WYLI5Aedtu0tkxcjNWMlUrKeR2rGyh4dJOBIhp9DS0U9Lmwo\nQKPyulutkzgHZVIJcWM9Hbxvia3jpABN9nXKyGDkV9ZgbnT3gP7cqlo89b4m62mYOZ09JgL5lTVg\nwGiOUb1zUtXGkgqMGzcOF8orMC94KPd14E7Qe6MVQ5gupGkPAriTSU7ckImI+HikCGGUYWHAILe0\nAkKBFy5ca8GicdGQBA6mEVTEoxg2U4na1E7tnu9pc5K1Hfi2durrfg4/NTeiWdSO8WtnURNVP0FB\nKiFuzBWnKtl6xn3S0jQcWZuO+OFDkVtaAR8vBjW3lZiVvoYL/swN8NcGolEBfpCE3o1VH2zjgtro\nO9t6LMsir7wKTe3tWDnpHqNMsnRHNub/dRWkq1bozVjNK6/CE9ERiA70B8uy2FlSDklggNF7aQQV\n8QS62+epGSsduu2tGyxWXCjD0HmTbTrxyvBa1gS59gbD2s8hNWMlxMmj7V4n8Ty03U+IG7NnO97Z\ntN36stAw+I0dB59J9+Gvu/dg5pNPAtBkUeuLi3mzPkIvzY8cw+fQXnNHJ4O3ymux4vR3aFDcRNzQ\nIbzXqZdptviSsrYg3zcQu9vasUF2AfcHDecC1F0VmjpVw/fSCCrSU72xNe7I7nltsNiSPBrqORJE\nrHsc12S1aK1s4F7DMAyK68utejZrO/B72qlPTVT9D2VSCekl1swZtWc73hVMZV+1GVE/E41RZaou\nsydJDb3RiiWSMOSolFg0LubO0ak8x6beUuDInW3/V+6sQy6T4dMd23D+zud0c9RoRA0U6F3f1iw0\nHatKDPXW3M6eHnOqi+/Eqoi5D+DsOinEo4cjJGEc/KKCofQTIP2I5WezNng0fF1rZQMaikrh3daJ\n1IyVFrf+qYmq/6EglZBeYEvXvq3b8b1Ju81f5QXklVchJSaq+7jTimqMGizGjBdW8AZ6uiUCIoEA\nDMMgYWQI73V8vLwQ58WiUGfr3vBzkstkPTr+lY5VJXysPZLUGRzVPW8qqPSXhCAqJQHV0tOo/ed3\nAAA2YRiyC80/m7XBo+7rWisb0HSuBjGLHrE62Dd1MteqpGX2fhTEzVGQSkgvMHVqkifUS5rLLmob\nvaID/fHPmlrklJZDJBBAqVYjYaRmRBTfM8plMpR//x3yvL2gVKtxo6MTLMsaXafpdjs6u9QIHjQI\nH12sRCgr4FsigJ5noT35/xFxHldtOTvzKFC+oLKloh4tZfWoyiuCur0TqlvtmLR2Lmryv4LguvlQ\nwdrgUfd1DUWlXIAKWBfsOzKbTDwDBamE9AJnd+1by9btbEvZRZVYjIr6OnxZ3wgBo6k//VXQCL2u\nfcNnlMtk2P3CcwiFttYOuNnRyWVQQ3x98fSYaFReb8W5q01GEwXkMpnJNfckC23L/yMqC+g/TGUN\nVc03kZqx0iFBpaNLCgwD3odj78NbuQcQ/kyC5voV9bhyqgyTNz7J/dkqzzmGtqpGRMx9AGXpB/Wu\n06i8jvranzAiaATuCgxGWlIKMhONj3XNLsyD4nCu3uehDTK92zrtCvadMYuVuC8KUgnpBa7o2uej\nG0w1q1ToariM1NGhVm9nW8ouxk59GKff2YxF42L0Bu0D4BqaDJ9x/YuHMP62CimxMejs8obsl2iI\nff4PF641Y6PsAhTtmvrWovoG7roAUNXShi5lB7YuT8PIeybqBYa6z9moVEIoEGCIt7dNAaS1/4+o\nLKB/4csaNknPQtl1E0i+xyFBpSNLCvgC3t0FRzEr+H58vPEQugYK0NF6E5M3PqV3v5hFj6A85xj8\no0PADh+IfYfysbviKPyS4zCQCUMEOxE1+V9BHS/U1K0mLkP2K5u4ez63OxMK704IBvhA3d6Jkt2Z\neG9Bul6nPtWXEkuou5+QXuCsrn1zR6hywdStFswRqLFECAQo21HV0gbAuoMCLB2jWnLyOBbeqR/V\nfm9udASK6ht4n1EqBQpOrMWp+vdwSzUAfz75Ahb8Zw3EPk+j00uAlbv3IDwmBjkl5RB6eXHX1WZV\nF8fG4EVJuN4xqrrPeU9bM4b9dAlLhLB43Kohvv9H0sYmNDVd0/t8nX3gAnEvfMd/BjYxCE1NcNig\nfVtLCnSnDcz5yyIk/3UJN3kg6+N/8HbU17ZdwZ6Vb2L8oFEQBYp57ycQeWsaFf1EePPgTqPrRMx9\nAI1fXjR61td2vov2AAFiFk9D9NNTEbN4GtoDBHht57vcaxw5rYD0XZRJJaQXOKNr31JGjy8LmiyJ\n1JtdaqnkwFJ2kW+LvKqlDZVtCrxdeQni0WHc16VSICWFBQtvHP35Pkzcu5v73opTz+GRqKuIi49H\n4F13obGuFk3t3RMDDLOquhldsOCe09zrLNWVGv4/au5UoUupxPKBAjCMmvt8lT5CMH4D9d5Lo676\nNsMt55Ss56F2YJ2qLV3supnStqpGXFX9gsjkB6G+kzUt+3spxjL62Vft2uLjJmL/2zuR/NclvFM0\n1O2dqMn/CiFTY3H1/34wGcgaPmv1tZ8gWZ6k9+cuMvlBVGwo5F5D9aXEGhSkEtJLHN21b2kr3lSN\npUjQ3XxkqeQgaWma2Y55wyC28norzv5yFa//elL369em418PfoC/rQ6DWs3w3kfNCvDfyvVIfmwH\nHnw0HN//59+ICfDD+m/PYV5MJNf9b/gs3goFwIJ7TrOvs4Lu/6OMZalI1jnhSvv5ri+rATs4wuWl\nG8R9OHo0ki1d7LqlAXzNSIxYyBuA6q5t1fzlRvcrzzmG9us3IQ4fgYaTJbjZeI0/kFWqjK4nHDyQ\nf07yYP1/zFF9KbGEglRC+ghLjT6msqDtKjX335ZGNGmzizs2vwFFbS1udHZgaEQk933DIPbk5QYs\njtXPZIpa/wcvpd+NLpY/QOXWBm8cOLoIHZdewZYH7+f+8swpKUdp83VuLJXus6jEYoAF95xKtZr3\nL1Z7AkhTn29Q0AgUNLfZPeqKeD5TdarqJgYpWc9D2XgDAqEA3kMG2XwUqKUso25pgDarqStkaixq\ndp1ExMKpJgNe3fs1tjej8lINRj31GzAMg6tnqxGzeBraqhpRLT2NyOQHuevU5H+F4Cljja4XJg7i\n/XMXJg6y+zMm/RMFqYT0EZa24vmyoNLGJigjo/UG7AOarKG5TnXt0H3d7CjXKKSzRd4mFOmt53Dt\nb7Di1J/QZWZ0lK4uVoDPql/Db0e+jxnhX4NhGCyKjUH6+TJIG5v0jkTVDQy1z5kwMgTSimokSyJ7\nHECa+nwD77oLiUvS3O7ABeI6hkGlqvkmlF03IViegGaeLXhbjgK1RDeLq81q6v4e9YsKhu8X9QjM\n/9FswKt7P5m8GNs+3Ytvy2SQrNVs2/tHa05vK885Bq9mJQQ3VRgoGoBrF89gZPgoZBfmIe3OdVbN\nX44V0r9jWPIkvaB9y/zn7fh0SX/GsNqqZTc3bdo0AMDRo0d7eSWEuCdtTaphRk+SsgAlJ49zHf2d\nXV0IEgk1wdQS/QDU1DUSdTrVM5alIpknWJP6BmC1QZ2n7mu/bRyLBf9eDbWVAaouAaPG7t9l4FfB\nFwEAB7oEmPHCCnyqGxgu0e/u135P093vjSHeAt5n7unnm0hd/MRAasZKtNw5Y75s51G9LXhA8+cl\nQFqHbas39/heXE3qrDs1qWer9bKdbQVyZCbaV+uZkvU81HMk3K91T4iK9A3G5a5WrmHM8F7aQFcb\nGDtyzitxb46M1yiTSkgfwdeMFTt9Jirydnc3U3kBBc0KJK7iD6ysGWBvy/xQ3ext/PAK/Db0HI7+\nfJ/NzxbuX4TBPmcA+HLZYXM1vc44pctdj6glrmXNkH1LW/A9aaoypDd7VNAB36aBYHach2DIIKub\nkUw9k7kToliWRbv0NNqqGuEfHWI0JovqTYkjUJBKSB9iGJxlLEu16dQkawJQW2a86gZ29WfP4J7h\nf8bRn7+2+bn2T8/F4UuN8PYaCnkX02v1no4Ifmnwv+eydsi+pS14R88D5QsItYHnlsO5EBearoM1\n90yWToiKTH6Qm6Wq/ZqjT95yFGee4EWch+akEtKHWZprakglFsOwAsgwADU14zU24WHeGa1x8fF4\n5YNsDBgtwQ9X34U9Vpz6M/4QEY1tZVX4BV4o3J5t1axTd2M4q9aWua2k92UX5vHOHDWch6o7AzQk\nYRyqpaddOg9UG3i2JI+Geo4ELcmjkX5kK2TyYpueSXcmrKkTogSi7lyXuTFZ2jmuqRkredfhTLZ8\nHsS9UJBKSB9mTdCpy9IhA9osoMJHiPVlNdjafANS3wBIUhZoygrMBF91V6Nw/Gf7MoYnfo5HcVMM\nJgb4YdnQwQ4N7swdgOBoNPjfs1k7ZF83uBvygwJh2i34O8P/zdWIOiKgszaYtuaZ4uMmIvuVTZgU\nMob3Z4m6vZP775pdJ3Gt6Zremt0hQLTl8yDuhbb7CenDLM01NWRuxJTeYQF+A8EOjtA0Di1Js6qW\ndfJd1QgctwE7S9eCteFHj4BR480p7+P+oFJcuNZl8vr2cPWRprbU8xL3Y8s8VHtqMq0tJ7DElhOr\nrH0mU2O2Qpq8cfHtI2AGCxEyNRaCqGC9NTvyiFd72XqCF3EflEklpA+Li49H4oZM5PsG4kCXAFLf\nAKu60YfeaMULkjCsGx+D5QMF2PfHpdiwaKHJLKA1ZQWNSiVG+x3BW1Peh4BRW7V+bYCaGPYVckrL\nkTAyxOT17aHNbFa1tGFnSRnyyqtwvaEBO7Ne69F1TbE1s03ci7OP8nRUxk+sEvL+PuMLpq19Jr7j\nYLckPY+7w8Iw5i/TMebZR/UbqO6sWTdAbK1sQNnOo6jKK8J35cUuy6ba8nkQ90KZVEL6OG2zj3ar\n/vBbW1BopmGHLyu6UBKBN8+dNxmIWtNMJRQI7swr/QYMw+ClL/9kdhwVAxUeHfUyritPYGeJGs3t\n7dzxrXzXt4e3QoGqtjacu9rEHZ/Ksix2VdRALpM5PJtqa2abuBdnH+VpLuOn2/hj6XAAw6xnS0U9\nfvrka0SFRSA1Y6Xe67XPtO6NbzAorBIB8MJLiX9EW/M9vM9vmB1WHM7lXfMV5XWkZqxEaXkpJLOj\n0VbVaDQdwJ4ssT1sOcGLuBeXBqlXrlzBa6+9hm+//RYDBgzA9OnT8eKLL0IoFLpyGYT0O7Zsa5va\nkr7R2cnbpdzcqcJ8K4KvId7eYO5kUGeEazr8TQWqDFT466RNeHb8ZbBsFKSNTVApldz9HRXcqcRi\nnLxYYnQq1kJJRI9LCfjQGCvP58zRSqa23pWNbVwZQBvP4QArpH/HFjxvFHhuy9+Ln5ob0Sxqx/i1\ns0yWEFSUTsSRXRMxYwbwYR6QkgIcPgzk5QHJyfatufJSDURrZyG4SoRq6WmoFO2IWTytV7b9nf2P\nC+I8Lg1S//znPyMgIAB79+5FS0sL0tPTIRAIsHLlSlcugxCPcV4uw/7CbLDeCjAqMZ5ISsOEONsD\nGmtqRrVMZUXFPt7IK6/ijiNlWRbSimp0iQYAgKaswEzwZXjdGeFfg2VZvHTqOfzv4wLk6fzl+HrG\nT+j48UccUNwZwJ+lCUYdFdxps8rKK4345Xa7TRMQesoZM1xJ32Aq4+cnFHBf4xsFNSx5ErJ2/APS\nTTu4a2mD6dSMlRDfOVhA+3rd4FAq1fy5U6uBTz8FBg/uXk/KnR1/c4Eq35prc4u4Y1W146kuFX7X\nq3WhNLfVM7ksSK2pqcH58+dx+vRpDBkyBIAmaN20aRMFqYTwOC+XYe+RdDycrM1OtmBvQTqATJsD\nVXsH8Gv/0smvrMEfIsJxqLoWOaXlEAkEUKrVSBgZgqgATbC7+oNtZoMvvusq/b/Ah9tnIOX/RUMo\nBPbvB77+Gpg6NRyA8bUcEdwZNoB9OHAAb4aY6kSJq5nK+G05nAu1hcMBahVXuF/L5MV4bee7qL72\nEzpEQBy3ArUvAAAgAElEQVQTZvR6hXenXoDKR60Gnpqnxo6CT7B59Xgu82g4c3SqXzQOrv8MrL8Q\nTGsHWKUKjawKV7+rhFqpQkjCOIgCxfx/zppv9vyDI32Wy4LU4cOH48MPP+QCVEDzG/TGjRuuWgIh\nHmV/YTYXoAKav1genuWH/dJsTIizbRvangH8Oza/gZvlFzGYYTA1VBOMCkUiLI4dY/Qe3WDX1LB6\n01vd0dx7hUJg6lSbHs1mhlnlqaEhkFZU36mXpTpR0vs0TT7djT7WHA7QdPkKUrKeh6r5Fiqv/Qyv\n8ABIliehPOcY7+uvyCYjZbXpAJV7bZcAxw7OwzOCncj9m+ZruhMI6irqceb0N4ha9zj356ci5xiC\np4xFgGSkZjRV/lcQjx6O6p3HEbn4Ye511dLTGNClhkxe7PCtdxre3ze4LEgdPHgwHnzwQe7XLMti\nz549eOCBB1y1BEI8CuvN3zHPetu+DW3PKKpNe/dBLpPh0x3b8IPiBs76ijFIMsZs1tFU7WtVygKU\nnDwOb4UCrFiMGS+s6LU6TMOssrYZa2NJBcaNG0d1oqTXmBpBtSB2GnYXHIXfrDjucIDI5Ae5P8vl\nOccQnfoo1JKRAMuidc1FTE6eAYZhEJIwDjX5XyFi7gPdr18nwoWDKRYDVC1W7YXz+YuxUr0XkXE/\n6I2UavzyolH5gWTRIyjPOYYAyUgwDIOIuQ/gwoYCjBINQXnOMQhE3lyG1S8q2OF1qY4a5UV6X691\n92/atAllZWU4ePBgby2BELfGqMRgWePsJ6OyfRva3oYdw/pJuUxmNtjlq32N82Jx6r238ExUuNmm\nLb4MrPaajjxClC+rHBXgB0no3Vjl4EYpQmxhaqbocen3yEzSlAFcb7mMq2WVuH2lFQOGitFy8TKi\nF/wWAZKR3Hv8Y0Zy19DWhFbkHoOgtRMh7Ym4kP8s1GqGfxEmsGovHN3/FFqYWkQ92f1eU+UHuidR\nMQyD6LAI+Pr6ImCOxOjajq5LdYfZrMQxeiVI3bx5Mz7++GO88847iIyM7I0lkH7EUc1Hzma4zomx\nD+N4wW48PKs7IDxe0IZ5SfZtQzuiYcdSsMtX+/plfSM33gngb9riy8BKV61Ai1KJ1NGhDh20T2Og\niLsyN4JKtxEq8K3fcK+r3HOSC1C1ujr0SwL8o0PgFxWMig2FCIqsw0NT2nDyhD9sNfKROlzvOgqW\nncNd21T5QUvZZVTuOQm1UoXgKWMxWhQIqGD1YQg9QcP7+w6XB6kbN26EVCrF5s2b8eijj7r69qSf\ncWTzkeF1HRn48q3zeMFu3CdZgNP5J9AluAFGJca8pFW9HmCbC3b5spRCLy+L3fN8Gdjk4GHYfPYH\n5NxUcI1aU0YGo7CHo6FoDBRxV9ac/mQYgPEFicFTxqI855jeTNKqvCIEz/sVbkQFw8drDXDiXZvX\n9+s3/4ublx/UuzbfvcpzjiH6//2Wq0mt2nUCCyfPRUyUxCXzSm05GYy4N5cGqe+//z6kUinefvtt\nPPbYY668NemnHNl8pOWMwNfUOk9Jj2PjasdsQZtqaHIkvixlzW2l2TpWgD8DW9XShqEDB+gN2c+v\nrMF1MwcAAMChfftw8O034Q8WrWAw+4WXMPPJJ/VeQ2OgiDuyZui8YQDGV6N6TVYLcdhwlOccw62f\nm+HjPxB3z5gE/+gQqDu8UFr0F7vW981Lj+GBd/+Fqn1f4szafRAF+EJ5XQGVshPlOccgalWhq02J\n4HmT9coPohb+Fsel3+PJmXNdMq/UluH91GDl3lwWpFZXV2Pr1q1ITU1FfHw8mpqauO8NGzbMVcsg\n/Ywjm4+0nBH4OmOdulx1Rj1flnJW+hoU5O02ub0ul8lwoaICt6FGR1cXEkaGIDrQHycvNxgN2Z8b\nHYH15TUm739o3z6ceWcz1o3pnuW6653NAGAUqBLibqwZOm8YgPlFBePG0QpcyfwPfhG1QxgwCCFT\nY+EXFYyqXScQOngYgv7yMPf+a7JgNJwIs2t9DSdGo+4zb4hDhxllTkMSxmHIDwoArKaBS4fuVrup\neaWODBatHd5PDVbuz2VB6tGjR9HV1YWtW7di69atAMBlVy5evOiqZRA34MoaUUc2H3HvNxNQ2vts\nzlinLluG+fcUX5ZSHhPDu72uDZ7XjYnQy5ayYNGm7uL9nMNHhpq898G33+QCVO3rF8ZEYf07b1KQ\nStyCbjCmar4FdYcaouDBeoGZuaHzvAHYgnRkF+ZhwL1CXDl1Eb98W4GGolIETxmLa3vPYIRO5nXE\nr+rxqy3/xrcvPgbWwq6ELkbQhV9t+Td++WY3Jm98Uu/PWMyiR3Bm9SeID4nB0GFDbd5qd0awaM3w\nfmqwcn8uC1KXLl2KpUuXuup2xE05q0bUlCeS0rC3IL3HzUe6wWfphQqMmjAU4ZIA7vssy+Jao9Lu\nZ3PUOk2xZZi/M8TFxwNLUjXlBjcUKNyeDSxN4w2etdnSkWPG8pYJiIKDTN7HHyxvYOvPsibeYRtX\nlEyQvsswGAPL4pL0NIbfMxKqqGCrAzO+AExxOBcBkjCjJipBUJDm1CqdrW/BzzkY90QtSg+mglV7\nWV44o8YDc/fg5tk9GBjkz/9nbEworvuwmBN7Hzcuy9q6094KFqnByv312ggq0j85Y6vcHE1wmIkD\n+dvsbj4yDKynzI7AoV1VAIBwSQAXUPoI/bgg09Znc8Q6zbF2mL+zgjDDcoOK+jpsWvA0xEIhmPEx\neq9lGAbjJRLMeGGFzV34rWB4A9tWnV/b+4yuKpkgfRdfMBaZrGlE8o8Ogd+sOLz2j3cxdNhQm7e9\nTTUL3RUYhLTEFL3MK7z8EZqhht8D/8E3Lz1mNlBlvNR4ZM4n+O/eBUjNuIBvFVW8f8a6OlQY9vQk\nvXFZ1tad9lawSA1W7o+CVOJSzq695DMhLh4T4uxvkvnw4yzMWKIffM5cGIVt62swbvxQLqDMP7wF\nDKM/HduWZ+vpOs2xZuySM4Mw3Yxp5fVWyK5ew1Pho/DRxUp8fLFCrxZVGzzb04U/+4WXsOudzXgw\naDi+rG+E0MsLFa1t+M0zz/b4GV1ZMkH6JlPBmHamaFtVI5p8b0CQfK/N297aWlU2bhgav7wIL6E3\nlDVNWDMrzSjzmpL1PNQMg7tnaP6xbSpQZbzUmJCcg81/u4+7R8nuTKNGrZr8rxCSME5vXFYqwJU1\nZBfmIQ0w+QymgsWKCxeRkvW80xqabGmwIr2DglTiUs6uvXS083IZmm+VgWHG6n2dYRiMGy9Bxqo8\n7mv7C+1/NmfX6VoT8DkzCNMtNyiqb8CUkcGQXb2Gjb+ZbFSLKu9iuODZ1i78mU8+iZ/qLuH4ASmW\n6kwFKDh1EvLf/U+PnrG3SyaI5zMVjKmVKgBAw8kSxCyeZte2d3zcRCyomoa3Tx3Qa2raXXAUMXKJ\nXoCnuw7dQHVw+Bncl/U5vn7+EdyqfwDhv3kDuX9L5N4bHzcR7y1IR2bOeziz4mP4TxyNrg7NyVH+\n0SFcFnLfoXxsLMiGKGIYum5q5qSmHzEdbPMFi1W7TmD4vMlQS0Y6raHJ2gYr0nsoSCUu5ezaS0fb\nX5iNQWL9LeRLla04c7IBt9tEWJORygWU9j6bq+p0LQV8zgzCdMsNRAIB74B/bS3qy1u39yhz23ap\nFkvHSXgD0Z48o7UlE4SYwheMVUtPIyRhHFiWBavo2bb38ZLvEfZMgsUg13AdoxIr0Sz/Fnc/rsL1\nC1V46IMudN5oQshPvwAAUjNW6pUf7H97Z3d97dP6WciZsdPw1vcHMH7tLL1M69D4cGQX8gfbhsFi\nxYWLGG4wxspZNarWNFiR3kNBKnEpZ9deOhrrrcDkqSH4PL8G0+dGoK6qDaVnr2L24hgTAaXtz+bq\nOl1TnBmE6ZYbKNVqkwP+x0skPS4tMDVztbykAoN9hPiwU4mpoZrSAsD6Z6STqkhPGQZjquabCO8c\nBOEPCojP1mHMoJG89Z7W1khaW9upu47G9maUnL8An1GB+OWbgXeyoiPAspeh/L4NKwr/jmHJk7jy\ngxXSv2MLnjeZhcwuzEO4QaAcMfcBlOccw1D/u81+NtpgMSXrebNjrEj/QUEqcTln1l46GqMSY3SU\nZiuuILcCl+tu4LlXJ5kMKPmezdJWfm/U6fJxZhCmW27QERKK2ooKiwP+7WUYbFdeb8XZX65ibayE\ney5pRTUAICrAz+pnpJOqiCOYy9zJ5MU9qpG0pRFIdx1cVtTgvkzHLQxLfkjv592w5EnI2vEPSDft\nMDllgO/nmZfQG8rGNqOsLN/Wuu5ztFY2oKGoFF5CbwiqWyGTF9N2fD9CQSohZuhu4YdF+6Pw4wqb\nAkprtvLdpU7XliDMng553XKDQ/v2Ife9t/FMVJjDA2LDYJvvUIBkSSQ2lFQgJvRumwJNOqmKOFNP\nayTtbQQydd+0rWvhp/NzSRswdjbfQGrGSt4g01SgfPNiA65PiIIgebTFpjDdJrBrslq9Glsatt+/\nCF599dVXe3sR1vjoo48AAAsWLOjllZD+JCgoBCP8Y/FFQQkulXWitvw6Jk0NNPoB/JPcD48kPG70\n/ldffw63VD+i+mIL5N9fhe9gISY+6Id/HbzAvX5Y4F347NMjCBsr0qtlffLxlxEUFOKyZwWAoJAQ\nJMz4Xzw0czYSZjyOoBDj+2s75J/wZhHrxSK2sx2fHjkC/7GxvK/nM3b8eIy4dzIK5CUoU3ZCLvbD\n4yte1gsW5TIZdmx4FV9J9+HEf/6NwNC7rLp+UEgI/MfGctf+pbUNDwwL1HsNwzBoGhaE9A9zrV4z\nIa4QEhSM0MDhOH+xBArvDpy/WIq7AkcgJCjYqvfG+o9CScEpXP2yHD8fPocA38Eoq6q0eA2++1aU\nVyLosfFcRrPpXA2in05A0JSxaI/1x5FPDyPWf5Tede8KHIEjnx6GaOwI7udZbW4RRqp8MST1Ab1/\nLIrGjsCFg1/i8YTf8T5HfvZHiFj+qFXvIe7DkfEaZVKJwzi6Q92VJ1OZo7uFf14us7o56rxcBta3\nDrMXdneZf56vOdKT9fbWu74n1ek6agqAblZSm5k9fCczGzv1YVTk7UbyED9UtbXh5MUSbPvuGwyS\njMH8v1rOfOpe+69PJTuttID0L644510mL8ZzuzOh8O6EYIAP1O2dKNmdifcWpPPei29Nqb+fh/Qj\nWxGR9rjFrOW+Q/l48+BOdPp64WarApH/LwEBkjC0sCx8FTUo+cfniF0+HQ1FpVxGEzDdzMSXld32\nVDq2HM6F2oamsPi4iZCMH2PTe0jfQ0EqcQhHd6i7+mQq3fuaC4xtCSj3F2Zj5sIIvR/q0+dG4ODO\ncgSJ9RsIPKVOVy6Tob64GExMuN7XezIFgG92ae57b+GhYUNQ1dKGc1ebuO16lmVtmt0ql8nQ1XAZ\nUmU7kiWR3TWpjU1IyqKGJ2I9V53z/trOd9E+TICY5AS9CQCv7XwXB97JsWpNfte64LfkHosB5b5D\n+dh8Jh9R6x7n7lWecwyXCr9H+B/uR+Tih1H8ihTlOcfQ0XrT6skDfLWq4kLbB+fTsH1ixXlohFi2\nvzCb/7SlQvsCL0dfzxrawPih5BYkzFHjoeQW7D2SjvNymd7rJsTFY8Mr2chYlYeNq7eZDJpNNUTd\nusHiiaQ0pz2Hs2iDSb9OJViDY0Z7kpks3J7N1Y8Cms/omahwfFnfiKL6BsyN1g/0Zw3xQ+EO634f\nFG7PRuroUEwaMRy5pRXYU1aJnSXlaAoIpIYnYpPswjyu1hPQCfwK8yy80zbV137iBuVr7xOZ/CCq\nr/1k9ZouKa5YFVC+eXAnohb+Vu/9MYsewYBAXzSdq0FbVSPGxsTg1+IoiG508f651w7cT81YCZm8\n2ORzpSWloK1Azl1DWy+blpTi0PeQvoUyqcQhHN2hbu56zioDcPQoKFMNUUN8x7jtVr4pcpkMb6Qt\nxboxEajyAvIra7jgsadNT6Zml2rHVPH9PrA2a6u9dnSgPzdyCgAOdAnsWivpv1x1dKdw8EDe+wgH\nD7R6TR03bls1yor1F/K+XzDARzM2audR/EochexXNvFOHrBl4L49TWE0bJ9QkEocwtEd6qaud61R\n6bQyAEcH2qaG+z87P7NH63QWUx372gxqJNRgGIYL9nJLKyAUeEHe0oawaAkKt2cDVnT5GzI1n7Xm\nthKhQp8e1ZPSAH7iKK7aeg4TB/H+ng8TB1m9psiho9BWILfY5c+0dvDeS61Uad53owNpKZqspSMG\n7tszOJ+G7fdvtN1PHOKJpDQcL2jT25Y5XtBm97a2qev5CAVOKwPQBMbG21n2BtoT4uIxLzETp/MD\nUXRAgFPSAMxLdG5Nrb24utBbLZgjUCP5VguOrE3nAtdZQ/zQ0dW93Rcd6I9FsTFIiYlCtNgXy4YO\n1nuPLZKWpqGgWf//dUFzG2alr4EyMhq7KmqMvpe0xLrfV6aube37CdFy1dbzqvnL0SQ9q3efJulZ\nrJq/nHuNTF6M1IyV+Pl6I2pzi4zW9MriP2OBZBpq13+Gqrf/hZr1n2GBZJpRBvKl2YtRteuE3vtr\n8r/iTsAa4ztS7z3xcROR/com5K16B5LxY7gAVYuamoijMazh38puatq0aQCAo0eP9vJKiCnn5TIc\n+LS7ocgR3f2G18s/vAUJc9RGry06IEDGqp7VhnHNWoad+24aWDpSxrJUJPNkHKW+AfC+ocAcgRqV\n11tx7mqT3ja/tKIak0YM1zu9SeobgNVWdPnrZm6bVSp0dnUhSCTUZHGXdGdk5TIZPtWd3XonwLR2\nTivf+6keldhDJi/Gtk+7t56d0d1v6T6Gg/dbKurx075vED06HEGiQC5o5hvOn5nI392f9clWdAwR\nQhgwCCFTY+EXFWzy9VqpGSvRcmfmqRbLsgiQ1mHb6s0O/0yI53BkvEZBKvEoazJS8VCycTB1ShqA\njat7foSoowNtT5E1PwVzBMbB/4EuAVS+Yi6Arbzeii/rG+HjxaD4eivSxkr0aj2171n1kfl/MGgz\nt4anWyVa0bXfk/cS4umsCQ7tCSBl8mJk5ryHqqYfoe5Sw0sFjJZEIPhO4GsYrJo6pWqBZBqOl3zv\n1DFdxL05Ml6jmlTiUUzVefLNKbWHp4yCcjRztZtJS9IgXbUCycHDEB3oj6gAP0gbmxA+OgJRA/Ub\nkKyt9+zJrFVHzWklxBNZ08Blb5OXItgbQY/dj2uyWkTMfcDsqC2+pqaZsdOwu+IoNxKrrqIeT296\nEVFhpoNdQsyhIJV4FE8bfO8pDI8SNezYb1EqkVNaDpFAgHaVGiqxGFNnzUFB3m6T7zHHVEe/NV37\npt6rbLxiwxMT4plMNUspG9uQmrESCu8OVFwow9AJYr2aUUtNXtmFefBLjkN5zjHeof1L178Myfgx\netlRw6am1IyVXIDaWtmAa7JajF87y6lzZUnfRkEq8TiOzna6y8lWvSkuPh7YkIl83drNO2faZyxL\nReroUON61aLjSDLxHkt60nVv6r2Xqiohl8loy5/0adpz7XW32S9vK4JomC8Ed7b4I2ZHo2rXCQBA\ngGSkye5+Xdrsq0DkzZuFVUf6Qz1HYjbY1M3gWntCFSHmUJBK+rXeOtnKHekeJarLXNbT1Hss4cvc\n5lbV4r7nXrTqvbl/SsUzUeHce/Mra/BU+CgU0pY/6eP4ttnh5Q8k658wFbXwt6hZ/xmGjh9r1XxR\nbYZWrVTxjqVqq2xA2c6jCEkYZzLY1M3ymgp2qfuf2IKCVNInWZsddfQAf1dwdebXGbNG4+LjUZWy\nABszNyJ8oAgdXV2YMjIY8rzdOASg5ORxk537cfHx2DVkODenValWI2FkCKID/XHezqNZCfEkhtvs\nKVnP855xLxk/Fnmr3oFMXozswjwoDueabGbSZmiDp4xFTf5XXE2q9ljW6PlT4RcVjJr8rwAA3jzB\npm6W11SwS0eaEltQkEr6HFuyowplIxhG/yQXhmGgULpnfWNvZH4t1avaq+TkcayJH6+fbbneglPv\nvaXJkgoYsLdaULA2HTDo3BeIxXhmnMToL8DmTlWP1kSIJzJ30ADXhX+nVtRiI9Sne1Ff2ogza/bB\nRzwAA4YNRsjUWPhHhwAAdxLV3eIoo3XoZnlDO8SozS1C+DMJZg8UIMQcClJJn2NLdrSuth4sG2H0\nw72u9rLN93VFhrM3Mr/m6lV7gq+M4Mv6RiwaF2Oxc79DrYa04hKSJZF6M1s77g7r0ZoI8QRcZvTO\nmKeHY+/D7oKjvCdMaRui2qoa0VBUCoHIG+r2Try2810ceCdH77raDG1K1vNQz5Ggcs9JRD89Ve81\nhidRGdLN8srkxXSkKekRClJJn2PqeNNLDWexJiNVL3gcETQCn+fXYPrc7gH1n+fXYETQCIv30Q1K\n25pVuN11GXNSQ23OcNoS3Pbk6NaeBNH21p6aw1dGIPTy4n0+w67/YJEI94ww3vL/QSRy6BoJcTd8\nmdHdBUexQDINJ/LPGAWEisO5aKtqRNO5Gq6RiWVZ1Ow6CZm8mDdotFSfangSlSl0pCnpKQpSSZ+j\nOd7UuIZyoJ8KDyXrB4/DA+/CXfFqFORWwEfohQ6lGpOmBOPnc3eZvYfxtjvwubQddVVtCIv2tzrD\naev2valns3R0qzs2iPGVEdTcVvL+pWhY/6oSixHlpdI7SIBlWZz1tb9O1pDuiViWTrUixFW0mVHD\nrvnj0u95B/WLVUKUnCxBzOJpeu+JWDjVZKe9ufrUtgI5MnWOaCXEmbx6ewGEONoTSWk4XqB/Xvvn\n+TWYnBDSHTwWZnOvrZUzmPWMBL9/OhqzF8WgVs7giSTzZ7vvL8zmDhQAND/0pydH4kxRA/caazKc\nfNfRXZ81z3ZoVw2amq7hvFzmsPu4Qlx8PBI3ZCLfNxAHugSQ+gZgVvoaFDTrP19Bcxt3FKpW0tI0\nq15nL+2pVsm3WjBHoEbyrRYcWZsOucz0Z0yIK9g6qD8tKQWsgv89ZxvKkJqxEjJ5sd734uMmIjNx\nGcJknRjR7I3a9Z+hfbcMAdI6s0elEuJolEklfY7uwP/a+jMY6KfC5IQQhEVrsm51VW0oKa/A6qwU\nMCox7otdgNP5J9AluIFrjUr4CP2Qf3gL9hea3hI3te0uFHWfwGRNhpP1VqCuqg1nihogFAnQoVRj\nckIIWG/+P5raZ9u54w1cu3kRgwYzmDw1BKOjBGYzoz0pE3AmvjICeUyMxfpXZ9XJatGpVsRdmWuS\n4hMfNxFjBo3k36Hw80FL8miTjVSu2Ko3rK+lU6mILgpSiUczVWepHfi/JiMVDyV3b49fqmxF6dmr\nSF0rAcOowbItOF6wG/N+nwkAmi3xWX7c93QDP917lV6owKgJQxEuCeDWwrIsOtrV3H9bc1zrtUYl\nFKqrmL0oprsmVloNZZOvyfdMiIuHn58/EpeM0ftLx1x5gb1lAr3B2vpXZ9TJavXkRCxCnIlvmL+l\nrvlV85cbvacm/yuEJIzr1SH71k4eIP0XBanEY1lTZ/lEUhr2FqRzW91nTjZg9uIY421vqSbYMdU5\nD6Tp3WvK7Agc2lUFAAiXBIBlWfxb2gRvZTSKDgisPq7VRyjA9ORIo7KBwzvMP7utmVHDz8HaILq/\ncsZsWEIcgW+Yv6Wued33nKm/CJWfD0ISxnFjpXpryL6p+lo6lYpoUZBKPJY145h0t/67BDdwu01k\nNrgz9T2+e81cGIVt62swbvxQMCoxFlgRlBryG+INhlEb3dNviMDEO+68xsbMqOHnYG0Q7Y5c0dDk\nrNmwhDiCPVvx2vekZqxEy53jU7XsHbLf0616W+trSf9DQSpxC/aMR7I2m6jd+geANRmpZoM7U98z\nda9x4yXIWJVn+wNrr2Ei2GxrNj+U3p7MqO7n4Km4hqYhfmaH/feUs2teCektaUkpeG5bJhTenRAM\n8IG6vVOTjV2QbtN1HLFVb2t9Lel/KEglvc7e8Uj21FlOjH0Yh3ZtxsyFUVxwd2hXFR6bvBLRUTEm\nA7/9hdk238uawPuJpDTslq7A75KH6dWk3u4agPNymcnn70uZUVu4sqHJmTWvhPQmUYAvQpMncT9z\nmqRnbb6GI7bq7amvJf0LBamk19l7ipI92cTikuOIf3C40VzU4nPHMXvmkzAd+Nl2L2sD7wlx8bi1\nMxAHc8r1uvtHR1l+/r6QGbUVNTQR0jPZhXkYdidABTR/foYlT7K5DtQRW/X21NeS/oWCVNLr7B2P\nZE82kfVWIFwSoNeVDwA/nVdw1zQV+LU2+GLbxgr4ioUIFIfh2fmmM722BN5Dg0VImDOGd61EHzU0\nEdIzjqoDddRWPZ1KRcyhIJX0OlPb9iUlpUbHmBqyNZtoT4mANiv6h+V+YJgYLotqji2BtyeNh3KE\nnjQ+UUMTIT3jqOCStuqJK7j8xKmOjg6kp6fjvvvuw5QpU5Cbm+vqJRA3w3tClLQa0+cFa44xPZJu\n9jQlU87LZViTkYrVWSlYk5GK83IZ772OF7SZPWFKe1pTXVUbDuwsw2d5VWi+3oCtO18z+R5N4Mnq\nfc1U4GnPmjxVT09y4julKtHBTVOE9BUyeTFSM1YiJet57mSptKQUtBXI9X7etBXIkZaUYtO1tadS\nBeb/CMGBSjqNijgFwxr+TepkGzduxNmzZ5GVlYWff/4ZL7/8Ml5//XX87ne/M/u+adOmAQCOHj3q\nimUSFzsvl+HAp9tQe1mGgX5KTJ7afUIUy7I4JQ3AxtXWN8ZwNaGGNaSJmqH9Bz7tLhGwNElgdVYK\n7r6nGaXnmjB9boROw1UN/vjkdt73Hjy0D//5/i3MfCbc6P58r9c+v7Vr8lQZy1KRzLNdL/UNwGo6\nyYkQh+G67w0ynZmJmkzntk+760DplCfiSI6M11y63X/79m0cOHAAO3fuxJgxYzBmzBg8++yz2LNn\nj8UglfRt2m371VkpSJhjPDfU1vpM3ZrQS5WtOFPUAB+hF159YylefXk7NrxiW4nAmZMlRocAzFwY\nwQZPFtkAACAASURBVFtjevDQPnxcsBHDQrzx7pozCBgmws1WBvNnrbHQrZ/KTQPYX6g5QKCvBarU\n+ESIa5jtvl+9mepAiUdwaZBaVlYGtVqNiRO7/3BMmjQJ27ZRBoVoOKo+U1sT+vXRn3H5kkLv2FFr\nxlvpeiIpDZu2fWNVjel5uQz/+f4tLFs7vnukVH4NEhKH6kwQMGbvGC5PQ41PhLgGDconfYFLg9Sr\nV68iICAA3t7dtx06dCiUSiWuX7+OwMBAVy6HuCFHHd/JqMSorajDmS+v4E/r7jXqsv9wRxb8/QKs\nOjzgenM8/EWxYFk1VJ3euCiTYMKvLvIGz/sLs7ktfu39ps+NwEd/v4Cmhmq8sCYJYlEwdz/tLNVL\n9cUY6KdEXRUQFu1v9Rguc+w5IMHZqPGJENegQfmkL3D5dr9QKNT7mvbXHR0drlwKcVM9HVKvDcyu\nXv8Z5z6qRljMYN5sQvPNi5ixZIzFrKVUCqSkAFOmfADRJ8/g66IV+O7EvXh5y3voUn5hFDyb7Orv\nYvFi1r1696usWoDvK3bj4WQ/TGHCuawr0B2o2juGyl0zs3SSEyGOZepoUuq+J32BS4NUkUhkFIxq\nfz1w4EBXLoW4MXuH1OsHZoORv30QOju6NBlPg2zCoMGMxRmm2gBVrQZOnAjAiROHuO9lvfgn3HN/\nPZSKVyEuDMbE2IdRXHIcVT9exLnVrRgyfAAGiX24wfxiP6HR/batfxOp6yKMsq4Hc8oRFu2vl6m1\nNStq7wEJruCKk5x6MuaKEE9h6WhSGpRPPJ1Lg9SgoCC0tLSgq6sLXl6a6VdNTU0YMGAA/Pz8XLkU\n0gdt3fka1AMa8FneL+hQqnH7lgoJiaPweX6NXlf+/u3luP+REL33arOW2mCwpHQ8/in9I9guAe+9\nWNYb58+8julPvY+HEr/CoV2bEf/gcEx5UsJlRMfGD0Xp2av4z8FaPDY73Oh+vv4sb9ZVKBLolTnY\nkxW194CEvoAbczXED4yAAXurBQVr0wEaVUX6GEtHk9KgfOLpXDondezYsfD29kZxcTH3tTNnzmD8\n+PGuXAbpg87LZWB96zB7cQx+/3Q0Zi+KgcDHC/LvrmJs/FDkvnke7649g22v/4AfKzt4Z5hea1Ri\n75F0qMVjUZhvOkDV6lIL8MZLf8LJIw9g5sIonP2yEUB3RvTsl42YnhyJluZ2jI7S/0cYy7K42crw\nruNydRdOSQO4cVXaOa1GWdFC09lIW+a09jWF27O5mldA83nNGuKHwh396whZ0vdRcxTp61wapA4Y\nMABJSUlYt24d5HI5/vvf/yI3NxcLFixw5TJIH7S/MBszF+pvnaf8cRyu1t9EwfbLCBgqxnPrJyEt\nfSJe2hwH2emrqK1oAdA9PN9HKICX6H/wxorn0KU2H6Bq6QaqQlH3e7QZUYZhMDrKH5/n1xgN60+Z\n/RLvEP/1f83BxtXbuCypPVnR/nRAgCFvBf/nRWOuSF8jVgl5/zFKzVGkr3D5sah/+9vfsH79eixY\nsACDBw/G888/j0cffdTVyyAejK8+UxvIaWeiCkUCdCjVUNzohLfQ2yiAnbkwCtvW12Dc+KFcc9bm\n9/4P/3jL+gBVSxuoJiafB3ATgOYvig6lGizLorleCG/1CGxbX4vR4SMhFgVxzWDR8hiLTWL2jOXq\naQOaJ6MxV6S/oOYo0te5/MQpe9GJU/2HuSYhUydJtTb4YuLvFLgou6ZXf3rgw3KwLIu5S8ca3afo\ngAAZq/K4Xy9/aR7O/fACvjl6n81rHh19Ais3b0F0rFivJlV2+hqYm6OxbPErdgeI5k7P6g9Bp620\nNamGY67o+FTSF8nkxXR6FHErHnviFCGWWGoSMtW1fngH8PknP3FD9LXfm/NsDP6+5gxvh79hJrJL\n3YH437yIb45+afO6Yyb8CV//l8XxT1W4dkUFvwAf3LrRiclTQzA6SoC9Bek4dnwqir47CF9/TT1q\nyuyXTA73N9Ta4IttGyvgKxYiUByGZ+dTgGoKjbki/Qk1R5G+jIJU4lYsjU4yVZ/pN0SAcFGU0ffq\nqtrQfluFgpxyzNI5depQbi0eu+9FvdcGDBuEzw9vsmvdbS25CA1fhraWFgwZIcLQEUJMnhqCsGh/\nAEB4HIvvju9D6jqdNezaDABmA1Vt0P6H5X5gmBgui0rMc8WYK0IIIc5FQSpxK5aahEzVZ5aUlIJh\nvVFbMRThkgAAwKXKVpSevYqXt/wadVVtOJhTjrbrSgi8vZCQOAr/OvY+Tn59CH5DvMGoxPju1HCc\n/9b2rX4A+O7EvZj1zP+iUv5PzHl2jNFg/rNfNmLu0hieutg3zQap7jzvlBDSd5g6FICQ3kRBKnEr\nlpqE+I5N/VxajenzgjE6yg+HdlUBAMIlAThzsgGzF2sCw7Bof25A/kd/v4BjhZcQMGQAZiwCGEYN\nlm3BL7e+R8TETcjZ8rJNzVNeAjVefvN93POrMtRVat5nOJjfR+jFG3z7+psvCe/P804JIbazJ9i0\ndCgAIb3FpSOoCLHE0uikCXHxmJeYidP5gfj4rVoc3FmOcZOGc8eIzlwYhS/2XkPRAQFuXOcPDFmW\nhX/gAG77X/v1Wc9Eg2Hy8fKb78NLoLZqvdoA9bczvuY6+nXvpR3Mf6n8Bu+omJutjOEl9dfbj+ed\nEkJsow02W5JHQz1Hgpbk0Ug/shUyebHZ92UX5nETAgCdQwEK88y+jxBnoyCVuBXdILTogEBvqL3u\naza8ko3Rd0dhzrNjuLpPQPPDddx4CTJW5SFq1H38MwQHCyEcIDB52tNvZ3xtVaDKeKnx8pb3uAD1\nc2k1Jid0n2SlO5g/LjIRB3PK9YLvgznlSLh/ttl79Od5p4QQ29gbbNKhAMRd0XY/cTsT4uIxIc5y\n04s1pQG7pSvwu+Rh3aOIcsoxeWoIzhQ18Hb8azOhv53xNcCyyHrpOd6TpwQCIPP1H6G4WYaiAwK0\nNatwu2sAd7KU7mD+CXHxWJORikn3BqMgtwI+Qi90KNWYNCUYP5+rtfhZ9Nd5p4QQ29gbbIpVQrTw\n/DykQwFIb6MglXgsvvrUQ7m1GBM8C2syUsF6K3DhzC+4cqURAUMHoEOpRqvOEaWf59fozVQ9tKsK\nk6YEA9D8gFYqPsO0/+nC0S+eR7REjt27GLyeeQ8OHwby8oDk5HAA3cH0ebnMZDDJeisQLglAuCSA\nO3BA/t1V1FdfwXm5zGzQaW3QTgjp3+wNNulQAOKuaJg/8WgHD+3DnkMbMTJchM6OLkyaEgzZ6auI\nf3A4wiUBXJf9uHuHISzaX9Pxf64J0+dGoK6qDWeKGnCrjcUQ3zGY+puZ+KH0BLoEN+5kRi9jTmoo\n5N+Nw5iJ5Tj1f9fxxGOvQ3FjIqZO1dyf7+ABAEZf21+YjYeSW1BX1cbd39bB/OYOOSCEEK4ByiDY\nzEy03ABFhwIQR3FkvEZBKvFoazJS8VCy8Zb/wZxyzFk8hvfXtRUt+GJvM0aHj0Rd7WWMCBoBRj0Q\nPkIBN46qqeka/rBcYHTdU9IAbFytGf108NA+/Of7tzDzmXDuL4Qjexrwy5XrWPjSOL0g9D7JAnxf\nsRvN17snDpi6Lh86dYoQYg0KNklvc2S8Ro1TxKOZGtEkFAn0fu0j1PxWZ1kWtXIGKbNfhEisROq6\nCNz7aBdEwy5hxhIgYY4aDyW3gPWtQ11Vm9F1taOfzstl+LhgIxegar+f+HQIBgcI9L728Cw/FJcc\nx7zETNy+IbJrpNSHH2dxAarudfcXUhkAIaRbfNxEZL+yCXmr3sG21ZspQCUejYJU4tFMjWjSHQXF\nsiwaqgV60wKKS45zQd+ZogZMT440GLQfgTMnG4yuq23K2l+YjdAI/oBTOEBg9LVLDWexvzAbgb5h\nNo+UOi+XoflWGc1LJYQQ0q9QkEo8Gt+IpoLccr0GqH9Lm/Dqy9uRsSoPG1dvw4S4eL0MrFDEP47q\n1g3W5Ogn1luBzo4u/gC5XW30tYF+KjyU3ILbXZfxb2mTTSOl9hdmY5CYoXmphBBC+hXq7icu5ejm\nH90RTbWXZRjop0RouBjnTl2B/Lur6GhXQ9AeZXQP3fFVHUq10Tiq2ooWtDYx2PPOJdxs60DQ0Egs\nW9xd/8moxJg0JdhoQsCBD8uh7Oi+nrYednJCCBiGwZzUUPzzH2qczg+0eqQU663A5KkhPNMIavDH\nJ7fb/dkRQggh7oyCVOIyXPNPsrb5pwV7C9IBWNf8YyrA1Y5oWp2VgoQ5xgP4iw4IjK6hUDZi64Yq\nTH9qFCYnhGiOVr2z5V9b0QLZ6av4U0aMXpOSrieS0rD3SDrGxg9FQW4FvH0Y1FW2wVvIYOKvg/Du\n2jPwCxDhSv1NhN7tizNFDTj/7S/oUKrhJQjFhlesryVlVGKMjlIBQPec1TvBNwBu3BZ1/BNCCOlL\nKEglTqUbWJZeqEDqugjj5h9pNibEme5s117HUoBrabi//jUGgmXH45+5l8Aqh4FRh+HIDgaDhwhQ\neuEaUtdFmV2nNoP76hupCIlk0NnRhd/Pj0bDjzdw+ZICf94wufsAgdwK3PtQEDcSa//2CqS9NAPD\nA++yKqjUnQcbFu2vMzFgjt5nUltRh7WbnkZ4WBTEomAKWAkhhHg0ClKJ0xgGltdvq+1u/tlfmM1d\nR/s+w8CRb7j/8YI2zEtaZfIaf3gmzGj80+qsFDCMfkaWYRgolFeMspbjxkfrZW/PFDVg9qIYvXvM\nekaCgznlCJcEgGEYPLFU8+uHksX4YNdSMDtHY9niV0wGlKZOndJ9nkuVrbgou4Zla8fblaUmhBBC\n3A0FqcRpDINCbaORqUynOaZGTekGuJaOELXmGoDpjGztpUosWyvSCwJbG33Bst2NV6aasAxHYmlf\nN3NhBA7uLMfeI+YDSr5Tp/IPdz8PX3BsbZaaEEIIcUcUpBKnMQwKJycYN//oZjrNsbSVr2XuCFFr\nygH2F2ajqbUWm1ZUInjU/2/v7gOqru/+j78OIAcnIoLITXMpiDcpJmk2M9uYXWvVHGUZlaaZpV5X\nVtNZGnkz05x2o6tVU8uUNi20KNus37acLlOzLFBSUQGbmkmoIWJ4EDi/P+icOHAOcA6Hcw6c5+Of\nxvdwvud9YMDLz837014dOgbXnGL10RndcGe3eiFw08vSlqxS6+htxYX6m7DstcSyfGxpWeVKoKz9\nfhyFY1pUAfC27NwcLd+4VmVBFQqtDOaAATQZIRUtpm4otKynXDH/iC7rn9ikne0WjU3lN0VD97Bd\nmhAhs3mI3l9fqL7Jkfp0S5FOn2inHveG274/g0EVVeek02H66x+/1Ddflcosg15bdkAdwgI0+Gex\nurRnmN5ek2/TEuv99YUafG2s9eMKU5VLgbL2+7HXoYAWVQCawx3h0npUa1rNUa0lZrPSs/6sRWr8\nqFaAkIoWYy8UHsk16PczVzi9TrKxqfzm3mPOwsn11qvecHu83nr1oEZP6q2nZnwis7mHTQg8cqhE\nMp7RTRNC9d/8CO3/rMraIcDSIir7Hx30Pz97RHuytypv50kd+fKwbrizmzWwWwKrK4Gy9vsJqLhE\nb68+YnNEq7MhHgAs3BUul29ca72HVPO7NWxUkpZnrtUKQioaQUhFi3FHsKx7v7pT+c70XW3ocxs6\nXtVgMKhH77B6IfD914/phju76a1XD+rE0TI9+PtB9U6t+igzXLfecoduveUOaw2r/rJE2zYd0I86\nGqyjra4Gytpfk7252W77WgPwb+4Kl2VBFXZ/t5YFXXRrvWibCKloUQ2tEW0uZ/qu2vvcjMwZ+m5V\nZ0XGGLX/i0Mafmt8velyyzR6QKBBMkXZNOHv1LGrDmSf1q339tbf1uY3aU3ogKRkPffUGzWB8t0V\nOrrnnI595p5A2ZJfawD+xV3hMrQyWCV2liKFVrZzS51o2wipaLWa0paqoc/9ZVoXvbXqoK69rY+6\nDYjU22vydcs9PX8YKV1faD1VatDwGB3//Mc2TfhHjb1Sd02Ld3hqVUNT+M0dFQaAluSucDkldYzS\ns/6ssFFJ1t+tpVm5mpX6v+4uGW0QIRWtVlNbSjX0ucEhNa2hevSq2RS1Yv4RRceG69Chmt3935Vd\n1KDhMTqSa9BdqVNsnn9pjzjrPZvTuUBq/mlcAOBO7gqXyUkDtUj/qxXr1+lcYIVCK9tpViqbptA0\nhFS0Wk1tS9XQ59ZuDdWjV7gu6x+phbPWWqfjqwPP6fjnobrLzqhmqDHGes/uiZ0kSW+9elDlZ43q\nHjfQqSl8Z0aFAaCluTNcJicN1HJCKVxASEWr5UxbKnuf+35mgbUVlGQbcJuyvrPuPS/tGabCvdJd\ndzk/+unMqDAAeALhEt5GSEWr5Uz3gLqfW3qmUuXVIbq0Z5gkudSyyZ3dC5wZFQYAwB8YzGaz2dtF\nNMWIESMkSZs3b/ZyJWgrak/pe3ujknVNat1R4RtZkwoAaD3cmdcIqfB7vrKr3pdCMwAArnBnXmO6\nH37Nl3bV0+cUAIAfEFLh19y9q95XRmUBAGjtCKnwa+7cVe9Lo7IAALR2hFT4NUe76kvPVGrOwslO\njYjS6xQAAPcJ8HYBgDeNTp2iLVmlsuwfNJvN+mfmKZVXf6Vr0kp07W1VuiatROveS9fe3OwG70Wv\nUwAA3MejI6nnzp3T4sWLtXXrVlVXV+vnP/+50tPT1bFjR0+WAVjZ63X63akq3fZAoNMjovQ6BQDA\nfTw6kjp37lwdOnRIr7zyil599VUVFBRozpw5niwBqGdAUrKeeHy5Fs5aqwWzVygyxujSiKi9Udkt\nWaUanTqlxWoHAKCt8thIanl5uf71r3/p9ddfV9++fSVJ6enpGjt2rCoqKhQcHOypUoAG1R0R/fLw\nWe3+z9c6XVShUXcnKzoyQf878XENSEqut5v/yn7jtX391mafQAUAgL/zWEgNCAjQ8uXL1adPH+s1\ns9msqqoqfffdd4RUNJu72j+NTp2idVk1pz/9N79U+z8r1q0Te1tPgno/s0B/ynhQvxo2VZ8eyrDZ\nzb8lK0N3/Ybd/AAANJfHpvuNRqOuueYatWvXznrttddeU+/evRUeHu6pMtBGWdo/ObvZyZ4BScm6\n68ZF2r6+s/7fupO6IS3BZn3qDWkJMgSVae1bz1qPMbU8ljIqTBs20pAfAIDmcutIqslkUlFRkd3H\noqKi1L59e+vHf/3rX/WPf/xDq1atcmcJ8FPubv9kOf1p9uIxMhiqbB4zGAwKDglUh05mdvMDANBC\n3BpS9+zZo3HjxtX7wy1JL7zwgvU817Vr1+rJJ5/U448/rqFDh7qzBPgpR+2fir897nS/U5t7ONix\nX3GhSufPtqvZvc9ufgAA3M6tIXXIkCHKy8tr8HNWrVqlp59+WrNmzdLYsWPd+fLwY/bC5JFDJZLx\njK5JC3X5BKjRqVOUkTlDv0zrYrMm1VwZqjG3TtWWrAzrlL9lN/9dqbNa6F0CAOA/PNon9e2339Yz\nzzyjxx9/XHfffbcnXxptXO3NTtYw+fox/e/c/s1aAjAgKVnj9YxWvbxEZ84fUVlphWIiE/Tg97v7\nE3N72/RYZTc/AADu4bGQevbsWS1YsEA333yzbrjhBp06dcr6WEREhAICOPwKrrPXlL9H955uWTM6\nIClZzz31hsPHBiSxUQoAAHfzWEjdvn27ysvL9c477+idd96RJOt6vs2bNysuLs5TpaCNqhsY5yyc\nzAlQAAC0Uh4LqTfeeKNuvPFGT70cYHcJAGtGAQBoHTy6JhXwJHtLAFgzCgBA60BIRZvGmlEAAFon\ndisBAADA5xBSAQAA4HMIqQAAAPA5hFQAAAD4HEIqAAAAfA4hFQAAAD6HkAoAAACfQ0gFAACAzyGk\nAgAAwOcQUgEAAOBzCKkAAADwOYRUAAAA+BxCKgAAAHxOkLcLAFrK3txsbdi4XOagMhkqQzU6dYoG\nJCV7uywAANAEhFS0SXtzs7XuvXSlpIXJYDDIbC7Ruqx0SYsIqgAAtAJM96NN2rBxuVJG1QRUSTIY\nDEoZFaYNG5d7uTIAANAUhFS0SeagMmtAtTAYDDIHlXmpIgAA4AxCKtokQ2WozGazzTWz2SxDZaiX\nKgIAAM4gpKJNGp06RVuySq1B1Ww2a0tWqUanTvFyZQAAoCnYOIU2qWZz1CK9uX6FqgPPyVAZqrtS\nZ7FpCgCAVoKQijZrQFKyBiSxUQoAgNaI6X4AAAD4HEIqAAAAfA4hFQAAAD6HkAoAAACfQ0gFAACA\nz2F3P+CEvbnZ2rBxec2JVpWhGp06hbZWAAC0AEIq0ER7c7O17r10paSF1Ryxai7Ruqx0SYsIqgAA\nuBnT/UATbdi4XCmjagKqJBkMBqWMCtOGjfRiBQDA3QipQBOZg8qsAdXCYDDIHFTmpYoAAGi7CKlA\nExkqQ2U2m22umc1mGSpDvVQRAABtFyEVaKLRqVO0JavUGlTNZrO2ZJVqdOoUL1cGAEDbw8YpoIlq\nNkct0pvrV6g68JwMlaG6K3UWm6YAAGgBhFTACQOSkjUgiY1SAAC0NK9N98+fP1933323t14eAAAA\nPswrIfXzzz/XG2+8UW+nNAAAACB5IaRevHhR8+bNU3Iy6/gAAABgn8dD6ooVK9S7d29dffXVnn5p\nAAAAtBIeDakFBQV64403lJ6e7smXBQAAQCvj1t39JpNJRUVFdh+LiorSvHnz9PDDDysiIsKdLwsA\nAIA2xq0hdc+ePRo3bpzdDVHTp09XdXW1Ro8e7c6XBNAGZefkannGRpWZghRqrNSU8alKHpjk7bIA\nAB7k1pA6ZMgQ5eXl2X1s3Lhx+uKLL6wbpi5evKjq6mpdccUVeu+99xQTE+POUgC0Utk5uUpf9p7C\nEtNkMBhUYjYrfVmWFk0TQRUA/IjHmvk/88wzMplM1o8zMjKUm5urZ555Rl27dvVUGQB83PKMjdaA\nKkkGg0FhiaO0PCNTKwipAOA3PBZS6wbR8PBwGY1GdevWzVMlAGgFykxB9ZYMGQwGlZk4IA8A/InX\nTpwCAHtCjZUym80218xms0KNlV6qCADgDV4LqVOnTtVrr73mrZcH4KOmjE9V6eEsa1A1m80qPZyl\nKeNTvVwZAMCTmD8D4DWOdvEvmiateG29zl0IVKixUrOmsbsfAPwNIRWAVzS2i385oRQA/BprUgF4\nRc0u/lF2dvFv9HJlAABfQEgF4BXs4gcANIS/BgC8ItRYqRKz2SaotrVd/JycBQCuYyQVgFe09V38\nljW3JZFpqrrkNpVEpil92XvKzsn1dmkA0CowkgrAK3xhF39LjnRychYANA8hFYDXeHMXf2PdBZqL\nNbcA0DxM9wPwSy3dXYCTswCgefgnPQC/1BIjnbWXD5jKTunbk5nq0q9mpNay5nbWtLax5hYAWhoh\nFYBfcnd3gbrLBwLNZpk+XSHDkZcVaIxwuOaWDgAAYB/T/QD8kru7C9hbPnDJlZPVKSxMa1+apRXL\nZtsNqHQAAAD7CKkA/FJNd4Eb1fnMegV+9abCT2dq0bQbXRrFzM7JVc6BE04vH+DULQBwjOl+AH7L\nHd0FLKOhJkOYzHaWD5jKTmrytIV2p/PpAAAAjjGSCgDNYBkNjU38mQp3r7dZPvDVpyv07cUuDqfz\n6QAAAI4RUgGgGSyjoZ2iE9XlJ1fo0I7VOvzxX3Xogyd0SUSAdXe/VH86v62fugUAzcGcEgA0Q+0u\nAZ2iE9UpOlFms1nhpzNVZgpSVQPT+b5w6hYA+CpCKgA0w5TxqUpflmXdAFW7H+ryjI2Ntrny5qlb\nAODLmO4HgGZoqEsA0/kA4DpGUgGgmRyNhjKdDwCuI6QCQAtiOh8AXENIBQBxPCkA+BpCKgCv8KVQ\naGnIH5ZY0y6qxGxW+rIsLZoma02u1utL7xMAWhM2TgHwOF85sz47J1eTpy3Uvb99qsHjSV2t197z\nJs15Q7eN+63H3ysAtDaEVAAe5wtn1tcOkAEdExo8ntTVeu09L/6qe5RfHOKVUA4ArQkhFYDH+cKZ\n9bUDZFWlqcHjSV2t19HzAoNCPB7KAaC1YU0qAI+rfUqThSfOrK+9PnT//oOKGZCvTtGJik28VoW7\n1yt+8O31GvI3p15Hz6uqNHk8lANAa8NIKgCP80aT+7rrQ3tdN1fFRz/T2aLD6hSdqC4/uUIHt7+q\nI9uW2jTkb0699p5XuHu9YhOv9UgoB4DWjH/GA/A4bzS5r5neT7NZH5owOE0Ht69Sp+hEhXXtKZ3d\nq0XT7qpXh6v1Wp635E8v68B/z8vQrqNiE3+msK49bUZqAQD1EVIBeIWnm9w7Wh9q1DkFfvVmo8HT\n1XqTBybpjVVPKTsnVytee1fnLuxR6OnPOHkKABpBSAXgFxytDx3YJ1Yrls1q8dfn5CkAcA4hFUCr\n40qD/CnjU5W+LMu6o7/u5igAgG8hpAJoVZpyOpQ93lgHCwBwHSEVQKtibwNUTc/RTK1owkYmptwB\noHUgpAJoVZpzEIArywTcwVuvCwCtmcdD6vPPP6/MzExVVlbq+uuv1+zZsxUcHOzpMgC0Uq421nd1\nmUBzNfS6kgivAOCAR5v5r1y5Um+88YaWLVumV155RR9//LFefPFFT5YAoJVztbF+7WNQpdrLBFr2\naFJHr7v4+b/YHC5QEpmm9GXvKTsnt0XrAYDWwmMjqdXV1VqzZo1mzpypIUOGSJIeeughvf32254q\nAUAb0NAGqIam1ZuzTKA5HL3ukRNl6n7N/S6trQUAf+CxkHr48GGVlJRoxIgR1mu//vWv9etf/9pT\nJQDwIc1Zp2lvA1Rj0/muLhNwpn6p/vS9o9etuHDOK6EZAFoLj033Hzt2TJ06ddLnn3+uW265RT//\n+c+1aNEiVVRUeKoEAD7CEijdOdXd2HR+7WUCZ4sOK++jV5S39TmVlJQ6/br26n9wXoZmLNlY7z2l\nDO1nd3lCwk8irdcsmhOaAaCtces/2U0mk4qKiuw+VlZWpvLyci1dulTp6emqqqrS3LlzVV1d4iIY\nHwAAIABJREFUrdmzZ7uzDAA+rjltpBxpbDrfskxg0dKXdOrbH6n3sInWpv7ObqCyV39ZRZAuubL+\ne1ry0hNK+EmkDEdeVqAxwro8QRKHCwBAA9waUvfs2aNx48bV+0MhSc8++6wuXLig2bNna/DgwZKk\nmTNnasaMGYRUwM+0xPrQpkznJw9MkllmXaysUv6utaqqNCk28VqZw5I0afoS9erTv0lLD+zVHxgU\nYvc9mcyhyi8OUWhwpf40/yab+3K4AAA45taQOmTIEOXl5dl97NNPP5XBYFB8fLz1Wo8ePWQymXTm\nzBlFRES4sxQAPszd60Olph17mp2Tq/9+20G9h91j/Zz8XWtVUX5Wl/1inqqa2CLKXv1VlRdktvOe\ngn8Urt7D7lXB7kw9uXSV3nztj9bHOVwAABzz2JrUvn37ql27djYhtqCgQB06dFB4eLinygDgZdk5\nuSopKVXe1ueU99ErOlt0uMltpBpSM51/ozqfWa/Ar96UCl9WB/PXemblJk2ettC60Sn+qntspuR7\nXjVGge2MTrWIstcGKzS4Uqf2ZdpcK9y9XrGJ18pgMChhcJoKjp5u5lcPAPyHx7aRhoaGavTo0Vqw\nYIEWL16s6upqPfvssxo9erQCAjzarhWAl/ywA/9+9e1ZM5JZuGuNOpT8Q4umT2z2VLdlZNL6Or1G\n2YyOBsuk9l3qT8kHBoXUu9Zgi6hls+tP1c8fL6nm2rbdR9SuQ5RiE69Vp+hE6/ODQzo26/0BgD/x\naK+Txx57TE8//bQmTZokSfrNb36j6dOne7IEAF5kb8NR/FX3KPx0pjWguuMIUUcbswr/PV+RYYd0\nMn+bAoOMqqo0KabncFVVXrB5flNaRDmaql8+MElpEx6VEu6tN/XfPS7UqfcBAP7MoyE1KChIjz32\nmB577DFPviwAH9HYhil3HV3q6HXatzequHC7eg+717om9fDO1QqoKLauJ63bIsqVdbOzHr5bM5Zk\nqku/NOs9T+3L1DMz727yewAAf0fXaAAe09iGKXe1pnL0OuXlJvX8he2a1MShE6TCl9X5jO0ue8n1\nFlHJA5P0zMw6ywFmprp1tBgA2jpCKgCPaWwHvrtaUzl6nbhuPezePzAkQsuXzqp3n+a0iHK0HMBd\no8UA0NYRUgF4jKWhvqPg567WVI5eZ3nGRrv3r7xwRpOnLaw3stkSLaJa4iADAGiLCKkAPKqh4Gdv\nBPTIJ6s1feyVbnmdKao/hf/lzhdVcuqEOnTtq+rK84rpOVzpy96zO7Lpjmn6ljjIAADaIn4rAvAZ\nyQOTNP6mfC340wIZw3uourJCMT2HK2NTrnr3ynVpOrxusBx/Uz9t/bhmhNVUdlJVCtHAkU9aQ2vh\n7vWK7Jas5RkbbUY23TVN72i0+FDeFxrzf4tZowoA3zOYLZ2nfdyIESMkSZs3b/ZyJQBa0uRpC1US\nmVYvxIWfrulP2pC6gTRlaD9lbDpUb23qomk3KnlgksPXOrj9VXXvGqToCKP1XiUlpVLC/S7VVbfG\nmrD7Q035O9coKn6YwmN61asRAFoTd+Y1RlIBtChnp8hdnQ63N9K54Pkn1P9Xcx2u/3T0WgFBwTpc\nUChj4lzrvfbsekIhJ1+19le1NOp3dpq+7nrZQ3lfKKrfXQqP6WW3RgDwV4RUAC2msSlyewHW1c1T\n9jYkGTvHNxh4Hb3WuaKDSrhqnPV66Tf5Cun0Y5v+qoW718tsNutSJzd1SbbrZcf832JVfR9Q7dUI\nAP6K80gBtJia4DjKzkjmRmuALYlMU9Ult6kkMk3py95TytB+Kj2cJctKJMv095TxDfcntTcqWl1Z\nobormmoH3pSh/ZS/c43Na+391zMyV5ap+MtPlPfRKp0tOqyvD39oDaiW9xE/+HYdy3690boaE2qs\nbLBGAPBX/FMdQItpaOreUSumLTsztWhaqtP9Se2Nisb0HK4jn6xWjyET7PZl3bJzn6Lih+nQjtUK\nCAzW+bNfq0OnWPX8nxk2I6YXL5TZfR+JCd3t1uXMEofGese6ck8AaAsIqQBaTENT9w0FWFf6k9oL\ne4bSXE0fe6V1N3/dwFtmClL4Jb2s60HzPlqlnleNUfGX7RXZ7YICgwy6dODt2vHGWrtHpEZHGOvV\n4WwXgMZ6x7pyTwBoCwipAFpM3eBYcvKQjmW/rp4J3XXiWJ7i425tduN+i4bC3h2332L3OXVDdGCQ\nUce+6KiPN0Qrttd5/XR0kT7eEKOvD83Rvvc3qt8NSY0ekepKs/7GQjkHAADwR4RUAC2mdnA89vUZ\nnTlvtO62jww7pPyda9Rz6D2NBj9nXq+hsGe/RdUPIfrUsV46lhstc7VBJ/JClbUg1Prc/TtTVV7y\nki7teUDnzpWqa3S0lmds1BTZjmY6GiEuOmNy+X1xAAAAf8RvOAAtyhIcJ09bqNBaPUktU+xH/j1f\niX36N3ntqavsTZlnbMrS+Jt6aevH6/XFF/10bG+azGaD3eebzYE6kjdV1cGZ6n1NmDrG9LI77W4q\nO6n8j15RYFCItVVVWNeeOlzwpbJzXDuQwF3HxQJAa8LufgAeYW80MDymlxL79Nfal2ZpxbLZLbq+\n0lGngS079ynlqsf18b9vltkc2OA9zNUG/Xdvmg5sK7G5x/KMjZJqgvC3F7uo97CJSvzpWPUedq+K\nj36m/VtfUrfkO62f56wp41Nd6ngAAK0ZIRWAR3ir1VJ2Tq4mT1uoz/Z9rYPbX9XZosPWxwwGg/bt\n668xY8yqqmra/czVBh3de4eO5oZa72GZdl+esVFd+tmuHU0YnKbqqosKj+nl8vR8zbKJG9X5zHoF\nfvWmwk9nciIVgDaP6X4AbtFYi6Smtlpyd02WKf6eP/uhpZQkdYpOVFFhiHZ8MNLhFL9D5gB9vCFa\nIaGViupebg3ajtaOhoR2aXYgd6XjAQC0ZoykAmg2R435s3NyrZ/jjdFAe1P88YNv19eHP5TZbFaw\n6XXF/jjPpXvH9jqviB+Xq3DXGuu0u6PR4qrKC0zPA4CTGEkF0GxNbZHk6dFARyObQVWlCj+dqVkz\nUvWH9v9PG17p5/S9u3R/TjnvH1B8TJA1aNsbLS7ctUY9oy7o8VqB3N6osySa9QNALYRUAM3WnBZJ\nLXmSkqNd8YP6xWrFstmqqJC2/8O1CaVTXz6soWlfK/Doy9Zr9nq1rlxwR6ON+e/+7dMy/ihcPa++\nn2b9APA9pvsBNJurm6KaskygOVKG9lP+zjU2u+Lzd65RytB+ys7J1fWpy3XiWB+X7v31oQ46c7y9\ngkIibK4nD0zS8qWPa8akmyRJz6zcpMnTFlrfk70lCEHtI6wB1XKtdtcAAPBHjKQCaDZXN0W19ElK\nW3buU1T8MB3asVoBgcGqqjQppudwvfn3f+q84ZC6DhuloZ2K9fGGmgb+TWUIMOuno4tqNk2drh/E\nGzrG1N6oc2BQCM36AaAOfgMCaLamnD9vT0svEygzBSn8kl7WgwMsDua+rl7XPSCDwaCfJJVJUtOD\nqqFKPx1drG79z7l0NGqoUfWWIFRVXpCZZv0AYIOQCsAtXNkU5epJSg2NVNYOqo7uHxzS0eaaJaju\nXB8lNdTQ31ClbglLVXHmSxX++6x+N+VWu0G8ofA9Y9JNmjx3tXoMmWAdda68eEH5u9aq51VjrNeO\nfLJaK564s8GvAwC0ZYRUAF5jb5nAqX2Zqmp3SmP+b7HDEdKmLBPIzslVSUmp8nKfk6FdqGITf6aw\nrj1VejhL3WND641cdut/ToW7N+ibI1PtjqgaDFVKvGKdBt58s7XWjE1Z6t2r/lGnDYXv5IFJmj42\nXwv+tEDG8B4ylZ2WITBIUd2H6NCO1TIEtpPp20LNeWgUm6YA+DVCKgCvqbtMoPLCGZm+q1bglQ+o\nqoER0saWCfww0nq/+vb8oRVUh5J/aNH0iZJULxwf+HC54np3lDH0dR3de4fCovJ05c3HlP9Bdx3/\nb2/9/JcbFXn1T5u0fraxNbp33H6LevfqqRWvvatzF34sU9lJBX23VZHdI74P5o8SUAH4PUIqAK+q\nvUxg8rSFUnzjG6kaWyZgb6Q1/qp7FH460xr+xt+Ur/nPzVf7iARVV1YorvcInczfpr7Dw5Uw+IQi\nftxO/83Zrb6DNivj1ae09NVsGQwDbGo3GAwqOmOy+54aW6PLCVIA0DBCKgCf0dSNVI2NVDblPm+/\nv1MDbphn83mdohN1cPur6nNNL0k1p1MV/nu+fvYzacb8I4pPrB+Mvzp2xO57IYQCQPPQJxWAz2hq\nv9XGjlht7D7ZObnKzS+2G2QDg4w2H1/SrYckqWt0tAp3r7fpuVq4e726Rke74Z0DAOpiJBVAi3H2\nNCln+q3WHanMzsnV5GkLVWYKUmV5qb4qXKFLrpxsvc9Xn66QIgJ0052/U/F5o4JDo+22faqqNNl8\nHB1RE1p/HNNZVZXJ9Xqu/jjoc3d8qQAAdRBSAbSI7JxcPTgvQ2UVQQoMClFV5QXtm5ehP80f7zCo\nutpvtW5LKpnNMl7IVPWhlxQcGq3TJ/N1obKLlDBRX21/Vb2HTVDpN/kq3L1e8YNvtwbZgztWK6bn\ncEk1AbVg56t6+cm7JFkC9HvqdfUEmwA9pZEDCwAAriGkAmgRTy5dpQuGLuo9LM0a6gp2Z+rJpav0\n5mt/dPg8V9Zy2tso1aVfmsJPZ2rK+Js0dupT6v+ridbpfIPBoE7RiZJkHRk9W3RIAUFGHdv3voq/\n/ERVlRdU9k2eTV2uBGgAgGsIqQBaRMHR09ZTnaSa4JgwOE2HPnjC7a9Vd6PU2aLD+vrwhwqqKtWk\n6Utkbhdufbyq0mSd5u8UnahO0Ykym836eMN09b1qrMJjelnXm/a8epKWZ2y0dhZgMxQAeA4hFUCL\nqHuqk1QTVINDOrr9tWq3pDpbdFinjn6u3sPu/WEaf/urKjl5SOExvRSbeG29af6C3ZkyhoTWGkU1\nKTbxWnWKTlTZV3vdWquz63QBwF8RUgG0CHunOpnNZnWPC3X7a9XecPX14Q+tAVWqCca9h92rz/42\nX4NGzrOOnH729/nq0ClOhoBAxSZeq4TBaTq4fZUSfzrWpt7Gjmh1hqPjXMfflK8tO/c1KbgScgH4\nC1pQAWgRsx6+W6f2Zdq0bDq1L1OzHrrb7a9VuyVVUFWp3RHcDuFxOrRjtfZ+8Ecd+M9LNgFVkg5u\nf1Wm86eU99ErOlt0+IeNUePdtzGqZu3sKJsAbQ5L0tK/fqqSyDRVXXKbSiLTlL7sPWXn5NZ7viXk\nNuVz6z5v8rSFGvN/izV52sJGPx8AfIFHR1JLS0v1xBNPaNu2bQoJCVFqaqqmT5/uyRIAeEjywCQ9\nM7PORqOZLTfqZ1kvOnnaQrunUQVfPKGuEdEqDgpT0ohl1qn+/F1rVVF+Vpf9/P+s12ofodqUzgK1\nRzZThvZzOCpq75CBk/nb6o38Ojpu1d4GMUefW7s+e6O3dY+aBQBf49GQ+vvf/15nzpzRunXrdPr0\naU2fPl2RkZEaP368J8sA4CHe2GjkqNfqyqUztTxjo0IjbUNez6vG6OD2VQ0eoepI3QD435OH9HTG\ndvUceo/dQGjvONeAoGC7I79FZ0zWvq+WsNvUE7lqcyXYAoAv8Oh0/4cffqgJEyYoISFBQ4YM0ciR\nI7Vz505PlgCgjWvoNCpHIS8wKKTetYaCn0Xd6fuT+dusAdVyn5pAuFFSTYAuPZxlswTC9G2h3dOx\nDhd8WW9av7L8TJNO5KrNlWALAL7Ao7+lwsPD9e677+qqq67S2bNntW3bNl1//fWeLAFAG+JoE5Gj\nEVx7I5k1p0xdsPm8pm6YqhsALT1Ya6sdCO31Wp3z0ChlbLId+T3yyWp1S76zXtitOvSizh9u2olc\nFqayk8r/6JXvD1So6VoQ1rWnWzeEAUBL8GhInTdvnh599FFdccUVqq6u1rBhw/TAAw94sgQAbYQr\nay1ThvbT0xlrrKOdlvZU58+eVMnJQzqZv00BQcEyfVuou1OvrDfdXve+dUNv7R6sFnUDr70A3btX\nrk1wjepgUseYXjafYzAYZAyN0eOTbmzygQLZObn69mL9AxXO/XezZs1nmRUA3+bWkGoymVRUVGT3\nsaioKBUWFiopKUlTp07VN998o/nz5+vll1/W5MmT3VkGAD/gylrLLTv3KSp+mPb+61ldLD+ndu07\nKiAgSNXfndDpIx+q97CJ1jC3YesaRcUPU/glvRwG4LrrX2N6Dlf+TtsQ3NhIp1Q/uDra/BVqrHRq\nne/yjI3q0s/2a5QwOE0qfJlNUwB8nltD6p49ezRu3Lh6012SNGPGDC1dulQffvihIiMjJUnl5eWa\nP3++7r//fgUE0A0LQH2OpvRdWWtZZgqSIcig0M7dFP8/PzTzP/TRK4rsca3thqqh9+jg9lcVHtPL\nYQCuO31/qbFS94wfrK0fN+/oVEebv2ZNS3WqT6rDNbghEU7VAwDe4NaQOmTIEOXl5dl97P3331fn\nzp2tAVWSLrvsMp0/f14lJSWKiOCXJgBbDU3pO1pf2tBay1Bjpfbl/cc6YirVhLZe19xnDaQWNRuq\njDYf2wvAyQOTNFnS4uf+opzCMn2SfVAJP4nU401oX+WIvbWrltFYZ5Y4uPI1AgBf4bHhy65du6qk\npERnzpyxXisoKNCPfvQjAioAu+w1v7fslre3U76x5vtTxqfKfLHM7uhiQFCwzbWaDVUmm4/thbvs\nnFzNWLJRSrhfPYZPV6/r5urL0i56cF5Gs5rmJw9M0vKlj2vtS7O0Ytnsmo8b+Ho4er/Ofo0AwFd4\nbOPUwIEDlZCQoJkzZ2rmzJk6c+aMnn76aY0dO7bxJwPwSw1N6Tsabaw9omhvarxPtx/Z3dxkaQVl\nbfK/c41ieg63Pu5obamjdZ8Ht6/S8oyNbu1F6uwSh6Z8jQDAV3kspAYGBmrlypV68sknNWbMGHXo\n0EGpqamaOnWqp0oA0Mo0Nl3d0CYiR0sFxt80tF7Lp9LDWZrz0CibtaSPjB+srR9n69xXexsMdw31\nXnV3L1JXpu+9caACALiDR1tQRUdH6/nnn/fkSwJoxRraQNQYR7v/t+zM1KJpqXZHF++4/Rabe9T9\n2J6Geq+GGpv+K7YpG6Ka8/UAgNaGI0cA+KzmTFc3tlTA1dFFS5g8fvJbfVNUpM5dYnVs13xdOugu\nhcf0svYiDQ2ubPLaz6b2fGX6HoA/IaQC8GmuBsqW2NluCZPmsCt0ujxb8b+YIoPBoM4DzCr8eJVO\n7FmnwACD3d39DY2UOtPzlel7AP6C5qQA2qSW2Nlu2V1/Mn+b4gffbhMq4386UUMH9VH21r/ozdf+\nWC+gpi97TyWRaaq65DaVRKYpfdl71t3/rvR8BYC2jpAKoE2qmRq/UZ3PrFfgV28q/HSmFk27sVlT\n45YwGRhkdCpUNtY6KtRYaQ3TFvQzBeDv+Gc6gDbL3VPjliUEVZUmu22sHIXKxkZK2RAFAPURUgH4\nHWeOFq3NEiZjeg5X4e711in/xkJlU1ppsSEKAGwZzHXnmHzUiBEjJEmbN2/2ciUAWrMfdtL/MGp5\nal+mOrc7JWNoTKOhNTsnVytee1fHvj6j4qIixXXroegIY6PPqfuapYezNP6mXtqyc5/TYRkAfJU7\n8xohFYBfmTxtoUoi0+qNah7cvkp9rrnPGiCbu361Lku4tYyUpgztp4xNh+oFV3e/LgB4kjvzGtP9\nAPxKQydEWf63o/ZP9jR16UDd9bGTpy1sctspAPBH7O4H4Fcc7aSvqjRZP25q+6fsnFw9OC9Du/LK\nlHfsgnbllenBeRnW1lINoe0UADSMkArAr9jrn1qwO1OxiddaP6ep7Z+eXLpKFwzh6j1sohJ/Ola9\nh03UBUO4nly6qtHn0nYKABpGSAXgV2r3Tz2ybam+2PycTGWnFda1p6SaoHjkk9VNavpfcPS0Egbb\nTtknDE5TwdHTjT63JQ4bAIC2hHklAH7Hsj70h2NOk3Rox2oZAtvJ9G2h5jw0qkmbl4JDOtqdsg8O\n6dikGmg7BQCOEVIB+K0fguK7iuwe9v3Gp0ebHBS7x4baberfPS60ya/vzsMGAKAtIaQC8GvNCYqz\nHr5bM5Zkqku/NJueq8/MvNvNVQKA/yGkAoCLkgcm6ZmZdabsZzJlDwDuQEgFgGZgyh4AWga7+wEA\nAOBzCKkAAADwOYRUAAAA+BzWpAKAm2Tn5Gp5xkaVmYK+b2fFJioAcBUjqQDgBpaDAUoi01R1yW0q\niUxT+rL3lJ2T6+3SAKBVIqQCgBssz9iosMRRNkekhiWO0vKMjV6uDABaJ0IqALhBmSnI7hGpZSZW\nVQGAKwipAOAGocZKmc1mm2tms1mhxkovVQQArRshFQDcYMr4VJUezrIGVbPZrNLDWZoyPtXLlQFA\n68Q8FAC4QfLAJC2aVueI1Gns7gcAVxFSAcBNOCIVANyHkAoALnDUE5VeqQDgHoRUAHCSpSdqWGKa\nDAaDSsxmpS/L0vib8pWx6VC964umiaAKAE5i4xQAOMlRT9Rnl79Fr1QAcBNCKgA4yVFPVHNQJ3ql\nAoCbEFIBwEmOeqIaKs/SKxUA3ISQCgBOctQT9XdTbqVXKgC4CXNQAOCkhnqi9u6VS69UAHCDFgup\nEydO1MiRI3XzzTdbr5WUlGjOnDnavn27IiIi9NBDD+k3v/lNS5UAAC3GUU9UeqUCgHu4fbrfbDZr\nwYIF2rFjR73HZs2apfPnz2vDhg2aMmWKZs+erdzcXHeXAAAAgFbOrSOpRUVFeuSRR3T8+HGFhYXZ\nPHbs2DFt3bpVW7ZsUWxsrBISEpSTk6N169bpD3/4gzvLAAAAQCvn1pHU/fv3Ky4uTllZWerQoYPN\nY3v27FFcXJxiY2Ot1wYNGqScnBx3lgAAAIA2wK0jqSkpKUpJSbH7WHFxsbp27WpzLTIyUidPnnRn\nCQAAAGgDnAqpJpNJRUVFdh+LiopS+/btHT63vLxc7dq1s7kWHBysixcvOlMCAAAA/IBTIXXPnj0a\nN25cvRNVJOmFF17QiBEjHD7XaDTWC6QVFRUKCQlxpgQAAAD4AadC6pAhQ5SXl+fSC0VHR6u4uNjm\n2qlTpxQVFeXS/QAAANB2eezEqcsvv1wnTpywWS7w2WefaeDAgZ4qAQAAAK2Ex0Jqt27ddM011+iR\nRx7RwYMHtWHDBm3atEljxozxVAkAAABoJVospNpbt7pkyRKFhoYqLS1NK1eu1KJFi9S/f/+WKgEA\nAACtVIsdi7p58+Z61yIiIvTSSy+11EsCAACgjfDYdD8AAADQVIRUAAAA+BxCKgAAAHwOIRUAAAA+\nh5AKAAAAn0NIBQAAgM8hpAIAAMDnEFIBAADgcwipAAAA8DmEVAAAAPgcQioAAAB8DiEVAAAAPoeQ\nCgAAAJ9DSAUAAIDPIaQCAADA5xBSAQAA4HMIqQAAAPA5hFQAAAD4HEIqAAAAfA4hFQAAAD6HkAoA\nAACfQ0gFAACAzyGkAgAAwOcQUgEAAOBzCKkAAADwOYRUAAAA+BxCKgAAAHwOIRUAAAA+h5AKAAAA\nn0NIBQAAgM8hpAIAAMDnEFIBAADgcwipAAAA8DmEVAAAAPgcQioAAAB8TouF1IkTJ+qdd96xuVZY\nWKiJEydq0KBBuu6667RixYqWenkAAAC0Ym4PqWazWQsWLNCOHTtsrl+4cEGTJk1STEyM3nrrLc2d\nO1cZGRlat26du0sAAABAK+fWkFpUVKTx48dry5YtCgsLs3ns008/VWlpqebPn6/u3bvr2muv1T33\n3KO///3v7iwBAAAAbYBbQ+r+/fsVFxenrKwsdejQweaxyy67TC+++KKCgoJsrp87d86dJQAAAKAN\nCGr8U5ouJSVFKSkpdh+LjIxUZGSk9WOTyaT169drxIgR7iwBAAAAbYBTIdVkMqmoqMjuY1FRUWrf\nvn2T7mM2mzVz5kx99913mjRpUpOe880336iqqopQCwAA4KO+/vrrerPmrnLqLnv27NG4ceNkMBjq\nPfbCCy80KUBWVVXp0Ucf1X/+8x+tWbPGZnS1IUajURUVFc6UCwAAAA8KDAxUcHCwW+7lVEgdMmSI\n8vLyXH6xyspK/fa3v9WOHTv08ssv6/LLL2/yc3fv3u3y6wIAAKB1ceua1MbMmTNHO3fu1KpVq5Sc\nnOzJlwYAAEAr4rGQun37dr399ttasGCBunXrplOnTkmSAgICFBER4akyAAAA0Aq0WEitu271n//8\npwwGg+bOnWtzPS4uTps3b26pMgAAANAKGcxms9nbRQAAAAC1uf1YVAAAAKC5CKkAAADwOYRUAAAA\n+BxCKgAAAHwOIRUAAAA+p1WF1IkTJ+qdd96xubZmzRr16dNHffv2tf73qaee8lKF/s3e96ekpEQP\nPvigrrjiCl133XV69913vVQdDhw4YPOz0qdPH912223eLstvVVRUKD09XVdeeaWGDx+u1atXe7sk\nfO+DDz6o93fl4Ycf9nZZfq+iokIjR47Up59+ar12/PhxTZgwQcnJyfr1r3+t7du3e7FC/2Xve7Nw\n4cJ6P0dr16516r4ePXHKVWazWQsXLtSOHTs0cuRIm8cKCgo0ZswYPfDAA7J002rfvr03yvRbDX1/\nZs2apYqKCm3YsEHZ2dmaPXu2evTooaSkJC9V67/y8/N12WWX6ZVXXrH+rAQFtYpfAW3SkiVLtH//\nfv3lL3/R8ePHNXPmTF1yySX65S9/6e3S/F5+fr5+8YtfaOHChdafFaPR6OWq/FtFRYWmT5+u/Px8\nm+sPPPCA+vTpo7feeksffPCBpk6dqvfff18xMTFeqtT/OPreFBYWasaMGbrlllus10JDQ526t8//\nhSoqKtIjjzyi48ePKywsrN7jBQUFuuWWWzi1yksa+v4cO3ZMW7du1ZYtWxQbG6uEhATl5ORo3bp1\n+sMf/uCliv1XQUGB4uPj+VnxAeXl5XrzzTe1atUq66j2fffdp7/+9a+EVB9QUFCgxMTLUrVvAAAG\nBUlEQVREflZ8REFBgX73u9/Vu75z504dO3ZM69evl9Fo1KRJk7Rz5069+eabmjp1qhcq9T+OvjeW\nx+677z5FRka6fH+fn+7fv3+/4uLilJWVpQ4dOtR7vKCgQN27d/d8YZDU8Pdnz549iouLU2xsrPXa\noEGDlJOT4+kyIX5WfEleXp6qqqo0cOBA67VBgwZp7969XqwKFgUFBerRo4e3y8D3PvnkEw0dOlSZ\nmZmqff7Q3r171a9fP5tRbv7GeJaj701ZWZmKioqa/TfH50dSU1JSlJKSYvex06dP6+zZs8rKytLM\nmTMVEhKi2267Tffee6+Hq/RfDX1/iouL1bVrV5trkZGROnnypCdKQx0FBQWqrq7WyJEjVVZWpuHD\nh+vRRx91evoFzVdcXKzw8HCb5RaRkZEymUz69ttv1blzZy9WhyNHjmjbtm3685//rOrqav3qV7/S\nQw89pHbt2nm7NL9055132r3u6G9MUVGRJ8qCHH9vCgsLZTAY9Oc//1kffvihwsPDNWHCBN18881O\n3d/rIdVkMjn8P1RUVFSD60stX4SoqCitWLFC+/fv18KFCxUYGKjx48e3VMl+pTnfn/Ly8nq/1IOD\ng3Xx4kW31ogaDX2vIiIidPToUf3kJz/R4sWLVVpaqkWLFmnmzJl68cUXPVwpysvLFRwcbHPN8nFF\nRYU3SsL3Tpw4oQsXLshoNOq5557T8ePHtXDhQplMJqWnp3u7PNTi6OeInyHvKywsVEBAgBISEnT3\n3Xfrk08+0Zw5cxQaGqrrrruuyffxekjds2ePxo0bJ4PBUO+xF154QSNGjHD43CuvvFIff/yxOnXq\nJElKTEzUmTNn9PrrrxNS3aQ53x+j0VgvkFZUVCgkJMTtdaLx79WuXbsUEhKiwMBASdLixYt16623\nqri4WFFRUZ4u168ZjcZ6f0gtH7Px07vi4uK0a9cu6xr7Pn36qLq6Wo8++qgee+wxuz9f8A6j0aiz\nZ8/aXONvjG+4+eab9Ytf/ML6c9SrVy99+eWXev3111tXSB0yZIjy8vJcfr4loFrEx8cz1O9Gzfn+\nREdHq7i42ObaqVOnCEQtxNnvVUJCgqSazW98TzwrOjpaJSUlqq6uVkBAzdaAU6dOKSQkxO4GUXhW\n3e9BQkKCTCaTSkpKWIrhQ6Kjo+vtKOdvjO+o+3MUHx+vXbt2OXUPn9841ZANGzboV7/6lc21AwcO\nKD4+3ksVobbLL79cJ06csPlHw2effWazWQSeUVBQoCuuuEJfffWV9dr+/fsVFBSkSy+91IuV+ae+\nffsqKCjIZoPH7t271b9/fy9WBUn66KOPdNVVV8lkMlmv7d+/X+Hh4QRUH3P55Zdr//79NrMS/I3x\nDc8//7wmTJhgc+3AgQNOb0hs1SF12LBhOnXqlJYsWaKjR49q06ZNWrVqlSZNmuTt0iCpW7duuuaa\na/TII4/o4MGD2rBhgzZt2qQxY8Z4uzS/Ex8fr+7du2vOnDk6fPiwdu/erblz5yotLU0dO3b0dnl+\nJyQkRKmpqZo3b55yc3P1wQcfaPXq1SxT8gHJyclq3769Hn/8cR05ckT/+c9/9PTTT+v+++/3dmmo\nY8iQIYqNjdWsWbOUn5+vlStXKjc3l0NKfEBKSoo+/fRTrV69WseOHdO6dev07rvv6r777nPqPq0q\npNZdCxQXF6eVK1cqOztbqampWrZsmR555BFdf/31XqrQv9lbq7VkyRKFhoYqLS1NK1eu1KJFixgt\n8gLLLsvQ0FCNHTtWU6dO1dVXX61Zs2Z5uzS/9dhjj6l///4aP368FixYoIcfftiptVpoGR06dNCq\nVav07bff6rbbbtOcOXN0xx130DXGR9T+OxMQEKCXXnpJxcXFuvXWW/W3v/1NL774Io38vaT29yYp\nKUnPP/+83nnnHY0cOVJr167Vs88+qwEDBjh3T3PtxlYAAACAD2hVI6kAAADwD4RUAAAA+BxCKgAA\nAHwOIRUAAAA+h5AKAAAAn0NIBQAAgM8hpAIAAMDnEFIBAADgcwipAAAA8DmEVAAAAPgcQioAAAB8\nzv8HKD3yRMAdZO8AAAAASUVORK5CYII=\n", 69 | "text/plain": [ 70 | "" 71 | ] 72 | }, 73 | "metadata": {}, 74 | "output_type": "display_data" 75 | } 76 | ], 77 | "source": [ 78 | "model.plot()" 79 | ] 80 | } 81 | ], 82 | "metadata": { 83 | "anaconda-cloud": {}, 84 | "kernelspec": { 85 | "display_name": "Python [py35]", 86 | "language": "python", 87 | "name": "Python [py35]" 88 | }, 89 | "language_info": { 90 | "codemirror_mode": { 91 | "name": "ipython", 92 | "version": 3 93 | }, 94 | "file_extension": ".py", 95 | "mimetype": "text/x-python", 96 | "name": "python", 97 | "nbconvert_exporter": "python", 98 | "pygments_lexer": "ipython3", 99 | "version": "3.5.2" 100 | } 101 | }, 102 | "nbformat": 4, 103 | "nbformat_minor": 0 104 | } 105 | -------------------------------------------------------------------------------- /unsupervised/kmeans.py: -------------------------------------------------------------------------------- 1 | from utils.base_estimator import BaseEstimator 2 | 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import seaborn as sns 6 | 7 | np.random.seed(2046) 8 | 9 | def euclidean_distance(x, y): 10 | return np.sqrt(np.sum((x - y) ** 2)) 11 | 12 | class KMeans(BaseEstimator): 13 | """Partition a dataset into K clusters. 14 | Finds clusters by repeatedly assigning each data point to the cluster with 15 | the nearest centroid and iterating until the assignments converge (meaning 16 | they don't change during an iteration) or the maximum number of iterations 17 | is reached. 18 | Init centroids by randomly select k values from the dataset 19 | For better method to improve convergence rates and avoid degenerate cases. 20 | See: Arthur, D. and Vassilvitskii, S. 21 | "k-means++: the advantages of careful seeding". ACM-SIAM symposium 22 | on Discrete algorithms. 2007 23 | Parameters 24 | ---------- 25 | K : int, default 8 26 | The number of clusters into which the dataset is partitioned. 27 | max_iters: int, default 300 28 | The maximum iterations of assigning points to the nearest cluster. 29 | Short-circuited by the assignments converging on their own. 30 | """ 31 | 32 | def __init__(self, K=8, max_iters=300): 33 | self.K = K 34 | self.max_iters = max_iters 35 | # an array of cluster that each data point belongs to 36 | self.labels = [] 37 | # an array of center value of cluster 38 | self.centroids = [] 39 | 40 | def _init_cetroids(self): 41 | """Set the initial centroids.""" 42 | indices = np.random.choice(self.n_samples, self.K, replace=False) 43 | self.centroids = self.X[indices] 44 | 45 | def _dist_from_centers(self): 46 | return np.array([min([euclidean_distance(x, c) for c in self.centroids]) for x in self.X]) 47 | 48 | def fit(self, X=None): 49 | """Perform the clustering on the given dataset.""" 50 | self._setup_input(X, y_required=False) 51 | 52 | self._init_cetroids() 53 | 54 | for i in range(self.max_iters): 55 | new_centroids = [] 56 | # update clusters base on new centroids 57 | new_labels = np.apply_along_axis(self._closest_cluster, 1, self.X) 58 | # update centroids base on new clusters 59 | for k in range(self.K): 60 | centroid = np.mean(self.X[new_labels == k], axis=0) 61 | new_centroids.append(centroid) 62 | 63 | if self._is_converged(self.centroids, new_centroids): 64 | print('Converged on iteration %s' % (i + 1)) 65 | break 66 | 67 | # not converged yet, update centroids / labels to new centroids / labels 68 | self.labels = new_labels 69 | self.centroids = new_centroids 70 | 71 | def _predict(self, X=None): 72 | return np.apply_along_axis(self._closest_cluster, 1, X) 73 | 74 | def _closest_cluster(self, data_point): 75 | """ Return the closest cluster index and distance given data point""" 76 | 77 | closest_index = 0 78 | closest_distance = float("inf") 79 | 80 | for cluster_i, centroid in enumerate(self.centroids): 81 | distance = euclidean_distance(data_point, centroid) 82 | if distance < closest_distance: 83 | closest_distance = distance 84 | closest_index = cluster_i 85 | 86 | return closest_index 87 | 88 | def _is_converged(self, centroids, new_centroids): 89 | return True if sum([euclidean_distance(centroids[i], new_centroids[i]) for i in range(self.K)]) == 0 else False 90 | 91 | def plot(self, data=None): 92 | if data is None: 93 | data = self.X 94 | 95 | for k in range(self.K): 96 | points = data[self.labels == k].T 97 | plt.scatter(*points, c=sns.color_palette("hls", self.K + 1)[k]) 98 | 99 | for point in self.centroids: 100 | plt.scatter(*point, marker='x', linewidths=10) 101 | -------------------------------------------------------------------------------- /unsupervised/pca.py: -------------------------------------------------------------------------------- 1 | from utils.base_estimator import BaseEstimator 2 | 3 | import numpy as np 4 | 5 | np.random.seed(2046) 6 | 7 | 8 | class PCA(BaseEstimator): 9 | """ 10 | In general, PCA aims to find the directions of maximum variance 11 | in high-dimensional data and projects it onto a new subspace 12 | with equal or fewer dimensions that the original one. 13 | Also, PCA directions are highly sensitive to data scaling, and we need to 14 | standardize the features prior to PCA if the features were measured on 15 | different scales and we want to assign equal importance to all features 16 | """ 17 | 18 | def __init__(self, n_components, solver='svd'): 19 | """Principal Component Analysis (PCA) implementation. 20 | Transforms a dataset of possibly correlated values into n linearly 21 | uncorrelated components. 22 | The components are ordered such that the first 23 | has the largest possible variance and each following component as the 24 | largest possible variance given the previous components. This causes 25 | the early components to contain most of the variability in the dataset. 26 | Parameters 27 | ---------- 28 | n_components : int 29 | solver : str, default 'svd' 30 | {'svd', 'eigen'} 31 | """ 32 | self.solver = solver 33 | self.n_components = n_components 34 | self.components = None 35 | self.variance_ratio = None 36 | self.mean = None 37 | self.std = None 38 | 39 | def fit(self, X): 40 | self.mean = X.mean(axis=0) 41 | self.std = X.std(axis=0) 42 | # standardize input then decompose 43 | X_std = (X - self.mean) / self.std 44 | self._decompose(X_std) 45 | 46 | return self 47 | 48 | def _decompose(self, X): 49 | if self.solver == 'svd': 50 | _, s, V = np.linalg.svd(X, full_matrices=True) 51 | s = (s ** 2) / len(X) 52 | V = V.T 53 | elif self.solver == 'eigen': 54 | X_cov = X.T.dot(X) / (len(X) - 1) 55 | s, V = np.linalg.eig(X_cov) 56 | 57 | # sort eigenvalues (and eigenvectors accordingly) descending 58 | sorted_idx = np.argsort(s)[::-1] 59 | s = s[sorted_idx] 60 | V = V[:, sorted_idx] 61 | 62 | variance_ratio = s / s.sum() 63 | self.variance_ratio = variance_ratio 64 | print('Explained variance ratio: %s' % variance_ratio[0:self.n_components]) 65 | 66 | self.components = V[:, 0:self.n_components] 67 | 68 | def transform(self, X): 69 | X_std = (X - self.mean) / self.std 70 | return np.dot(X_std, self.components) 71 | 72 | def _predict(self, X): 73 | return self.transform(X) 74 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hungtuchen/ml-playground/f273ea923cd442f5a51b73c14eb220f3d2b11151/utils/__init__.py -------------------------------------------------------------------------------- /utils/base_estimator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class BaseEstimator(object): 5 | 6 | def _setup_input(self, X, y=None, y_required = True): 7 | """Ensure inputs to an estimator are in the expected format. 8 | Ensures X and y are stored as numpy ndarrays by converting from an 9 | array-like object if necessary. Enables estimators to define whether 10 | they require a set of y target values or not with y_required, e.g. 11 | kmeans clustering requires no target labels and is fit against only X. 12 | Parameters 13 | ---------- 14 | X : array-like 15 | Feature dataset. 16 | y : array-like 17 | Target values. By default is required, but if y_required = false 18 | then may be omitted. 19 | y_required: boolean, default True 20 | """ 21 | if not isinstance(X, np.ndarray): 22 | X = np.array(X) 23 | 24 | if X.size == 0: 25 | raise ValueError('Number of features must be > 0') 26 | 27 | if X.ndim == 1: 28 | self.n_samples, self.n_features = 1, X.shape 29 | else: 30 | self.n_samples, self.n_features = X.shape[0], np.prod(X.shape[1:]) 31 | 32 | self.X = X 33 | 34 | if y_required: 35 | if y is None: 36 | raise ValueError('Missed required argument y') 37 | 38 | if not isinstance(y, np.ndarray): 39 | y = np.array(y) 40 | 41 | if y.size == 0: 42 | raise ValueError('Number of targets must be > 0') 43 | 44 | self.y = y 45 | 46 | def fit(self, X, y=None): 47 | self._setup_input(X, y) 48 | 49 | def predict(self, X=None): 50 | if not isinstance(X, np.ndarray): 51 | X = np.array(X) 52 | 53 | if self.X is not None: 54 | return self._predict(X) 55 | else: 56 | raise ValueError('You must call `fit` before `predict`') 57 | 58 | def _predict(self, X=None): 59 | raise NotImplementedError() 60 | -------------------------------------------------------------------------------- /utils/metrics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def squared_error(actual, predicted): 4 | return (actual - predicted) ** 2 5 | 6 | def mean_squared_error(actual, predicted): 7 | return np.mean(squared_error(actual, predicted)) 8 | -------------------------------------------------------------------------------- /utils/plotting.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | import numpy as np 3 | import pandas as pd 4 | from collections import namedtuple 5 | from matplotlib import pyplot as plt 6 | from mpl_toolkits.mplot3d import Axes3D 7 | 8 | EpisodeStats = namedtuple("Stats",["episode_lengths", "episode_rewards"]) 9 | 10 | def plot_cost_to_go_mountain_car(env, estimator, num_tiles=20): 11 | x = np.linspace(env.observation_space.low[0], env.observation_space.high[0], num=num_tiles) 12 | y = np.linspace(env.observation_space.low[1], env.observation_space.high[1], num=num_tiles) 13 | X, Y = np.meshgrid(x, y) 14 | Z = np.apply_along_axis(lambda _: -np.max(estimator.predict(_)), 2, np.dstack([X, Y])) 15 | 16 | fig = plt.figure(figsize=(10, 5)) 17 | ax = fig.add_subplot(111, projection='3d') 18 | surf = ax.plot_surface(X, Y, Z, rstride=1, cstride=1, 19 | cmap=matplotlib.cm.coolwarm, vmin=-1.0, vmax=1.0) 20 | ax.set_xlabel('Position') 21 | ax.set_ylabel('Velocity') 22 | ax.set_zlabel('Value') 23 | ax.set_title("Mountain \"Cost To Go\" Function") 24 | fig.colorbar(surf) 25 | plt.show() 26 | 27 | 28 | def plot_value_function(V, title="Value Function"): 29 | """ 30 | Plots the value function as a surface plot. 31 | """ 32 | min_x = min(k[0] for k in V.keys()) 33 | max_x = max(k[0] for k in V.keys()) 34 | min_y = min(k[1] for k in V.keys()) 35 | max_y = max(k[1] for k in V.keys()) 36 | 37 | x_range = np.arange(min_x, max_x + 1) 38 | y_range = np.arange(min_y, max_y + 1) 39 | X, Y = np.meshgrid(x_range, y_range) 40 | 41 | # Find value for all (x, y) coordinates 42 | Z_noace = np.apply_along_axis(lambda _: V[(_[0], _[1], False)], 2, np.dstack([X, Y])) 43 | Z_ace = np.apply_along_axis(lambda _: V[(_[0], _[1], True)], 2, np.dstack([X, Y])) 44 | 45 | def plot_surface(X, Y, Z, title): 46 | fig = plt.figure(figsize=(20, 10)) 47 | ax = fig.add_subplot(111, projection='3d') 48 | surf = ax.plot_surface(X, Y, Z, rstride=1, cstride=1, 49 | cmap=matplotlib.cm.coolwarm, vmin=-1.0, vmax=1.0) 50 | ax.set_xlabel('Player Sum') 51 | ax.set_ylabel('Dealer Showing') 52 | ax.set_zlabel('Value') 53 | ax.set_title(title) 54 | ax.view_init(ax.elev, -120) 55 | fig.colorbar(surf) 56 | plt.show() 57 | 58 | plot_surface(X, Y, Z_noace, "{} (No Usable Ace)".format(title)) 59 | plot_surface(X, Y, Z_ace, "{} (Usable Ace)".format(title)) 60 | 61 | 62 | 63 | def plot_episode_stats(stats, smoothing_window=10, noshow=False): 64 | # Plot the episode length over time 65 | fig1 = plt.figure(figsize=(10,5)) 66 | plt.plot(stats.episode_lengths) 67 | plt.xlabel("Epsiode") 68 | plt.ylabel("Epsiode Length") 69 | plt.title("Episode Length over Time") 70 | if noshow: 71 | plt.close(fig1) 72 | else: 73 | plt.show(fig1) 74 | 75 | # Plot the episode reward over time 76 | fig2 = plt.figure(figsize=(10,5)) 77 | rewards_smoothed = pd.Series(stats.episode_rewards).rolling(smoothing_window, min_periods=smoothing_window).mean() 78 | plt.plot(rewards_smoothed) 79 | plt.xlabel("Epsiode") 80 | plt.ylabel("Epsiode Reward (Smoothed)") 81 | plt.title("Episode Reward over Time (Smoothed over window size {})".format(smoothing_window)) 82 | if noshow: 83 | plt.close(fig2) 84 | else: 85 | plt.show(fig2) 86 | 87 | # Plot time steps and episode number 88 | fig3 = plt.figure(figsize=(10,5)) 89 | plt.plot(np.cumsum(stats.episode_lengths), np.arange(len(stats.episode_lengths))) 90 | plt.xlabel("Time Steps") 91 | plt.ylabel("Episode") 92 | plt.title("Episode per time step") 93 | if noshow: 94 | plt.close(fig3) 95 | else: 96 | plt.show(fig3) 97 | 98 | return fig1, fig2, fig3 99 | -------------------------------------------------------------------------------- /utils/policy.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def make_epsilon_greedy_policy(Q, epsilon, nA): 4 | """ 5 | Creates an epsilon-greedy policy based on a given Q-function and epsilon. 6 | 7 | Args: 8 | Q: A dictionary that maps from state -> action-values. 9 | Each value is a numpy array of length nA (see below) 10 | epsilon: The probability to select a random action . float between 0 and 1. 11 | nA: Number of actions in the environment. 12 | 13 | Returns: 14 | A function that takes the state as an argument and returns 15 | the probabilities for each action in the form of a numpy array of length nA. 16 | 17 | """ 18 | def policy_fn(state): 19 | # 1 / epsilon for non-greedy actions 20 | probs = (epsilon / nA) * np.ones(nA) 21 | 22 | greedy_action = Q[state].argmax() 23 | # (1 / epsilon + (1 - epsilon)) for greedy action 24 | probs[greedy_action] += 1.0 - epsilon 25 | 26 | return probs 27 | 28 | return policy_fn 29 | 30 | def make_greedy_policy(Q, nA): 31 | """ 32 | Creates a greedy policy based on Q values. 33 | 34 | Args: 35 | Q: A dictionary that maps from state -> action values 36 | nA: Number of actions in the environment. 37 | 38 | Returns: 39 | A function that takes an observation as input and returns a vector 40 | of action probabilities. 41 | """ 42 | 43 | def policy_fn(observation): 44 | probs = np.zeros(nA) 45 | greedy_action = Q[observation].argmax() 46 | probs[greedy_action] = 1.0 47 | 48 | return probs 49 | 50 | return policy_fn 51 | --------------------------------------------------------------------------------