├── Machine Learning Specialization-Capstone ├── wk2.ipynb ├── wk3.ipynb ├── wk4.ipynb ├── wk5.ipynb ├── wk6-1.ipynb └── wk6-2.ipynb ├── Machine Learning Specialization-Classification ├── WK1 │ ├── WK1.ipynb │ ├── module-2-assignment-test-idx.json │ └── module-2-assignment-train-idx.json ├── WK2 │ ├── WK2-1.ipynb │ ├── WK2-2.ipynb │ ├── important_words.json │ ├── module-4-assignment-train-idx.json │ └── module-4-assignment-validation-idx.json ├── WK3 │ ├── WK3-1.ipynb │ ├── WK3-2.ipynb │ ├── module-5-assignment-1-train-idx.json │ ├── module-5-assignment-1-validation-idx.json │ ├── module-5-assignment-2-test-idx.json │ └── module-5-assignment-2-train-idx.json ├── WK4 │ ├── WK4.ipynb │ ├── module-6-assignment-train-idx.json │ └── module-6-assignment-validation-idx.json ├── WK5 │ ├── WK5-1.ipynb │ ├── WK5-2.ipynb │ ├── module-8-assignment-1-train-idx.json │ ├── module-8-assignment-1-validation-idx.json │ ├── module-8-assignment-2-test-idx.json │ ├── module-8-assignment-2-train-idx.json │ └── module-8-boosting-assignment-1-blank.ipynb ├── WK6 │ ├── WK6.ipynb │ ├── module-9-assignment-test-idx.json │ └── module-9-assignment-train-idx.json └── WK7 │ ├── WK7.ipynb │ ├── amazon_baby_subset.zip │ ├── important_words.json │ ├── module-10-assignment-train-idx.json │ └── module-10-assignment-validation-idx.json ├── Machine Learning Specialization-Clustering ├── WK2 │ ├── WK2-1.ipynb │ ├── WK2-2.ipynb │ ├── people_wiki_map_index_to_word.json │ └── people_wiki_word_count.npz ├── WK3 │ ├── WK3.ipynb │ └── people_wiki_map_index_to_word.json ├── WK4 │ ├── WK4-1.ipynb │ └── WK4-2.ipynb ├── WK5 │ └── 5_lda_blank.ipynb ├── WK6 │ └── WK6.ipynb └── empty.md ├── Machine Learning Specialization-Regression ├── WK1 │ ├── Philadelphia_Crime_Rate_noNA.csv │ ├── PhillyCrime.ipynb │ ├── WK1.ipynb │ ├── kc_house_data.csv │ ├── kc_house_test_data.csv │ └── kc_house_train_data.csv ├── WK2 │ ├── WK2-1.ipynb │ ├── WK2-2.ipynb │ ├── kc_house_test_data.csv │ ├── kc_house_train_data.csv │ └── numpy-tutorial.ipynb ├── WK3 │ ├── Untitled.ipynb │ ├── WK3.ipynb │ ├── kc_house_data.csv │ ├── wk3_kc_house_set_1_data.csv │ ├── wk3_kc_house_set_2_data.csv │ ├── wk3_kc_house_set_3_data.csv │ ├── wk3_kc_house_set_4_data.csv │ ├── wk3_kc_house_test_data.csv │ ├── wk3_kc_house_train_data.csv │ └── wk3_kc_house_valid_data.csv ├── WK4 │ ├── WK4-1.ipynb │ ├── WK4-2.ipynb │ ├── kc_house_data.csv │ ├── kc_house_test_data.csv │ ├── kc_house_train_data.csv │ ├── wk3_kc_house_set_1_data.csv │ ├── wk3_kc_house_set_2_data.csv │ ├── wk3_kc_house_set_3_data.csv │ ├── wk3_kc_house_set_4_data.csv │ ├── wk3_kc_house_test_data.csv │ ├── wk3_kc_house_train_data.csv │ ├── wk3_kc_house_train_valid_shuffled.csv │ └── wk3_kc_house_valid_data.csv ├── WK5 │ ├── Overfitting_Demo_Ridge_Lasso.ipynb │ ├── WK5-1.ipynb │ ├── WK5-2.ipynb │ ├── kc_house_data.csv │ ├── kc_house_test_data.csv │ ├── kc_house_train_data.csv │ ├── wk3_kc_house_test_data.csv │ ├── wk3_kc_house_train_data.csv │ └── wk3_kc_house_valid_data.csv └── WK6 │ ├── WK6.ipynb │ ├── kc_house_data_small.csv │ ├── kc_house_data_small_test.csv │ ├── kc_house_data_small_train.csv │ ├── kc_house_data_validation.csv │ └── week-6-local-regression-assignment-blank.ipynb ├── README.md └── _config.yml /Machine Learning Specialization-Classification/WK2/important_words.json: -------------------------------------------------------------------------------- 1 | ["baby", "one", "great", "love", "use", "would", "like", "easy", "little", "seat", "old", "well", "get", "also", "really", "son", "time", "bought", "product", "good", "daughter", "much", "loves", "stroller", "put", "months", "car", "still", "back", "used", "recommend", "first", "even", "perfect", "nice", "bag", "two", "using", "got", "fit", "around", "diaper", "enough", "month", "price", "go", "could", "soft", "since", "buy", "room", "works", "made", "child", "keep", "size", "small", "need", "year", "big", "make", "take", "easily", "think", "crib", "clean", "way", "quality", "thing", "better", "without", "set", "new", "every", "cute", "best", "bottles", "work", "purchased", "right", "lot", "side", "happy", "comfortable", "toy", "able", "kids", "bit", "night", "long", "fits", "see", "us", "another", "play", "day", "money", "monitor", "tried", "thought", "never", "item", "hard", "plastic", "however", "disappointed", "reviews", "something", "going", "pump", "bottle", "cup", "waste", "return", "amazon", "different", "top", "want", "problem", "know", "water", "try", "received", "sure", "times", "chair", "find", "hold", "gate", "open", "bottom", "away", "actually", "cheap", "worked", "getting", "ordered", "came", "milk", "bad", "part", "worth", "found", "cover", "many", "design", "looking", "weeks", "say", "wanted", "look", "place", "purchase", "looks", "second", "piece", "box", "pretty", "trying", "difficult", "together", "though", "give", "started", "anything", "last", "company", "come", "returned", "maybe", "took", "broke", "makes", "stay", "instead", "idea", "head", "said", "less", "went", "working", "high", "unit", "seems", "picture", "completely", "wish", "buying", "babies", "won", "tub", "almost", "either"] -------------------------------------------------------------------------------- /Machine Learning Specialization-Classification/WK5/module-8-boosting-assignment-1-blank.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Exploring Ensemble Methods" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "In this assignment, we will explore the use of boosting. We will use the pre-implemented gradient boosted trees in GraphLab Create. You will:\n", 15 | "\n", 16 | "* Use SFrames to do some feature engineering.\n", 17 | "* Train a boosted ensemble of decision-trees (gradient boosted trees) on the LendingClub dataset.\n", 18 | "* Predict whether a loan will default along with prediction probabilities (on a validation set).\n", 19 | "* Evaluate the trained model and compare it with a baseline.\n", 20 | "* Find the most positive and negative loans using the learned model.\n", 21 | "* Explore how the number of trees influences classification performance.\n", 22 | "\n", 23 | "Let's get started!\n", 24 | "\n", 25 | "## Fire up Graphlab Create" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": null, 31 | "metadata": {}, 32 | "outputs": [], 33 | "source": [ 34 | "import graphlab" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": {}, 40 | "source": [ 41 | "# Load LendingClub dataset\n", 42 | "\n", 43 | "We will be using the [LendingClub](https://www.lendingclub.com/) data. As discussed earlier, the [LendingClub](https://www.lendingclub.com/) is a peer-to-peer leading company that directly connects borrowers and potential lenders/investors. \n", 44 | "\n", 45 | "Just like we did in previous assignments, we will build a classification model to predict whether or not a loan provided by lending club is likely to default.\n", 46 | "\n", 47 | "Let us start by loading the data." 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "loans = graphlab.SFrame('lending-club-data.gl/')" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": {}, 62 | "source": [ 63 | "Let's quickly explore what the dataset looks like. First, let's print out the column names to see what features we have in this dataset. We have done this in previous assignments, so we won't belabor this here." 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "loans.column_names()" 73 | ] 74 | }, 75 | { 76 | "cell_type": "markdown", 77 | "metadata": {}, 78 | "source": [ 79 | "## Modifying the target column\n", 80 | "\n", 81 | "The target column (label column) of the dataset that we are interested in is called `bad_loans`. In this column **1** means a risky (bad) loan **0** means a safe loan.\n", 82 | "\n", 83 | "As in past assignments, in order to make this more intuitive and consistent with the lectures, we reassign the target to be:\n", 84 | "* **+1** as a safe loan, \n", 85 | "* **-1** as a risky (bad) loan. \n", 86 | "\n", 87 | "We put this in a new column called `safe_loans`." 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": { 94 | "collapsed": true 95 | }, 96 | "outputs": [], 97 | "source": [ 98 | "loans['safe_loans'] = loans['bad_loans'].apply(lambda x : +1 if x==0 else -1)\n", 99 | "loans = loans.remove_column('bad_loans')" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "## Selecting features\n", 107 | "\n", 108 | "In this assignment, we will be using a subset of features (categorical and numeric). The features we will be using are **described in the code comments** below. If you are a finance geek, the [LendingClub](https://www.lendingclub.com/) website has a lot more details about these features.\n", 109 | "\n", 110 | "The features we will be using are described in the code comments below:" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": null, 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [ 119 | "target = 'safe_loans'\n", 120 | "features = ['grade', # grade of the loan (categorical)\n", 121 | " 'sub_grade_num', # sub-grade of the loan as a number from 0 to 1\n", 122 | " 'short_emp', # one year or less of employment\n", 123 | " 'emp_length_num', # number of years of employment\n", 124 | " 'home_ownership', # home_ownership status: own, mortgage or rent\n", 125 | " 'dti', # debt to income ratio\n", 126 | " 'purpose', # the purpose of the loan\n", 127 | " 'payment_inc_ratio', # ratio of the monthly payment to income\n", 128 | " 'delinq_2yrs', # number of delinquincies \n", 129 | " 'delinq_2yrs_zero', # no delinquincies in last 2 years\n", 130 | " 'inq_last_6mths', # number of creditor inquiries in last 6 months\n", 131 | " 'last_delinq_none', # has borrower had a delinquincy\n", 132 | " 'last_major_derog_none', # has borrower had 90 day or worse rating\n", 133 | " 'open_acc', # number of open credit accounts\n", 134 | " 'pub_rec', # number of derogatory public records\n", 135 | " 'pub_rec_zero', # no derogatory public records\n", 136 | " 'revol_util', # percent of available credit being used\n", 137 | " 'total_rec_late_fee', # total late fees received to day\n", 138 | " 'int_rate', # interest rate of the loan\n", 139 | " 'total_rec_int', # interest received to date\n", 140 | " 'annual_inc', # annual income of borrower\n", 141 | " 'funded_amnt', # amount committed to the loan\n", 142 | " 'funded_amnt_inv', # amount committed by investors for the loan\n", 143 | " 'installment', # monthly payment owed by the borrower\n", 144 | " ]" 145 | ] 146 | }, 147 | { 148 | "cell_type": "markdown", 149 | "metadata": {}, 150 | "source": [ 151 | "## Skipping observations with missing values\n", 152 | "\n", 153 | "Recall from the lectures that one common approach to coping with missing values is to **skip** observations that contain missing values.\n", 154 | "\n", 155 | "We run the following code to do so:" 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": null, 161 | "metadata": {}, 162 | "outputs": [], 163 | "source": [ 164 | "loans, loans_with_na = loans[[target] + features].dropna_split()\n", 165 | "\n", 166 | "# Count the number of rows with missing data\n", 167 | "num_rows_with_na = loans_with_na.num_rows()\n", 168 | "num_rows = loans.num_rows()\n", 169 | "print 'Dropping %s observations; keeping %s ' % (num_rows_with_na, num_rows)" 170 | ] 171 | }, 172 | { 173 | "cell_type": "markdown", 174 | "metadata": {}, 175 | "source": [ 176 | "Fortunately, there are not too many missing values. We are retaining most of the data." 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": {}, 182 | "source": [ 183 | "## Make sure the classes are balanced" 184 | ] 185 | }, 186 | { 187 | "cell_type": "markdown", 188 | "metadata": {}, 189 | "source": [ 190 | "We saw in an earlier assignment that this dataset is also imbalanced. We will undersample the larger class (safe loans) in order to balance out our dataset. We used `seed=1` to make sure everyone gets the same results." 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": null, 196 | "metadata": {}, 197 | "outputs": [], 198 | "source": [ 199 | "safe_loans_raw = loans[loans[target] == 1]\n", 200 | "risky_loans_raw = loans[loans[target] == -1]\n", 201 | "\n", 202 | "# Undersample the safe loans.\n", 203 | "percentage = len(risky_loans_raw)/float(len(safe_loans_raw))\n", 204 | "safe_loans = safe_loans_raw.sample(percentage, seed = 1)\n", 205 | "risky_loans = risky_loans_raw\n", 206 | "loans_data = risky_loans.append(safe_loans)\n", 207 | "\n", 208 | "print \"Percentage of safe loans :\", len(safe_loans) / float(len(loans_data))\n", 209 | "print \"Percentage of risky loans :\", len(risky_loans) / float(len(loans_data))\n", 210 | "print \"Total number of loans in our new dataset :\", len(loans_data)" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "metadata": {}, 216 | "source": [ 217 | "**Checkpoint:** You should now see that the dataset is balanced (approximately 50-50 safe vs risky loans)." 218 | ] 219 | }, 220 | { 221 | "cell_type": "markdown", 222 | "metadata": {}, 223 | "source": [ 224 | "**Note:** There are many approaches for dealing with imbalanced data, including some where we modify the learning algorithm. These approaches are beyond the scope of this course, but some of them are reviewed in this [paper](http://ieeexplore.ieee.org/xpl/login.jsp?tp=&arnumber=5128907&url=http%3A%2F%2Fieeexplore.ieee.org%2Fiel5%2F69%2F5173046%2F05128907.pdf%3Farnumber%3D5128907 ). For this assignment, we use the simplest possible approach, where we subsample the overly represented class to get a more balanced dataset. In general, and especially when the data is highly imbalanced, we recommend using more advanced methods." 225 | ] 226 | }, 227 | { 228 | "cell_type": "markdown", 229 | "metadata": {}, 230 | "source": [ 231 | "## Split data into training and validation sets" 232 | ] 233 | }, 234 | { 235 | "cell_type": "markdown", 236 | "metadata": {}, 237 | "source": [ 238 | "We split the data into training data and validation data. We used `seed=1` to make sure everyone gets the same results. We will use the validation data to help us select model parameters." 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "metadata": {}, 245 | "outputs": [], 246 | "source": [ 247 | "train_data, validation_data = loans_data.random_split(.8, seed=1)" 248 | ] 249 | }, 250 | { 251 | "cell_type": "markdown", 252 | "metadata": {}, 253 | "source": [ 254 | "# Gradient boosted tree classifier" 255 | ] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "metadata": {}, 260 | "source": [ 261 | "Gradient boosted trees are a powerful variant of boosting methods; they have been used to win many [Kaggle](https://www.kaggle.com/) competitions, and have been widely used in industry. We will explore the predictive power of multiple decision trees as opposed to a single decision tree.\n", 262 | "\n", 263 | "**Additional reading:** If you are interested in gradient boosted trees, here is some additional reading material:\n", 264 | "* [GraphLab Create user guide](https://dato.com/learn/userguide/supervised-learning/boosted_trees_classifier.html)\n", 265 | "* [Advanced material on boosted trees](http://homes.cs.washington.edu/~tqchen/pdf/BoostedTree.pdf)\n", 266 | "\n", 267 | "\n", 268 | "We will now train models to predict `safe_loans` using the features above. In this section, we will experiment with training an ensemble of 5 trees. To cap the ensemble classifier at 5 trees, we call the function with **max_iterations=5** (recall that each iterations corresponds to adding a tree). We set `validation_set=None` to make sure everyone gets the same results." 269 | ] 270 | }, 271 | { 272 | "cell_type": "code", 273 | "execution_count": null, 274 | "metadata": {}, 275 | "outputs": [], 276 | "source": [ 277 | "model_5 = graphlab.boosted_trees_classifier.create(train_data, validation_set=None, \n", 278 | " target = target, features = features, max_iterations = 5)" 279 | ] 280 | }, 281 | { 282 | "cell_type": "markdown", 283 | "metadata": {}, 284 | "source": [ 285 | "# Making predictions\n", 286 | "\n", 287 | "Just like we did in previous sections, let us consider a few positive and negative examples **from the validation set**. We will do the following:\n", 288 | "* Predict whether or not a loan is likely to default.\n", 289 | "* Predict the probability with which the loan is likely to default." 290 | ] 291 | }, 292 | { 293 | "cell_type": "code", 294 | "execution_count": null, 295 | "metadata": {}, 296 | "outputs": [], 297 | "source": [ 298 | "# Select all positive and negative examples.\n", 299 | "validation_safe_loans = validation_data[validation_data[target] == 1]\n", 300 | "validation_risky_loans = validation_data[validation_data[target] == -1]\n", 301 | "\n", 302 | "# Select 2 examples from the validation set for positive & negative loans\n", 303 | "sample_validation_data_risky = validation_risky_loans[0:2]\n", 304 | "sample_validation_data_safe = validation_safe_loans[0:2]\n", 305 | "\n", 306 | "# Append the 4 examples into a single dataset\n", 307 | "sample_validation_data = sample_validation_data_safe.append(sample_validation_data_risky)\n", 308 | "sample_validation_data" 309 | ] 310 | }, 311 | { 312 | "cell_type": "markdown", 313 | "metadata": {}, 314 | "source": [ 315 | "### Predicting on sample validation data\n", 316 | "\n", 317 | "For each row in the **sample_validation_data**, write code to make **model_5** predict whether or not the loan is classified as a **safe loan**.\n", 318 | "\n", 319 | "**Hint:** Use the `predict` method in `model_5` for this." 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "execution_count": null, 325 | "metadata": {}, 326 | "outputs": [], 327 | "source": [] 328 | }, 329 | { 330 | "cell_type": "markdown", 331 | "metadata": {}, 332 | "source": [ 333 | "**Quiz Question:** What percentage of the predictions on `sample_validation_data` did `model_5` get correct?\n", 334 | "\n", 335 | "### Prediction probabilities\n", 336 | "\n", 337 | "For each row in the **sample_validation_data**, what is the probability (according **model_5**) of a loan being classified as **safe**? \n", 338 | "\n", 339 | "**Hint:** Set `output_type='probability'` to make **probability** predictions using `model_5` on `sample_validation_data`:" 340 | ] 341 | }, 342 | { 343 | "cell_type": "code", 344 | "execution_count": null, 345 | "metadata": {}, 346 | "outputs": [], 347 | "source": [] 348 | }, 349 | { 350 | "cell_type": "markdown", 351 | "metadata": {}, 352 | "source": [ 353 | "**Quiz Question:** According to **model_5**, which loan is the least likely to be a safe loan?\n", 354 | "\n", 355 | "**Checkpoint:** Can you verify that for all the predictions with `probability >= 0.5`, the model predicted the label **+1**?" 356 | ] 357 | }, 358 | { 359 | "cell_type": "markdown", 360 | "metadata": { 361 | "collapsed": true 362 | }, 363 | "source": [ 364 | "## Evaluating the model on the validation data" 365 | ] 366 | }, 367 | { 368 | "cell_type": "markdown", 369 | "metadata": {}, 370 | "source": [ 371 | "Recall that the accuracy is defined as follows:\n", 372 | "$$\n", 373 | "\\mbox{accuracy} = \\frac{\\mbox{# correctly classified examples}}{\\mbox{# total examples}}\n", 374 | "$$\n", 375 | "\n", 376 | "Evaluate the accuracy of the **model_5** on the **validation_data**.\n", 377 | "\n", 378 | "**Hint**: Use the `.evaluate()` method in the model." 379 | ] 380 | }, 381 | { 382 | "cell_type": "code", 383 | "execution_count": null, 384 | "metadata": {}, 385 | "outputs": [], 386 | "source": [] 387 | }, 388 | { 389 | "cell_type": "markdown", 390 | "metadata": {}, 391 | "source": [ 392 | "Calculate the number of **false positives** made by the model." 393 | ] 394 | }, 395 | { 396 | "cell_type": "code", 397 | "execution_count": null, 398 | "metadata": {}, 399 | "outputs": [], 400 | "source": [] 401 | }, 402 | { 403 | "cell_type": "markdown", 404 | "metadata": {}, 405 | "source": [ 406 | "**Quiz Question**: What is the number of **false positives** on the **validation_data**?" 407 | ] 408 | }, 409 | { 410 | "cell_type": "markdown", 411 | "metadata": {}, 412 | "source": [ 413 | "Calculate the number of **false negatives** made by the model." 414 | ] 415 | }, 416 | { 417 | "cell_type": "code", 418 | "execution_count": null, 419 | "metadata": {}, 420 | "outputs": [], 421 | "source": [] 422 | }, 423 | { 424 | "cell_type": "markdown", 425 | "metadata": {}, 426 | "source": [ 427 | "## Comparison with decision trees\n", 428 | "\n", 429 | "In the earlier assignment, we saw that the prediction accuracy of the decision trees was around **0.64** (rounded). In this assignment, we saw that **model_5** has an accuracy of **0.67** (rounded).\n", 430 | "\n", 431 | "Here, we quantify the benefit of the extra 3% increase in accuracy of **model_5** in comparison with a single decision tree from the original decision tree assignment.\n", 432 | "\n", 433 | "As we explored in the earlier assignment, we calculated the cost of the mistakes made by the model. We again consider the same costs as follows:\n", 434 | "\n", 435 | "* **False negatives**: Assume a cost of \\$10,000 per false negative.\n", 436 | "* **False positives**: Assume a cost of \\$20,000 per false positive.\n", 437 | "\n", 438 | "Assume that the number of false positives and false negatives for the learned decision tree was\n", 439 | "\n", 440 | "* **False negatives**: 1936\n", 441 | "* **False positives**: 1503\n", 442 | "\n", 443 | "Using the costs defined above and the number of false positives and false negatives for the decision tree, we can calculate the total cost of the mistakes made by the decision tree model as follows:\n", 444 | "\n", 445 | "```\n", 446 | "cost = $10,000 * 1936 + $20,000 * 1503 = $49,420,000\n", 447 | "```\n", 448 | "\n", 449 | "The total cost of the mistakes of the model is $49.42M. That is a **lot of money**!.\n", 450 | "\n", 451 | "**Quiz Question**: Using the same costs of the false positives and false negatives, what is the cost of the mistakes made by the boosted tree model (**model_5**) as evaluated on the **validation_set**?" 452 | ] 453 | }, 454 | { 455 | "cell_type": "code", 456 | "execution_count": null, 457 | "metadata": {}, 458 | "outputs": [], 459 | "source": [] 460 | }, 461 | { 462 | "cell_type": "markdown", 463 | "metadata": {}, 464 | "source": [ 465 | "**Reminder**: Compare the cost of the mistakes made by the boosted trees model with the decision tree model. The extra 3% improvement in prediction accuracy can translate to several million dollars! And, it was so easy to get by simply boosting our decision trees." 466 | ] 467 | }, 468 | { 469 | "cell_type": "markdown", 470 | "metadata": {}, 471 | "source": [ 472 | "## Most positive & negative loans.\n", 473 | "\n", 474 | "In this section, we will find the loans that are most likely to be predicted **safe**. We can do this in a few steps:\n", 475 | "\n", 476 | "* **Step 1**: Use the **model_5** (the model with 5 trees) and make **probability predictions** for all the loans in the **validation_data**.\n", 477 | "* **Step 2**: Similar to what we did in the very first assignment, add the probability predictions as a column called **predictions** into the validation_data.\n", 478 | "* **Step 3**: Sort the data (in descreasing order) by the probability predictions.\n", 479 | "\n", 480 | "Start here with **Step 1** & **Step 2**. Make predictions using **model_5** for examples in the **validation_data**. Use `output_type = probability`." 481 | ] 482 | }, 483 | { 484 | "cell_type": "code", 485 | "execution_count": null, 486 | "metadata": {}, 487 | "outputs": [], 488 | "source": [] 489 | }, 490 | { 491 | "cell_type": "markdown", 492 | "metadata": {}, 493 | "source": [ 494 | "**Checkpoint:** For each row, the probabilities should be a number in the range **[0, 1]**. We have provided a simple check here to make sure your answers are correct." 495 | ] 496 | }, 497 | { 498 | "cell_type": "code", 499 | "execution_count": null, 500 | "metadata": {}, 501 | "outputs": [], 502 | "source": [ 503 | "print \"Your loans : %s\\n\" % validation_data['predictions'].head(4)\n", 504 | "print \"Expected answer : %s\" % [0.4492515948736132, 0.6119100103640573,\n", 505 | " 0.3835981314851436, 0.3693306705994325]" 506 | ] 507 | }, 508 | { 509 | "cell_type": "markdown", 510 | "metadata": {}, 511 | "source": [ 512 | "Now, we are ready to go to **Step 3**. You can now use the `prediction` column to sort the loans in **validation_data** (in descending order) by prediction probability. Find the top 5 loans with the highest probability of being predicted as a **safe loan**." 513 | ] 514 | }, 515 | { 516 | "cell_type": "code", 517 | "execution_count": null, 518 | "metadata": {}, 519 | "outputs": [], 520 | "source": [] 521 | }, 522 | { 523 | "cell_type": "markdown", 524 | "metadata": {}, 525 | "source": [ 526 | "** Quiz Question**: What grades are the top 5 loans?\n", 527 | "\n", 528 | "Let us repeat this excercise to find the top 5 loans (in the **validation_data**) with the **lowest probability** of being predicted as a **safe loan**:" 529 | ] 530 | }, 531 | { 532 | "cell_type": "code", 533 | "execution_count": null, 534 | "metadata": {}, 535 | "outputs": [], 536 | "source": [] 537 | }, 538 | { 539 | "cell_type": "markdown", 540 | "metadata": {}, 541 | "source": [ 542 | "**Checkpoint:** You should expect to see 5 loans with the grade ['**D**', '**C**', '**C**', '**C**', '**B**'] or with ['**D**', '**C**', '**B**', '**C**', '**C**']." 543 | ] 544 | }, 545 | { 546 | "cell_type": "markdown", 547 | "metadata": {}, 548 | "source": [ 549 | "## Effect of adding more trees" 550 | ] 551 | }, 552 | { 553 | "cell_type": "markdown", 554 | "metadata": {}, 555 | "source": [ 556 | "In this assignment, we will train 5 different ensemble classifiers in the form of gradient boosted trees. We will train models with 10, 50, 100, 200, and 500 trees. We use the **max_iterations** parameter in the boosted tree module. \n", 557 | "\n", 558 | "Let's get sarted with a model with **max_iterations = 10**:" 559 | ] 560 | }, 561 | { 562 | "cell_type": "code", 563 | "execution_count": null, 564 | "metadata": {}, 565 | "outputs": [], 566 | "source": [ 567 | "model_10 = graphlab.boosted_trees_classifier.create(train_data, validation_set=None, \n", 568 | " target = target, features = features, max_iterations = 10, verbose=False)" 569 | ] 570 | }, 571 | { 572 | "cell_type": "markdown", 573 | "metadata": {}, 574 | "source": [ 575 | "Now, train 4 models with **max_iterations** to be:\n", 576 | "* `max_iterations = 50`, \n", 577 | "* `max_iterations = 100`\n", 578 | "* `max_iterations = 200`\n", 579 | "* `max_iterations = 500`. \n", 580 | "\n", 581 | "Let us call these models **model_50**, **model_100**, **model_200**, and **model_500**. You can pass in `verbose=False` in order to suppress the printed output.\n", 582 | "\n", 583 | "**Warning:** This could take a couple of minutes to run." 584 | ] 585 | }, 586 | { 587 | "cell_type": "code", 588 | "execution_count": null, 589 | "metadata": {}, 590 | "outputs": [], 591 | "source": [ 592 | "model_50 = \n", 593 | "model_100 = \n", 594 | "model_200 = \n", 595 | "model_500 = " 596 | ] 597 | }, 598 | { 599 | "cell_type": "markdown", 600 | "metadata": {}, 601 | "source": [ 602 | "## Compare accuracy on entire validation set" 603 | ] 604 | }, 605 | { 606 | "cell_type": "markdown", 607 | "metadata": {}, 608 | "source": [ 609 | "Now we will compare the predicitve accuracy of our models on the validation set. Evaluate the **accuracy** of the 10, 50, 100, 200, and 500 tree models on the **validation_data**. Use the `.evaluate` method." 610 | ] 611 | }, 612 | { 613 | "cell_type": "code", 614 | "execution_count": null, 615 | "metadata": {}, 616 | "outputs": [], 617 | "source": [] 618 | }, 619 | { 620 | "cell_type": "markdown", 621 | "metadata": {}, 622 | "source": [ 623 | "**Quiz Question:** Which model has the **best** accuracy on the **validation_data**?\n", 624 | "\n", 625 | "**Quiz Question:** Is it always true that the model with the most trees will perform best on test data?" 626 | ] 627 | }, 628 | { 629 | "cell_type": "markdown", 630 | "metadata": {}, 631 | "source": [ 632 | "## Plot the training and validation error vs. number of trees" 633 | ] 634 | }, 635 | { 636 | "cell_type": "markdown", 637 | "metadata": {}, 638 | "source": [ 639 | "Recall from the lecture that the classification error is defined as\n", 640 | "\n", 641 | "$$\n", 642 | "\\mbox{classification error} = 1 - \\mbox{accuracy} \n", 643 | "$$\n", 644 | "\n", 645 | "In this section, we will plot the **training and validation errors versus the number of trees** to get a sense of how these models are performing. We will compare the 10, 50, 100, 200, and 500 tree models. You will need [matplotlib](http://matplotlib.org/downloads.html) in order to visualize the plots. \n", 646 | "\n", 647 | "First, make sure this block of code runs on your computer." 648 | ] 649 | }, 650 | { 651 | "cell_type": "code", 652 | "execution_count": null, 653 | "metadata": { 654 | "collapsed": true 655 | }, 656 | "outputs": [], 657 | "source": [ 658 | "import matplotlib.pyplot as plt\n", 659 | "%matplotlib inline\n", 660 | "def make_figure(dim, title, xlabel, ylabel, legend):\n", 661 | " plt.rcParams['figure.figsize'] = dim\n", 662 | " plt.title(title)\n", 663 | " plt.xlabel(xlabel)\n", 664 | " plt.ylabel(ylabel)\n", 665 | " if legend is not None:\n", 666 | " plt.legend(loc=legend, prop={'size':15})\n", 667 | " plt.rcParams.update({'font.size': 16})\n", 668 | " plt.tight_layout()" 669 | ] 670 | }, 671 | { 672 | "cell_type": "markdown", 673 | "metadata": {}, 674 | "source": [ 675 | "In order to plot the classification errors (on the **train_data** and **validation_data**) versus the number of trees, we will need lists of these accuracies, which we get by applying the method `.evaluate`. \n", 676 | "\n", 677 | "**Steps to follow:**\n", 678 | "\n", 679 | "* **Step 1:** Calculate the classification error for model on the training data (**train_data**).\n", 680 | "* **Step 2:** Store the training errors into a list (called `training_errors`) that looks like this:\n", 681 | "```\n", 682 | "[train_err_10, train_err_50, ..., train_err_500]\n", 683 | "```\n", 684 | "* **Step 3:** Calculate the classification error of each model on the validation data (**validation_data**).\n", 685 | "* **Step 4:** Store the validation classification error into a list (called `validation_errors`) that looks like this:\n", 686 | "```\n", 687 | "[validation_err_10, validation_err_50, ..., validation_err_500]\n", 688 | "```\n", 689 | "Once that has been completed, the rest of the code should be able to evaluate correctly and generate the plot.\n", 690 | "\n", 691 | "\n", 692 | "Let us start with **Step 1**. Write code to compute the classification error on the **train_data** for models **model_10**, **model_50**, **model_100**, **model_200**, and **model_500**." 693 | ] 694 | }, 695 | { 696 | "cell_type": "code", 697 | "execution_count": null, 698 | "metadata": {}, 699 | "outputs": [], 700 | "source": [] 701 | }, 702 | { 703 | "cell_type": "markdown", 704 | "metadata": {}, 705 | "source": [ 706 | "Now, let us run **Step 2**. Save the training errors into a list called **training_errors**" 707 | ] 708 | }, 709 | { 710 | "cell_type": "code", 711 | "execution_count": null, 712 | "metadata": { 713 | "collapsed": true 714 | }, 715 | "outputs": [], 716 | "source": [ 717 | "training_errors = [train_err_10, train_err_50, train_err_100, \n", 718 | " train_err_200, train_err_500]" 719 | ] 720 | }, 721 | { 722 | "cell_type": "markdown", 723 | "metadata": {}, 724 | "source": [ 725 | "Now, onto **Step 3**. Write code to compute the classification error on the **validation_data** for models **model_10**, **model_50**, **model_100**, **model_200**, and **model_500**." 726 | ] 727 | }, 728 | { 729 | "cell_type": "code", 730 | "execution_count": null, 731 | "metadata": { 732 | "collapsed": true 733 | }, 734 | "outputs": [], 735 | "source": [] 736 | }, 737 | { 738 | "cell_type": "markdown", 739 | "metadata": {}, 740 | "source": [ 741 | "Now, let us run **Step 4**. Save the training errors into a list called **validation_errors**" 742 | ] 743 | }, 744 | { 745 | "cell_type": "code", 746 | "execution_count": null, 747 | "metadata": {}, 748 | "outputs": [], 749 | "source": [ 750 | "validation_errors = [validation_err_10, validation_err_50, validation_err_100, \n", 751 | " validation_err_200, validation_err_500]" 752 | ] 753 | }, 754 | { 755 | "cell_type": "markdown", 756 | "metadata": {}, 757 | "source": [ 758 | "Now, we will plot the **training_errors** and **validation_errors** versus the number of trees. We will compare the 10, 50, 100, 200, and 500 tree models. We provide some plotting code to visualize the plots within this notebook. \n", 759 | "\n", 760 | "Run the following code to visualize the plots." 761 | ] 762 | }, 763 | { 764 | "cell_type": "code", 765 | "execution_count": null, 766 | "metadata": {}, 767 | "outputs": [], 768 | "source": [ 769 | "plt.plot([10, 50, 100, 200, 500], training_errors, linewidth=4.0, label='Training error')\n", 770 | "plt.plot([10, 50, 100, 200, 500], validation_errors, linewidth=4.0, label='Validation error')\n", 771 | "\n", 772 | "make_figure(dim=(10,5), title='Error vs number of trees',\n", 773 | " xlabel='Number of trees',\n", 774 | " ylabel='Classification error',\n", 775 | " legend='best')" 776 | ] 777 | }, 778 | { 779 | "cell_type": "markdown", 780 | "metadata": {}, 781 | "source": [ 782 | "**Quiz Question**: Does the training error reduce as the number of trees increases?\n", 783 | "\n", 784 | "**Quiz Question**: Is it always true that the validation error will reduce as the number of trees increases?" 785 | ] 786 | }, 787 | { 788 | "cell_type": "code", 789 | "execution_count": null, 790 | "metadata": { 791 | "collapsed": true 792 | }, 793 | "outputs": [], 794 | "source": [] 795 | } 796 | ], 797 | "metadata": { 798 | "kernelspec": { 799 | "display_name": "Python 3", 800 | "language": "python", 801 | "name": "python3" 802 | }, 803 | "language_info": { 804 | "codemirror_mode": { 805 | "name": "ipython", 806 | "version": 3 807 | }, 808 | "file_extension": ".py", 809 | "mimetype": "text/x-python", 810 | "name": "python", 811 | "nbconvert_exporter": "python", 812 | "pygments_lexer": "ipython3", 813 | "version": "3.6.5" 814 | } 815 | }, 816 | "nbformat": 4, 817 | "nbformat_minor": 1 818 | } 819 | -------------------------------------------------------------------------------- /Machine Learning Specialization-Classification/WK7/amazon_baby_subset.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfirefire/Machine-Learning-Specialization/aae0574b51386a812dda764e5612976ece77e41d/Machine Learning Specialization-Classification/WK7/amazon_baby_subset.zip -------------------------------------------------------------------------------- /Machine Learning Specialization-Classification/WK7/important_words.json: -------------------------------------------------------------------------------- 1 | ["baby", "one", "great", "love", "use", "would", "like", "easy", "little", "seat", "old", "well", "get", "also", "really", "son", "time", "bought", "product", "good", "daughter", "much", "loves", "stroller", "put", "months", "car", "still", "back", "used", "recommend", "first", "even", "perfect", "nice", "bag", "two", "using", "got", "fit", "around", "diaper", "enough", "month", "price", "go", "could", "soft", "since", "buy", "room", "works", "made", "child", "keep", "size", "small", "need", "year", "big", "make", "take", "easily", "think", "crib", "clean", "way", "quality", "thing", "better", "without", "set", "new", "every", "cute", "best", "bottles", "work", "purchased", "right", "lot", "side", "happy", "comfortable", "toy", "able", "kids", "bit", "night", "long", "fits", "see", "us", "another", "play", "day", "money", "monitor", "tried", "thought", "never", "item", "hard", "plastic", "however", "disappointed", "reviews", "something", "going", "pump", "bottle", "cup", "waste", "return", "amazon", "different", "top", "want", "problem", "know", "water", "try", "received", "sure", "times", "chair", "find", "hold", "gate", "open", "bottom", "away", "actually", "cheap", "worked", "getting", "ordered", "came", "milk", "bad", "part", "worth", "found", "cover", "many", "design", "looking", "weeks", "say", "wanted", "look", "place", "purchase", "looks", "second", "piece", "box", "pretty", "trying", "difficult", "together", "though", "give", "started", "anything", "last", "company", "come", "returned", "maybe", "took", "broke", "makes", "stay", "instead", "idea", "head", "said", "less", "went", "working", "high", "unit", "seems", "picture", "completely", "wish", "buying", "babies", "won", "tub", "almost", "either"] -------------------------------------------------------------------------------- /Machine Learning Specialization-Classification/WK7/module-10-assignment-validation-idx.json: -------------------------------------------------------------------------------- 1 | [9, 14, 18, 32, 38, 53, 64, 102, 105, 117, 123, 134, 142, 170, 176, 184, 205, 217, 221, 223, 229, 233, 239, 263, 264, 284, 285, 326, 328, 350, 352, 356, 390, 403, 410, 411, 425, 430, 431, 451, 456, 466, 469, 492, 505, 509, 519, 528, 542, 543, 546, 551, 555, 562, 574, 577, 585, 596, 600, 626, 635, 637, 647, 657, 661, 671, 714, 715, 716, 722, 733, 737, 740, 742, 781, 790, 793, 802, 848, 851, 853, 861, 864, 865, 866, 869, 871, 893, 899, 907, 909, 923, 924, 930, 951, 965, 988, 995, 997, 999, 1010, 1043, 1055, 1056, 1061, 1074, 1078, 1116, 1118, 1122, 1124, 1129, 1136, 1143, 1156, 1158, 1177, 1185, 1195, 1200, 1203, 1217, 1244, 1252, 1254, 1262, 1263, 1287, 1290, 1297, 1311, 1325, 1335, 1336, 1354, 1358, 1359, 1367, 1428, 1432, 1444, 1452, 1482, 1492, 1499, 1530, 1538, 1562, 1580, 1589, 1594, 1611, 1631, 1636, 1644, 1648, 1651, 1667, 1684, 1700, 1703, 1706, 1724, 1735, 1741, 1748, 1755, 1767, 1770, 1771, 1773, 1807, 1809, 1818, 1824, 1830, 1857, 1870, 1881, 1884, 1911, 1912, 1918, 1920, 1936, 1946, 1951, 1959, 1963, 1966, 1981, 2003, 2012, 2024, 2047, 2056, 2085, 2088, 2093, 2105, 2109, 2111, 2119, 2127, 2129, 2136, 2145, 2146, 2155, 2159, 2187, 2243, 2247, 2248, 2252, 2257, 2268, 2283, 2284, 2288, 2294, 2318, 2321, 2337, 2341, 2345, 2367, 2377, 2389, 2396, 2401, 2417, 2419, 2436, 2437, 2439, 2451, 2459, 2464, 2471, 2494, 2496, 2499, 2500, 2519, 2540, 2549, 2559, 2563, 2571, 2574, 2585, 2609, 2617, 2636, 2640, 2653, 2662, 2669, 2675, 2677, 2701, 2710, 2714, 2715, 2718, 2722, 2724, 2726, 2728, 2730, 2741, 2745, 2747, 2766, 2767, 2772, 2776, 2781, 2811, 2815, 2817, 2824, 2827, 2830, 2831, 2836, 2837, 2839, 2840, 2841, 2866, 2874, 2893, 2894, 2900, 2926, 2935, 2940, 2947, 2948, 2965, 2966, 3005, 3008, 3021, 3045, 3059, 3060, 3069, 3088, 3089, 3095, 3102, 3112, 3114, 3115, 3117, 3143, 3147, 3159, 3196, 3200, 3221, 3247, 3254, 3255, 3263, 3308, 3313, 3321, 3324, 3337, 3345, 3347, 3351, 3364, 3366, 3367, 3376, 3387, 3393, 3398, 3413, 3414, 3416, 3423, 3426, 3429, 3443, 3457, 3470, 3471, 3494, 3549, 3583, 3587, 3609, 3612, 3656, 3664, 3682, 3694, 3698, 3707, 3741, 3748, 3784, 3789, 3807, 3810, 3815, 3830, 3832, 3836, 3837, 3850, 3872, 3885, 3907, 3909, 3911, 3915, 3930, 3934, 3940, 3944, 3945, 3947, 3953, 3955, 3958, 3959, 3966, 3967, 3973, 3975, 3979, 3981, 3982, 3984, 3999, 4003, 4020, 4027, 4043, 4064, 4065, 4066, 4068, 4075, 4082, 4091, 4093, 4118, 4121, 4122, 4137, 4152, 4161, 4179, 4200, 4210, 4213, 4231, 4233, 4254, 4258, 4275, 4278, 4294, 4298, 4316, 4341, 4342, 4346, 4354, 4359, 4361, 4363, 4376, 4382, 4384, 4387, 4389, 4403, 4417, 4420, 4425, 4449, 4450, 4472, 4503, 4507, 4508, 4509, 4525, 4526, 4529, 4535, 4543, 4544, 4554, 4589, 4592, 4594, 4595, 4602, 4611, 4614, 4626, 4641, 4650, 4685, 4703, 4718, 4727, 4729, 4740, 4742, 4747, 4760, 4763, 4764, 4770, 4792, 4796, 4797, 4801, 4807, 4813, 4853, 4865, 4867, 4869, 4875, 4878, 4882, 4900, 4917, 4918, 4924, 4925, 4944, 4947, 4955, 4964, 4965, 4971, 4994, 4997, 5019, 5021, 5025, 5057, 5061, 5067, 5086, 5091, 5098, 5100, 5114, 5124, 5166, 5167, 5171, 5173, 5178, 5184, 5186, 5194, 5202, 5209, 5213, 5232, 5238, 5260, 5261, 5266, 5268, 5271, 5280, 5282, 5290, 5293, 5296, 5310, 5319, 5335, 5341, 5345, 5359, 5361, 5372, 5382, 5388, 5393, 5395, 5441, 5454, 5469, 5481, 5482, 5486, 5490, 5501, 5506, 5527, 5534, 5536, 5561, 5597, 5614, 5616, 5626, 5632, 5635, 5640, 5649, 5654, 5684, 5686, 5688, 5697, 5711, 5719, 5722, 5725, 5729, 5741, 5756, 5784, 5786, 5787, 5792, 5805, 5821, 5825, 5829, 5858, 5866, 5875, 5880, 5881, 5883, 5905, 5925, 5927, 5943, 5982, 5987, 5996, 5997, 6003, 6009, 6013, 6015, 6023, 6029, 6040, 6042, 6061, 6064, 6074, 6075, 6078, 6079, 6082, 6085, 6102, 6108, 6111, 6119, 6139, 6160, 6161, 6166, 6184, 6193, 6226, 6234, 6243, 6251, 6255, 6263, 6265, 6279, 6282, 6285, 6290, 6297, 6305, 6309, 6319, 6358, 6369, 6381, 6389, 6391, 6392, 6394, 6429, 6454, 6473, 6482, 6499, 6502, 6504, 6540, 6541, 6584, 6587, 6602, 6605, 6606, 6627, 6633, 6634, 6635, 6685, 6688, 6689, 6698, 6705, 6722, 6730, 6737, 6738, 6744, 6745, 6777, 6780, 6784, 6803, 6814, 6828, 6832, 6851, 6855, 6861, 6868, 6875, 6903, 6904, 6905, 6907, 6930, 6933, 6943, 6944, 6949, 6959, 6975, 6976, 6979, 6985, 7002, 7005, 7006, 7010, 7020, 7027, 7036, 7044, 7049, 7059, 7073, 7082, 7091, 7108, 7109, 7120, 7126, 7148, 7164, 7204, 7217, 7219, 7222, 7226, 7241, 7244, 7260, 7281, 7284, 7287, 7300, 7305, 7314, 7317, 7323, 7328, 7330, 7374, 7379, 7396, 7397, 7399, 7401, 7407, 7412, 7417, 7426, 7433, 7440, 7441, 7452, 7457, 7465, 7487, 7518, 7543, 7544, 7545, 7546, 7556, 7567, 7577, 7585, 7592, 7593, 7613, 7617, 7620, 7623, 7644, 7666, 7668, 7671, 7735, 7737, 7742, 7758, 7760, 7762, 7763, 7771, 7774, 7776, 7795, 7797, 7800, 7810, 7822, 7849, 7871, 7937, 7948, 7958, 7971, 7972, 7981, 8004, 8032, 8045, 8063, 8067, 8077, 8093, 8096, 8111, 8130, 8137, 8156, 8162, 8176, 8191, 8199, 8202, 8207, 8234, 8248, 8273, 8277, 8291, 8306, 8308, 8323, 8333, 8338, 8346, 8348, 8349, 8355, 8357, 8364, 8365, 8370, 8376, 8390, 8404, 8423, 8442, 8449, 8451, 8452, 8461, 8482, 8491, 8499, 8503, 8527, 8540, 8553, 8559, 8563, 8566, 8580, 8585, 8622, 8666, 8670, 8684, 8749, 8753, 8800, 8805, 8807, 8812, 8826, 8832, 8845, 8852, 8872, 8873, 8880, 8881, 8895, 8898, 8909, 8917, 8944, 8947, 8952, 8960, 8965, 8986, 8991, 8999, 9002, 9003, 9017, 9020, 9037, 9049, 9079, 9083, 9092, 9093, 9144, 9147, 9150, 9151, 9164, 9188, 9191, 9208, 9233, 9234, 9246, 9274, 9291, 9296, 9302, 9332, 9339, 9356, 9360, 9385, 9392, 9411, 9418, 9421, 9422, 9434, 9437, 9453, 9469, 9479, 9489, 9491, 9499, 9501, 9510, 9522, 9523, 9527, 9531, 9536, 9541, 9545, 9547, 9552, 9565, 9574, 9576, 9581, 9583, 9595, 9596, 9599, 9601, 9602, 9621, 9633, 9655, 9689, 9693, 9735, 9745, 9751, 9768, 9786, 9788, 9795, 9796, 9798, 9808, 9862, 9873, 9876, 9888, 9898, 9901, 9908, 9918, 9920, 9922, 9933, 9942, 9943, 9946, 9949, 9951, 9963, 9969, 9974, 9981, 10000, 10007, 10008, 10009, 10012, 10032, 10036, 10045, 10064, 10071, 10073, 10080, 10091, 10093, 10095, 10099, 10108, 10109, 10119, 10123, 10126, 10167, 10182, 10186, 10188, 10192, 10211, 10229, 10235, 10248, 10254, 10267, 10268, 10274, 10278, 10281, 10282, 10314, 10322, 10323, 10336, 10339, 10341, 10356, 10364, 10370, 10376, 10381, 10384, 10386, 10394, 10420, 10427, 10444, 10452, 10453, 10454, 10457, 10460, 10462, 10465, 10490, 10507, 10512, 10517, 10519, 10522, 10524, 10527, 10563, 10571, 10572, 10580, 10610, 10631, 10641, 10642, 10658, 10665, 10669, 10670, 10672, 10692, 10709, 10719, 10733, 10762, 10765, 10770, 10771, 10806, 10809, 10813, 10824, 10828, 10852, 10867, 10868, 10878, 10880, 10883, 10909, 10910, 10925, 10930, 10951, 10959, 10960, 10962, 10982, 11006, 11031, 11045, 11067, 11070, 11083, 11085, 11089, 11092, 11117, 11134, 11139, 11147, 11159, 11174, 11178, 11180, 11191, 11203, 11206, 11221, 11222, 11224, 11231, 11252, 11273, 11277, 11294, 11299, 11305, 11308, 11309, 11313, 11331, 11333, 11338, 11341, 11345, 11349, 11352, 11374, 11381, 11409, 11419, 11422, 11425, 11437, 11443, 11449, 11451, 11463, 11481, 11486, 11490, 11506, 11536, 11543, 11544, 11545, 11558, 11563, 11577, 11581, 11587, 11588, 11589, 11593, 11612, 11621, 11635, 11656, 11657, 11678, 11690, 11699, 11706, 11713, 11717, 11730, 11733, 11744, 11748, 11758, 11769, 11772, 11795, 11803, 11827, 11840, 11846, 11853, 11854, 11874, 11878, 11888, 11891, 11903, 11904, 11912, 11925, 11929, 11937, 11954, 11967, 11986, 11996, 11997, 11998, 12003, 12005, 12010, 12015, 12023, 12029, 12032, 12038, 12040, 12041, 12049, 12055, 12095, 12107, 12122, 12125, 12130, 12150, 12192, 12198, 12211, 12213, 12223, 12225, 12230, 12231, 12232, 12245, 12263, 12293, 12333, 12346, 12368, 12386, 12396, 12398, 12401, 12416, 12418, 12420, 12429, 12450, 12457, 12464, 12473, 12489, 12508, 12511, 12529, 12539, 12552, 12559, 12574, 12602, 12604, 12607, 12626, 12680, 12682, 12692, 12700, 12726, 12749, 12763, 12796, 12808, 12816, 12818, 12825, 12849, 12865, 12869, 12873, 12875, 12892, 12899, 12901, 12902, 12904, 12906, 12923, 12927, 12929, 12933, 12936, 12963, 12965, 12967, 12968, 12975, 12978, 12991, 12999, 13026, 13028, 13042, 13043, 13054, 13055, 13085, 13108, 13110, 13134, 13142, 13150, 13180, 13190, 13204, 13221, 13222, 13235, 13247, 13252, 13265, 13266, 13329, 13332, 13340, 13350, 13353, 13356, 13362, 13366, 13373, 13379, 13400, 13403, 13419, 13433, 13441, 13449, 13472, 13480, 13483, 13492, 13499, 13511, 13520, 13524, 13528, 13532, 13550, 13561, 13572, 13591, 13596, 13607, 13612, 13614, 13616, 13626, 13636, 13638, 13642, 13646, 13652, 13654, 13669, 13672, 13684, 13705, 13706, 13712, 13714, 13717, 13733, 13747, 13758, 13775, 13788, 13796, 13803, 13811, 13827, 13835, 13843, 13852, 13853, 13862, 13863, 13869, 13874, 13887, 13904, 13919, 13921, 13923, 13926, 13930, 13931, 13948, 13951, 13952, 13969, 13978, 13983, 13985, 13986, 13988, 13992, 13999, 14025, 14029, 14030, 14057, 14071, 14072, 14082, 14084, 14099, 14109, 14119, 14141, 14145, 14154, 14157, 14174, 14185, 14191, 14202, 14206, 14208, 14219, 14265, 14268, 14271, 14289, 14292, 14293, 14298, 14303, 14305, 14318, 14320, 14339, 14340, 14358, 14359, 14365, 14393, 14404, 14425, 14426, 14431, 14437, 14459, 14466, 14479, 14484, 14487, 14502, 14506, 14519, 14522, 14535, 14560, 14567, 14580, 14584, 14586, 14605, 14630, 14637, 14653, 14665, 14671, 14675, 14678, 14700, 14711, 14731, 14733, 14735, 14744, 14782, 14809, 14817, 14827, 14866, 14867, 14880, 14902, 14918, 14924, 14933, 14954, 14955, 14963, 14966, 14975, 14983, 14987, 14990, 14995, 15010, 15017, 15044, 15064, 15076, 15077, 15080, 15101, 15107, 15117, 15123, 15130, 15136, 15141, 15143, 15152, 15162, 15172, 15185, 15212, 15216, 15225, 15259, 15272, 15285, 15301, 15309, 15311, 15316, 15322, 15324, 15348, 15357, 15359, 15362, 15403, 15404, 15421, 15422, 15424, 15433, 15434, 15438, 15439, 15442, 15458, 15463, 15466, 15473, 15489, 15491, 15499, 15529, 15534, 15541, 15543, 15545, 15550, 15556, 15563, 15576, 15581, 15588, 15597, 15615, 15655, 15666, 15673, 15675, 15687, 15698, 15700, 15711, 15714, 15718, 15724, 15725, 15753, 15783, 15792, 15833, 15838, 15840, 15842, 15847, 15860, 15897, 15900, 15903, 15913, 15916, 15929, 15931, 15945, 15959, 15984, 15991, 15996, 16025, 16040, 16058, 16067, 16068, 16079, 16085, 16107, 16111, 16129, 16134, 16141, 16148, 16159, 16161, 16164, 16166, 16168, 16201, 16211, 16212, 16246, 16267, 16270, 16275, 16279, 16287, 16295, 16297, 16306, 16311, 16312, 16323, 16333, 16339, 16354, 16358, 16367, 16370, 16379, 16384, 16390, 16407, 16414, 16416, 16438, 16448, 16467, 16486, 16491, 16507, 16511, 16533, 16536, 16541, 16544, 16553, 16558, 16574, 16575, 16610, 16622, 16634, 16669, 16674, 16687, 16697, 16700, 16704, 16715, 16719, 16725, 16735, 16738, 16739, 16752, 16753, 16755, 16765, 16787, 16800, 16802, 16805, 16810, 16811, 16812, 16815, 16819, 16821, 16825, 16853, 16873, 16878, 16930, 16950, 16961, 16963, 16982, 16996, 17015, 17018, 17024, 17027, 17032, 17049, 17054, 17060, 17062, 17075, 17077, 17101, 17103, 17110, 17111, 17116, 17128, 17144, 17146, 17148, 17161, 17185, 17186, 17187, 17199, 17204, 17216, 17221, 17222, 17225, 17232, 17250, 17252, 17257, 17282, 17285, 17297, 17306, 17307, 17310, 17334, 17336, 17364, 17395, 17411, 17413, 17415, 17450, 17479, 17489, 17490, 17520, 17526, 17527, 17528, 17537, 17542, 17547, 17554, 17560, 17564, 17569, 17579, 17594, 17596, 17610, 17611, 17616, 17618, 17622, 17626, 17647, 17688, 17696, 17703, 17714, 17720, 17735, 17744, 17756, 17759, 17762, 17771, 17774, 17787, 17788, 17792, 17793, 17798, 17801, 17808, 17827, 17829, 17835, 17845, 17855, 17871, 17879, 17886, 17893, 17895, 17897, 17898, 17904, 17906, 17918, 17931, 17940, 17941, 17950, 17959, 17967, 17971, 17973, 17978, 17987, 17993, 17994, 18007, 18019, 18023, 18036, 18048, 18062, 18078, 18083, 18128, 18133, 18146, 18147, 18154, 18159, 18167, 18189, 18194, 18201, 18205, 18209, 18213, 18214, 18217, 18224, 18234, 18239, 18249, 18254, 18283, 18289, 18290, 18312, 18323, 18330, 18332, 18337, 18375, 18382, 18386, 18387, 18392, 18397, 18410, 18412, 18432, 18433, 18437, 18445, 18459, 18465, 18468, 18471, 18481, 18522, 18526, 18546, 18552, 18554, 18561, 18564, 18568, 18569, 18572, 18577, 18586, 18589, 18598, 18611, 18612, 18627, 18628, 18654, 18659, 18664, 18674, 18680, 18681, 18700, 18721, 18723, 18730, 18765, 18780, 18785, 18788, 18793, 18795, 18843, 18857, 18858, 18862, 18867, 18880, 18881, 18895, 18901, 18931, 18934, 18938, 18958, 18987, 18988, 19019, 19025, 19034, 19035, 19038, 19041, 19050, 19054, 19064, 19075, 19083, 19087, 19099, 19109, 19116, 19130, 19138, 19140, 19146, 19152, 19153, 19159, 19168, 19173, 19182, 19184, 19195, 19202, 19206, 19216, 19221, 19236, 19257, 19260, 19269, 19273, 19276, 19281, 19284, 19292, 19337, 19339, 19343, 19347, 19353, 19358, 19362, 19390, 19396, 19398, 19422, 19430, 19447, 19449, 19451, 19453, 19487, 19489, 19498, 19502, 19506, 19517, 19522, 19524, 19525, 19536, 19547, 19548, 19556, 19596, 19602, 19604, 19608, 19618, 19622, 19629, 19641, 19647, 19658, 19672, 19681, 19690, 19706, 19709, 19710, 19712, 19719, 19720, 19738, 19757, 19764, 19772, 19776, 19780, 19783, 19806, 19816, 19818, 19832, 19848, 19858, 19859, 19867, 19880, 19889, 19907, 19909, 19933, 19935, 19936, 19944, 19948, 19967, 19973, 19982, 19993, 20006, 20018, 20033, 20090, 20095, 20107, 20117, 20119, 20144, 20145, 20153, 20154, 20157, 20163, 20176, 20183, 20189, 20215, 20218, 20224, 20272, 20274, 20281, 20290, 20295, 20302, 20306, 20308, 20325, 20334, 20346, 20348, 20353, 20369, 20370, 20379, 20380, 20383, 20385, 20387, 20394, 20400, 20415, 20444, 20448, 20450, 20454, 20465, 20468, 20476, 20488, 20491, 20505, 20512, 20523, 20544, 20553, 20570, 20585, 20599, 20622, 20624, 20633, 20638, 20647, 20651, 20657, 20664, 20677, 20700, 20702, 20710, 20713, 20715, 20717, 20722, 20724, 20725, 20746, 20766, 20782, 20800, 20827, 20834, 20840, 20848, 20855, 20856, 20864, 20868, 20872, 20883, 20891, 20893, 20897, 20904, 20927, 20935, 20941, 20944, 20948, 20949, 20953, 20963, 20993, 21004, 21005, 21008, 21012, 21013, 21023, 21029, 21045, 21047, 21050, 21059, 21064, 21074, 21087, 21096, 21100, 21115, 21117, 21123, 21132, 21140, 21157, 21173, 21174, 21177, 21179, 21188, 21190, 21195, 21207, 21221, 21230, 21236, 21238, 21245, 21253, 21254, 21256, 21261, 21269, 21291, 21313, 21325, 21342, 21346, 21347, 21375, 21383, 21384, 21385, 21391, 21404, 21408, 21426, 21433, 21445, 21496, 21497, 21510, 21515, 21521, 21523, 21524, 21527, 21545, 21554, 21571, 21572, 21588, 21619, 21640, 21649, 21655, 21657, 21658, 21662, 21663, 21670, 21673, 21682, 21688, 21699, 21704, 21710, 21719, 21726, 21728, 21735, 21739, 21758, 21759, 21794, 21803, 21805, 21811, 21823, 21831, 21838, 21852, 21854, 21861, 21879, 21882, 21883, 21892, 21910, 21920, 21925, 21932, 21937, 21949, 21951, 21963, 21968, 21972, 21973, 21981, 21983, 21995, 22012, 22018, 22019, 22030, 22039, 22043, 22051, 22052, 22069, 22082, 22085, 22086, 22087, 22102, 22109, 22110, 22130, 22150, 22157, 22172, 22176, 22184, 22186, 22187, 22199, 22205, 22210, 22233, 22236, 22238, 22239, 22241, 22276, 22284, 22286, 22288, 22319, 22327, 22337, 22342, 22345, 22346, 22362, 22375, 22379, 22384, 22395, 22396, 22400, 22408, 22422, 22434, 22450, 22467, 22471, 22473, 22474, 22477, 22478, 22482, 22484, 22490, 22512, 22518, 22520, 22532, 22533, 22538, 22550, 22563, 22603, 22604, 22612, 22617, 22621, 22625, 22628, 22629, 22660, 22664, 22678, 22680, 22683, 22692, 22706, 22730, 22732, 22738, 22742, 22757, 22802, 22826, 22845, 22846, 22850, 22863, 22892, 22914, 22921, 22923, 22947, 22955, 22958, 22964, 22967, 22995, 23013, 23016, 23026, 23028, 23034, 23046, 23051, 23075, 23077, 23100, 23111, 23138, 23141, 23149, 23150, 23163, 23184, 23185, 23197, 23201, 23204, 23207, 23226, 23228, 23243, 23247, 23248, 23253, 23262, 23263, 23267, 23274, 23276, 23278, 23286, 23307, 23309, 23310, 23350, 23359, 23367, 23375, 23381, 23382, 23383, 23389, 23409, 23420, 23422, 23423, 23430, 23431, 23442, 23446, 23470, 23479, 23485, 23491, 23504, 23508, 23527, 23528, 23529, 23535, 23545, 23548, 23563, 23570, 23582, 23607, 23614, 23622, 23631, 23640, 23646, 23647, 23675, 23683, 23689, 23700, 23706, 23714, 23715, 23716, 23724, 23726, 23730, 23731, 23748, 23752, 23769, 23800, 23802, 23852, 23853, 23856, 23864, 23889, 23904, 23910, 23915, 23916, 23920, 23929, 23936, 23947, 23959, 23968, 23969, 23976, 23991, 24030, 24031, 24038, 24055, 24074, 24075, 24077, 24082, 24083, 24084, 24094, 24097, 24102, 24117, 24157, 24168, 24180, 24181, 24192, 24221, 24239, 24244, 24258, 24279, 24322, 24347, 24350, 24351, 24357, 24358, 24368, 24370, 24389, 24396, 24405, 24415, 24420, 24438, 24442, 24457, 24473, 24497, 24505, 24529, 24541, 24565, 24567, 24596, 24608, 24624, 24632, 24636, 24642, 24647, 24659, 24666, 24673, 24690, 24693, 24705, 24711, 24752, 24761, 24771, 24784, 24785, 24786, 24808, 24833, 24836, 24851, 24860, 24871, 24877, 24883, 24925, 24935, 24952, 24959, 24965, 24978, 24980, 24982, 24988, 24990, 24991, 25003, 25004, 25018, 25019, 25027, 25036, 25061, 25062, 25078, 25084, 25087, 25092, 25099, 25105, 25107, 25147, 25152, 25170, 25183, 25184, 25185, 25186, 25190, 25194, 25199, 25207, 25210, 25231, 25241, 25251, 25254, 25258, 25263, 25280, 25287, 25289, 25291, 25292, 25339, 25350, 25377, 25383, 25391, 25392, 25394, 25410, 25414, 25429, 25430, 25453, 25457, 25466, 25467, 25470, 25476, 25479, 25490, 25491, 25493, 25514, 25523, 25524, 25527, 25562, 25564, 25573, 25575, 25598, 25604, 25611, 25661, 25677, 25688, 25695, 25696, 25709, 25712, 25714, 25766, 25768, 25769, 25772, 25791, 25794, 25799, 25806, 25810, 25833, 25840, 25847, 25850, 25868, 25871, 25873, 25877, 25885, 25908, 25922, 25927, 25938, 25939, 25999, 26008, 26014, 26028, 26042, 26053, 26095, 26102, 26122, 26128, 26143, 26162, 26175, 26183, 26199, 26203, 26219, 26221, 26229, 26234, 26244, 26248, 26258, 26260, 26281, 26282, 26287, 26295, 26312, 26317, 26328, 26329, 26330, 26344, 26364, 26379, 26382, 26383, 26390, 26393, 26397, 26398, 26402, 26405, 26409, 26415, 26450, 26453, 26460, 26463, 26465, 26479, 26488, 26505, 26513, 26516, 26517, 26550, 26554, 26565, 26585, 26589, 26596, 26601, 26607, 26614, 26622, 26632, 26639, 26653, 26655, 26667, 26679, 26680, 26687, 26706, 26720, 26722, 26741, 26742, 26750, 26757, 26761, 26770, 26782, 26796, 26821, 26824, 26828, 26835, 26847, 26855, 26863, 26868, 26883, 26896, 26911, 26913, 26922, 26923, 26933, 26935, 26942, 26947, 26952, 26963, 26967, 26975, 27003, 27014, 27015, 27025, 27034, 27036, 27059, 27062, 27075, 27083, 27092, 27093, 27100, 27122, 27131, 27134, 27136, 27150, 27166, 27181, 27192, 27207, 27219, 27222, 27223, 27229, 27237, 27257, 27265, 27298, 27300, 27301, 27302, 27305, 27311, 27318, 27329, 27343, 27344, 27373, 27379, 27399, 27401, 27444, 27456, 27458, 27478, 27493, 27509, 27529, 27530, 27550, 27557, 27567, 27570, 27576, 27582, 27599, 27616, 27617, 27640, 27642, 27670, 27673, 27680, 27690, 27695, 27699, 27700, 27702, 27706, 27708, 27716, 27745, 27760, 27763, 27780, 27781, 27797, 27832, 27838, 27845, 27846, 27852, 27860, 27864, 27880, 27893, 27900, 27901, 27905, 27909, 27911, 27918, 27923, 27927, 27965, 27966, 27975, 27984, 27993, 28004, 28041, 28052, 28064, 28066, 28069, 28083, 28095, 28105, 28115, 28120, 28122, 28125, 28135, 28152, 28161, 28185, 28196, 28207, 28220, 28225, 28232, 28258, 28266, 28281, 28334, 28363, 28379, 28389, 28395, 28399, 28409, 28425, 28450, 28461, 28464, 28465, 28481, 28483, 28485, 28495, 28496, 28502, 28510, 28533, 28544, 28552, 28555, 28556, 28557, 28561, 28563, 28572, 28574, 28577, 28589, 28592, 28605, 28607, 28618, 28626, 28634, 28638, 28639, 28645, 28647, 28651, 28662, 28764, 28775, 28816, 28824, 28837, 28842, 28853, 28869, 28871, 28874, 28891, 28908, 28910, 28915, 28920, 28951, 28956, 28965, 28979, 28997, 29001, 29005, 29007, 29027, 29037, 29052, 29058, 29062, 29066, 29082, 29097, 29105, 29126, 29150, 29158, 29176, 29178, 29183, 29186, 29189, 29206, 29209, 29217, 29242, 29248, 29250, 29260, 29266, 29313, 29314, 29327, 29328, 29349, 29365, 29366, 29376, 29381, 29388, 29392, 29401, 29424, 29431, 29434, 29443, 29449, 29456, 29468, 29471, 29474, 29496, 29516, 29519, 29524, 29529, 29556, 29557, 29559, 29567, 29578, 29583, 29587, 29611, 29614, 29661, 29662, 29668, 29675, 29694, 29696, 29701, 29705, 29714, 29734, 29752, 29757, 29759, 29765, 29768, 29771, 29780, 29784, 29792, 29794, 29841, 29881, 29886, 29904, 29943, 29958, 29980, 29983, 29986, 29989, 30002, 30008, 30014, 30023, 30031, 30037, 30042, 30049, 30053, 30059, 30072, 30094, 30101, 30104, 30115, 30143, 30158, 30178, 30182, 30188, 30193, 30198, 30212, 30242, 30246, 30261, 30278, 30279, 30282, 30284, 30299, 30300, 30303, 30322, 30333, 30334, 30352, 30359, 30374, 30394, 30411, 30419, 30430, 30439, 30450, 30457, 30459, 30468, 30473, 30477, 30499, 30500, 30506, 30516, 30525, 30526, 30539, 30540, 30541, 30550, 30557, 30561, 30579, 30583, 30604, 30614, 30642, 30651, 30663, 30672, 30674, 30696, 30712, 30739, 30744, 30750, 30765, 30784, 30787, 30798, 30805, 30808, 30810, 30811, 30818, 30828, 30831, 30833, 30860, 30863, 30865, 30886, 30891, 30893, 30910, 30925, 30940, 30967, 30969, 30984, 30986, 31019, 31026, 31030, 31034, 31053, 31055, 31068, 31093, 31099, 31101, 31117, 31121, 31126, 31136, 31138, 31145, 31150, 31160, 31172, 31176, 31183, 31193, 31200, 31201, 31208, 31236, 31263, 31277, 31284, 31303, 31311, 31315, 31319, 31330, 31369, 31372, 31381, 31388, 31392, 31393, 31398, 31442, 31447, 31449, 31454, 31458, 31475, 31478, 31482, 31525, 31553, 31574, 31588, 31598, 31616, 31622, 31625, 31638, 31646, 31668, 31675, 31684, 31695, 31697, 31708, 31709, 31711, 31720, 31727, 31739, 31757, 31769, 31776, 31781, 31782, 31786, 31795, 31796, 31806, 31824, 31826, 31837, 31840, 31851, 31861, 31863, 31875, 31886, 31889, 31899, 31931, 31936, 31937, 31944, 31946, 31978, 31980, 31995, 31999, 32000, 32019, 32027, 32045, 32056, 32057, 32061, 32068, 32081, 32085, 32094, 32103, 32110, 32123, 32142, 32149, 32154, 32161, 32163, 32167, 32175, 32220, 32222, 32226, 32240, 32245, 32246, 32261, 32266, 32274, 32315, 32318, 32335, 32344, 32351, 32358, 32362, 32370, 32381, 32422, 32445, 32451, 32458, 32475, 32487, 32506, 32518, 32521, 32528, 32533, 32538, 32545, 32547, 32569, 32577, 32581, 32583, 32593, 32601, 32608, 32610, 32613, 32652, 32658, 32663, 32690, 32755, 32756, 32761, 32789, 32795, 32802, 32820, 32845, 32852, 32859, 32862, 32873, 32875, 32879, 32880, 32883, 32892, 32903, 32918, 32920, 32922, 32938, 32939, 32946, 32954, 32968, 32971, 32974, 32978, 32989, 32992, 32993, 33014, 33017, 33019, 33052, 33058, 33064, 33074, 33076, 33077, 33082, 33086, 33118, 33130, 33142, 33146, 33153, 33175, 33185, 33193, 33234, 33236, 33256, 33282, 33287, 33322, 33323, 33326, 33338, 33360, 33363, 33426, 33428, 33429, 33444, 33445, 33449, 33458, 33465, 33475, 33483, 33503, 33512, 33518, 33532, 33537, 33554, 33572, 33574, 33576, 33579, 33590, 33604, 33609, 33614, 33617, 33620, 33624, 33625, 33650, 33673, 33681, 33685, 33691, 33700, 33714, 33723, 33728, 33732, 33737, 33739, 33749, 33759, 33775, 33806, 33833, 33837, 33840, 33842, 33843, 33847, 33885, 33886, 33907, 33917, 33929, 33931, 33932, 33936, 33941, 33954, 33963, 33964, 33983, 33999, 34020, 34024, 34041, 34042, 34048, 34049, 34054, 34056, 34070, 34085, 34098, 34103, 34125, 34133, 34141, 34151, 34154, 34164, 34165, 34183, 34209, 34212, 34218, 34245, 34246, 34256, 34264, 34286, 34290, 34316, 34340, 34345, 34364, 34374, 34378, 34380, 34382, 34383, 34390, 34393, 34404, 34407, 34411, 34421, 34427, 34448, 34468, 34469, 34475, 34485, 34488, 34489, 34501, 34502, 34509, 34510, 34527, 34538, 34539, 34552, 34573, 34574, 34613, 34619, 34626, 34635, 34647, 34652, 34656, 34662, 34667, 34686, 34687, 34688, 34693, 34695, 34707, 34717, 34724, 34727, 34743, 34749, 34755, 34761, 34766, 34780, 34791, 34796, 34821, 34857, 34864, 34878, 34882, 34902, 34905, 34909, 34913, 34941, 34952, 34958, 34961, 34976, 34996, 35023, 35031, 35033, 35036, 35043, 35065, 35098, 35102, 35118, 35123, 35137, 35149, 35158, 35160, 35173, 35177, 35178, 35182, 35191, 35199, 35215, 35243, 35265, 35268, 35269, 35277, 35282, 35287, 35292, 35308, 35313, 35314, 35316, 35325, 35328, 35332, 35341, 35345, 35350, 35363, 35378, 35388, 35397, 35405, 35407, 35408, 35423, 35427, 35443, 35456, 35462, 35469, 35471, 35474, 35480, 35483, 35518, 35521, 35534, 35536, 35543, 35547, 35552, 35554, 35575, 35583, 35603, 35608, 35615, 35632, 35682, 35683, 35745, 35755, 35767, 35779, 35781, 35793, 35795, 35811, 35820, 35826, 35827, 35833, 35836, 35866, 35879, 35901, 35904, 35907, 35909, 35913, 35931, 35936, 35937, 35941, 35951, 35957, 35966, 35976, 35983, 35989, 36002, 36008, 36012, 36014, 36023, 36030, 36047, 36056, 36071, 36081, 36120, 36124, 36132, 36149, 36155, 36167, 36168, 36197, 36202, 36224, 36230, 36241, 36245, 36276, 36281, 36283, 36305, 36312, 36324, 36336, 36349, 36357, 36360, 36367, 36392, 36419, 36425, 36427, 36428, 36435, 36444, 36463, 36464, 36477, 36479, 36489, 36499, 36507, 36508, 36526, 36540, 36544, 36582, 36583, 36592, 36596, 36610, 36660, 36695, 36703, 36708, 36709, 36715, 36723, 36734, 36736, 36772, 36776, 36796, 36801, 36803, 36812, 36820, 36845, 36861, 36882, 36906, 36926, 36954, 36961, 36969, 37014, 37048, 37056, 37074, 37094, 37105, 37113, 37131, 37142, 37154, 37160, 37183, 37198, 37200, 37202, 37245, 37252, 37267, 37276, 37299, 37311, 37342, 37345, 37346, 37380, 37389, 37403, 37406, 37407, 37410, 37422, 37429, 37434, 37435, 37451, 37487, 37496, 37504, 37506, 37513, 37516, 37519, 37537, 37544, 37549, 37568, 37573, 37574, 37580, 37586, 37589, 37591, 37599, 37600, 37624, 37657, 37663, 37665, 37684, 37697, 37705, 37707, 37730, 37766, 37791, 37797, 37816, 37830, 37834, 37837, 37855, 37863, 37868, 37871, 37873, 37880, 37885, 37894, 37906, 37910, 37917, 37919, 37920, 37928, 37932, 37934, 37979, 37984, 37985, 37993, 37995, 38000, 38005, 38015, 38021, 38071, 38098, 38109, 38130, 38139, 38140, 38148, 38165, 38197, 38208, 38226, 38234, 38250, 38267, 38307, 38316, 38331, 38336, 38343, 38348, 38350, 38358, 38370, 38371, 38378, 38381, 38392, 38399, 38402, 38410, 38415, 38420, 38424, 38426, 38427, 38431, 38450, 38455, 38466, 38490, 38496, 38498, 38515, 38521, 38536, 38540, 38544, 38556, 38561, 38564, 38571, 38584, 38586, 38601, 38603, 38628, 38645, 38653, 38666, 38674, 38676, 38680, 38683, 38697, 38708, 38719, 38732, 38769, 38771, 38793, 38795, 38826, 38832, 38837, 38842, 38851, 38861, 38880, 38883, 38899, 38908, 38922, 38928, 38930, 38938, 38941, 38947, 38955, 38956, 38957, 38970, 38976, 38987, 38998, 39007, 39019, 39041, 39043, 39053, 39063, 39066, 39075, 39095, 39118, 39128, 39154, 39158, 39163, 39178, 39220, 39247, 39254, 39256, 39274, 39286, 39287, 39296, 39297, 39299, 39315, 39337, 39355, 39366, 39369, 39394, 39395, 39430, 39451, 39459, 39488, 39493, 39508, 39542, 39544, 39549, 39553, 39559, 39562, 39566, 39571, 39578, 39587, 39621, 39646, 39658, 39662, 39664, 39668, 39683, 39684, 39686, 39692, 39704, 39705, 39717, 39731, 39733, 39737, 39741, 39746, 39755, 39786, 39804, 39805, 39812, 39819, 39837, 39838, 39851, 39852, 39869, 39882, 39888, 39890, 39906, 39914, 39917, 39921, 39943, 39945, 39961, 39963, 39965, 39972, 39977, 39979, 39984, 39990, 39997, 40004, 40014, 40017, 40019, 40028, 40064, 40067, 40081, 40085, 40091, 40103, 40106, 40117, 40135, 40149, 40152, 40167, 40169, 40173, 40178, 40184, 40186, 40203, 40205, 40213, 40219, 40243, 40247, 40258, 40272, 40321, 40327, 40330, 40382, 40390, 40405, 40418, 40424, 40432, 40448, 40461, 40470, 40487, 40488, 40490, 40498, 40503, 40512, 40524, 40560, 40563, 40593, 40616, 40659, 40663, 40664, 40666, 40672, 40677, 40679, 40700, 40728, 40746, 40747, 40760, 40775, 40800, 40810, 40830, 40847, 40849, 40866, 40901, 40904, 40920, 40922, 40931, 40943, 40962, 40967, 40969, 40978, 40985, 41006, 41009, 41014, 41018, 41041, 41043, 41058, 41086, 41117, 41133, 41138, 41143, 41144, 41157, 41173, 41175, 41179, 41198, 41205, 41216, 41232, 41272, 41288, 41289, 41314, 41322, 41326, 41364, 41367, 41383, 41394, 41430, 41434, 41457, 41464, 41483, 41518, 41524, 41526, 41529, 41537, 41539, 41556, 41562, 41574, 41578, 41583, 41588, 41593, 41635, 41644, 41650, 41670, 41682, 41693, 41705, 41720, 41730, 41732, 41734, 41737, 41741, 41757, 41766, 41777, 41839, 41855, 41866, 41868, 41871, 41875, 41879, 41890, 41909, 41914, 41921, 41924, 41927, 41930, 41938, 41939, 41945, 41954, 41969, 41974, 41981, 41984, 42003, 42009, 42016, 42021, 42023, 42027, 42040, 42049, 42052, 42061, 42065, 42069, 42072, 42092, 42098, 42101, 42106, 42109, 42112, 42120, 42123, 42128, 42145, 42146, 42156, 42172, 42173, 42181, 42188, 42194, 42202, 42207, 42215, 42227, 42234, 42238, 42258, 42259, 42266, 42268, 42274, 42280, 42284, 42287, 42310, 42335, 42344, 42348, 42378, 42380, 42391, 42413, 42429, 42434, 42436, 42441, 42446, 42447, 42455, 42464, 42465, 42482, 42491, 42496, 42497, 42509, 42557, 42561, 42566, 42577, 42592, 42593, 42594, 42606, 42628, 42634, 42637, 42647, 42664, 42676, 42682, 42695, 42710, 42717, 42718, 42722, 42726, 42739, 42762, 42768, 42779, 42786, 42796, 42803, 42808, 42817, 42821, 42830, 42836, 42843, 42845, 42847, 42861, 42866, 42876, 42883, 42893, 42896, 42907, 42915, 42920, 42936, 42938, 42942, 42953, 42971, 42982, 42986, 42987, 42988, 42991, 42998, 43002, 43009, 43016, 43018, 43019, 43042, 43044, 43058, 43072, 43087, 43090, 43103, 43107, 43109, 43113, 43120, 43121, 43125, 43141, 43150, 43162, 43167, 43170, 43179, 43187, 43206, 43216, 43223, 43231, 43234, 43235, 43247, 43263, 43270, 43271, 43272, 43279, 43299, 43310, 43322, 43350, 43364, 43381, 43390, 43394, 43410, 43412, 43425, 43446, 43454, 43463, 43473, 43482, 43500, 43509, 43521, 43523, 43530, 43551, 43568, 43585, 43591, 43605, 43615, 43629, 43641, 43664, 43666, 43680, 43690, 43710, 43735, 43741, 43742, 43766, 43771, 43781, 43784, 43786, 43820, 43829, 43834, 43839, 43845, 43864, 43888, 43906, 43910, 43912, 43921, 43939, 43970, 43974, 43976, 44012, 44030, 44042, 44044, 44054, 44068, 44078, 44081, 44083, 44100, 44104, 44123, 44128, 44134, 44143, 44145, 44160, 44183, 44199, 44200, 44210, 44213, 44240, 44272, 44308, 44314, 44339, 44377, 44396, 44398, 44412, 44467, 44492, 44502, 44514, 44520, 44525, 44527, 44536, 44543, 44547, 44576, 44577, 44580, 44585, 44586, 44613, 44614, 44615, 44627, 44629, 44654, 44658, 44659, 44661, 44669, 44677, 44683, 44701, 44707, 44718, 44728, 44737, 44738, 44751, 44776, 44781, 44792, 44817, 44832, 44845, 44854, 44859, 44881, 44891, 44900, 44920, 44922, 44962, 44973, 44981, 45000, 45014, 45016, 45021, 45025, 45050, 45054, 45095, 45104, 45112, 45120, 45127, 45128, 45140, 45154, 45158, 45176, 45192, 45199, 45202, 45213, 45219, 45237, 45241, 45255, 45262, 45263, 45266, 45271, 45280, 45282, 45285, 45295, 45296, 45301, 45318, 45334, 45341, 45350, 45358, 45365, 45382, 45390, 45402, 45403, 45414, 45417, 45423, 45438, 45449, 45503, 45506, 45507, 45520, 45525, 45534, 45554, 45565, 45579, 45583, 45584, 45585, 45597, 45602, 45619, 45624, 45626, 45634, 45635, 45652, 45656, 45657, 45661, 45664, 45665, 45673, 45685, 45695, 45696, 45705, 45712, 45720, 45726, 45735, 45741, 45744, 45753, 45756, 45758, 45785, 45805, 45813, 45815, 45817, 45825, 45850, 45852, 45897, 45904, 45916, 45942, 45944, 45955, 45956, 45957, 45975, 45978, 45982, 45991, 45994, 45998, 46020, 46021, 46026, 46040, 46049, 46055, 46060, 46065, 46068, 46074, 46078, 46094, 46100, 46118, 46125, 46143, 46144, 46145, 46149, 46181, 46186, 46195, 46205, 46208, 46231, 46236, 46242, 46269, 46276, 46279, 46286, 46303, 46306, 46330, 46333, 46346, 46368, 46371, 46375, 46383, 46386, 46388, 46393, 46414, 46415, 46431, 46432, 46437, 46443, 46453, 46468, 46473, 46477, 46480, 46483, 46513, 46515, 46529, 46536, 46537, 46540, 46560, 46561, 46563, 46575, 46580, 46593, 46607, 46618, 46634, 46648, 46653, 46656, 46662, 46667, 46701, 46704, 46729, 46738, 46743, 46770, 46795, 46798, 46808, 46811, 46812, 46819, 46839, 46842, 46854, 46868, 46885, 46889, 46893, 46912, 46917, 46941, 46944, 46951, 46954, 46975, 46977, 46997, 47000, 47009, 47032, 47034, 47042, 47047, 47051, 47060, 47077, 47111, 47124, 47126, 47140, 47153, 47158, 47159, 47164, 47173, 47198, 47200, 47208, 47211, 47219, 47224, 47241, 47245, 47249, 47260, 47262, 47264, 47265, 47266, 47272, 47280, 47308, 47311, 47316, 47328, 47332, 47346, 47357, 47365, 47405, 47419, 47447, 47499, 47517, 47521, 47544, 47567, 47572, 47576, 47579, 47598, 47623, 47627, 47628, 47632, 47635, 47639, 47642, 47645, 47648, 47657, 47665, 47666, 47682, 47707, 47713, 47718, 47726, 47739, 47744, 47748, 47761, 47768, 47779, 47789, 47796, 47839, 47840, 47853, 47884, 47922, 47939, 47947, 47949, 47954, 47974, 47995, 48000, 48023, 48033, 48035, 48036, 48039, 48041, 48068, 48069, 48073, 48090, 48103, 48106, 48108, 48133, 48134, 48146, 48152, 48161, 48196, 48202, 48203, 48204, 48230, 48243, 48248, 48257, 48277, 48287, 48300, 48303, 48333, 48348, 48371, 48379, 48383, 48389, 48392, 48394, 48396, 48399, 48401, 48411, 48440, 48457, 48462, 48488, 48490, 48491, 48504, 48510, 48526, 48537, 48538, 48561, 48567, 48570, 48594, 48599, 48603, 48648, 48649, 48663, 48668, 48676, 48684, 48689, 48710, 48719, 48737, 48747, 48758, 48763, 48796, 48798, 48809, 48829, 48840, 48860, 48873, 48904, 48907, 48912, 48917, 48918, 48935, 48939, 48951, 48973, 48985, 49000, 49022, 49039, 49059, 49077, 49079, 49080, 49086, 49099, 49110, 49136, 49147, 49161, 49167, 49168, 49169, 49171, 49173, 49180, 49192, 49194, 49203, 49214, 49223, 49258, 49265, 49280, 49291, 49294, 49319, 49328, 49334, 49335, 49338, 49339, 49352, 49364, 49370, 49384, 49386, 49388, 49404, 49423, 49449, 49450, 49468, 49476, 49479, 49498, 49499, 49501, 49511, 49548, 49559, 49569, 49585, 49591, 49597, 49608, 49616, 49623, 49691, 49694, 49709, 49712, 49731, 49734, 49752, 49804, 49811, 49815, 49825, 49827, 49830, 49833, 49837, 49850, 49862, 49869, 49878, 49881, 49916, 49923, 49943, 49958, 49963, 49964, 49970, 49975, 49976, 49977, 49986, 49996, 50007, 50022, 50024, 50027, 50035, 50048, 50058, 50064, 50075, 50080, 50088, 50101, 50108, 50109, 50116, 50131, 50134, 50144, 50146, 50161, 50169, 50172, 50177, 50191, 50230, 50231, 50237, 50244, 50255, 50258, 50271, 50276, 50278, 50284, 50307, 50308, 50317, 50323, 50326, 50352, 50367, 50387, 50394, 50395, 50396, 50402, 50409, 50420, 50424, 50433, 50436, 50448, 50462, 50497, 50519, 50534, 50535, 50538, 50561, 50576, 50587, 50597, 50598, 50618, 50656, 50658, 50660, 50670, 50679, 50686, 50692, 50697, 50703, 50711, 50715, 50734, 50735, 50774, 50784, 50792, 50793, 50796, 50798, 50800, 50803, 50819, 50828, 50832, 50835, 50846, 50853, 50863, 50874, 50876, 50877, 50881, 50885, 50890, 50896, 50900, 50907, 50912, 50913, 50914, 50919, 50939, 50946, 50956, 50970, 50995, 50997, 51029, 51032, 51035, 51045, 51051, 51052, 51061, 51096, 51102, 51104, 51126, 51152, 51172, 51187, 51196, 51199, 51208, 51210, 51231, 51237, 51246, 51252, 51267, 51270, 51277, 51290, 51292, 51293, 51299, 51305, 51311, 51318, 51329, 51337, 51348, 51375, 51387, 51390, 51404, 51411, 51413, 51416, 51422, 51454, 51461, 51478, 51479, 51485, 51486, 51491, 51496, 51509, 51519, 51523, 51535, 51541, 51551, 51562, 51568, 51572, 51584, 51608, 51616, 51628, 51634, 51648, 51657, 51662, 51679, 51711, 51718, 51726, 51727, 51735, 51746, 51752, 51754, 51764, 51770, 51794, 51808, 51810, 51822, 51824, 51843, 51845, 51854, 51869, 51871, 51874, 51878, 51885, 51891, 51903, 51931, 51932, 51937, 51940, 51943, 51949, 51951, 51959, 51960, 51965, 51969, 51973, 51988, 51999, 52040, 52045, 52079, 52085, 52094, 52106, 52108, 52115, 52124, 52145, 52154, 52180, 52191, 52192, 52204, 52214, 52236, 52238, 52249, 52250, 52268, 52273, 52277, 52300, 52306, 52320, 52325, 52326, 52351, 52356, 52406, 52424, 52426, 52447, 52450, 52456, 52463, 52483, 52503, 52507, 52508, 52522, 52532, 52543, 52551, 52569, 52573, 52578, 52579, 52598, 52604, 52619, 52627, 52640, 52643, 52657, 52663, 52669, 52686, 52687, 52693, 52695, 52699, 52718, 52721, 52727, 52734, 52741, 52762, 52774, 52780, 52783, 52788, 52798, 52802, 52812, 52813, 52818, 52835, 52839, 52845, 52848, 52849, 52872, 52903, 52952, 52972, 52986, 52995, 53006, 53019, 53029, 53031, 53045, 53055, 53063] -------------------------------------------------------------------------------- /Machine Learning Specialization-Clustering/WK2/people_wiki_word_count.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfirefire/Machine-Learning-Specialization/aae0574b51386a812dda764e5612976ece77e41d/Machine Learning Specialization-Clustering/WK2/people_wiki_word_count.npz -------------------------------------------------------------------------------- /Machine Learning Specialization-Clustering/empty.md: -------------------------------------------------------------------------------- 1 | idk 2 | -------------------------------------------------------------------------------- /Machine Learning Specialization-Regression/WK1/Philadelphia_Crime_Rate_noNA.csv: -------------------------------------------------------------------------------- 1 | HousePrice,"HsPrc ($10,000)",CrimeRate,MilesPhila,PopChg,Name,County 140463,14.0463,29.7,10,-1,Abington,Montgome 113033,11.3033,24.1,18,4,Ambler,Montgome 124186,12.4186,19.5,25,8,Aston,Delaware 110490,11.049,49.4,25,2.7,Bensalem,Bucks 79124,7.9124,54.1,19,3.9,Bristol B.,Bucks 92634,9.2634,48.6,20,0.6,Bristol T.,Bucks 89246,8.9246,30.8,15,-2.6,Brookhaven,Delaware 195145,19.5145,10.8,20,-3.5,Bryn Athyn,Montgome 297342,29.7342,20.2,14,0.6,Bryn Mawr,Montgome 264298,26.4298,20.4,26,6,Buckingham,Bucks 134342,13.4342,17.3,31,4.2,Chalfont,Bucks 147600,14.76,50.3,9,-1,Cheltenham,Montgome 77370,7.737,34.2,10,-1.2,Clifton,Delaware 170822,17.0822,33.7,32,2.4,Collegeville,Montgome 40642,4.0642,45.7,15,0,Darby Bor.,Delaware 71359,7.1359,22.3,8,1.6,Darby Town,Delaware 104923,10.4923,48.1,21,6.9,Downingtown,Chester 190317,19.0317,19.4,26,1.9,Doylestown,Bucks 215512,21.5512,71.9,26,5.8,E. Bradford,Chester 178105,17.8105,45.1,25,2.3,E. Goshen,Chester 131025,13.1025,31.3,19,-1.8,E. Norriton,Montgome 149844,14.9844,24.9,22,6.4,E. Pikeland,Chester 170556,17.0556,27.2,30,4.6,E. Whiteland,Chester 280969,28.0969,17.7,14,2.9,Easttown,Chester 114233,11.4233,29,30,1.3,Falls Town,Bucks 74502,7.4502,21.4,15,-3.2,Follcroft,Delaware 475112,47.5112,28.6,12,,Gladwyne,Montgome 97167,9.7167,29.3,10,0.2,Glenolden,Delaware 114572,11.4572,17.5,20,5.2,Hatboro,Montgome 436348,43.6348,16.5,10,-0.7,Haverford,Delaware 389302,38.9302,17.8,20,1.5,Horsham,Montgome 122392,12.2392,17.3,10,1.9,Jenkintown,Montgome 130436,13.0436,31.2,17,-0.4,L Southampton,Delaware 272790,27.279,14.5,20,-5.1,L. Gwynedd,Montgome 194435,19.4435,15.7,32,15,L. Makefield,Bucks 299621,29.9621,28.6,10,1.4,L. Merion,Montgome 210884,21.0884,20.8,20,0.1,L. Moreland,Montgome 112471,11.2471,29.3,35,3.4,Lansdale,Montgome 93738,9.3738,19.3,7,-0.4,Lansdown,Delaware 121024,12.1024,39.5,35,26.9,Limerick,Montgome 156035,15.6035,13,23,6.3,Malvern,Chester 185404,18.5404,24.1,10,0.9,Marple,Delaware 126160,12.616,38,20,-2.4,Media,Delaware 143072,14.3072,40.1,23,1.6,Middletown,Bucks 96769,9.6769,36.1,15,5.1,Morrisville,Bucks 94014,9.4014,26.6,14,0.5,Morton,Delaware 118214,11.8214,25.1,25,5.7,N. Wales,Montgome 157446,15.7446,14.6,15,3.1,Narberth,Montgome 150283,15.0283,18.2,15,0.9,Nether,Delaware 153842,15.3842,15.3,23,8.5,Newtown,Bucks 197214,19.7214,15.2,25,2.1,Newtown B.,Bucks 206127,20.6127,17.4,15,2.7,Newtown T.,Delaware 71981,7.1981,73.3,19,4.9,Norristown,Montgome 169401,16.9401,7.1,22,1.5,Northampton,Bucks 99843,9.9843,12.5,12,-3.7,Norwood,Delaware 60000,6,45.8,18,-1.4,"Phila, Far NE",Phila 28000,2.8,44.9,5.5,-8.4,"Phila, N",Phila 60000,6,65,9,-4.9,"Phila, NE",Phila 61800,6.18,49.9,9,-6.4,"Phila, NW",Phila 38000,3.8,54.8,4.5,-5.1,"Phila, SW",Phila 38000,3.8,53.5,2,-9.2,"Phila, South",Phila 42000,4.2,69.9,4,-5.7,"Phila, West",Phila 96200,9.62,366.1,0,4.8,"Phila,CC",Phila 103087,10.3087,24.6,24,3.9,Phoenixville,Chester 147720,14.772,58.6,25,1.5,Plymouth,Montgome 78175,7.8175,53.2,41,2.2,Pottstown,Montgome 92215,9.2215,17.4,14,7.8,Prospect Park,Delaware 271804,27.1804,15.5,17,1.2,Radnor,Delaware 119566,11.9566,14.5,12,-2.9,Ridley Park,Delaware 100231,10.0231,24.1,15,1.9,Ridley Town,Delaware 95831,9.5831,21.2,32,3.2,Royersford,Montgome 229711,22.9711,9.8,22,5.3,Schuylkill,Chester 74308,7.4308,29.9,7,1.8,Sharon Hill,Delaware 259506,25.9506,7.2,40,17.4,Solebury,Bucks 159573,15.9573,19.4,15,-2.1,Springfield,Montgome 147176,14.7176,41.1,12,-1.7,Springfield,Delaware 205732,20.5732,11.2,12,-0.2,Swarthmore,Delaware 215783,21.5783,21.2,20,1.1,Tredyffin,Chester 116710,11.671,42.8,20,12.9,U. Chichester,Delaware 359112,35.9112,9.4,36,4,U. Makefield,Bucks 189959,18.9959,61.7,22,-2.1,U. Merion,Montgome 133198,13.3198,19.4,22,-2,U. Moreland,Montgome 242821,24.2821,6.6,21,1.6,U. Providence,Delaware 142811,14.2811,15.9,20,-1.6,U. Southampton,Bucks 200498,20.0498,18.8,36,11,U. Uwchlan,Chester 199065,19.9065,13.2,20,7.8,Upper Darby,Montgome 93648,9.3648,34.5,8,-0.7,Upper Darby,Delaware 163001,16.3001,22.1,50,8,Uwchlan T.,Chester 436348,43.6348,22.1,15,1.3,Villanova,Montgome 124478,12.4478,71.9,22,4.6,W. Chester,Chester 168276,16.8276,31.9,26,5.9,W. Goshen,Chester 114157,11.4157,44.6,38,14.6,W. Whiteland,Chester 130088,13.0088,28.6,19,-0.2,Warminster,Bucks 152624,15.2624,24,19,23.1,Warrington,Bucks 174232,17.4232,13.8,25,4.7,Westtown,Chester 196515,19.6515,29.9,16,1.8,Whitemarsh,Montgome 232714,23.2714,9.9,21,0.2,Willistown,Chester 245920,24.592,22.6,10,0.3,Wynnewood,Montgome 130953,13.0953,13,24,5.2,Yardley,Bucks -------------------------------------------------------------------------------- /Machine Learning Specialization-Regression/WK1/WK1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import pandas as pd" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 2, 17 | "metadata": { 18 | "collapsed": false 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "house=pd.read_csv('kc_house_data.csv', dtype= {'bathrooms':float, 'waterfront':int, 'sqft_above':int, 'sqft_living15':float, 'grade':int, 'yr_renovated':int, 'price':float, 'bedrooms':float, 'zipcode':str, 'long':float, 'sqft_lot15':float, 'sqft_living':float, 'floors':str, 'condition':int, 'lat':float, 'date':str, 'sqft_basement':int, 'yr_built':int, 'id':str, 'sqft_lot':int, 'view':int})" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 3, 28 | "metadata": { 29 | "collapsed": true 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "train=pd.read_csv('kc_house_train_data.csv', dtype= {'bathrooms':float, 'waterfront':int, 'sqft_above':int, 'sqft_living15':float, 'grade':int, 'yr_renovated':int, 'price':float, 'bedrooms':float, 'zipcode':str, 'long':float, 'sqft_lot15':float, 'sqft_living':float, 'floors':str, 'condition':int, 'lat':float, 'date':str, 'sqft_basement':int, 'yr_built':int, 'id':str, 'sqft_lot':int, 'view':int})\n", 34 | "test=pd.read_csv('kc_house_test_data.csv', dtype= {'bathrooms':float, 'waterfront':int, 'sqft_above':int, 'sqft_living15':float, 'grade':int, 'yr_renovated':int, 'price':float, 'bedrooms':float, 'zipcode':str, 'long':float, 'sqft_lot15':float, 'sqft_living':float, 'floors':str, 'condition':int, 'lat':float, 'date':str, 'sqft_basement':int, 'yr_built':int, 'id':str, 'sqft_lot':int, 'view':int})" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 4, 40 | "metadata": { 41 | "collapsed": false 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "def closed_form(features,labels):\n", 46 | " numerator=(features*labels).sum()-(features.sum())*(labels.sum())/len(labels)\n", 47 | " denominator=((features**2).sum())-(features.sum())*(features.sum())/len(labels)\n", 48 | " slope=numerator/denominator\n", 49 | " intercept=(labels.mean())-slope*(features.mean())\n", 50 | " return intercept,slope" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 5, 56 | "metadata": { 57 | "collapsed": false 58 | }, 59 | "outputs": [], 60 | "source": [ 61 | "sqft_intercept,sqft_slope=closed_form(train['sqft_living'],train['price'])\n", 62 | "bed_intercept,bed_slope=closed_form(train['bedrooms'],train['price'])" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 6, 68 | "metadata": { 69 | "collapsed": true 70 | }, 71 | "outputs": [], 72 | "source": [ 73 | "def predict(feature,slope,intercept):\n", 74 | " price=feature*slope+intercept\n", 75 | " return price" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 7, 81 | "metadata": { 82 | "collapsed": true 83 | }, 84 | "outputs": [], 85 | "source": [ 86 | "pre_price=predict(train['sqft_living'],sqft_slope,sqft_intercept)" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 8, 92 | "metadata": { 93 | "collapsed": false 94 | }, 95 | "outputs": [], 96 | "source": [ 97 | "test['prediction']=pre_price" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 9, 103 | "metadata": { 104 | "collapsed": false 105 | }, 106 | "outputs": [ 107 | { 108 | "data": { 109 | "text/html": [ 110 | "
\n", 111 | "\n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | "
iddatepricebedroomsbathroomssqft_livingsqft_lotfloorswaterfrontview...sqft_abovesqft_basementyr_builtyr_renovatedzipcodelatlongsqft_living15sqft_lot15prediction
0011410151620140528T000000310000.03.01.01430.0199011.500...14300192709802847.7558-122.2291780.012697.0285595.351691
1929730005520150124T000000650000.04.03.02950.05000203...1980970197909812647.5714-122.3752140.04000.0677518.138777
2120200020020141103T000000233000.03.02.01710.046971.500...17100194109800247.3048-122.2181030.04705.0169992.227442
3856275032020141110T000000580500.03.02.52320.03980200...23200200309802747.5391-122.0702580.03980.0505523.246603
4758920019320141110T000000535000.03.01.01090.030001.500...10900192909811747.6889-122.3751570.05080.0426574.771506
\n", 261 | "

5 rows × 22 columns

\n", 262 | "
" 263 | ], 264 | "text/plain": [ 265 | " id date price bedrooms bathrooms sqft_living \\\n", 266 | "0 0114101516 20140528T000000 310000.0 3.0 1.0 1430.0 \n", 267 | "1 9297300055 20150124T000000 650000.0 4.0 3.0 2950.0 \n", 268 | "2 1202000200 20141103T000000 233000.0 3.0 2.0 1710.0 \n", 269 | "3 8562750320 20141110T000000 580500.0 3.0 2.5 2320.0 \n", 270 | "4 7589200193 20141110T000000 535000.0 3.0 1.0 1090.0 \n", 271 | "\n", 272 | " sqft_lot floors waterfront view ... sqft_above \\\n", 273 | "0 19901 1.5 0 0 ... 1430 \n", 274 | "1 5000 2 0 3 ... 1980 \n", 275 | "2 4697 1.5 0 0 ... 1710 \n", 276 | "3 3980 2 0 0 ... 2320 \n", 277 | "4 3000 1.5 0 0 ... 1090 \n", 278 | "\n", 279 | " sqft_basement yr_built yr_renovated zipcode lat long \\\n", 280 | "0 0 1927 0 98028 47.7558 -122.229 \n", 281 | "1 970 1979 0 98126 47.5714 -122.375 \n", 282 | "2 0 1941 0 98002 47.3048 -122.218 \n", 283 | "3 0 2003 0 98027 47.5391 -122.070 \n", 284 | "4 0 1929 0 98117 47.6889 -122.375 \n", 285 | "\n", 286 | " sqft_living15 sqft_lot15 prediction \n", 287 | "0 1780.0 12697.0 285595.351691 \n", 288 | "1 2140.0 4000.0 677518.138777 \n", 289 | "2 1030.0 4705.0 169992.227442 \n", 290 | "3 2580.0 3980.0 505523.246603 \n", 291 | "4 1570.0 5080.0 426574.771506 \n", 292 | "\n", 293 | "[5 rows x 22 columns]" 294 | ] 295 | }, 296 | "execution_count": 9, 297 | "metadata": {}, 298 | "output_type": "execute_result" 299 | } 300 | ], 301 | "source": [ 302 | "test.head()" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": 10, 308 | "metadata": { 309 | "collapsed": true 310 | }, 311 | "outputs": [], 312 | "source": [ 313 | "pre_2650=predict(2650,sqft_slope,sqft_intercept)" 314 | ] 315 | }, 316 | { 317 | "cell_type": "code", 318 | "execution_count": 11, 319 | "metadata": { 320 | "collapsed": false 321 | }, 322 | "outputs": [ 323 | { 324 | "data": { 325 | "text/plain": [ 326 | "700074.8459475137" 327 | ] 328 | }, 329 | "execution_count": 11, 330 | "metadata": {}, 331 | "output_type": "execute_result" 332 | } 333 | ], 334 | "source": [ 335 | "pre_2650" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": 12, 341 | "metadata": { 342 | "collapsed": true 343 | }, 344 | "outputs": [], 345 | "source": [ 346 | "def cal_rss(slope,intercept,feature,real_label):\n", 347 | " rss=((real_label-(slope*feature+intercept))**2).sum()\n", 348 | " return rss" 349 | ] 350 | }, 351 | { 352 | "cell_type": "code", 353 | "execution_count": 13, 354 | "metadata": { 355 | "collapsed": false 356 | }, 357 | "outputs": [], 358 | "source": [ 359 | "sqftrss=cal_rss(sqft_slope,sqft_intercept,train['sqft_living'],train['price'])\n", 360 | "bedrss=cal_rss(bed_slope,bed_intercept,train['bedrooms'],train['price'])" 361 | ] 362 | }, 363 | { 364 | "cell_type": "code", 365 | "execution_count": 14, 366 | "metadata": { 367 | "collapsed": false 368 | }, 369 | "outputs": [ 370 | { 371 | "data": { 372 | "text/plain": [ 373 | "(1201918354177286.2, 2143244498162069.0)" 374 | ] 375 | }, 376 | "execution_count": 14, 377 | "metadata": {}, 378 | "output_type": "execute_result" 379 | } 380 | ], 381 | "source": [ 382 | "sqftrss,bedrss" 383 | ] 384 | }, 385 | { 386 | "cell_type": "code", 387 | "execution_count": 15, 388 | "metadata": { 389 | "collapsed": true 390 | }, 391 | "outputs": [], 392 | "source": [ 393 | "def invert_sqft(slope,intercept,output):\n", 394 | " sqft=(output-intercept)/slope\n", 395 | " return sqft" 396 | ] 397 | }, 398 | { 399 | "cell_type": "code", 400 | "execution_count": 16, 401 | "metadata": { 402 | "collapsed": false 403 | }, 404 | "outputs": [ 405 | { 406 | "data": { 407 | "text/plain": [ 408 | "3004.3962451522766" 409 | ] 410 | }, 411 | "execution_count": 16, 412 | "metadata": {}, 413 | "output_type": "execute_result" 414 | } 415 | ], 416 | "source": [ 417 | "invert_sqft(sqft_slope,sqft_intercept,800000)" 418 | ] 419 | }, 420 | { 421 | "cell_type": "code", 422 | "execution_count": null, 423 | "metadata": { 424 | "collapsed": true 425 | }, 426 | "outputs": [], 427 | "source": [] 428 | } 429 | ], 430 | "metadata": { 431 | "kernelspec": { 432 | "display_name": "Python 2", 433 | "language": "python", 434 | "name": "python2" 435 | }, 436 | "language_info": { 437 | "codemirror_mode": { 438 | "name": "ipython", 439 | "version": 2 440 | }, 441 | "file_extension": ".py", 442 | "mimetype": "text/x-python", 443 | "name": "python", 444 | "nbconvert_exporter": "python", 445 | "pygments_lexer": "ipython2", 446 | "version": "2.7.11" 447 | } 448 | }, 449 | "nbformat": 4, 450 | "nbformat_minor": 0 451 | } 452 | -------------------------------------------------------------------------------- /Machine Learning Specialization-Regression/WK2/WK2-1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 50, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import pandas as pd\n", 12 | "import math\n", 13 | "from sklearn.linear_model import LinearRegression" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 4, 19 | "metadata": { 20 | "collapsed": true 21 | }, 22 | "outputs": [], 23 | "source": [ 24 | "train=pd.read_csv('kc_house_train_data.csv',dtype = {'bathrooms':float, 'waterfront':int, 'sqft_above':int, 'sqft_living15':float, 'grade':int, 'yr_renovated':int, 'price':float, 'bedrooms':float, 'zipcode':str, 'long':float, 'sqft_lot15':float, 'sqft_living':float, 'floors':str, 'condition':int, 'lat':float, 'date':str, 'sqft_basement':int, 'yr_built':int, 'id':str, 'sqft_lot':int, 'view':int})" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 6, 30 | "metadata": { 31 | "collapsed": true 32 | }, 33 | "outputs": [], 34 | "source": [ 35 | "test=pd.read_csv('kc_house_test_data.csv',dtype = {'bathrooms':float, 'waterfront':int, 'sqft_above':int, 'sqft_living15':float, 'grade':int, 'yr_renovated':int, 'price':float, 'bedrooms':float, 'zipcode':str, 'long':float, 'sqft_lot15':float, 'sqft_living':float, 'floors':str, 'condition':int, 'lat':float, 'date':str, 'sqft_basement':int, 'yr_built':int, 'id':str, 'sqft_lot':int, 'view':int})" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 7, 41 | "metadata": { 42 | "collapsed": true 43 | }, 44 | "outputs": [], 45 | "source": [ 46 | "train['bed_sqr']=train['bedrooms']**2" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 9, 52 | "metadata": { 53 | "collapsed": true 54 | }, 55 | "outputs": [], 56 | "source": [ 57 | "test['bed_sqr']=test['bedrooms']**2" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 10, 63 | "metadata": { 64 | "collapsed": true 65 | }, 66 | "outputs": [], 67 | "source": [ 68 | "train['bed_bath']=train['bedrooms']*train['bathrooms']\n", 69 | "test['bed_bath']=test['bedrooms']*test['bathrooms']" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 17, 75 | "metadata": { 76 | "collapsed": true 77 | }, 78 | "outputs": [], 79 | "source": [ 80 | "train['log_sqft']=list(map(lambda x: math.log(x),train['sqft_living']))\n", 81 | "test['log_sqft']=list(map(lambda x: math.log(x),test['sqft_living']))\n", 82 | "train['lat_long_sum']=train['lat']+train['long']\n", 83 | "test['lat_long_sum']=test['lat']+test['long']" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 18, 89 | "metadata": {}, 90 | "outputs": [ 91 | { 92 | "data": { 93 | "text/html": [ 94 | "
\n", 95 | "\n", 108 | "\n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | "
iddatepricebedroomsbathroomssqft_livingsqft_lotfloorswaterfrontview...yr_renovatedzipcodelatlongsqft_living15sqft_lot15bed_sqrbed_bathlog_sqftlat_long_sum
0712930052020141013T000000221900.03.01.001180.05650100...09817847.5112-122.2571340.05650.09.03.007.073270-74.7458
1641410019220141209T000000538000.03.02.252570.07242200...19919812547.7210-122.3191690.07639.09.06.757.851661-74.5980
2563150040020150225T000000180000.02.01.00770.010000100...09802847.7379-122.2332720.08062.04.02.006.646391-74.4951
3248720087520141209T000000604000.04.03.001960.05000100...09813647.5208-122.3931360.05000.016.012.007.580700-74.8722
4195440051020150218T000000510000.03.02.001680.08080100...09807447.6168-122.0451800.07503.09.06.007.426549-74.4282
\n", 258 | "

5 rows × 25 columns

\n", 259 | "
" 260 | ], 261 | "text/plain": [ 262 | " id date price bedrooms bathrooms sqft_living \\\n", 263 | "0 7129300520 20141013T000000 221900.0 3.0 1.00 1180.0 \n", 264 | "1 6414100192 20141209T000000 538000.0 3.0 2.25 2570.0 \n", 265 | "2 5631500400 20150225T000000 180000.0 2.0 1.00 770.0 \n", 266 | "3 2487200875 20141209T000000 604000.0 4.0 3.00 1960.0 \n", 267 | "4 1954400510 20150218T000000 510000.0 3.0 2.00 1680.0 \n", 268 | "\n", 269 | " sqft_lot floors waterfront view ... yr_renovated zipcode \\\n", 270 | "0 5650 1 0 0 ... 0 98178 \n", 271 | "1 7242 2 0 0 ... 1991 98125 \n", 272 | "2 10000 1 0 0 ... 0 98028 \n", 273 | "3 5000 1 0 0 ... 0 98136 \n", 274 | "4 8080 1 0 0 ... 0 98074 \n", 275 | "\n", 276 | " lat long sqft_living15 sqft_lot15 bed_sqr bed_bath log_sqft \\\n", 277 | "0 47.5112 -122.257 1340.0 5650.0 9.0 3.00 7.073270 \n", 278 | "1 47.7210 -122.319 1690.0 7639.0 9.0 6.75 7.851661 \n", 279 | "2 47.7379 -122.233 2720.0 8062.0 4.0 2.00 6.646391 \n", 280 | "3 47.5208 -122.393 1360.0 5000.0 16.0 12.00 7.580700 \n", 281 | "4 47.6168 -122.045 1800.0 7503.0 9.0 6.00 7.426549 \n", 282 | "\n", 283 | " lat_long_sum \n", 284 | "0 -74.7458 \n", 285 | "1 -74.5980 \n", 286 | "2 -74.4951 \n", 287 | "3 -74.8722 \n", 288 | "4 -74.4282 \n", 289 | "\n", 290 | "[5 rows x 25 columns]" 291 | ] 292 | }, 293 | "execution_count": 18, 294 | "metadata": {}, 295 | "output_type": "execute_result" 296 | } 297 | ], 298 | "source": [ 299 | "train.head()" 300 | ] 301 | }, 302 | { 303 | "cell_type": "code", 304 | "execution_count": 28, 305 | "metadata": {}, 306 | "outputs": [ 307 | { 308 | "name": "stdout", 309 | "output_type": "stream", 310 | "text": [ 311 | "bedrooms^2=12.4466777015843, bed_bath=7.5039016315913925, log=7.550274679645921, lat+long=-74.65333355403185\n" 312 | ] 313 | } 314 | ], 315 | "source": [ 316 | "print('bedrooms^2='+str(test['bed_sqr'].mean())+', bed_bath='+str(test['bed_bath'].mean())+', log='+str(test['log_sqft'].mean())+', lat+long='+str(test['lat_long_sum'].mean()))" 317 | ] 318 | }, 319 | { 320 | "cell_type": "code", 321 | "execution_count": 131, 322 | "metadata": {}, 323 | "outputs": [ 324 | { 325 | "data": { 326 | "text/plain": [ 327 | "['sqft_living', 'bedrooms', 'lat', 'long', 'bathrooms']" 328 | ] 329 | }, 330 | "execution_count": 131, 331 | "metadata": {}, 332 | "output_type": "execute_result" 333 | } 334 | ], 335 | "source": [ 336 | "y_train=train[['price']]\n", 337 | "x_model1=train[['sqft_living']+['bedrooms']+['lat']+['long']+['bathrooms']]\n", 338 | "x_model2=train[['sqft_living']+['bedrooms']+['lat']+['long']+['bed_bath']+['bathrooms']]\n", 339 | "x_model3=train[['sqft_living']+['bedrooms']+['lat']+['long']+['bed_bath']+['bed_sqr']+['log_sqft']+['lat_long_sum']+['bathrooms']]\n", 340 | "list(x_model1)" 341 | ] 342 | }, 343 | { 344 | "cell_type": "code", 345 | "execution_count": 132, 346 | "metadata": {}, 347 | "outputs": [ 348 | { 349 | "data": { 350 | "text/plain": [ 351 | "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)" 352 | ] 353 | }, 354 | "execution_count": 132, 355 | "metadata": {}, 356 | "output_type": "execute_result" 357 | } 358 | ], 359 | "source": [ 360 | "model_1=LinearRegression()\n", 361 | "model_2=LinearRegression()\n", 362 | "model_3=LinearRegression()\n", 363 | "model_1.fit(x_model1,y_train)\n", 364 | "model_2.fit(x_model2,y_train)\n", 365 | "model_3.fit(x_model3,y_train)" 366 | ] 367 | }, 368 | { 369 | "cell_type": "code", 370 | "execution_count": 133, 371 | "metadata": {}, 372 | "outputs": [ 373 | { 374 | "name": "stdout", 375 | "output_type": "stream", 376 | "text": [ 377 | "sqft_living 312.258646273\n", 378 | "bedrooms -59586.5331536\n", 379 | "lat 658619.263931\n", 380 | "long -309374.351268\n", 381 | "bathrooms 15706.7420827\n", 382 | "--------------\n", 383 | "sqft_living 306.610053459\n", 384 | "bedrooms -113446.36807\n", 385 | "lat 654844.629503\n", 386 | "long -294298.969138\n", 387 | "bed_bath 25579.6520008\n", 388 | "bathrooms -71461.3082928\n", 389 | "--------------\n", 390 | "sqft_living 529.422819647\n", 391 | "bedrooms 34514.229578\n", 392 | "lat 534085.610867\n", 393 | "long -406750.710861\n", 394 | "bed_bath -8570.50439463\n", 395 | "bed_sqr -6788.58667034\n", 396 | "log_sqft -561831.484076\n", 397 | "lat_long_sum 127334.900006\n", 398 | "bathrooms 67060.7813189\n" 399 | ] 400 | } 401 | ], 402 | "source": [ 403 | "def get_coef(x,coef):\n", 404 | " for i in range(len(x)):\n", 405 | " print (x[i],coef[0][i])\n", 406 | "get_coef(list(x_model1),model_1.coef_)\n", 407 | "print ('--------------')\n", 408 | "get_coef(list(x_model2),model_2.coef_)\n", 409 | "print ('--------------')\n", 410 | "get_coef(list(x_model3),model_3.coef_)" 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": 140, 416 | "metadata": {}, 417 | "outputs": [ 418 | { 419 | "name": "stdout", 420 | "output_type": "stream", 421 | "text": [ 422 | "price 9.678800e+14\n", 423 | "dtype: float64 price 9.584196e+14\n", 424 | "dtype: float64 price 9.034365e+14\n", 425 | "dtype: float64\n" 426 | ] 427 | } 428 | ], 429 | "source": [ 430 | "RSS_train1=((model_1.predict(x_model1.values)-y_train)**2).sum()\n", 431 | "RSS_train2=((model_2.predict(x_model2.values)-y_train)**2).sum()\n", 432 | "RSS_train3=((model_3.predict(x_model3.values)-y_train)**2).sum()\n", 433 | "print (RSS_train1,RSS_train2,RSS_train3)" 434 | ] 435 | }, 436 | { 437 | "cell_type": "code", 438 | "execution_count": 141, 439 | "metadata": { 440 | "collapsed": true 441 | }, 442 | "outputs": [], 443 | "source": [ 444 | "y_test=test[['price']]\n", 445 | "x_tm1=test[['sqft_living']+['bedrooms']+['lat']+['long']+['bathrooms']]\n", 446 | "x_tm2=test[['sqft_living']+['bedrooms']+['lat']+['long']+['bed_bath']+['bathrooms']]\n", 447 | "x_tm3=test[['sqft_living']+['bedrooms']+['lat']+['long']+['bed_bath']+['bed_sqr']+['log_sqft']+['lat_long_sum']+['bathrooms']]" 448 | ] 449 | }, 450 | { 451 | "cell_type": "code", 452 | "execution_count": 139, 453 | "metadata": {}, 454 | "outputs": [ 455 | { 456 | "name": "stdout", 457 | "output_type": "stream", 458 | "text": [ 459 | "price 2.255005e+14\n", 460 | "dtype: float64 price 2.233775e+14\n", 461 | "dtype: float64 price 2.592363e+14\n", 462 | "dtype: float64\n" 463 | ] 464 | } 465 | ], 466 | "source": [ 467 | "RSS_test1=((model_1.predict(x_tm1.values)-y_test)**2).sum()\n", 468 | "RSS_test2=((model_2.predict(x_tm2.values)-y_test)**2).sum()\n", 469 | "RSS_test3=((model_3.predict(x_tm3.values)-y_test)**2).sum()\n", 470 | "print (RSS_test1,RSS_test2,RSS_test3)" 471 | ] 472 | }, 473 | { 474 | "cell_type": "code", 475 | "execution_count": 149, 476 | "metadata": {}, 477 | "outputs": [ 478 | { 479 | "name": "stdout", 480 | "output_type": "stream", 481 | "text": [ 482 | "\n" 483 | ] 484 | }, 485 | { 486 | "data": { 487 | "text/html": [ 488 | "
\n", 489 | "\n", 502 | "\n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | "
bedrooms
03.0
13.0
22.0
34.0
43.0
\n", 532 | "
" 533 | ], 534 | "text/plain": [ 535 | " bedrooms\n", 536 | "0 3.0\n", 537 | "1 3.0\n", 538 | "2 2.0\n", 539 | "3 4.0\n", 540 | "4 3.0" 541 | ] 542 | }, 543 | "execution_count": 149, 544 | "metadata": {}, 545 | "output_type": "execute_result" 546 | } 547 | ], 548 | "source": [ 549 | "print(type(train[['bedrooms']]))\n", 550 | "train[['bedrooms']].head()" 551 | ] 552 | }, 553 | { 554 | "cell_type": "code", 555 | "execution_count": 148, 556 | "metadata": {}, 557 | "outputs": [ 558 | { 559 | "name": "stdout", 560 | "output_type": "stream", 561 | "text": [ 562 | "\n" 563 | ] 564 | }, 565 | { 566 | "data": { 567 | "text/plain": [ 568 | "0 3.0\n", 569 | "1 3.0\n", 570 | "2 2.0\n", 571 | "3 4.0\n", 572 | "4 3.0\n", 573 | "Name: bedrooms, dtype: float64" 574 | ] 575 | }, 576 | "execution_count": 148, 577 | "metadata": {}, 578 | "output_type": "execute_result" 579 | } 580 | ], 581 | "source": [ 582 | "print(type(train['bedrooms']))\n", 583 | "train['bedrooms'].head()" 584 | ] 585 | }, 586 | { 587 | "cell_type": "code", 588 | "execution_count": null, 589 | "metadata": { 590 | "collapsed": true 591 | }, 592 | "outputs": [], 593 | "source": [] 594 | } 595 | ], 596 | "metadata": { 597 | "kernelspec": { 598 | "display_name": "Python 3", 599 | "language": "python", 600 | "name": "python3" 601 | }, 602 | "language_info": { 603 | "codemirror_mode": { 604 | "name": "ipython", 605 | "version": 3 606 | }, 607 | "file_extension": ".py", 608 | "mimetype": "text/x-python", 609 | "name": "python", 610 | "nbconvert_exporter": "python", 611 | "pygments_lexer": "ipython3", 612 | "version": "3.6.3" 613 | } 614 | }, 615 | "nbformat": 4, 616 | "nbformat_minor": 2 617 | } 618 | -------------------------------------------------------------------------------- /Machine Learning Specialization-Regression/WK2/WK2-2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import pandas as pd\n", 12 | "import numpy as np\n", 13 | "from math import sqrt" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 3, 19 | "metadata": { 20 | "collapsed": true 21 | }, 22 | "outputs": [], 23 | "source": [ 24 | "train=pd.read_csv('kc_house_train_data.csv',dtype = {'bathrooms':float, 'waterfront':int, 'sqft_above':int, 'sqft_living15':float, 'grade':int, 'yr_renovated':int, 'price':float, 'bedrooms':float, 'zipcode':str, 'long':float, 'sqft_lot15':float, 'sqft_living':float, 'floors':str, 'condition':int, 'lat':float, 'date':str, 'sqft_basement':int, 'yr_built':int, 'id':str, 'sqft_lot':int, 'view':int})" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 4, 30 | "metadata": { 31 | "collapsed": true 32 | }, 33 | "outputs": [], 34 | "source": [ 35 | "test=pd.read_csv('kc_house_test_data.csv',dtype = {'bathrooms':float, 'waterfront':int, 'sqft_above':int, 'sqft_living15':float, 'grade':int, 'yr_renovated':int, 'price':float, 'bedrooms':float, 'zipcode':str, 'long':float, 'sqft_lot15':float, 'sqft_living':float, 'floors':str, 'condition':int, 'lat':float, 'date':str, 'sqft_basement':int, 'yr_built':int, 'id':str, 'sqft_lot':int, 'view':int})" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 5, 41 | "metadata": { 42 | "collapsed": true 43 | }, 44 | "outputs": [], 45 | "source": [ 46 | "def get_numpy_data(dataframe,features,output):\n", 47 | " dataframe['constant']=1\n", 48 | " features=['constant']+features\n", 49 | " features_matrix=dataframe[features].values\n", 50 | " output=dataframe['price'].values\n", 51 | " return features_matrix,output" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 6, 57 | "metadata": { 58 | "collapsed": true 59 | }, 60 | "outputs": [], 61 | "source": [ 62 | "def predict(features_matrix,weights):\n", 63 | " prediction=np.dot(features_matrix,weights)\n", 64 | " return prediction" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 7, 70 | "metadata": { 71 | "collapsed": true 72 | }, 73 | "outputs": [], 74 | "source": [ 75 | "def derivative(errors,feature):\n", 76 | " derivative=2*np.dot(feature,errors)\n", 77 | " return derivative" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 8, 83 | "metadata": { 84 | "collapsed": true 85 | }, 86 | "outputs": [], 87 | "source": [ 88 | "def gradient_descent(feature_matrix,output,initial_weights,step,tolerance):\n", 89 | " converged=False\n", 90 | " weights=np.array(initial_weights)\n", 91 | " while not converged:\n", 92 | " predictions=np.dot(feature_matrix,weights)\n", 93 | " errors=predictions-output\n", 94 | " gradient_sum_square=0\n", 95 | " for i in range(len(weights)):\n", 96 | " derivative=2*np.dot(feature_matrix[:,i],errors)\n", 97 | " gradient_sum_square+=derivative**2\n", 98 | " weights[i]-=step*derivative\n", 99 | " grad_mag=sqrt(gradient_sum_square)\n", 100 | " if grad_mag\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mpandas\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mDataframe\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mpolynomial_dataframe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeature_col\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdegree\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[0mnew_frame\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataframe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", 16 | "\u001b[1;31mImportError\u001b[0m: cannot import name 'Dataframe'" 17 | ] 18 | } 19 | ], 20 | "source": [ 21 | "from pandas import Dataframe as df\n", 22 | "import pandas as pd\n", 23 | "import numpy as np\n", 24 | "def polynomial_dataframe(feature_col,degree):\n", 25 | " new_frame=pd.Dataframe()\n", 26 | " new_frame['pwr_1']=feature_col\n", 27 | " if degree>1:\n", 28 | " for i in range(2,degree+1):\n", 29 | " col_name=str(pwr_+i)\n", 30 | " new_frame[col_name]=feature_col**i\n", 31 | " return new_frame" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 7, 37 | "metadata": {}, 38 | "outputs": [ 39 | { 40 | "data": { 41 | "text/plain": [ 42 | "pandas.core.frame.DataFrame" 43 | ] 44 | }, 45 | "execution_count": 7, 46 | "metadata": {}, 47 | "output_type": "execute_result" 48 | } 49 | ], 50 | "source": [ 51 | "train=pd.read_csv('wk3_kc_house_set_1_data.csv')\n", 52 | "\n", 53 | "type(train)\n" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": null, 59 | "metadata": { 60 | "collapsed": true 61 | }, 62 | "outputs": [], 63 | "source": [] 64 | } 65 | ], 66 | "metadata": { 67 | "kernelspec": { 68 | "display_name": "Python 3", 69 | "language": "python", 70 | "name": "python3" 71 | }, 72 | "language_info": { 73 | "codemirror_mode": { 74 | "name": "ipython", 75 | "version": 3 76 | }, 77 | "file_extension": ".py", 78 | "mimetype": "text/x-python", 79 | "name": "python", 80 | "nbconvert_exporter": "python", 81 | "pygments_lexer": "ipython3", 82 | "version": "3.6.3" 83 | } 84 | }, 85 | "nbformat": 4, 86 | "nbformat_minor": 2 87 | } 88 | -------------------------------------------------------------------------------- /Machine Learning Specialization-Regression/WK4/WK4-2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np\n", 10 | "import pandas as pd\n", 11 | "sales=pd.read_csv('kc_house_data.csv')" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 2, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "def get_numpy_data(data_frame, features, output):\n", 21 | " data_frame['constant']=1\n", 22 | " features=['constant']+features\n", 23 | " feature_matrix=data_frame[features].values\n", 24 | " output_array=data_frame[output].values\n", 25 | " return (feature_matrix, output_array)\n", 26 | "\n", 27 | "def predict(feature_matrix,weights):\n", 28 | " prediction=np.dot(feature_matrix,weights)\n", 29 | " return prediction" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 6, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "def feature_derivative_ridge(errors, feature, weight, l2_penalty, feature_is_constant):\n", 39 | " if feature_is_constant==False:\n", 40 | " derivative=2*np.dot(feature,errors)+2*l2_penalty*weight\n", 41 | " else:\n", 42 | " derivative=2*np.dot(feature,errors)\n", 43 | " return derivative" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 7, 49 | "metadata": {}, 50 | "outputs": [ 51 | { 52 | "name": "stdout", 53 | "output_type": "stream", 54 | "text": [ 55 | "-56554166815950.0\n", 56 | "-56554166815950.0\n", 57 | "-22446749330.0\n", 58 | "-22446749330.0\n" 59 | ] 60 | } 61 | ], 62 | "source": [ 63 | "(example_features, example_output) = get_numpy_data(sales, ['sqft_living'], 'price')\n", 64 | "my_weights = np.array([1., 10.])\n", 65 | "test_predictions = predict(example_features, my_weights)\n", 66 | "errors = test_predictions - example_output # prediction errors\n", 67 | "\n", 68 | "# next two lines should print the same values\n", 69 | "print (feature_derivative_ridge(errors, example_features[:,1], my_weights[1], 1, False))\n", 70 | "print (np.sum(errors*example_features[:,1])*2+20.)\n", 71 | "\n", 72 | "\n", 73 | "# next two lines should print the same values\n", 74 | "print (feature_derivative_ridge(errors, example_features[:,0], my_weights[0], 1, True))\n", 75 | "print (np.sum(errors)*2.)" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 8, 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "data": { 85 | "text/plain": [ 86 | "array([ 1., 10.])" 87 | ] 88 | }, 89 | "execution_count": 8, 90 | "metadata": {}, 91 | "output_type": "execute_result" 92 | } 93 | ], 94 | "source": [ 95 | "my_weights" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": 10, 101 | "metadata": {}, 102 | "outputs": [], 103 | "source": [ 104 | "def ridge_regression_gradient_descent(feature_matrix, output, initial_weights, step_size, l2_penalty, max_iterations=100):\n", 105 | " weights = np.array(initial_weights) # make sure it's a numpy array\n", 106 | " #while not reached maximum number of iterations:\n", 107 | " # compute the predictions using your predict_output() function\n", 108 | " \n", 109 | " for iter in range(max_iterations):\n", 110 | " predictions=predict(feature_matrix,weights)\n", 111 | " errors=predictions-output\n", 112 | " # compute the errors as predictions - output\n", 113 | " \n", 114 | " for i in range(len(weights)): # loop over each weight\n", 115 | " # Recall that feature_matrix[:,i] is the feature column associated with weights[i]\n", 116 | " # compute the derivative for weight[i].\n", 117 | " #(Remember: when i=0, you are computing the derivative of the constant!)\n", 118 | " \n", 119 | " if i==0:\n", 120 | " de=feature_derivative_ridge(errors,feature_matrix[:,0],weights[0],l2_penalty,True)\n", 121 | " weights[0]-=de*step_size\n", 122 | " else:\n", 123 | " de=feature_derivative_ridge(errors,feature_matrix[:,i],weights[i],l2_penalty,False)\n", 124 | " weights[i]-=de*step_size\n", 125 | " # subtract the step size times the derivative from the current weight \n", 126 | " return weights" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 11, 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "train=pd.read_csv('kc_house_train_data.csv')\n", 136 | "test=pd.read_csv('kc_house_test_data.csv')" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": 12, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [ 145 | "simple_features = ['sqft_living']\n", 146 | "my_output = 'price'\n", 147 | "(simple_feature_matrix, output) = get_numpy_data(train, simple_features, my_output)\n", 148 | "(simple_test_feature_matrix, test_output) = get_numpy_data(test, simple_features, my_output)\n" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 47, 154 | "metadata": {}, 155 | "outputs": [], 156 | "source": [ 157 | "initial_weights=np.array([0.0,0.0])\n", 158 | "weights_0_penalty=ridge_regression_gradient_descent(simple_feature_matrix,output,initial_weights,1e-12,0.0,max_iterations=1000)" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": 48, 164 | "metadata": {}, 165 | "outputs": [ 166 | { 167 | "data": { 168 | "text/plain": [ 169 | "array([-1.63113515e-01, 2.63024369e+02])" 170 | ] 171 | }, 172 | "execution_count": 48, 173 | "metadata": {}, 174 | "output_type": "execute_result" 175 | } 176 | ], 177 | "source": [ 178 | "weights_0_penalty" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": 49, 184 | "metadata": {}, 185 | "outputs": [], 186 | "source": [ 187 | "weights_high_penalty=ridge_regression_gradient_descent(simple_feature_matrix,output,initial_weights,1e-12,1e11,max_iterations=1000)" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": 50, 193 | "metadata": {}, 194 | "outputs": [ 195 | { 196 | "data": { 197 | "text/plain": [ 198 | "array([ 9.76730382, 124.57217567])" 199 | ] 200 | }, 201 | "execution_count": 50, 202 | "metadata": {}, 203 | "output_type": "execute_result" 204 | } 205 | ], 206 | "source": [ 207 | "weights_high_penalty" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": 51, 213 | "metadata": {}, 214 | "outputs": [ 215 | { 216 | "data": { 217 | "text/plain": [ 218 | "[,\n", 219 | " ,\n", 220 | " ,\n", 221 | " ,\n", 222 | " ,\n", 223 | " ]" 224 | ] 225 | }, 226 | "execution_count": 51, 227 | "metadata": {}, 228 | "output_type": "execute_result" 229 | }, 230 | { 231 | "data": { 232 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAD8CAYAAABO3GKQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztvX18XFW1//9eM5NJWwRTQpHSUksvXKVQbCDWRri90WqhvWrz/fV6L1zuK708lSCgiBqpilbh20JBrShCgoVv41WRa5Wn28pDaRTNQFsoyoOUFtBay0NpKSDQpEnW749zzuTM5MxkZjKP6Xq/XueVc/be5+x9TmbOZ/Zea68tqophGIZhFIpQqRtgGIZhjGxMaAzDMIyCYkJjGIZhFBQTGsMwDKOgmNAYhmEYBcWExjAMwygoJjSGYRhGQTGhMQzDMApKRkIjIp8XkadE5EkR+ZmIjBKRo0XkERHZKiI/F5GoW7baPd7m5k/2XWexm75FRE7zpZ/upm0Tkct96VnXYRiGYZQXMlRkABGZAPwOmKqq74jI7cAaYB7wS1W9TURuAv6gqjeKyGeAE1W1RUTOAP6Pqv67iEwFfgbMAI4EHgD+0a3mWeDjwA5gI3Cmqj7t1pVxHenu47DDDtPJkydn/4QMwzAOYB599NFXVXXccK4RyaLcaBHZD4wBXgQ+CvyHm78KWALcCMx39wF+AfxARMRNv01Vu4EXRGQbjugAbFPV5wFE5DZgvoj8Kds6NI1qTp48mU2bNmV4u4ZhGAaAiPxluNcYcuhMVf8GXAdsxxGY14FHgb2q2usW2wFMcPcnAH91z+11y9f605POSZVem0MdhmEYRpkxpNCIyFicHsTROENeBwFzA4p6vQlJkZev9HR1JCAii0Rkk4hs2rVrV8AphmEYRqHJxBngY8ALqrpLVfcDvwQ+DNSIiDf0NhHY6e7vAI4CcPPfDezxpyedkyr91RzqSEBV21W1XlXrx40b1hCjYRiGkSOZCM12YKaIjHFtLbOBp4H1wL+6ZRYCd7r7d7nHuPkPuraTu4AzXI+xo4FjgQ04xv9jXQ+zKHAGcJd7TrZ1GIZhGGXGkM4AqvqIiPwCeAzoBTYD7cD/AreJyFVu2kr3lJXAj11j/x4c4UBVn3K9yJ52r3ORqvYBiMjFwL1AGLhFVZ9yr/XlbOowDMMwyo8h3ZtHCvX19WpeZ4ZhGNkhIo+qav1wrmGRAYYgFouxbNkyYrFYqZtiGEYFYe+OATKdR3NAEovFmD17Nj09PUSjUdatW0dDQ0Opm2UYRplj745ErEeThs7OTnp6eujr66Onp4fOzs5SN8kwjArA3h2JmNCkobGxkWg0SjgcJhqN0tjYWOomGYZRAdi7IxEbOktDQ0MD69ato7Ozk8bGxgO662sYRubYuyMR8zozDMMwUmJeZ4ZhGEbZY0JjGIZhFBQTGsMwDKOgmNAYhmEYBcWExjAMwygoJjSGYRhGQTGhMQzDMAqKCY1hGIZRUExoDMMwjIJiQmMYhmEUFBMawzAMo6AMKTQi8j4Redy3vSEil4rIoSJyv4hsdf+OdcuLiFwvIttE5I8icpLvWgvd8ltFZKEv/WQRecI953oRETc96zryjS1eZBilx76HFY6qZrwBYeAl4L3AcuByN/1y4Bp3fx6wFhBgJvCIm34o8Lz7d6y7P9bN2wA0uOesBea66VnVkW47+eSTNVu6urp09OjRGg6HdfTo0drV1ZX1NQzDGB72PSwtwCbNQieCtmyHzmYDz6nqX4D5wCo3fRXQ5O7PBzrcNj4M1IjIeOA04H5V3aOqrwH3A6e7eYeoasy9qY6ka2VTR16xxYsMo/TY97DyyVZozgB+5u6/R1VfBHD/Hu6mTwD+6jtnh5uWLn1HQHoudSQgIotEZJOIbNq1a1cWt+lgixcZRumx72Hlk/HCZyISBT4FLB6qaECa5pCeSx2JCartQDs469EMcc1B2OJFhlF67HtY+WSzwuZc4DFVfdk9fllExqvqi+6w1Stu+g7gKN95E4GdbnpjUnqnmz4xoHwudeSdhoYG+2AbRomx72Flk83Q2ZkMDJsB3AV4nmMLgTt96c2uZ9hM4HV32OteYI6IjHW9x+YA97p5b4rITNfbrDnpWtnUYRiGYZQZGfVoRGQM8HHgAl/y1cDtInIusB34tJu+BscrbBvwNnA2gKruEZErgY1uuW+p6h53/0Lg/wGjcbzJ1uZSh2EYhlF+iOPoNfKpr6/XTZs2lboZhmEYFYWIPKqq9cO5hkUGMAzDMAqKCY1hGIZRUExoDMMwjIJiQmMYhmEUFBMawzAMo6CY0BiGYRgFxYTGMAzDKCgmNIZhGEZBMaExDMMwCooJjWEYhlFQTGgMwzCMgmJCYxiGYRQUExrDMAyjoJjQGIZhGAXFhMYwDMMoKCY0hmEYRkHJSGhEpEZEfiEiz4jIn0SkQUQOFZH7RWSr+3esW1ZE5HoR2SYifxSRk3zXWeiW3yoiC33pJ4vIE+4517tLOpNLHYZhGEZ5kWmP5nvAr1X1/cAHgD8BlwPrVPVYYJ17DDAXONbdFgE3giMawDeADwEzgG94wuGWWeQ773Q3Pas6DMMwjPJjSKERkUOAWcBKAFXtUdW9wHxglVtsFdDk7s8HOtThYaBGRMYDpwH3q+oeVX0NuB843c07RFVj6qwr3ZF0rWzqMAzDMMqMTHo0U4BdwK0isllEfiQiBwHvUdUXAdy/h7vlJwB/9Z2/w01Ll74jIJ0c6jAMwzDKjEyEJgKcBNyoqnXAWwwMYQUhAWmaQ3o6MjpHRBaJyCYR2bRr164hLmkYhmEUgkyEZgewQ1UfcY9/gSM8L3vDVe7fV3zlj/KdPxHYOUT6xIB0cqgjAVVtV9V6Va0fN25cBrdqGIZh5JshhUZVXwL+KiLvc5NmA08DdwGe59hC4E53/y6g2fUMmwm87g573QvMEZGxrhPAHOBeN+9NEZnpeps1J10rmzoMwzCMMiOSYblLgJ+ISBR4HjgbR6RuF5Fzge3Ap92ya4B5wDbgbbcsqrpHRK4ENrrlvqWqe9z9C4H/B4wG1robwNXZ1GEYhmGUH+I4eo186uvrddOmTaVuhmEYRkUhIo+qav1wrmGRAQzDMIyCYkJjGIZhFBQTGsMwDKOgmNAYhmEYBcWEZghisRjLli0jFouVuimGUVHYd8fwyNS9+YAkFosxe/Zsenp6iEajrFu3joaGhlI3yzDKHvvuGH6sR5OGzs5Oenp66Ovro6enh87OzlI3yTAqAvvuGH5MaNLQ2NhINBolHA4TjUZpbGwsdZMMoyKw747hx4bO0tDQ0MC6devo7OyksbHRuv6GkSH23TH8WGQAwzAMIyUWGaAImOeMMZKwz7NRCmzoLA2xWIyPfOQjcc+Z9evX2xCAURbEYrGsh6XME8woFdajSUNHRwfd3d2oKt3d3XR0dJS6SYYRF4wrrriC2bNnZ9w7MU8wo1SY0BhGhZGrYJgnmFEqTGjS0NzcTDQaRUSIRqM0NzeXukmGkbNgeJ5gV155pQ2bGUXFvM6GIJexcMMoNPa5NIpFPrzOzBnAMCqQhoaGnATGBMooBRkJjYj8GXgT6AN6VbVeRA4Ffg5MBv4M/JuqviYiAnwPZ6nlt4H/UtXH3OssBL7mXvYqVV3lpp/MwFLOa4DPqarmUkc+MS8dYyRhn2ejVGRjo/mIqk73daEuB9ap6rHAOvcYYC5wrLstAm4EcEXjG8CHgBnAN0RkrHvOjW5Z77zTc6kj35iXjjGSsM+zUSqG4wwwH1jl7q8CmnzpHerwMFAjIuOB04D7VXWPqr4G3A+c7uYdoqoxdQxGHUnXyqaOvGJeOsZIwj7PRqnI1EajwH0iokCbqrYD71HVFwFU9UUROdwtOwH4q+/cHW5auvQdAenkUMeLGd5PRli8JmMkYZ9no1RkKjSnqOpO90V/v4g8k6asBKRpDunpyOgcEVmEM7TGpEmThrhkMLkaXQ2jHLHPs1EKMho6U9Wd7t9XgF/h2Fhe9oar3L+vuMV3AEf5Tp8I7BwifWJAOjnUkdzudlWtV9X6cePGZXKrhmEYRp4ZUmhE5CAROdjbB+YATwJ3AQvdYguBO939u4BmcZgJvO4Of90LzBGRsa4TwBzgXjfvTRGZ6XqTNSddK5s6DMMwjDIjk6Gz9wC/cjSACPBTVf21iGwEbheRc4HtwKfd8mtw3I634bgenw2gqntE5Epgo1vuW6q6x92/kAH35rXuBnB1NnUYhmEY5YdFBjAMwzBSYuvRGIZhGGWPCc0Q2EJRhlFY7Ds28rFYZ2mwkB2GUVjsO3ZgYD2aNFjIDsMoLPYdOzAwoUmDhewwjMJi37EDAxs6S4OF7DCMwmLfsQMDE5oheOKJJ+js7KS2tta+BEZOBK0BY+vCDGBhcUY+JjRpaG9v54ILLgDgvvvuA2DRokWlbJJRYQQZuwEzgBsHFGajScPq1avTHhvGUAQZu80AbhxomNCkYcGCBWmPDWMogozdZgA3DjRs6CwN3jDZ6tWrWbBggQ2bGVmTythtBnDjQMJinRmGYRgpsVhnhmEYRtljQjMEFofJKGfs82lUAmajSYPFYTLKGft8GpWC9WjSYG6oRjljn0+jUjChSYO5oRrljH0+jUohY6ERkbCIbBaRe9zjo0XkERHZKiI/F5Gom17tHm9z8yf7rrHYTd8iIqf50k9307aJyOW+9KzryCeea+qVV15pwxJG2WGfT6NSyNi9WUQuA+qBQ1T1EyJyO/BLVb1NRG4C/qCqN4rIZ4ATVbVFRM4A/o+q/ruITAV+BswAjgQeAP7RvfyzwMeBHcBG4ExVfTrbOtK139ybDcMwsqdo7s0iMhH4F+BH7rEAHwV+4RZZBTS5+/PdY9z82W75+cBtqtqtqi8A23BEZwawTVWfV9Ue4DZgfo515B3z6jFKhX32jJFCpl5nK4BW4GD3uBbYq6q97vEOYIK7PwH4K4Cq9orI6275CcDDvmv6z/lrUvqHcqzj1QzvJyPMq8coFfbZM0YSQ/ZoROQTwCuq+qg/OaCoDpGXr/Sh6o8jIotEZJOIbNq1a1fAKekxrx6jVNhnzxhJZDJ0dgrwKRH5M86w1kdxejg1IuL1iCYCO939HcBRAG7+u4E9/vSkc1Klv5pDHQmoaruq1qtq/bhx4zK41UTMq8coFfbZM0YSQwqNqi5W1YmqOhk4A3hQVc8C1gP/6hZbCNzp7t/lHuPmP6iOx8FdwBmux9jRwLHABhzj/7Guh1nUreMu95xs68grDQ0NrFixgtmzZ7NixQobujiAKba9xDzKjJHEcCIDfBm4TUSuAjYDK930lcCPRWQbTi/jDABVfcr1Insa6AUuUtU+ABG5GLgXCAO3qOpTudSRb2KxGBdddBG9vb08+OCDTJs2zb7wI5xUq2GavcQwhoGqHhDbySefrNnS1NSkOLYfBbSpqSnraxiVQ1dXl44ePVrD4bCOHj1au7q6VFV16dKlGg6HFdBwOKxLly4tWVsMo9gAm3SY71+LDJCGnTt3pj02RhZ+A/y+ffvo6OgASmMvMWcAYyRhQpOG5BeKGWRHNo2NjUQizmiyqnLLLbcQi8VKYi8xZwBjJGHRm9NQU1OT9tgoT5LtLEF2lyAaGhqYO3cud9xxBwB9fX10dnbS0NAQ34pFqpU5DaMSMaFJw1NPPZX22Cg/kg33K1as4LOf/Wz8eP369Slf2rFYjLVr18aPw+FwSXsSxRY3wygUNnSWht/85jdpj43yo7Ozk+7ubvr6+uju7mblypV0d3ejqnR3d8ftLqnO7e11AlGICOecc4696A0jD5jQpOGwww4DzsILVOAcG+VMbW0t/f39APT39zNq1KiMz/XbRUaNGkVzc3OhmmkYBWftWjjqKBCBrVtL2xYTmjTMnDkT+I57dJh7bJQzu3fvJhRyPtahUIipU6cSjUYREaLRaFrxsEmSRiXT3Q1LlzrCIgLz5sGOHU5eYUIOZ07GywRUOrksExCLxfjwh/8BOJxo9Cg6O2+3l0+ZEzS5EjCjujEi2bEDPv95+MUvBuf94z9CWxsM18yYj2UCTGiGYOzY/ezdW8U992zkX/7lgwVomZFvMvUyM4xKZP16WLQItm0bnPdv/wbf/jZMnJi/+vIhNOZ1NgRVVVUAfPCDJjKVgnlrGSOJ/fvhBz+Ayy4Lzl+61Mmrri5uu7LBbDRDsH//fgA2btxY4pYYhcIWGDPKjZdfhoULHdtKNJooMhMnwpo1oOpsixeXt8iA9WjSEovF2LvXsdF88pOf5Pe//5X9Ui4ixRgCy3fAzFzbnO97TXc9G1osT2IxuOACeOKJwXmf+AR873swZUrx25UXhhssrVK2XIJqHnPMMQovu78bxukxxxyT9TWM3ChWUMl8BszMtc35vtd017NgneVDb6/qTTd5/ZLB2xVXqL71VqlbmZ+gmtajScP27dvTHhuFIyioZCa/zLP9te7NnfF6NMOJBOBNFu3v76e7uzuwzanOCwqg2dnZSW1tLbt3786q95Hu2WXyXI3CsWcPfOUrjjdYMocdBu3t0NRUenfkfGNCk4Z3vetd7NmTeGwUh2QBqK2tZdmyZQnxy4LcmLMdBstnTLHkyaK1tbU53+vs2bPjohUKhaiurs54WC+deOZTWI3MeOwxuPBC2LBhcN7HPgbf/z68//3Fb1cxMaFJw1tvvZX22CgcfgGora3l0ksvTYhftnr16viL2N8LyOXXer681LzJop447N69O6PzksXO63X4RSvb+0klnhass/CoQkeH44Lc0zM4/4tfhK9/HQ4+uPhtKxnDHXurlC0XG82ECRMSbDQTJkzI+hpGerq6unTp0qVpbQV+O0ooFNJIJKKhUCh+7NkahrI/ZFLXcO8lH/YP7zpB92iUJ6+/rnrppcG2ljFjVH/yE9X+/lK3MjfIg41m6AIwCtgA/AF4Cvimm3408AiwFfg5EHXTq93jbW7+ZN+1FrvpW4DTfOmnu2nbgMt96VnXkWrLRWhmzZqVIDSzZs3K+hpGavwv1KqqKm1ra0vI80TB/wKvqqqKv4BFRI855piU5wXVVWgjeL7EzLtOW1tbQcXRyJ2nnlJtbAwWlw9/WPXxx0vdwvxQLKER4F3ufpX7Yp8J3A6c4abfBFzo7n8GuMndPwP4ubs/1RWraldAngPC7vYcMAWIumWmuudkVUe6LRehGT9+fILQjB8/PutrGKlZunRpXDQAjUQiKXsmXV1d2tLSok1NTRqNRhPEJhqNDvKsSn45++sKhUIJ3mWF7ukYI4P+ftXbb1d997uDxeWii1T37Cl1K/NPUYQmoTCMAR4DPgS8CkTc9AbgXnf/XqDB3Y+45cTtzSz2Xete97z4uTrQ61nsnpNVHenanovQRCKRBKGJRCJZX8NITVdXl1ZVVcWFxhOAIJdjv/hUV1frcccdFz8P0JaWlvg1g3oubW1tCeW9XlCp3H3zIW4mkIXn739XXbw4WFhEVH/0I9W+vlK3srDkQ2gycgYQkTDwKHAMcIPbA9mrqr1ukR3ABHd/AvBXAFXtFZHXgVo3/WHfZf3n/DUp/UPuOdnW8WpSuxcBiwAmTZqUya0m4BljUx0bw6OhoYHPf/7zXHfddagq1dXVcS+oZM+ojo4O9u3b5/0Y4eAUllS/++6+ffvo6OigoaEhpaG+EO6+Q7lY52OSaL4nmhoDbNsGl1wCv/714LyTToIbb4QZM4rfrkomI6FR1T5guojUAL8Cjgsq5v4N8gDXNOlBYXDSlU9XR2KCajvQDk5QzYBz0hKJRBK8Rrz15I3h4b2Ia2tr+f73vw84z3bFihXxl6XfMwrglltuiYtMJBLh3HPPZfPmzezfv5+qqqp4+P/GxkYikQh9fX2oKrfccgvNzc3U1tYSDocBJ37d9u3bicViNDY2Eg6H6e/vT7uiZqbzczIRgHyIWynmw4zkiAJ33+14ib300uC8c86Bq6+GceOK366RQlZvTlXdKyKdODaaGhGJuD2OicBOt9gO4Chgh4hEgHcDe3zpHv5zgtJfzaGOvNKT5JuYfGxkj/9FLCL09/fT39+PiCS4A/tdjpctW0ZfXx/grHx59tlnM23aNEKhECISX3/GO+/ss8+mra0NVaWvr4+Ojg5uueUW9u/fH+/V3HzzzaxatYoVK1Yg7uw4STFLzhMkT9TSvdQzEYB8zGUp9nyYkdaD2rcPrr3WcTMO4vrrnbkv9tsyPwwZVFNExrk9GURkNPAx4E/AeuBf3WILgTvd/bvcY9z8B91xvruAM0SkWkSOBo7F8WbbCBwrIkeLSBTHuH+Xe062dRhljv9F7PUiwuFw2pdl0MqX3rLLqkpvb298Hg1Ac3Mzo0aNil/3pZdeiv9I6O/vp7e3Ny4Eq1evTnkdj46ODnp6elBVenp60i4H7W9rqnvKxwJrxV6kLVX0gkpi+3ZYsMCZdT96dKLIvP/98JvfDFhgLrnERCavDGXEAU4ENgN/BJ4Evu6mT8ERim3A/wDVbvoo93ibmz/Fd62v4th3tgBzfenzgGfdvK/60rOuI9WWizPAmDFjEpwBxowZk/U1jESSje+Zuu8mG767urq0urpaRUSrq6vTzplpaWlJcAQIh8MJ9Q/lDJB8vud4kGlbRwKVGiPtgQdUjz462Jh/5pmqf/tbqVtY/lBsr7NK3nIRmtbW1gShaW1tzfoaxmCGehGny/fPL4lGo4HuzUHn+EUpWdwyaU86UcvlHiuRSrinnh7V664LFhZQXb5ctbu71K2sLExoCiw0NmGzcGQysbK6ulpbWloSBCFo4mYmUZfT1ZdLj2qosl47o9Fowj2UgkoQiOGwc6fqf/xHsLBMnqx6772lbmFlY0JTYKGJRqMJQhONRrO+hjGYdMMw/jk0+CZktrS0aEtLy6BQNNkOwWXajuEQdA+lGm6q1CGvoXjoIdWpU4PFpalJ9YUXSt3CkUM+hMbMXWno7e1Ne2zkRjrPLM+Y7s2ZUXUM8G1tbXEjOxAPrrl79+5BQTczNY4XykU41T2UIiT/SFkWoK8PbroJLr44OH/JEmhtdYz8RvlhSzmnoaamJu2xMUA2yyGn88zyvKkuuOACotFo3OVY1fEKO+ecczj//PNZuHAh06ZNY/HixezevTv+Mu3u7mbJkiUZtaO2tpZQKEQoFEpox3CXdvbfQ3V19ZBedYUkEy+4cuXVV+H88x0vsUgkUWQOPxzuvHOgH/ONb5jIlDXD7RJVypbL0JkT5mRg6Oy4447L+hoHAtkMz7S1temcOXO0tbV1yKEuz2Osuro6HniztbU1MA5attGOPQM/ro3Ha08mXmjZPptS20fKoQ2ZsnGj6sknBw+JnXaa6pYtpW7hgQdmoyms0DgvrgGhCYVCWV/jQMBvkxCRhLhj/hdcunhj6V6Era2tGgqFVEQSlgjwOwF0dXXpnDlzUjoIJNcR5PIcCoXi9XiCNWfOnKK+oCtJFPJBX5/qLbeohsPB4nL55apvvlnqVh7YmNAUWGicl9CA0DgdQCMZf+8A0Gg0qq2trXHPMK9nMGfOnISXu/cSH2oNGX/gTUCrqqoCy6e6VlB6stAkb36xGe7aMkOF+/eXGYmG+2Ree0314ouDheXgg1Vvu61y124ZieRDaMwZwBg2yWFfent7ue666+JBSLu7u+ns7GTBggXcd9998fMWLFgwpLG6o6OD/fv3D6pv6tSpNDc3x5d19mKnLVzoBIzw8rxreIZ5r47m5mZ+9KMfBTp4hEIhpkyZwvPPP5/16pZ+vLAt6ZZk9od2CYVC8YgJlWy4D+LJJ52QLr/73eC8WbPghhvghBOK3y6jOJjQGFmRKrBic3Mzq1atiod68WKTAYTDYWpra9m9ezetra08/vjjTJ8+Pe4x5o/ZVVtby7JlyxKCaSbz0EMPsXHjRpqbm2lvb+fiiy+Oh5HxXuZ1dXVx8bn11lud7rvbFq/t5513XlwcRQQRQVWpqqriS1/6UoInm99RINPAkskRp4MExC+0XvtFJK+G+1IEw1SF225zAlX+/e+D8z/7WfjmN8H8aw4QhtslqpTNhs6GT7qhKW/op6WlZdAaM7NmzdJoNJoy7Is3rJSc7p83k7yJiDY1NblrBiXm+efYRCKR+DAYSeFjkieHemUjkYi2tLQERhDIdGgreTiRFMNwuYbkGe7/rBC8+aZqa2vwkFgkonrrrSN/7ZaRCDZ0ZhSTVIEV/VF9Fy5cmLBuj4jw0EMPxX/Ve4Es/dfZvXs3ixcvZtmyZQnpMLAuTTgcZt68edx9993xX/933333oDWCQqEQ4XA4PgTl9VI8DjnkkPi+54bc2dnJ9u3bufnmm+NDf21tbYwaNSo+zBWLxViyZEl8GGyooS0v6Kf3DObPn8+MGTMG9Sr8bShEj6PQ82iefdZxO77//sF59fXO2i319XmrzqhQTGiMjAkKTe9/ke3bt4+nn346HrrfC8nvvei9IaEFCxbw0EMPJVwnFouxYcOGuHCoKnV1ddTV1bF69WqmT5/OG2+8kSAaXvRnz/5x2WWXUVNTkzCBU0TiwgTwne98h6ampkEv27q6upSTLIFBtpahhraSn1Vra2vKF7x/SYR8k+/lBFThrruc+S27dg3OX7QIli6F2tphVWPkkXJYR0j8X9yRTH19vW7atCmrc5zJgi8Dh7vbLg6U55UKv+Hds7Fccsklg9bqERE+8IEP8Kc//Yne3l5CoRB1dXWce+65TJs2LR5qv66ujs2bNwca5r0oAJ5YJfdOIpEIN9xwQ7wdu3fvjn+Z2tvb4wL13e9+N+5QICJ88IMf5KSTTqKuri7BDrNixQo2b97MrbfeSm9vbzzKQGdnJ1dccQV9fX2EQiE+9rGPsWTJkiG/tIX4gudyzeG245134JprHJtKED/8oSMw7r/LKCPysY6QiDyqqsPrlw537K1SNrPR5I/kcf+mpqYEO4h/C4fDOnny5PgclWg0qtXV1XG7iBeBOejcoTYvmnZbW1uCK3Wyrae1tXWQrQbXZpJqTk4q24w/0Gex57wU097ywguqn/pUsL3lhBNUf/e7glVt5BH/HLdMgs8Ggc2jMaEpBckf3paWloSZ+ek2EYm/8HMVGE8GciN7AAAgAElEQVQkvJe83yHAm2SZ/OVqa2vTY445JlAIMzXut7S0xJ0aPJEs5pyXfLw00vHrX6tOmhQsLv/5n06UZKOyyMePk3wIjcU6M7LGHz8rHA7z0ksvcdppp1GfgdXXs2+EQiHnl45LJBJJWJJ5+vTpVFdXx4374XA4HvfMc2H2bER+V+r+/n6mT5+eEN/Ls9k899xzg9rzyU9+MqNVKhsaGpg0aRJ9fX1xw/r+/fuLuuJkvuOW9fQ4Q2Iiznb66c4qlB7XXeeUUYUf/xjGjx9e+43iU+yVWFNhzgBG1ngf3o6ODlauXMkdd9wBQFVVFZFIJMHbSkTiBv5wOMwPf/hDpk2bxpIlS3jggQfinmHnnXdefInmZDvLmDFjuPvuu51fRgE2knA4nFBnTU1NgidXZ2cn3d3dCcLmtXfu3Lns3r07o/v2G9YjkQiqSl9fX9GCVebDQ23nTvjCF5w5LslMmQJtbfCxj+WhsUbZUEhnk4wZqssDHAWsB/4EPAV8zk0/FLgf2Or+HeumC3A9zjLLfwRO8l1roVt+K7DQl34y8IR7zvUMOClkXUeqzWKd5Z+lS5cmDH95cc68ra2tLT6XREQSVijNJPSMNxznLXSWqmxbW5uGw+G0yzonh7EBtKmpKethBb9dplLikv3mN6rve1/wkNi//qvqX/5S6hYa5QzFsNEA470XOXAw8CwwFVgOXO6mXw5c4+7PA9a6YjATeEQHRON59+9Yd98Tjg1Ag3vOWmCum55VHem2XITGGfsfEJpIJJL1NSqJTFehTF5O2XtxJ7/kk+OJVVVVJVyjpaVFZ82apTNmzBgUYLOlpSXB5hMKhQatVJncjnTLOnti5F3PW0wtnc2jUoQkmf37Vb/3vWBhAdWrrlJ9551St9KoFIoiNINOgDuBjwNbgPE6IEZb3P024Exf+S1u/plAmy+9zU0bDzzjS4+Xy7aOdO02Z4D0ZBqQ0vPo8mbft7a2JvRg/L/2g2bGe/mjR48e5Axw1llnJXh3JTsXTJ06VVtaWrS1tVVnzJiRMPvfv0RAqojLnrh5bfUb9zMN0JnJcyyFOL3yiurZZwcLy5FHqt59d1GbY4wgii40wGRgO3AIsDcp7zX37z3Aqb70dUA98EXga770K9y0euABX/o/Afe4+1nVEdDeRcAmYNOkSZNyecAHhNCkC7Gf7Ok0Z86cBIEIh8MJa8Kkcnn2L2ecvNRx0CYiOnbs2LRl/GWT64pEIvFeUtD9Brkr+8nFw6vYyyb/6Ed/1PHjXwwUl4aGPfqFL9xYcb0xo/zIh9Bk7HUmIu8CVgOXquob6YoGpGkO6Wmbk8k5qtquqvWqWj9u3LghLnlg4k3o8gzzHrXu1O7a2tr4TP9oNMq4ceM8EQec4JnLly+PB5D0Vrn0jPfgeJRdcMEFXHLJJSxZsoS9e/fGPc9Soaq89tprGd2DDvywiB/39vZy8cUXD1ol0x9Kpq+vj97eXiZNmjTIWBrk4TXUypupQvT46x7Oyp39/XDzzQNeYuedN40XXzwinv+VrzgBLLu6Yjz++ARWrLiY2bNn51yfYeSLjLzORKQKR2R+oqq/dJNfFpHxqvqiiIwHXnHTd+A4EHhMBHa66Y1J6Z1u+sSA8rnUYWSJ93L0i0xfXx+XXHIJAJdeemncM+x973sfP/nJTwZd484774x7YQGDZvB/4hOfoK6ujgsuuACA++67j1mzZjF16lTWrVvH1q1bc27/EUccwcsvv5xQn/8+/LG9ksP2iwiRSGSQx5g3k37FihXxaAPAkDOs04V7yXWG9muvwde+5sy+D8gFzicUuoOrrrqSxYsXA4WPb2YY2TKk0IgzeWEl8CdV/Y4v6y4cL7Kr3b93+tIvFpHbgA8Br7tCcS+wVETGuuXmAItVdY+IvCkiM4FHgGbg+7nUkf3tG97L0R/OHmD//v2sXLkyIf3xxx8PvIbXg/A45ZRTeOSRR+ju7gZgzZo1PPvsswnn/Pa3vyUWi3HkkUcOq/2vvPJKgnuzJ3iqGl+ewCNZVL12P/HEE4Hrw/gFITngp/fyTg7v4nc/Brjwwgvj9fvP7+joSOmm/Ic/OGu3BHVEPvIR+MEP4PXXE9vpF7V8xzczjGEz1NgacCrOsNQfgcfdbR5Qi2Mb2er+PdQtL8ANwHM4Lsv1vmudg+OSvA0425deDzzpnvMDBtybs64j1WbOAKnp6urSpqamBON7JBIJdAkO2pLLRqPRBBtNOBzW4447LqNr5bI1NTXFjfxdXV0JHmZ+L7RUTgh+O9OMGTMS2u0PS5Nsf/EcHoLcqpOdIaqqquKOB/4wPKNHj9bf/75Lf/xj1dGjg435l12m+vrrwf+3oVbtNBuNMVwoxjIBqvo7gm0iALMDyitwUYpr3QIMWslKVTcBg9bXU9Xd2dZhDE1QkMV7770XcCY/fvKTn+SII47g5ptvBojPyFff0Nh73/teJk2aFF/pcvny5fGJm95MfW/FSHBm4G/ZsmVQWP9kkofdMmHPnj0JIfg7Ojri9fb09LB8+fJ4vjfRtL29Pd4Wz860Zs2ahOCgycNqCxcu5KWXXuKIIxy7SEdHR7zX1t3dTUdHx6AFzTx6e3u54IILmDRpEtu3b6e9/Wf093+Nd975Iqeckng/o0ZBezv85386tphUpJuIVxaT9AzDY7hKVSmb9Wgckn+Zt7a26jHHHJMQf8yb1+LNTfHciL1879g7f8aMGQleZOFweJBrcigUyjq22bhx4zKKn+Zd3+tpzJo1K9Arzet1dHV1Deph+Xsy3uYtkuafPOrlVVdXa1NTU2B57xx/jyYajepPf/qYfvSjwb2WmTNVH3usVJ8Kw0gNFlSz8EJzJI/rl7hmxAiN32033Uvc7zKcqtxwgmIWYguHw2kjSQM6a9YsraqqGlSmtbU1YfJpOByOu0cvXbo08Bk0NTWlnSj6+9936Zw5bVpd/WaguMyY8aj++tcbSvExMEY6vb2qDz+s+o1vqH7hC87ypzmSD6GxWGdD8CALeB/P0U4tr5e6MXmgsbExvliY8xkKxp+XPNyVPJRWLkQiEe6888607fKv9ukRCoWoqamhs7OT5cuXx12zL730UqZNm0ZtbW3gkN8RRxxBZ2dnwjDk22/DsmVw1VXgBLtIHL5qa4Nzz/XWbjlp2PdsHMD09cGmTbB2rbNt2JC67Kc+BbNmFa9tSZjQDMFY9gIQLXE7hot/wTJPKPzeWt6xPxJyKspNYMBZonnKlCkpPeMg+P5EJB4JuqGhgRkzZsSXiO7u7mbJkiVMmTIlvlqoRzQapbm5mYaGBv72t2rOOedgnnlmcJ0f+ADcdBPMnJm3WzUOJLIRk2ROPBHmzYP580v+ATShOQDwu+x60ZS9Lm1TUxNvv/02CxYsYNq0aVx++eX89re/LXWTs+aNN95IKTKhUIhFixbx5ptvDpoHFA6HWbFiRdxw7rkGe3NtHnjgASKRSLwX6J2zaNEdfPrTDfztb5DcM/mv/4Krr4b3vCffd2mMSPr64NFHHSFZsyY3MZk7FxoaoKqqcO0cBiY0BwD+CXze+i7ebP+dO3fGl1devnw5v/vd70rd3KwYM2YMb7/9dvxYkty0VJ2lBV566SXuvvvuQeerasIyAQ0NDaxYsYJrr72W5557jv7+fvr6+qirm8mmTf8M/F/6+py5LIl8DriBlpbzufHGG1O2N92yyuWwtrtRIPxisnYtPPJI5ueeeKIjJHPnwoc/XLZikpbhGnkqZcvVGeBlalVBx1Fbcc4A/ujGyYExkz2mMvXuKtYmIhnP40m+jxkzZgyaF5SqbKpgms65ExT+J9CQP2nS27p+/eAI1X7Ps6D/R6pYaMWOk2YUgN5e1UceUV2yRPVDHxr8oUm3nXii6pe/rNrZqdrTU+o7SQBzBjBS4R8uC4fDzJw5k3379nHuueeyaNEiVq9enVB+qPktxUZV2b9/f9bn9ff3s3HjRiKRSMp7qqqq4vOf/zw1NTWDeg8rVz7PO+/8ATh20HmzZ79Kff1tzJ9/cvyc6upmbrnlFvbv309VVRXNzc0p25YuNIyFjakQ+vsTbSYHWs8kR0xoRgjJwy7+F1dfX1/c7vL4448zbdo0pk+fzn333VfiVhcG1cSQOH5mzJiRYJPp7YXvfhcuu8wrcVZC+aamDYwb92PC4V7X+H9xQr73rDMZ8koXGsbCxpQR/f3OMNeaNdmLybRpjpDMm3fAiUlahtslqpRtJA+dBa0Z41/UK3kbN26cu6hb6YfICrUF3XsoFNK2tjZ96SXV5ubgEYyjjlK95JL/1Tlz5mhra+ugSZfJi79lO8RlYWPKhL4+1Q0bVL/5TWe2bDbDXNOmqba2qq5fr9rdXeo7KTjkYejMiyk24qmvr9dNmzZldY6I8DK1HM5uDqeWXeymXJ6XvwfT2dnJFVdckWDsz2XYaSSR7I7sLMTaBpwYUPpuqqtbWb/eiY7kDTmGQqFBz3Hp0qU0NjYOCrwJBPZqzMBfQryeiTfM9fDDmZ/r9Uy8Ya5opU9wyB0ReVRV64dzDRs6q0CSIwyvWLEiPuwiIgeMyKSbeNrfDyIXohoYX5+vfx3C4Wv51rcWu+vShOns7GT79u3xiNWqOkiw9u7dG1/Ppr+/Px6JedWqVXR3dxMKhbjhhhtYtGhRzksDGFkwHDE54YQB1+ADXEwKjQlNBdLZ2Rl/0XV3d7N27VqmTZvGkUceydy5c2lpaSmbnlchOfnkk6mpqfHZmsYCS4EWwBnnGGAXzoKrd3DWWWcxatTx1NbWxgU6FArx3//932zZsiX+7CKRCPPmzUuINvDtb38bVaW/vz++GBwQ/3/09/dz8cUXM23atJQGfuvlZEl/Pzz22MA8k2zFxOuZnHKKiUmpGO7YW6VsI8FG44Xznzx5ckrbRFNTkx577LElt5EUY5sxY4bCdIWHA4fSRR5QeF/guZ5rc5CrNzju1d6yA/4YaP7z58yZEw/S6bd5eXnJbuVBS16bPcalr09140bVb31LtaEhO5vJCSeofulLqg8+eEDYTIoN5t584BCLxZg1a1ZKbyqPO+64Y9CkxZFHM3AzGzYE/Tq9lnPO+RvHHPMevvrVr+K8+wfj9QZXr17Njh07BuWPGjUqHmLmnHPOoa2tLaGXKCLs3buXO+64g5qaGi677DK++93v0tfXF48o8NBDDyWs0pluAbUDAn/PZO3a4JXdUmE9k8pmuEpVKVul92iWLl1a8h5E6baDFb6T4gft3xXOTCjvRV5O9jzzR6T2/02O5HzcccelmcQ5uH0iEu8dzZkzJ17Ov3Ba8rVSTdrMxOusq6srYaG3sqKvT3XTJuuZjCAoxjIBOAuVvQI86Us7FLgfZ+XL+4GxbroA1+OsoPlH4CTfOQvd8luBhb70k3FWydzmniu51pFuq3ShaWtrS/mSC9qv/O04hfUp3kkPKUxLe/7kyZN1+vTpg55PJBLRs846K2ENHv9WVVUVH+Lyv/S7uroSRCR580Qlk6GxIEHJdEgtaJ2boouNJyZXXqn64Q9nJybHH6/6xS+amFQQxRKaWThRA/1Csxy43N2/HLjG3Z8HrHXFYCbwiA6IxvPu37HuviccG3BiqYt77txc6hhqqzSh8b+MvCWGk19ukUgkviRwLuFaym/7tMJrKd5R1yvU5HTdILFJFgwRSbC5BL30Uy0F7fVohjPHxr9OUFBPyF8u+X5SlR0W+RCTdetU9+3Lf9uMopIPoclkKeffisjkpOT5QKO7vwroBL7spne4jXtYRGpEZLxb9n5V3QMgIvcDp4tIJ3CIqsbc9A6gCUdIsqpDVV8c6l4qheTwMSISXzLYT3V1NccffzzPPvsse/fuLUFLh8sY4KvAVwLy+nC8xG7FeafmjvNRGdjv6+tLSPPSFyxYELej+L36PDtKQ0MD69ato7Ozk7179/L4448zffr0QaFscllGOdPIAP7o0uCE08k5ikB/P2zePGAz6erK/Nzjj0+0mVRX59YG44AgV2eA93gvdlV9UUQOd9MnAH/1ldvhpqVL3xGQnksdI0ZoOjo64nM50sUge+utt9iQTUjxsuAfcEY+5wXkPQpcCGwsWO2hUIhQKISqJkzIFBE2b97MsmXL2Lt3b/y59/f3J4h4LiKSCX4RS+f23NDQwPr16+no6ACIOyykZDhiMnXqwDwTExNjGOTb6yzI3UlzSM+ljsEFRRbh/Cxm0qRJQ1y2PGhvb+fmm29O+MUdCoUyWpCsfPkX4GZgfEDeSpyR0VeL0hIR4d///d/ZunUrPT098TVsVJWVK1cGCvt1113Hs88+S2tra0FD+2cqYoPK9ffD448PzDPJVky8nsmpp5qYGIUhk/E1YDKJNpotwHh3fzywxd1vA85MLgecCbT50tvctPHAM770eLls6xjqHsrZRpPJ/JjK2qoVvpZmGP8iheA4bPncQqGQHnPMMRktgRDkfZa8VVdXFz20f9ze87vfqT76qOpVV6meckp2NpOpU5114x94wGwmRtZQwnk0d+F4kV3t/r3Tl36xiNwGfAh4XZ1hr3uBpSIy1i03B1isqntE5E0RmQk8gjNB4vu51JHjfZScWCzGP//zP4+AsDFHASuA/y8g72ngAqB4i6p5c4l6e3sJh8P+H02BZaPRKP39/Wn/DwUP7e/vmaxdC7//PQ04njJDYj0To5wZSomAn+HYP/bj2EPOBWqBdTiux+uAQ92yAtwAPIfjslzvu845OC7J24Czfen1wJPuOT9gwL056zrSbeXaowmalV4522yF51L8kP5vhfEladehhx46qBcza9Ysra6uHpQeiUS0qalJm5qaEubdhEIhraqqSiiflx5Nf7/qY4/l1DPZNW6c6mWXJfRMLOKzUWiw6M2ZU47Rm9vb27ngggvydr3CE8FZsvi6FPlfAr6H85ukvPDWoens7KS2tpbNmzcDUFdXx6WXXhp3vkguD8QN73V1dQmz/CGFjUY1sWeSzfLYxx0X75k8XFXFR+fOTRmU04J2GsXAojdXKN7L6frrry91UzLgCOBa4D8D8l7AGRK7v6gtyoUjjzwy0NjuhYRJ/gFx0kknJbgrD3qpP/AADaNH07B+PQ1r18JXgly0U+ATE049FUaNCiw2E9J6otmqnEalYEJTZNrb27nwwgvLbunkRD4MtAPHB+T9Cvg88Jeitmg4hMNhWltbA/MaGxvjyz57YlNdXe0syezrmUxsb+ftd95xTnrnHcfdNx0ZislQpPNEs1U5jUrBhKaIxGKxMhWZEE5o/RtS5H8DuAYYPGm0GNTU1GQ8IVVEEJGBsWERzj///NTh+VU5sa+Pj6syFzgVoLvbWZ/Ex1FBlb3//QPzTIYhJrmS6dwbwyg1JjRFpLOzs4xEphbHoe+8gLyXgPOBe4raolRkKjKhUIhPfepTzJ07l0svvTS+EFnd9On8YdUq1p53HnN6exO8uBpwYiCl5P3vT7CZrI/FyuqlXqgJpIaRT0xoCoz/V3RtbW2JW3MycBOOo18yvwYuwXEKrExO7O9n2j338Mknn2SRN8zV3w8tzkJoH0hx3jMirFVlDfB7ERg1KtCwPhOYacNThpE1JjQFxDMgd3d3x5cdLi4C/BeOvSXoX70M+L/AW0Vs0/D4ADDX3WYFFejthW2DxfLFmhr+5803WaPKI9Eoax58MC4kr8Vi/HrJEtbdf78TmsYX38wwjOFjQlNAOjs7ecf9ZV08kXk38C3gswF5r+NE5Lm9SG3JjRNxIqGlFJMUPAPcGwrxv/39/JYBi1J1dTXr16zhg8BbnZ18I2noq6GhgQULFsSXhO7v7y+D3qdhjBxMaArIU089VaSajgd+SPBr+TfARUCx2pIZJzLQM/nnLM57Bie091pIEJOqqirOPfdc6urq6Lz44oQZ/meffTZAWqP57t27CYVC9Pf3EwqF2L17dy63ZRhGACY0BeLLX/4yP/3pTwtYwxk4Q2IHB+R9D/g68EYB6x+afItJEOFwmPPPP39QFOOLLrqI/v5+qqurqaurG3JiY2NjI9XV1eYqbBgFwIQmz8RiMS6//HJ++9vf5vnKBwFX4CzJk0wPjpfYj3GipRSPXMVkC7AGR0weAvYFlJk8eTI9f/mLM58lgEgkwg033MCiRYsS0hctWsS0adPiPZhMJjaaq7BhFA4TmjwSi8X4p3/6pzyG9D8WJ/zbnIC8DThrtzyWp7pSMw1HSOaRvZj4eyZBYpKO008/nZUrVwYGumxqagoM2++R7PabycRGcxU2jMJgQpMnYrEYZ555Zh5EZj7OkNjhAXltOKtR7hlmHYPxxGQuA8uaZsJwxcTPWWedxc9//vP4kFdzczN1dXV85jOfiT9XEeFLX/oS11xzTcbXtd6KYZQWC6qZhkyDara3t3PRRRfR29ubQ8vCOPPR/w34TED+hTjCM3yvtXIQEz8iwqRJkzjooIP43Oc+x6JFiwJn78discxXlDQMI6/kI6imCU0aHKEZx+Hs4nDewy5eHiQ0sViMU089NUv35ffirN3S5EvrBqpxVj5oAbJYJdFHuYmJh4gwf/587rnnnvgaMT/84Q8H2VcMwygvLHpzUfAEJLi3snz58gxF5uM4yxm/NyDvP3DWc8ts4uQJDMwzaczoDIdncYRkDYUREz8iEl+COhQKceONN6bssRiGMbIxoRkGsViM1L2kKpwox6lsCZfhLCYaLGAnMNAz+UgWbfLEZC3ODJpCikk0GiUUCnHiiSdSU1PD9OnTeeMNx6W6ubkZGDx3xQzuhnHgYUIzBIfjTNwLJaV74WX27fO/yscD3wbODLjSNpy1Wx6Mp5yA48X1d+B9wOIM21QMMRERDjnkEMaOHcvo0aM56aST2LVrF+PGjWPXrl0sWLAgo2EvExXDMCpWaETkdJyZiWHgR6p6dSHrq06an+LNzVA9BcdYf1zAWbdzAp9lLi8P2TNJ7tdsZWCeSSHEREQ48sgjOeuss+js7OTII49k7ty5g1aQNAzDGC4VKTQiEsZZPOXjwA5go4jcpapP57uuV1yvs3eS0hsbG+nr2w0cwvE8xVyuZR5r+AidGV97K46Q3AA8T6pBtNyIRCKMGTOGsWPHUldXx0EHHcRvfvMbpkyZwtVXX21CYhhG0ahIoQFmANtU9XkAEbkNZwJK3oXGC+OiSXNXGhoa2HnoPzJ+z9a0Z3ti4vVMkgUrW6qqqqiqquK9730vu3fvZvTo0Zx66qlZDWcZhmEUk0oVmgnAX33HO4AP5buSqqoqmvfv5+fAq+6xn/HnzIfrruOdiRO59cUX+d/+ftarZiUmY8aMQVXj69aPGjWKww8/nEmTJjF16lTq6upsOMswjIqmUoVGAtIGTQgSkUU4cfGZNGlS1pV4IUtq9u+nqqqKnp6exALXXgvXXstooC4W4/XOTubX1rJ27Vp27txJY2Mjb7zxBi+99BIvvPACO3fuZMKECRxyyCHs27ePc88913oghmGMeCpywqaINABLVPU093gxgKouS3VOLhM2DcMwDnTyMWEz2Wu3UtgIHCsiR4tIFCdm/l0lbpNhGIYRQEUOnalqr4hcDNyL4958i6qW18pehmEYBlChQgOgqmtwppoYhmEYZUylDp0ZhmEYFYIJjWEYhlFQTGgMwzCMgmJCYxiGYRSUipxHkwsisgv4S46nH4YTHKCSsDYXB2tzcbA2F4egNr9XVccN56IHjNAMBxHZNNwJS8XG2lwcrM3FwdpcHArVZhs6MwzDMAqKCY1hGIZRUExoMqO91A3IAWtzcbA2Fwdrc3EoSJvNRmMYhmEUFOvRGIZhGAXFhGYIROR0EdkiIttE5PIStuMoEVkvIn8SkadE5HNu+qEicr+IbHX/jnXTRUSud9v9RxE5yXethW75rSKysAhtD4vIZhG5xz0+WkQecev/uRuBGxGpdo+3ufmTfddY7KZvEZHTCtzeGhH5hYg84z7vhnJ/ziLyefdz8aSI/ExERpXjcxaRW0TkFRF50peWt2crIieLyBPuOdeLSNDaVcNt77XuZ+OPIvIrEanx5QU+v1TvkVT/o3y32Zf3RRFRETnMPS7OM1ZV21JsOJGhnwOmAFHgD8DUErVlPHCSu38w8CwwFVgOXO6mXw5c4+7Pw1lBWoCZwCNu+qHA8+7fse7+2AK3/TLgp8A97vHtwBnu/k3Ahe7+Z4Cb3P0zgJ+7+1PdZ18NHO3+T8IFbO8q4Dx3PwrUlPNzxllx9gVgtO/5/lc5PmdgFnAS8KQvLW/PFtgANLjnrAXmFqC9c4CIu3+Nr72Bz48075FU/6N8t9lNPwon4v1fgMOK+YwL9nIZCZv7MO/1HS8GFpe6XW5b7gQ+DmwBxrtp44Et7n4bcKav/BY3/0ygzZeeUK4A7ZwIrAM+Ctzjfjhf9X1R48/Y/RI0uPsRt5wkP3d/uQK09xCcl7YkpZftc2ZgafND3ed2D3BauT5nYDKJL+68PFs37xlfekK5fLU3Ke//AD9x9wOfHyneI+m+C4VoM/AL4APAnxkQmqI8Yxs6S4/3BfbY4aaVFHeoow54BHiPqr4I4P493C2Wqu3FvqcVQCvQ7x7XAntVtTeg/njb3PzX3fLFbPMUYBdwqzjDfT8SkYMo4+esqn8DrgO2Ay/iPLdHKe/n7Cdfz3aCu5+cXkjOwflVzxDtCkpP913IKyLyKeBvqvqHpKyiPGMTmvQEjT2W1E1PRN4FrAYuVdU30hUNSNM06XlHRD4BvKKqj2bQrnR5xfw/RHCGHW5U1TrgLZzhnFSUvM2uTWM+znDNkcBBwNw09Ze8zRmSbTuL2n4R+SrQC/zES8qyXUVpr4iMAb4KfD0oO0Ub8tpmE5r07MAZ1/SYCOwsUVsQkSockfmJqv7STX5ZRMa7+eOBV9z0VG0v5j2dAnxKRP4M3IYzfLYCqBERb9E9f/3xtrn57wb2FLnNO4AdqvqIe/wLHOEp5+f8MeAFVfgB+3MAAAHLSURBVN2lqvuBXwIfpryfs598Pdsd7n5yet5xjeOfAM5Sdwwph/a+Sur/UT75B5wfIX9wv4sTgcdE5Igc2pzbM873+OtI2nB+3T7v/pM8I97xJWqLAB3AiqT0a0k0pC539/+FRCPfBjf9UBwbxFh3ewE4tAjtb2TAGeB/SDSAfsbdv4hEI/Xt7v7xJBpZn6ewzgAPAe9z95e4z7hsnzPwIeApYIzbjlXAJeX6nBlso8nbswU2umU9Q/W8ArT3dOBpYFxSucDnR5r3SKr/Ub7bnJT3ZwZsNEV5xgV7sYyUDccr41kcr5GvlrAdp+J0Uf8IPO5u83DGedcBW92/3odBgBvcdj8B1PuudQ6wzd3OLlL7GxkQmik4nivb3C9atZs+yj3e5uZP8Z3/VfdetjBMT6IM2jod2OQ+6zvcL1pZP2fgm8AzwJPAj92XXdk9Z+BnOHak/Ti/js/N57MF6t1n8BzwA5KcOvLU3m049gvve3jTUM+PFO+RVP+jfLc5Kf/PDAhNUZ6xRQYwDMMwCorZaAzDMIyCYkJjGIZhFBQTGsMwDKOgmNAYhmEYBcWExjAMwygoJjSGYRhGQTGhMQzDMAqKCY1hGIZRUP5/MHhp6xZa0ywAAAAASUVORK5CYII=\n", 233 | "text/plain": [ 234 | "
" 235 | ] 236 | }, 237 | "metadata": {}, 238 | "output_type": "display_data" 239 | } 240 | ], 241 | "source": [ 242 | "import matplotlib.pyplot as plt\n", 243 | "%matplotlib inline\n", 244 | "plt.plot(simple_feature_matrix,output,'k.',\n", 245 | " simple_feature_matrix,predict(simple_feature_matrix, weights_0_penalty),'b-',\n", 246 | " simple_feature_matrix,predict(simple_feature_matrix, weights_high_penalty),'r-')" 247 | ] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": 52, 252 | "metadata": {}, 253 | "outputs": [], 254 | "source": [ 255 | "rss_init=(((predict(simple_test_feature_matrix,initial_weights))-test_output)**2).sum()\n", 256 | "rss_no=(((predict(simple_test_feature_matrix,weights_0_penalty))-test_output)**2).sum()\n", 257 | "rss_high=(((predict(simple_test_feature_matrix,weights_high_penalty))-test_output)**2).sum()" 258 | ] 259 | }, 260 | { 261 | "cell_type": "code", 262 | "execution_count": 53, 263 | "metadata": {}, 264 | "outputs": [ 265 | { 266 | "name": "stdout", 267 | "output_type": "stream", 268 | "text": [ 269 | " 1784273286136298.0 \n", 270 | " 275723632153607.44 \n", 271 | " 694642101489902.5\n" 272 | ] 273 | } 274 | ], 275 | "source": [ 276 | "print('',rss_init,'\\n',rss_no,'\\n',rss_high)" 277 | ] 278 | }, 279 | { 280 | "cell_type": "code", 281 | "execution_count": 34, 282 | "metadata": {}, 283 | "outputs": [], 284 | "source": [ 285 | "model_features = ['sqft_living', 'sqft_living15']\n", 286 | "my_output = 'price'\n", 287 | "(feature_matrix, output) = get_numpy_data(train, model_features, my_output)\n", 288 | "(test_feature_matrix, test_output) = get_numpy_data(test, model_features, my_output)" 289 | ] 290 | }, 291 | { 292 | "cell_type": "code", 293 | "execution_count": 54, 294 | "metadata": {}, 295 | "outputs": [], 296 | "source": [ 297 | "multi_weights_0_p=ridge_regression_gradient_descent(feature_matrix,output,[0.0,0.0,0.0],1e-12,0,max_iterations=1000)" 298 | ] 299 | }, 300 | { 301 | "cell_type": "code", 302 | "execution_count": 55, 303 | "metadata": {}, 304 | "outputs": [], 305 | "source": [ 306 | "multi_weights_high_p=ridge_regression_gradient_descent(feature_matrix,output,[0.0,0.0,0.0],1e-12,1e11,max_iterations=1000)" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": 56, 312 | "metadata": {}, 313 | "outputs": [ 314 | { 315 | "name": "stdout", 316 | "output_type": "stream", 317 | "text": [ 318 | "[ -0.35743483 243.05416982 22.41481497] [ 6.74296579 91.48927365 78.43658766]\n" 319 | ] 320 | } 321 | ], 322 | "source": [ 323 | "print(multi_weights_0_p,multi_weights_high_p)" 324 | ] 325 | }, 326 | { 327 | "cell_type": "code", 328 | "execution_count": 61, 329 | "metadata": {}, 330 | "outputs": [], 331 | "source": [ 332 | "mrss_init=(((predict(test_feature_matrix,np.array([0.0,0.0,0.0])))-test_output)**2).sum()\n", 333 | "mrss_no=(((predict(test_feature_matrix,multi_weights_0_p))-test_output)**2).sum()\n", 334 | "mrss_high=(((predict(test_feature_matrix,multi_weights_high_p))-test_output)**2).sum()" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": 62, 340 | "metadata": {}, 341 | "outputs": [ 342 | { 343 | "name": "stdout", 344 | "output_type": "stream", 345 | "text": [ 346 | " 1784273286136298.0 \n", 347 | " 274067615918575.56 \n", 348 | " 500404800500841.75\n" 349 | ] 350 | } 351 | ], 352 | "source": [ 353 | "print('',mrss_init,'\\n',mrss_no,'\\n',mrss_high)" 354 | ] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": 59, 359 | "metadata": {}, 360 | "outputs": [ 361 | { 362 | "data": { 363 | "text/plain": [ 364 | "77465.47605823533" 365 | ] 366 | }, 367 | "execution_count": 59, 368 | "metadata": {}, 369 | "output_type": "execute_result" 370 | } 371 | ], 372 | "source": [ 373 | "predict(test_feature_matrix[0,:],multi_weights_0_p)-test_output[0]" 374 | ] 375 | }, 376 | { 377 | "cell_type": "code", 378 | "execution_count": 60, 379 | "metadata": {}, 380 | "outputs": [ 381 | { 382 | "data": { 383 | "text/plain": [ 384 | "-39546.46967805945" 385 | ] 386 | }, 387 | "execution_count": 60, 388 | "metadata": {}, 389 | "output_type": "execute_result" 390 | } 391 | ], 392 | "source": [ 393 | "predict(test_feature_matrix[0,:],multi_weights_high_p)-test_output[0]" 394 | ] 395 | }, 396 | { 397 | "cell_type": "code", 398 | "execution_count": null, 399 | "metadata": {}, 400 | "outputs": [], 401 | "source": [] 402 | } 403 | ], 404 | "metadata": { 405 | "kernelspec": { 406 | "display_name": "Python 3", 407 | "language": "python", 408 | "name": "python3" 409 | }, 410 | "language_info": { 411 | "codemirror_mode": { 412 | "name": "ipython", 413 | "version": 3 414 | }, 415 | "file_extension": ".py", 416 | "mimetype": "text/x-python", 417 | "name": "python", 418 | "nbconvert_exporter": "python", 419 | "pygments_lexer": "ipython3", 420 | "version": "3.6.5" 421 | } 422 | }, 423 | "nbformat": 4, 424 | "nbformat_minor": 2 425 | } 426 | -------------------------------------------------------------------------------- /Machine Learning Specialization-Regression/WK5/WK5-1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import pandas as pd \n", 10 | "import sklearn.linear_model" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 2, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "dtype_dict = {'bathrooms':float, 'waterfront':int, 'sqft_above':int, 'sqft_living15':float, 'grade':int, 'yr_renovated':int, 'price':float, 'bedrooms':float, 'zipcode':str, 'long':float, 'sqft_lot15':float, 'sqft_living':float, 'floors':float, 'condition':int, 'lat':float, 'date':str, 'sqft_basement':int, 'yr_built':int, 'id':str, 'sqft_lot':int, 'view':int}\n", 20 | "\n", 21 | "sales = pd.read_csv('kc_house_data.csv', dtype=dtype_dict)" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 3, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "from math import log, sqrt\n", 31 | "sales['sqft_living_sqrt'] = sales['sqft_living'].apply(sqrt)\n", 32 | "sales['sqft_lot_sqrt'] = sales['sqft_lot'].apply(sqrt)\n", 33 | "sales['bedrooms_square'] = sales['bedrooms']*sales['bedrooms']\n", 34 | "sales['floors_square'] = sales['floors']*sales['floors']" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [] 43 | } 44 | ], 45 | "metadata": { 46 | "kernelspec": { 47 | "display_name": "Python 3", 48 | "language": "python", 49 | "name": "python3" 50 | }, 51 | "language_info": { 52 | "codemirror_mode": { 53 | "name": "ipython", 54 | "version": 3 55 | }, 56 | "file_extension": ".py", 57 | "mimetype": "text/x-python", 58 | "name": "python", 59 | "nbconvert_exporter": "python", 60 | "pygments_lexer": "ipython3", 61 | "version": "3.6.5" 62 | } 63 | }, 64 | "nbformat": 4, 65 | "nbformat_minor": 2 66 | } 67 | -------------------------------------------------------------------------------- /Machine Learning Specialization-Regression/WK5/WK5-2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 247, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import pandas as pd\n", 10 | "import numpy as np\n", 11 | "from math import sqrt" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 308, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "def get_numpy_data(dataframe,features,output):\n", 21 | " dataframe['constant']=1\n", 22 | " features=['constant']+features\n", 23 | " feature_matrix=dataframe[features].values\n", 24 | " output_array=dataframe[output].values\n", 25 | " return (feature_matrix,output_array)" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 309, 31 | "metadata": {}, 32 | "outputs": [], 33 | "source": [ 34 | "sales=pd.read_csv('kc_house_data.csv')" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 374, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "def predict(feature_matrix,weights):\n", 44 | " predictions=np.dot(feature_matrix,weights)\n", 45 | " return predictions\n" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 311, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "def normalize_features(features):\n", 55 | " norms=np.linalg.norm(features,axis=0)\n", 56 | " normalized_features=features/norms\n", 57 | " return (normalized_features,norms)" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 377, 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "sqft_bed,output_s=get_numpy_data(sales,['sqft_living','bedrooms'],'price')" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 378, 72 | "metadata": {}, 73 | "outputs": [ 74 | { 75 | "name": "stdout", 76 | "output_type": "stream", 77 | "text": [ 78 | "[1.47013605e+02 3.34257264e+05 5.14075870e+02]\n" 79 | ] 80 | }, 81 | { 82 | "data": { 83 | "text/plain": [ 84 | "array([1.47013605e+02, 3.34257264e+05, 5.14075870e+02])" 85 | ] 86 | }, 87 | "execution_count": 378, 88 | "metadata": {}, 89 | "output_type": "execute_result" 90 | } 91 | ], 92 | "source": [ 93 | "norm_sb,sb_norms=normalize_features(sqft_bed)\n", 94 | "print(sb_norms)\n", 95 | "(((sqft_bed)**2).sum(axis=0))**0.5" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": 379, 101 | "metadata": {}, 102 | "outputs": [], 103 | "source": [ 104 | "initial_weights=[1,4,1]" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 380, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "predict_1=predict(norm_sb,initial_weights)" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 381, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "ro=[]\n", 123 | "for i in range(len(initial_weights)):\n", 124 | " r=((output_s-predict_1+initial_weights[i]*norm_sb[:,i])*norm_sb[:,i]).sum()\n", 125 | " ro.append(r)" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": 382, 131 | "metadata": {}, 132 | "outputs": [ 133 | { 134 | "data": { 135 | "text/plain": [ 136 | "[79400300.0145229, 87939470.82325175, 80966698.66623947]" 137 | ] 138 | }, 139 | "execution_count": 382, 140 | "metadata": {}, 141 | "output_type": "execute_result" 142 | } 143 | ], 144 | "source": [ 145 | "ro" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 383, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "def lasso_coordinate_descent_step(i, feature_matrix, output, weights, l1_penalty):\n", 155 | " # compute prediction\n", 156 | " prediction = predict(feature_matrix,weights)\n", 157 | " # compute ro[i] = SUM[ [feature_i]*(output - prediction + weight[i]*[feature_i]) ]\n", 158 | " ro_i = ((output-prediction+weights[i]*feature_matrix[:,i])*feature_matrix[:,i]).sum()\n", 159 | " if i == 0: # intercept -- do not regularize\n", 160 | " new_weight_i = ro_i\n", 161 | " elif ro_i < -l1_penalty/2.0:\n", 162 | " new_weight_i = ro_i+l1_penalty/2\n", 163 | " elif ro_i > l1_penalty/2.0:\n", 164 | " new_weight_i = ro_i-l1_penalty/2\n", 165 | " else:\n", 166 | " new_weight_i = 0.\n", 167 | " return new_weight_i" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": 384, 173 | "metadata": {}, 174 | "outputs": [ 175 | { 176 | "name": "stdout", 177 | "output_type": "stream", 178 | "text": [ 179 | "0.4255588466910251\n" 180 | ] 181 | } 182 | ], 183 | "source": [ 184 | "import math\n", 185 | "print (lasso_coordinate_descent_step(1, np.array([[3./math.sqrt(13),1./math.sqrt(10)],[2./math.sqrt(13),3./math.sqrt(10)]]), np.array([1., 1.]), np.array([1., 4.]), 0.1))" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": 331, 191 | "metadata": {}, 192 | "outputs": [], 193 | "source": [ 194 | "def lasso_cyclical_coordinate_descent(feature_matrix, output, initial_weights, l1_penalty, tolerance):\n", 195 | " weights=np.array(initial_weights)\n", 196 | " dif=[tolerance+1]*len(initial_weights)\n", 197 | " while True:\n", 198 | " for i in range(len(initial_weights)):\n", 199 | " old=weights[i]\n", 200 | " weights[i]=lasso_coordinate_descent_step(i,feature_matrix,output,weights,l1_penalty)\n", 201 | " dif[i]=abs(weights[i]-old)\n", 202 | " if max(dif)