├── README.md ├── Yelp_Dataset_-_Restaurant_Recommender.ipynb ├── .ipynb_checkpoints ├── Yelp_Dataset_Data_Preprocessing-checkpoint.ipynb └── Yelp_Dataset_-_NLP-checkpoint.ipynb └── Yelp_Dataset_-_NLP.ipynb /README.md: -------------------------------------------------------------------------------- 1 | # Yelp Challenge (Foodie Challenge) 2 | Yelp dataset challenge: NLP; sentiment analysis; Restaurant Recommendation System 3 | 4 | This projects use Yelp Dataset challenge data containing restaurant comments from Yelp users all over U.S., business information of restaurants registered on Yelp. 5 | 6 | In this project, I selected comments from year 2016 to year 2018 to train and test my model. All together, the raw data is of size ~1GB. There're three main sections of this projects: 7 | * data preprocessing 8 | * NLP and sentiment analysis (Naive Bayes Classifier) 9 | * recommendation system 10 | -------------------------------------------------------------------------------- /Yelp_Dataset_-_Restaurant_Recommender.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Yelp Data Challenge - Restaurant Recommender\n", 8 | "\n", 9 | "BitTiger DS501\n", 10 | "\n", 11 | "Nov 2017" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": { 18 | "collapsed": true 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "import pandas as pd\n", 23 | "import numpy as np\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "import pandas as pd\n", 26 | "from scipy import sparse\n", 27 | "from sklearn.metrics.pairwise import cosine_similarity\n", 28 | "% matplotlib inline\n", 29 | "plt.style.use(\"ggplot\")" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 2, 35 | "metadata": { 36 | "collapsed": true 37 | }, 38 | "outputs": [], 39 | "source": [ 40 | "df = pd.read_csv('dataset/last_2_years_restaurant_reviews.csv')" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 3, 46 | "metadata": {}, 47 | "outputs": [ 48 | { 49 | "data": { 50 | "text/html": [ 51 | "
\n", 52 | "\n", 53 | " \n", 54 | " \n", 55 | " \n", 56 | " \n", 57 | " \n", 58 | " \n", 59 | " \n", 60 | " \n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | "
business_idnamecategoriesavg_starscooldatefunnyreview_idstarstextusefuluser_id
0--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Steakhouses', 'Cajun/Creole', 'Restaurants']4.012016-05-1700Qc1THNHSapDL7cv-ZzW5g5What can I say.. Wowzers! Probably one of the ...04LxKRRIikhr65GfPDW626w
1--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Steakhouses', 'Cajun/Creole', 'Restaurants']4.002017-01-200L8lo5SKXfZRlbn1bpPiC9w5Went here for guys weekend. Unbelievable. Ravi...0nT8zgjoc-PbdBoQsFEXFLw
2--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Steakhouses', 'Cajun/Creole', 'Restaurants']4.0522016-09-25306eUT3IwwWPP3CZkAhxqOIw5One word my friends: tableside!!! Yes, tablesi...567RlyCglsIzhBn081inwvcg
3--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Steakhouses', 'Cajun/Creole', 'Restaurants']4.012017-02-1203cnTdE45VrsS0o4cVhfGog3Located inside my favorite hotel Venetian, Del...1rOIrilMC7VFwFVBeQNiKMw
4--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Steakhouses', 'Cajun/Creole', 'Restaurants']4.002016-10-300tYrSbjX3QgZGBZuQ3n8g6w5After the most incredible service, delicious m...2PiWlV_UC_-SXqyxQM9fAtw
\n", 148 | "
" 149 | ], 150 | "text/plain": [ 151 | " business_id name \\\n", 152 | "0 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 153 | "1 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 154 | "2 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 155 | "3 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 156 | "4 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 157 | "\n", 158 | " categories avg_stars cool \\\n", 159 | "0 ['Steakhouses', 'Cajun/Creole', 'Restaurants'] 4.0 1 \n", 160 | "1 ['Steakhouses', 'Cajun/Creole', 'Restaurants'] 4.0 0 \n", 161 | "2 ['Steakhouses', 'Cajun/Creole', 'Restaurants'] 4.0 52 \n", 162 | "3 ['Steakhouses', 'Cajun/Creole', 'Restaurants'] 4.0 1 \n", 163 | "4 ['Steakhouses', 'Cajun/Creole', 'Restaurants'] 4.0 0 \n", 164 | "\n", 165 | " date funny review_id stars \\\n", 166 | "0 2016-05-17 0 0Qc1THNHSapDL7cv-ZzW5g 5 \n", 167 | "1 2017-01-20 0 L8lo5SKXfZRlbn1bpPiC9w 5 \n", 168 | "2 2016-09-25 30 6eUT3IwwWPP3CZkAhxqOIw 5 \n", 169 | "3 2017-02-12 0 3cnTdE45VrsS0o4cVhfGog 3 \n", 170 | "4 2016-10-30 0 tYrSbjX3QgZGBZuQ3n8g6w 5 \n", 171 | "\n", 172 | " text useful \\\n", 173 | "0 What can I say.. Wowzers! Probably one of the ... 0 \n", 174 | "1 Went here for guys weekend. Unbelievable. Ravi... 0 \n", 175 | "2 One word my friends: tableside!!! Yes, tablesi... 56 \n", 176 | "3 Located inside my favorite hotel Venetian, Del... 1 \n", 177 | "4 After the most incredible service, delicious m... 2 \n", 178 | "\n", 179 | " user_id \n", 180 | "0 4LxKRRIikhr65GfPDW626w \n", 181 | "1 nT8zgjoc-PbdBoQsFEXFLw \n", 182 | "2 7RlyCglsIzhBn081inwvcg \n", 183 | "3 rOIrilMC7VFwFVBeQNiKMw \n", 184 | "4 PiWlV_UC_-SXqyxQM9fAtw " 185 | ] 186 | }, 187 | "execution_count": 3, 188 | "metadata": {}, 189 | "output_type": "execute_result" 190 | } 191 | ], 192 | "source": [ 193 | "df.head()" 194 | ] 195 | }, 196 | { 197 | "cell_type": "markdown", 198 | "metadata": {}, 199 | "source": [ 200 | "## 1. Clean data and get rating data " 201 | ] 202 | }, 203 | { 204 | "cell_type": "markdown", 205 | "metadata": {}, 206 | "source": [ 207 | "#### Select relevant columns in the original dataframe" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": 4, 213 | "metadata": {}, 214 | "outputs": [ 215 | { 216 | "name": "stdout", 217 | "output_type": "stream", 218 | "text": [ 219 | "\n", 220 | "Index: 136735 entries, ---udAKDsn0yQXmzbWQNSw to zzvqVZTYs5VKxPc-IkRQ4A\n", 221 | "Columns: 4268 entries, --9e1ONYQuAa-CB_Rrw7Tw to zwNC-Ow4eIMan2__bS9-rg\n", 222 | "dtypes: int64(4268)\n", 223 | "memory usage: 4.3+ GB\n" 224 | ] 225 | } 226 | ], 227 | "source": [ 228 | "# Get business_id, user_id, stars for recommender\n", 229 | "df_utility = pd.pivot_table(data=df, \n", 230 | " values='stars', \n", 231 | " index='user_id', \n", 232 | " columns='business_id', \n", 233 | " fill_value=0)\n", 234 | "\n", 235 | "df_utility.info()" 236 | ] 237 | }, 238 | { 239 | "cell_type": "markdown", 240 | "metadata": {}, 241 | "source": [ 242 | "#### There are many users that haven't given many reviews, exclude these users from the item-item similarity recommender\n", 243 | "\n", 244 | "**Q**: How do we recommend to these users anyways?" 245 | ] 246 | }, 247 | { 248 | "cell_type": "markdown", 249 | "metadata": { 250 | "collapsed": true 251 | }, 252 | "source": [ 253 | "**A**: We can use item-item based collaborative filtering. Business as item, user as user. So we need to calculate the similarity between business first and then recommend. Or we can use U-V decomposition to build recommender" 254 | ] 255 | }, 256 | { 257 | "cell_type": "markdown", 258 | "metadata": {}, 259 | "source": [ 260 | "#### Create utility matrix from records" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": 14, 266 | "metadata": {}, 267 | "outputs": [ 268 | { 269 | "data": { 270 | "text/plain": [ 271 | "<136735x4268 sparse matrix of type ''\n", 272 | "\twith 0 stored elements in LInked List format>" 273 | ] 274 | }, 275 | "execution_count": 14, 276 | "metadata": {}, 277 | "output_type": "execute_result" 278 | } 279 | ], 280 | "source": [ 281 | "num_users = df.user_id.value_counts()\n", 282 | "num_business = df.business_id.value_counts()\n", 283 | "#print(num_users,num_business)\n", 284 | "stars_mat = sparse.lil_matrix((num_users.shape[0], num_business.shape[0]))\n", 285 | "stars_mat" 286 | ] 287 | }, 288 | { 289 | "cell_type": "code", 290 | "execution_count": 17, 291 | "metadata": { 292 | "collapsed": true 293 | }, 294 | "outputs": [], 295 | "source": [ 296 | "df.set_index('user_id', inplace = True)" 297 | ] 298 | }, 299 | { 300 | "cell_type": "markdown", 301 | "metadata": {}, 302 | "source": [ 303 | "## 2. Item-Item similarity recommender" 304 | ] 305 | }, 306 | { 307 | "cell_type": "markdown", 308 | "metadata": {}, 309 | "source": [ 310 | "### Let's reuse the ItemItemRecommender class derived from previous exercise\n", 311 | "\n", 312 | "Hint: we need to make modification to accommodate the dense numpy array" 313 | ] 314 | }, 315 | { 316 | "cell_type": "code", 317 | "execution_count": 5, 318 | "metadata": {}, 319 | "outputs": [ 320 | { 321 | "data": { 322 | "text/plain": [ 323 | "(136735, 4268)" 324 | ] 325 | }, 326 | "execution_count": 5, 327 | "metadata": {}, 328 | "output_type": "execute_result" 329 | } 330 | ], 331 | "source": [ 332 | "df.set_index('user_id', inplace = True)\n", 333 | "utility_mat = df_utility.as_matrix()\n", 334 | "utility_mat.shape" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": 7, 340 | "metadata": {}, 341 | "outputs": [], 342 | "source": [ 343 | "# Item-Item Similarity Matrix\n", 344 | "item_sim_mat = cosine_similarity(utility_mat.T)\n", 345 | "\n", 346 | "least_to_most_sim_indexes = np.argsort(item_sim_mat, axis=1)\n", 347 | "\n", 348 | "# Neighborhoods\n", 349 | "neighborhood_size = 75\n", 350 | "neighborhoods = least_to_most_sim_indexes[:, -neighborhood_size:]" 351 | ] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": 8, 356 | "metadata": {}, 357 | "outputs": [ 358 | { 359 | "data": { 360 | "text/plain": [ 361 | "(4268, 75)" 362 | ] 363 | }, 364 | "execution_count": 8, 365 | "metadata": {}, 366 | "output_type": "execute_result" 367 | } 368 | ], 369 | "source": [ 370 | "neighborhoods.shape\n", 371 | "#" 372 | ] 373 | }, 374 | { 375 | "cell_type": "code", 376 | "execution_count": 38, 377 | "metadata": { 378 | "collapsed": true 379 | }, 380 | "outputs": [], 381 | "source": [ 382 | "# Let's pick a lucky user\n", 383 | "user_id = 3" 384 | ] 385 | }, 386 | { 387 | "cell_type": "code", 388 | "execution_count": 39, 389 | "metadata": {}, 390 | "outputs": [ 391 | { 392 | "name": "stdout", 393 | "output_type": "stream", 394 | "text": [ 395 | "[ 3. 4. 3. 4. 3. 3. 4.\n", 396 | " 3. 3. 3.55785356 3. 4. 3. 3.\n", 397 | " 4. 3.47318803 3. 3. 4. 4. 3.\n", 398 | " 3. 3. 4. 4. 4. 3. 3.\n", 399 | " 3. 3. 3. 3. 3. 4. 4.\n", 400 | " 3. 3. 4. 3. 3. 4. 4.\n", 401 | " 4. 3. 4. 3. 3. 4. 3.\n", 402 | " 3. 3. 4. 3. 3. 4. 3.\n", 403 | " 3. 3. 3. 3. 3. 3. 4.\n", 404 | " 3. 3. 4. 3. 3. 4. 3.\n", 405 | " 3. 3. 4. 3. 3.49174105 3. 3.\n", 406 | " 4. 3. 3. 3. 4. 4. 3.\n", 407 | " 3. 3. 4. 3. 3. 3. 4.\n", 408 | " 3. 3. 3. 3. 3. 4. 3.\n", 409 | " 3. 4. 3. 3. 4. 3. 3.\n", 410 | " 3. 3. 3. 3. 4. 3.88590038\n", 411 | " 3. 3.52389887 3. 3. 4. 3. 3.\n", 412 | " 4. 3. 4. 3. 3. 3. 3.\n", 413 | " 3.3187534 3. 3. 3. 4. 4. 3.\n", 414 | " 3. 3. 3. 3.49332573 4. 4. 4.\n", 415 | " 3. 3. 3. 3. 4. 3. 4.\n", 416 | " 3. 3.35313823 3. 3. 3. 3.\n", 417 | " 3.36801672 3. 3. 4. 4. 4. 4.\n", 418 | " 3. 3. 4. 3. 3. 3. 4.\n", 419 | " 3. 4. 3. 3. 4. 3. 4.\n", 420 | " 4. 3. 3. 3. 4. 4. 3.\n", 421 | " 4. 3. 3. 3. 3. 4. 3.\n", 422 | " 3. 3. 4. 3. 4. 3. 3.\n", 423 | " 3. ]\n", 424 | "[ 0. 0. 0. ..., 0. 0. 0.]\n", 425 | "Execution time: 0.049621 seconds\n" 426 | ] 427 | } 428 | ], 429 | "source": [ 430 | "from time import time\n", 431 | "n_users = utility_mat.shape[0]\n", 432 | "n_items = utility_mat.shape[1]\n", 433 | "\n", 434 | "start_time = time()\n", 435 | "items_rated_by_this_user = utility_mat[user_id].nonzero()[0]\n", 436 | "# Just initializing so we have somewhere to put rating preds\n", 437 | "out = np.zeros(n_items)\n", 438 | "for item_to_rate in range(n_items):\n", 439 | " relevant_items = np.intersect1d(neighborhoods[item_to_rate],\n", 440 | " items_rated_by_this_user,\n", 441 | " assume_unique=True) # assume_unique speeds up intersection op\n", 442 | " #print(neighborhoods[item_to_rate])\n", 443 | " #print(items_rated_by_this_user)\n", 444 | " if len(relevant_items) != 0:\n", 445 | " #print(relevant_items)\n", 446 | " #print(utility_mat[user_id, relevant_items], '*', item_sim_mat[item_to_rate, relevant_items])\n", 447 | " out[item_to_rate] = sum(utility_mat[user_id, relevant_items] * \\\n", 448 | " item_sim_mat[item_to_rate, relevant_items]) / \\\n", 449 | " item_sim_mat[item_to_rate, relevant_items].sum()\n", 450 | " else:\n", 451 | " out[item_to_rate] = np.nan\n", 452 | "\n", 453 | "\n", 454 | "print(out[~np.isnan(out)])\n", 455 | "pred_ratings = np.nan_to_num(out)\n", 456 | "print (pred_ratings)\n", 457 | "print(\"Execution time: %f seconds\" % (time()-start_time))" 458 | ] 459 | }, 460 | { 461 | "cell_type": "code", 462 | "execution_count": 46, 463 | "metadata": {}, 464 | "outputs": [ 465 | { 466 | "data": { 467 | "text/plain": [ 468 | "[44, 3604, 1264, 29, 918, 2372, 2893, 3476, 484, 3908]" 469 | ] 470 | }, 471 | "execution_count": 46, 472 | "metadata": {}, 473 | "output_type": "execute_result" 474 | } 475 | ], 476 | "source": [ 477 | "# Recommend n movies\n", 478 | "n = 10\n", 479 | "\n", 480 | "# Get item indexes sorted by predicted rating\n", 481 | "item_index_sorted_by_pred_rating = list(np.argsort(pred_ratings))\n", 482 | "\n", 483 | "# Find items that have been rated by user\n", 484 | "items_rated_by_this_user = utility_mat[user_id].nonzero()[0]\n", 485 | "\n", 486 | "# We want to exclude the items that have been rated by user\n", 487 | "unrated_items_by_pred_rating_it = [item for item in item_index_sorted_by_pred_rating\n", 488 | " if item not in items_rated_by_this_user]\n", 489 | "\n", 490 | "unrated_items_by_pred_rating_it[-n:]" 491 | ] 492 | }, 493 | { 494 | "cell_type": "code", 495 | "execution_count": 51, 496 | "metadata": {}, 497 | "outputs": [ 498 | { 499 | "name": "stdout", 500 | "output_type": "stream", 501 | "text": [ 502 | "recommend business name: Beer Park\n", 503 | "recommend business name: Leticia's Mexican Cocina\n", 504 | "recommend business name: Squeeze In\n", 505 | "recommend business name: Jaburritos\n", 506 | "recommend business name: Yong Kang Street\n", 507 | "recommend business name: Estiatorio Milos\n", 508 | "recommend business name: Off The Strip at the LINQ\n", 509 | "recommend business name: Gallagher's Steakhouse\n", 510 | "recommend business name: Lobster ME\n", 511 | "recommend business name: Pin-Up Pizza\n" 512 | ] 513 | } 514 | ], 515 | "source": [ 516 | "recommend_business_id = df_utility.columns[unrated_items_by_pred_rating_it[-n:]].values\n", 517 | "for business in recommend_business_id:\n", 518 | " print(\"recommend business name: \", df.loc[df.business_id == business].name.values[0])" 519 | ] 520 | }, 521 | { 522 | "cell_type": "markdown", 523 | "metadata": {}, 524 | "source": [ 525 | "## 3. Matrix Factorization recommender\n", 526 | "\n", 527 | "Take a look at Graphlab Create examples" 528 | ] 529 | }, 530 | { 531 | "cell_type": "code", 532 | "execution_count": 40, 533 | "metadata": {}, 534 | "outputs": [ 535 | { 536 | "name": "stdout", 537 | "output_type": "stream", 538 | "text": [ 539 | "9.89994527306\n", 540 | "2.46531960038\n" 541 | ] 542 | } 543 | ], 544 | "source": [ 545 | "from sklearn.decomposition import TruncatedSVD\n", 546 | "\n", 547 | "def fit_uvd(M,k):\n", 548 | " # use TruncatedSVD to realize UVD\n", 549 | " svd = TruncatedSVD(n_components=k, n_iter=7, random_state=0)\n", 550 | " svd.fit(M)\n", 551 | "\n", 552 | " V = svd.components_\n", 553 | " U = svd.transform(M) # effectively, it's doing: U = M.dot(V.T)\n", 554 | " # we can ignore svd.singular_values_ for our purpose\n", 555 | " \n", 556 | " # why we can do this?\n", 557 | " # recall: \n", 558 | " # SVD start from u*s*v=M => u*s=M*v.T, where M*v.T is our transformation above to get U in UVD\n", 559 | " # so the above U is effectively u*s in SVD\n", 560 | " # that's why U*V = u*s*v = M our original matrix\n", 561 | " # there are many ways to understand it!\n", 562 | " # here we by-passed singular values.\n", 563 | " \n", 564 | " return U,V\n", 565 | "\n", 566 | "# decompose\n", 567 | "U,V = fit_uvd(utility_mat,200)\n", 568 | "\n", 569 | "# reconstruct\n", 570 | "ratings_mat_fitted = U.dot(V) # U*V\n", 571 | "\n", 572 | "# calculate errs\n", 573 | "errs = np.array((utility_mat-ratings_mat_fitted).flatten()).squeeze()\n", 574 | "mask = np.array(utility_mat.flatten()).squeeze()>0\n", 575 | "\n", 576 | "mse = np.mean(errs[mask]**2)\n", 577 | "average_abs_err = abs(errs[mask]).mean()\n", 578 | "print (mse)\n", 579 | "print (average_abs_err)" 580 | ] 581 | }, 582 | { 583 | "cell_type": "code", 584 | "execution_count": 52, 585 | "metadata": {}, 586 | "outputs": [ 587 | { 588 | "data": { 589 | "text/plain": [ 590 | "[3908, 484, 3476, 2893, 2372, 918, 29, 1264, 3604, 44]" 591 | ] 592 | }, 593 | "execution_count": 52, 594 | "metadata": {}, 595 | "output_type": "execute_result" 596 | } 597 | ], 598 | "source": [ 599 | "# get recommendations for one user\n", 600 | "user_id = 3\n", 601 | "n = 10\n", 602 | "\n", 603 | "pred_ratings = ratings_mat_fitted[user_id,:]\n", 604 | "item_index_sorted_by_pred_rating = list(np.argsort(pred_ratings))[::-1]\n", 605 | "\n", 606 | "items_rated_by_this_user = utility_mat[user_id].nonzero()[0]\n", 607 | "\n", 608 | "unrated_items_by_pred_rating_uv = [item for item in item_index_sorted_by_pred_rating\n", 609 | " if item not in items_rated_by_this_user]\n", 610 | "\n", 611 | "unrated_items_by_pred_rating_uv[:n]" 612 | ] 613 | }, 614 | { 615 | "cell_type": "code", 616 | "execution_count": 53, 617 | "metadata": {}, 618 | "outputs": [ 619 | { 620 | "name": "stdout", 621 | "output_type": "stream", 622 | "text": [ 623 | "recommend business name: Culinary Dropout\n", 624 | "recommend business name: Shake Shack\n", 625 | "recommend business name: CUT by Wolfgang Puck\n", 626 | "recommend business name: Nine Fine Irishmen\n", 627 | "recommend business name: BabyStacks Cafe\n", 628 | "recommend business name: Dog Haus\n", 629 | "recommend business name: Rí Rá Irish Pub\n", 630 | "recommend business name: The Bootlegger Italian Bistro\n", 631 | "recommend business name: Scarpetta\n", 632 | "recommend business name: Delmonico Steakhouse\n" 633 | ] 634 | } 635 | ], 636 | "source": [ 637 | "recommend_business_id = df_utility.columns[unrated_items_by_pred_rating_uv[-n:]].values\n", 638 | "for business in recommend_business_id:\n", 639 | " print(\"recommend business name: \", df.loc[df.business_id == business].name.values[0])" 640 | ] 641 | }, 642 | { 643 | "cell_type": "markdown", 644 | "metadata": {}, 645 | "source": [ 646 | "## 4. Other recommenders (optional)\n", 647 | "\n", 648 | "What are other ways you can build a better recommender?\n", 649 | "\n", 650 | "* Other features (have you noticed there are other features in the Yelp dataset, e.g. tips, etc.?)\n", 651 | "* Popularity-based\n", 652 | "* Content-based\n", 653 | "* Hybrid" 654 | ] 655 | }, 656 | { 657 | "cell_type": "markdown", 658 | "metadata": { 659 | "collapsed": true 660 | }, 661 | "source": [ 662 | "## Note\n", 663 | "> We can change the neighborhood_size to 200 or even larger to improve the item-item based recommender\n", 664 | "\n", 665 | "> We can increase the latent factor from 200 to 500 to improve the U-V decomposition recommender" 666 | ] 667 | }, 668 | { 669 | "cell_type": "code", 670 | "execution_count": null, 671 | "metadata": { 672 | "collapsed": true 673 | }, 674 | "outputs": [], 675 | "source": [] 676 | } 677 | ], 678 | "metadata": { 679 | "anaconda-cloud": {}, 680 | "kernelspec": { 681 | "display_name": "Python 3", 682 | "language": "python", 683 | "name": "python3" 684 | }, 685 | "language_info": { 686 | "codemirror_mode": { 687 | "name": "ipython", 688 | "version": 3 689 | }, 690 | "file_extension": ".py", 691 | "mimetype": "text/x-python", 692 | "name": "python", 693 | "nbconvert_exporter": "python", 694 | "pygments_lexer": "ipython3", 695 | "version": "3.6.0" 696 | } 697 | }, 698 | "nbformat": 4, 699 | "nbformat_minor": 1 700 | } 701 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/Yelp_Dataset_Data_Preprocessing-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Yelp Data Challenge - Data Preprocessing" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Dataset Introduction\n", 15 | "\n", 16 | "[Yelp Dataset Challenge](https://www.yelp.com/dataset_challenge)\n", 17 | "\n", 18 | "The Challenge Dataset:\n", 19 | "\n", 20 | " 4.1M reviews and 947K tips by 1M users for 144K businesses\n", 21 | " 1.1M business attributes, e.g., hours, parking availability, ambience.\n", 22 | " Aggregated check-ins over time for each of the 125K businesses\n", 23 | " 200,000 pictures from the included businesses\n", 24 | "\n", 25 | "Cities:\n", 26 | "\n", 27 | " U.K.: Edinburgh\n", 28 | " Germany: Karlsruhe\n", 29 | " Canada: Montreal and Waterloo\n", 30 | " U.S.: Pittsburgh, Charlotte, Urbana-Champaign, Phoenix, Las Vegas, Madison, Cleveland\n", 31 | "\n", 32 | "Files:\n", 33 | "\n", 34 | " yelp_academic_dataset_business.json\n", 35 | " yelp_academic_dataset_checkin.json\n", 36 | " yelp_academic_dataset_review.json\n", 37 | " yelp_academic_dataset_tip.json\n", 38 | " yelp_academic_dataset_user.json\n", 39 | "\n", 40 | "Notes on the Dataset\n", 41 | "\n", 42 | " Each file is composed of a single object type, one json-object per-line.\n", 43 | " Take a look at some examples to get you started: https://github.com/Yelp/dataset-examples.\n", 44 | "\n" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": {}, 50 | "source": [ 51 | "## Read data from file and load to Pandas DataFrame\n", 52 | "\n", 53 | "**Warning**: Loading all the 1.8 GB data into Pandas at a time takes long time and a lot of memory!" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 2, 59 | "metadata": { 60 | "collapsed": true 61 | }, 62 | "outputs": [], 63 | "source": [ 64 | "import json\n", 65 | "import pandas as pd" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 3, 71 | "metadata": { 72 | "collapsed": true 73 | }, 74 | "outputs": [], 75 | "source": [ 76 | "file_business, file_checkin, file_review, file_tip, file_user = [\n", 77 | " 'dataset/business.json',\n", 78 | " 'dataset/checkin.json',\n", 79 | " 'dataset/review_after2016.json',\n", 80 | " 'dataset/tip.json',\n", 81 | " 'dataset/user.json'\n", 82 | "]" 83 | ] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "metadata": {}, 88 | "source": [ 89 | "#### Business Data" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": null, 95 | "metadata": { 96 | "collapsed": true 97 | }, 98 | "outputs": [], 99 | "source": [ 100 | "with open(file_business) as f:\n", 101 | " df_business = pd.DataFrame(json.loads(line) for line in f)" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": null, 107 | "metadata": { 108 | "collapsed": true 109 | }, 110 | "outputs": [], 111 | "source": [ 112 | "df_business.head(2)" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": null, 118 | "metadata": { 119 | "collapsed": true 120 | }, 121 | "outputs": [], 122 | "source": [ 123 | "df_business.info()" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": {}, 129 | "source": [ 130 | "#### Checkin Data" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": null, 136 | "metadata": { 137 | "collapsed": true 138 | }, 139 | "outputs": [], 140 | "source": [ 141 | "# with open(file_checkin) as f:\n", 142 | "# df_checkin = pd.DataFrame(json.loads(line) for line in f)\n", 143 | "# df_checkin.head(2)" 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": {}, 149 | "source": [ 150 | "#### Review Data" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": { 157 | "collapsed": true 158 | }, 159 | "outputs": [], 160 | "source": [ 161 | "# with open(file_review) as f:\n", 162 | "# df_review = pd.DataFrame(json.loads(line) for line in f)\n", 163 | "# df_review.head(2)" 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "metadata": {}, 169 | "source": [ 170 | "#### Tip Data" 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": null, 176 | "metadata": { 177 | "collapsed": true 178 | }, 179 | "outputs": [], 180 | "source": [ 181 | "# with open(file_tip) as f:\n", 182 | "# df_tip = pd.DataFrame(json.loads(line) for line in f)\n", 183 | "# df_tip.head(2)" 184 | ] 185 | }, 186 | { 187 | "cell_type": "markdown", 188 | "metadata": {}, 189 | "source": [ 190 | "#### User Data" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": null, 196 | "metadata": { 197 | "collapsed": true 198 | }, 199 | "outputs": [], 200 | "source": [ 201 | "# with open(file_user) as f:\n", 202 | "# df_user = pd.DataFrame(json.loads(line) for line in f)\n", 203 | "# df_user.head(2)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "markdown", 208 | "metadata": {}, 209 | "source": [ 210 | "## Filter data by city and category" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "metadata": {}, 216 | "source": [ 217 | "#### Create filters/masks\n", 218 | "\n", 219 | "* create filters that selects business \n", 220 | " * that are located in \"Las Vegas\"\n", 221 | " * that contains \"Restaurants\" in their category (first filter null categories)" 222 | ] 223 | }, 224 | { 225 | "cell_type": "code", 226 | "execution_count": null, 227 | "metadata": { 228 | "collapsed": true 229 | }, 230 | "outputs": [], 231 | "source": [ 232 | "temp = df_business[df_business['city'] == 'Las Vegas'].dropna(subset = ['categories'])\n", 233 | "df_filtered = temp[temp['categories'].apply(lambda x: True if 'Restaurants' in x else False)]\n", 234 | "del temp" 235 | ] 236 | }, 237 | { 238 | "cell_type": "markdown", 239 | "metadata": {}, 240 | "source": [ 241 | "#### Keep relevant columns\n", 242 | "\n", 243 | "* only keep some useful columns\n", 244 | " * business_id\n", 245 | " * name\n", 246 | " * categories\n", 247 | " * stars" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": null, 253 | "metadata": { 254 | "collapsed": true 255 | }, 256 | "outputs": [], 257 | "source": [ 258 | "selected_features = [u'business_id', u'name', u'categories', u'stars']" 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "execution_count": null, 264 | "metadata": { 265 | "collapsed": true 266 | }, 267 | "outputs": [], 268 | "source": [ 269 | "# Make a DataFrame that contains only the abovementioned columns, and name it as df_selected_business\n", 270 | "df_selected_business = df_filtered[selected_features]" 271 | ] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "execution_count": null, 276 | "metadata": { 277 | "collapsed": true 278 | }, 279 | "outputs": [], 280 | "source": [ 281 | "# Rename the column name \"stars\" to \"avg_stars\" to avoid naming conflicts with review dataset\n", 282 | "df_selected_business.rename(columns={\"stars\":\"avg_stars\"}, inplace = True)" 283 | ] 284 | }, 285 | { 286 | "cell_type": "code", 287 | "execution_count": null, 288 | "metadata": { 289 | "collapsed": true 290 | }, 291 | "outputs": [], 292 | "source": [ 293 | "# Inspect your DataFrame\n", 294 | "df_selected_business.head()" 295 | ] 296 | }, 297 | { 298 | "cell_type": "markdown", 299 | "metadata": {}, 300 | "source": [ 301 | "#### Save results to csv files" 302 | ] 303 | }, 304 | { 305 | "cell_type": "code", 306 | "execution_count": null, 307 | "metadata": { 308 | "collapsed": true 309 | }, 310 | "outputs": [], 311 | "source": [ 312 | "# Save to ./data/selected_business.csv for your next task\n", 313 | "df_selected_business.to_csv('dataset/selected_business.csv', index = False)" 314 | ] 315 | }, 316 | { 317 | "cell_type": "code", 318 | "execution_count": 15, 319 | "metadata": { 320 | "collapsed": true 321 | }, 322 | "outputs": [], 323 | "source": [ 324 | "# Try reload the csv file to check if everything works fine\n", 325 | "df_selected_business = pd.read_csv('dataset/selected_business.csv')" 326 | ] 327 | }, 328 | { 329 | "cell_type": "markdown", 330 | "metadata": {}, 331 | "source": [ 332 | "### Use the \"business_id\" column to filter review data\n", 333 | "\n", 334 | "* We want to make a DataFrame that contain and only contain the reviews about the business entities we just obtained" 335 | ] 336 | }, 337 | { 338 | "cell_type": "markdown", 339 | "metadata": {}, 340 | "source": [ 341 | "#### Load review dataset" 342 | ] 343 | }, 344 | { 345 | "cell_type": "code", 346 | "execution_count": 4, 347 | "metadata": {}, 348 | "outputs": [ 349 | { 350 | "data": { 351 | "text/html": [ 352 | "
\n", 353 | "\n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | "
business_idcooldatefunnyreview_idstarstextusefuluser_id
0uYHaNptLzDLoV_JZ_MuzUA02016-07-120VfBHSwC5Vz_pbFluy07i9Q5My girlfriend and I stayed here for 3 nights a...0cjpdDjZyprfyDG3RlkVG3w
1uYHaNptLzDLoV_JZ_MuzUA02016-10-0203zRpneRKDsOPq92tq7ybAA3If you need an inexpensive place to stay for a...0bjTcT8Ty4cJZhEOEo01FGA
\n", 395 | "
" 396 | ], 397 | "text/plain": [ 398 | " business_id cool date funny review_id \\\n", 399 | "0 uYHaNptLzDLoV_JZ_MuzUA 0 2016-07-12 0 VfBHSwC5Vz_pbFluy07i9Q \n", 400 | "1 uYHaNptLzDLoV_JZ_MuzUA 0 2016-10-02 0 3zRpneRKDsOPq92tq7ybAA \n", 401 | "\n", 402 | " stars text useful \\\n", 403 | "0 5 My girlfriend and I stayed here for 3 nights a... 0 \n", 404 | "1 3 If you need an inexpensive place to stay for a... 0 \n", 405 | "\n", 406 | " user_id \n", 407 | "0 cjpdDjZyprfyDG3RlkVG3w \n", 408 | "1 bjTcT8Ty4cJZhEOEo01FGA " 409 | ] 410 | }, 411 | "execution_count": 4, 412 | "metadata": {}, 413 | "output_type": "execute_result" 414 | } 415 | ], 416 | "source": [ 417 | "with open(file_review) as f:\n", 418 | " df_review = pd.DataFrame(json.loads(line) for line in f)\n", 419 | "df_review.head(2)" 420 | ] 421 | }, 422 | { 423 | "cell_type": "markdown", 424 | "metadata": {}, 425 | "source": [ 426 | "#### Prepare dataframes to be joined, - on business_id" 427 | ] 428 | }, 429 | { 430 | "cell_type": "code", 431 | "execution_count": 16, 432 | "metadata": {}, 433 | "outputs": [ 434 | { 435 | "data": { 436 | "text/html": [ 437 | "
\n", 438 | "\n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | "
namecategoriesavg_stars
business_id
LDMCrFlGIFUN6L-FEFgzWgEl Pollo Loco['Restaurants', 'American (Traditional)', 'Mex...3.0
QTH_XGh4rWYdd0fTW-tUDwBaja Fresh Mexican Grill['Mexican', 'Restaurants']3.5
\n", 468 | "
" 469 | ], 470 | "text/plain": [ 471 | " name \\\n", 472 | "business_id \n", 473 | "LDMCrFlGIFUN6L-FEFgzWg El Pollo Loco \n", 474 | "QTH_XGh4rWYdd0fTW-tUDw Baja Fresh Mexican Grill \n", 475 | "\n", 476 | " categories \\\n", 477 | "business_id \n", 478 | "LDMCrFlGIFUN6L-FEFgzWg ['Restaurants', 'American (Traditional)', 'Mex... \n", 479 | "QTH_XGh4rWYdd0fTW-tUDw ['Mexican', 'Restaurants'] \n", 480 | "\n", 481 | " avg_stars \n", 482 | "business_id \n", 483 | "LDMCrFlGIFUN6L-FEFgzWg 3.0 \n", 484 | "QTH_XGh4rWYdd0fTW-tUDw 3.5 " 485 | ] 486 | }, 487 | "execution_count": 16, 488 | "metadata": {}, 489 | "output_type": "execute_result" 490 | } 491 | ], 492 | "source": [ 493 | "# Prepare the business dataframe and set index to column \"business_id\", and name it as df_left\n", 494 | "df_left = df_selected_business.set_index('business_id')\n", 495 | "df_left.head(2)" 496 | ] 497 | }, 498 | { 499 | "cell_type": "code", 500 | "execution_count": 7, 501 | "metadata": {}, 502 | "outputs": [ 503 | { 504 | "data": { 505 | "text/html": [ 506 | "
\n", 507 | "\n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | " \n", 533 | " \n", 534 | " \n", 535 | " \n", 536 | " \n", 537 | " \n", 538 | " \n", 539 | " \n", 540 | " \n", 541 | " \n", 542 | " \n", 543 | " \n", 544 | " \n", 545 | " \n", 546 | " \n", 547 | " \n", 548 | " \n", 549 | " \n", 550 | " \n", 551 | " \n", 552 | " \n", 553 | " \n", 554 | " \n", 555 | " \n", 556 | "
cooldatefunnyreview_idstarstextusefuluser_id
business_id
uYHaNptLzDLoV_JZ_MuzUA02016-07-120VfBHSwC5Vz_pbFluy07i9Q5My girlfriend and I stayed here for 3 nights a...0cjpdDjZyprfyDG3RlkVG3w
uYHaNptLzDLoV_JZ_MuzUA02016-10-0203zRpneRKDsOPq92tq7ybAA3If you need an inexpensive place to stay for a...0bjTcT8Ty4cJZhEOEo01FGA
\n", 557 | "
" 558 | ], 559 | "text/plain": [ 560 | " cool date funny review_id \\\n", 561 | "business_id \n", 562 | "uYHaNptLzDLoV_JZ_MuzUA 0 2016-07-12 0 VfBHSwC5Vz_pbFluy07i9Q \n", 563 | "uYHaNptLzDLoV_JZ_MuzUA 0 2016-10-02 0 3zRpneRKDsOPq92tq7ybAA \n", 564 | "\n", 565 | " stars \\\n", 566 | "business_id \n", 567 | "uYHaNptLzDLoV_JZ_MuzUA 5 \n", 568 | "uYHaNptLzDLoV_JZ_MuzUA 3 \n", 569 | "\n", 570 | " text \\\n", 571 | "business_id \n", 572 | "uYHaNptLzDLoV_JZ_MuzUA My girlfriend and I stayed here for 3 nights a... \n", 573 | "uYHaNptLzDLoV_JZ_MuzUA If you need an inexpensive place to stay for a... \n", 574 | "\n", 575 | " useful user_id \n", 576 | "business_id \n", 577 | "uYHaNptLzDLoV_JZ_MuzUA 0 cjpdDjZyprfyDG3RlkVG3w \n", 578 | "uYHaNptLzDLoV_JZ_MuzUA 0 bjTcT8Ty4cJZhEOEo01FGA " 579 | ] 580 | }, 581 | "execution_count": 7, 582 | "metadata": {}, 583 | "output_type": "execute_result" 584 | } 585 | ], 586 | "source": [ 587 | "# Prepare the review dataframe and set index to column \"business_id\", and name it as df_right\n", 588 | "df_right = df_review.set_index('business_id')\n", 589 | "df_right.head(2)" 590 | ] 591 | }, 592 | { 593 | "cell_type": "markdown", 594 | "metadata": {}, 595 | "source": [ 596 | "#### Join! and reset index" 597 | ] 598 | }, 599 | { 600 | "cell_type": "code", 601 | "execution_count": 19, 602 | "metadata": {}, 603 | "outputs": [ 604 | { 605 | "name": "stdout", 606 | "output_type": "stream", 607 | "text": [ 608 | "(286608, 11)\n", 609 | "(1690348, 8)\n", 610 | "(5682, 3)\n" 611 | ] 612 | } 613 | ], 614 | "source": [ 615 | "# Join df_left and df_right. What type of join?\n", 616 | "df_joined = df_left.join(df_right, how = \"inner\")\n", 617 | "print(df_joined.shape)\n", 618 | "print(df_right.shape)\n", 619 | "print(df_left.shape)" 620 | ] 621 | }, 622 | { 623 | "cell_type": "code", 624 | "execution_count": 20, 625 | "metadata": { 626 | "collapsed": true 627 | }, 628 | "outputs": [], 629 | "source": [ 630 | "# You may want to reset the index \n", 631 | "df_joined.reset_index(inplace = True)" 632 | ] 633 | }, 634 | { 635 | "cell_type": "markdown", 636 | "metadata": {}, 637 | "source": [ 638 | "#### We further filter data by date, e.g. keep comments from last 2 years\n", 639 | "\n", 640 | "* Otherwise your laptop may crush on memory when running machine learning algorithms\n", 641 | "* Purposefully ignoring the reviews made too long time ago" 642 | ] 643 | }, 644 | { 645 | "cell_type": "code", 646 | "execution_count": 21, 647 | "metadata": { 648 | "collapsed": true, 649 | "scrolled": true 650 | }, 651 | "outputs": [], 652 | "source": [ 653 | "# Make a filter that selects date after 2015-01-20\n", 654 | "cond_last_years = df_joined['date'] > u'2016-01-20'" 655 | ] 656 | }, 657 | { 658 | "cell_type": "code", 659 | "execution_count": 22, 660 | "metadata": { 661 | "collapsed": true 662 | }, 663 | "outputs": [], 664 | "source": [ 665 | "# Filter the joined DataFrame and name it as df_final\n", 666 | "df_joined = df_joined[cond_last_years]" 667 | ] 668 | }, 669 | { 670 | "cell_type": "markdown", 671 | "metadata": {}, 672 | "source": [ 673 | "#### Take a glance at the final dataset\n", 674 | "\n", 675 | "* Do more EDA here as you like!" 676 | ] 677 | }, 678 | { 679 | "cell_type": "code", 680 | "execution_count": 23, 681 | "metadata": { 682 | "collapsed": true 683 | }, 684 | "outputs": [], 685 | "source": [ 686 | "import matplotlib.pyplot as plt\n", 687 | "\n", 688 | "% matplotlib inline" 689 | ] 690 | }, 691 | { 692 | "cell_type": "code", 693 | "execution_count": 33, 694 | "metadata": {}, 695 | "outputs": [ 696 | { 697 | "data": { 698 | "text/plain": [ 699 | "" 700 | ] 701 | }, 702 | "execution_count": 33, 703 | "metadata": {}, 704 | "output_type": "execute_result" 705 | }, 706 | { 707 | "data": { 708 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAD8CAYAAABgmUMCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFQlJREFUeJzt3X+w3XV95/HnyyAorTVBgmWS0Bts\nxso6dc2myK5dt1sqv9oS3Ck7OE7NUNrsrmB13U4N2ilOu87obivKtMs2LlkD64KIWrILLka0pTuz\n/AiK/IqYFFm4JiVxg+AWlQXf+8f53HK83CTnm9xzz7m5z8fMmfP9fr6fc77vfOZcXnx/p6qQJGlQ\nLxp1AZKk+cXgkCR1YnBIkjoxOCRJnRgckqRODA5JUicGhySpE4NDktSJwSFJ6uSoURcwDMcff3xN\nTEyMugxJmlfuvvvub1fV0oP1OyKDY2Jigm3bto26DEmaV5L870H6uatKktSJwSFJ6sTgkCR1MrTg\nSLIpyZ4k98+w7HeSVJLj23ySXJFkZ5J7k6zu67suyY72WjeseiVJgxnmFscngLOmNyZZAbwZeLSv\n+WxgVXutB65sfY8DLgPeAJwKXJZkyRBrliQdxNCCo6puA/bNsOhy4HeB/idIrQWurp7bgcVJTgTO\nBLZW1b6qegLYygxhJEmaO3N6jCPJucC3qupr0xYtAx7rm59sbftrn+m71yfZlmTb3r17Z7FqSVK/\nOQuOJMcC7wd+f6bFM7TVAdpf2Fi1sarWVNWapUsPev2KJOkQzeUWx6uAlcDXkjwCLAe+kuQn6W1J\nrOjruxzYdYB2SdKIzFlwVNV9VXVCVU1U1QS9UFhdVX8DbAHe3s6uOg14sqp2A7cAZyRZ0g6Kn9Ha\nhmpiw03DXoUkzVvDPB33WuB/Aa9OMpnkogN0vxl4GNgJfBx4B0BV7QP+ELirvf6gtUmSRmRo96qq\nqrceZPlE33QBF++n3yZg06wWJ0k6ZF45LknqxOCQJHVicEiSOjE4JEmdGBySpE4MDklSJwaHJKkT\ng0OS1InBIUnqxOCQJHVicEiSOjE4JEmdGBySpE4MDklSJwaHJKkTg0OS1InBIUnqxOCQJHVicEiS\nOjE4JEmdDC04kmxKsifJ/X1t/z7J15Pcm+RzSRb3Lbs0yc4kDyU5s6/9rNa2M8mGYdUrSRrMMLc4\nPgGcNa1tK/DaqvpZ4BvApQBJTgEuAP5e+8x/SLIoySLgT4GzgVOAt7a+kqQRGVpwVNVtwL5pbV+o\nqmfb7O3A8ja9Friuqn5QVd8EdgKnttfOqnq4qp4Brmt9JUkjMspjHL8BfL5NLwMe61s22dr21y5J\nGpGRBEeS9wPPAp+capqhWx2gfabvXJ9kW5Jte/funZ1CJUkvMOfBkWQd8CvA26pqKgQmgRV93ZYD\nuw7Q/gJVtbGq1lTVmqVLl85+4ZIkYI6DI8lZwHuBc6vq6b5FW4ALkhyTZCWwCrgTuAtYlWRlkqPp\nHUDfMpc1S5J+1FHD+uIk1wK/AByfZBK4jN5ZVMcAW5MA3F5V/7KqHkhyPfAgvV1YF1fVc+17LgFu\nARYBm6rqgWHVLEk6uKEFR1W9dYbmqw7Q/4PAB2dovxm4eRZLkyQdBq8clyR1YnBIkjoxOCRJnRgc\nkqRODA5JUicGhySpE4NDktSJwSFJ6sTgkCR1YnBIkjoxOCRJnRgckqRODA5JUicGhySpE4NDktSJ\nwSFJ6sTgkCR1YnBIkjoxOCRJnRgckqRODA5JUidDC44km5LsSXJ/X9txSbYm2dHel7T2JLkiyc4k\n9yZZ3feZda3/jiTrhlWvJGkww9zi+ARw1rS2DcCtVbUKuLXNA5wNrGqv9cCV0Asa4DLgDcCpwGVT\nYSNJGo2hBUdV3Qbsm9a8FtjcpjcD5/W1X109twOLk5wInAlsrap9VfUEsJUXhpEkaQ7N9TGOV1bV\nboD2fkJrXwY81tdvsrXtr12SNCLjcnA8M7TVAdpf+AXJ+iTbkmzbu3fvrBYnSXreXAfH420XFO19\nT2ufBFb09VsO7DpA+wtU1caqWlNVa5YuXTrrhUuSeuY6OLYAU2dGrQNu7Gt/ezu76jTgybYr6xbg\njCRL2kHxM1qbJGlEjhrWFye5FvgF4Pgkk/TOjvoQcH2Si4BHgfNb95uBc4CdwNPAhQBVtS/JHwJ3\ntX5/UFXTD7hLkubQ0IKjqt66n0Wnz9C3gIv38z2bgE2zWJok6TCMy8FxSdI8YXBIkjoxOCRJnRgc\nkqRODA5JUicGhySpE4NDktSJwSFJ6sTgkCR1YnBIkjoxOCRJnRgckqROBgqOJK8ddiGSpPlh0C2O\n/5jkziTvSLJ4qBVJksbaQMFRVT8PvI3e0/i2JfmvSd481MokSWNp4GMcVbUD+D3gvcA/Aa5I8vUk\n/2xYxUmSxs+gxzh+NsnlwHbgF4FfrarXtOnLh1ifJGnMDPoEwD8BPg68r6q+N9VYVbuS/N5QKpMk\njaVBg+Mc4HtV9RxAkhcBL6mqp6vqmqFVJ0kaO4Me4/gi8NK++WNbmyRpgRk0OF5SVf93aqZNHzuc\nkiRJ42zQ4PjbJKunZpL8A+B7B+h/QEn+dZIHktyf5NokL0myMskdSXYk+VSSo1vfY9r8zrZ84lDX\nK0k6fIMGx7uBTyf5qyR/BXwKuORQVphkGfDbwJqqei2wCLgA+DBweVWtAp4ALmofuQh4oqp+mt4Z\nXB8+lPVKkmbHoBcA3gX8DPCvgHcAr6mquw9jvUcBL01yFL1dXrvpndp7Q1u+GTivTa9t87TlpyfJ\nYaxbknQYBj2rCuDngIn2mdcnoaqu7rrCqvpWkj8CHqW3u+sLwN3Ad6rq2dZtEljWppcBj7XPPpvk\nSeAVwLe7rluSdPgGCo4k1wCvAu4BnmvNBXQOjiRL6G1FrAS+A3waOHuGrjX1kQMs6//e9cB6gJNO\nOqlrWZKkAQ26xbEGOKWqXvAf7EPwS8A3q2ovQJLPAv8IWJzkqLbVsRzY1fpP0rtH1mTbtfVyYN/0\nL62qjcBGgDVr1sxGnZKkGQx6cPx+4CdnaZ2PAqclObYdqzgdeBD4MvBrrc864MY2vaXN05Z/aZYC\nTJJ0CAbd4jgeeDDJncAPphqr6tyuK6yqO5LcAHwFeBb4Kr0thZuA65L829Z2VfvIVcA1SXbS29K4\noOs6JUmzZ9Dg+MBsrrSqLgMum9b8MHDqDH2/D5w/m+uXJB26gYKjqv4yyU8Bq6rqi0mOpXf9hSRp\ngRn0tuq/Re8aij9rTcuAPx9WUZKk8TXowfGLgTcCT8HfPdTphGEVJUkaX4MGxw+q6pmpmXZarGc2\nSdICNGhw/GWS99G7Tcib6V2099+GV5YkaVwNGhwbgL3AfcC/AG6m9/xxSdICM+hZVT+k9+jYjw+3\nHEnSuBv0XlXfZIZjGlV18qxXJEkaa13uVTXlJfQuyDtu9suRJI27QZ/H8X/6Xt+qqo/Se36GJGmB\nGXRX1eq+2RfR2wJ52VAqkiSNtUF3Vf1x3/SzwCPAP5/1aiRJY2/Qs6r+6bALkSTND4PuqnrPgZZX\n1UdmpxxJ0rjrclbVz9F7qBLArwK30Z4FLklaOLo8yGl1VX0XIMkHgE9X1W8OqzBJ0nga9JYjJwHP\n9M0/A0zMejWSpLE36BbHNcCdST5H7wrytwBXD60qSdLYGvSsqg8m+Tzwj1vThVX11eGVJUkaV4Pu\nqgI4Fniqqj4GTCZZOaSaJEljbNBHx14GvBe4tDW9GPgvwypKkjS+Bt3ieAtwLvC3AFW1i8O45UiS\nxUluSPL1JNuT/MMkxyXZmmRHe1/S+ibJFUl2Jrl32u1PJElzbNDgeKaqinZr9SQ/dpjr/RjwP6rq\nZ4DXAdvpPSzq1qpaBdza5gHOBla113rgysNctyTpMAwaHNcn+TNgcZLfAr7IIT7UKclPAG8CrgKo\nqmeq6jvAWmBz67YZOK9NrwWurp7bWw0nHsq6u5jYcBMTG24a9mokad4Z9KyqP2rPGn8KeDXw+1W1\n9RDXeTK9x9D+5ySvA+4G3gW8sqp2t/XtTnJC67+MH71CfbK17T7E9UuSDsNBgyPJIuCWqvol4FDD\nYvo6VwPvrKo7knyM53dLzVjCDG0veBphkvX0dmVx0kknzUKZkqSZHHRXVVU9Bzyd5OWztM5JYLKq\n7mjzN9ALksendkG19z19/Vf0fX45sGuGOjdW1ZqqWrN06dJZKlWSNN2gV45/H7gvyVbamVUAVfXb\nXVdYVX+T5LEkr66qh4DTgQfbax3wofZ+Y/vIFuCSJNcBbwCenNqlJUmae4MGx03tNVveCXwyydHA\nw8CF9LZ+rk9yEfAoveeaA9wMnAPsBJ5ufSVJI3LA4EhyUlU9WlWbD9Svq6q6h96t2qc7fYa+BVw8\nm+uXJB26gx3j+POpiSSfGXItkqR54GDB0X9G08nDLESSND8cLDhqP9OSpAXqYAfHX5fkKXpbHi9t\n07T5qqqfGGp1kqSxc8DgqKpFc1WIJGl+6PI8DkmSDA5JUjcGhySpE4NDktSJwSFJ6sTgkCR1YnBI\nkjoxOCRJnRgckqRODA5JUicGhySpE4NDktSJwSFJ6sTgkCR1YnBIkjoxOCRJnYwsOJIsSvLVJP+9\nza9MckeSHUk+leTo1n5Mm9/Zlk+MqmZJ0mi3ON4FbO+b/zBweVWtAp4ALmrtFwFPVNVPA5e3fpKk\nERlJcCRZDvwy8J/afIBfBG5oXTYD57XptW2etvz01l+SNAKj2uL4KPC7wA/b/CuA71TVs21+EljW\nppcBjwG05U+2/pKkEZjz4EjyK8Ceqrq7v3mGrjXAsv7vXZ9kW5Jte/funYVKJUkzGcUWxxuBc5M8\nAlxHbxfVR4HFSY5qfZYDu9r0JLACoC1/ObBv+pdW1caqWlNVa5YuXTrcf4EkLWBzHhxVdWlVLa+q\nCeAC4EtV9Tbgy8CvtW7rgBvb9JY2T1v+pap6wRaHJGlujNN1HO8F3pNkJ71jGFe19quAV7T29wAb\n5rKoiQ03zeXqJGnsHXXwLsNTVX8B/EWbfhg4dYY+3wfOn9PCJEn7NU5bHJKkecDgkCR1YnBIkjox\nOCRJnRgckqRODA5JUicGhySpE4NDktSJwSFJ6sTgkCR1YnBIkjoxOCRJnRgckqRODA5JUicGxwAm\nNtzkczkkqTE4JEmdGBySpE4MDklSJwaHJKkTg0OS1InBIUnqZM6DI8mKJF9Osj3JA0ne1dqPS7I1\nyY72vqS1J8kVSXYmuTfJ6rmuWZL0vFFscTwL/Juqeg1wGnBxklOADcCtVbUKuLXNA5wNrGqv9cCV\nc19yj9dySNIIgqOqdlfVV9r0d4HtwDJgLbC5ddsMnNem1wJXV8/twOIkJ85x2ZKkZqTHOJJMAK8H\n7gBeWVW7oRcuwAmt2zLgsb6PTbY2SdIIjCw4kvw48Bng3VX11IG6ztBWM3zf+iTbkmzbu3fvbJUp\nSZpmJMGR5MX0QuOTVfXZ1vz41C6o9r6ntU8CK/o+vhzYNf07q2pjVa2pqjVLly4dXvGStMCN4qyq\nAFcB26vqI32LtgDr2vQ64Ma+9re3s6tOA56c2qUlSZp7R41gnW8Efh24L8k9re19wIeA65NcBDwK\nnN+W3QycA+wEngYunNtyJUn95jw4qup/MvNxC4DTZ+hfwMVDLUqSNDCvHO/IZ3NIWugMDklSJwaH\nJKkTg0OS1InBIUnqZBSn4x4R+g+QP/KhXx5hJZI0t9zikCR1YnBIkjoxOGaB13VIWkgMDklSJwaH\nJKkTg0OS1InBMUu8h5WkhcLgkCR1YnDMsv6tDrdAJB2JvHJ8CAwMSUcytzgkSZ0YHEPmQXNJRxqD\nYwQME0nzmcExRwwKSUcKD47Poenh4a3ZJc1H82aLI8lZSR5KsjPJhlHXM2zuzpI0rubFFkeSRcCf\nAm8GJoG7kmypqgdHW9ns2V9I7K/dLRRJozIvggM4FdhZVQ8DJLkOWAscMcHR1fRAmQoSd39JGrb5\nEhzLgMf65ieBN4yolrE005ZJ111dMwXN9CCamu/vO0hYTWy4adaDrL+WmeqSNBypqlHXcFBJzgfO\nrKrfbPO/DpxaVe/s67MeWN9mXw08dBirPB749mF8fqFwnAbjOA3GcRrMMMfpp6pq6cE6zZctjklg\nRd/8cmBXf4eq2ghsnI2VJdlWVWtm47uOZI7TYBynwThOgxmHcZovZ1XdBaxKsjLJ0cAFwJYR1yRJ\nC9K82OKoqmeTXALcAiwCNlXVAyMuS5IWpHkRHABVdTNw8xytblZ2eS0AjtNgHKfBOE6DGfk4zYuD\n45Kk8TFfjnFIksaEwdFnod3W5GCSPJLkviT3JNnW2o5LsjXJjva+pLUnyRVt7O5Nsnq01Q9Xkk1J\n9iS5v6+t89gkWdf670iybhT/lmHazzh9IMm32u/qniTn9C27tI3TQ0nO7Gs/ov82k6xI8uUk25M8\nkORdrX08f1NV5au3u24R8NfAycDRwNeAU0Zd14jH5BHg+Glt/w7Y0KY3AB9u0+cAnwcCnAbcMer6\nhzw2bwJWA/cf6tgAxwEPt/clbXrJqP9tczBOHwB+Z4a+p7S/u2OAle3vcdFC+NsETgRWt+mXAd9o\n4zGWvym3OJ73d7c1qapngKnbmuhHrQU2t+nNwHl97VdXz+3A4iQnjqLAuVBVtwH7pjV3HZszga1V\nta+qngC2AmcNv/q5s59x2p+1wHVV9YOq+iawk97f5RH/t1lVu6vqK236u8B2enfMGMvflMHxvJlu\na7JsRLWMiwK+kOTudmU+wCurajf0fuzACa3d8es+Ngt5zC5pu1g2Te1+wXECIMkE8HrgDsb0N2Vw\nPC8ztC30U87eWFWrgbOBi5O86QB9Hb/929/YLNQxuxJ4FfD3gd3AH7f2BT9OSX4c+Azw7qp66kBd\nZ2ibs7EyOJ530NuaLDRVtau97wE+R2+XweNTu6Da+57W3fHrPjYLcsyq6vGqeq6qfgh8nN7vChb4\nOCV5Mb3Q+GRVfbY1j+VvyuB4nrc16ZPkx5K8bGoaOAO4n96YTJ2psQ64sU1vAd7ezvY4DXhyahN7\nAek6NrcAZyRZ0nbXnNHajmjTjn29hd7vCnrjdEGSY5KsBFYBd7IA/jaTBLgK2F5VH+lbNJ6/qVGf\nTTBOL3pnKnyD3hkc7x91PSMei5Ppnb3yNeCBqfEAXgHcCuxo78e19tB72NZfA/cBa0b9bxjy+FxL\nbzfL/6P3f3kXHcrYAL9B7yDwTuDCUf+75micrmnjcC+9/wCe2Nf//W2cHgLO7ms/ov82gZ+nt0vp\nXuCe9jpnXH9TXjkuSerEXVWSpE4MDklSJwaHJKkTg0OS1InBIUnqxOCQJHVicEiSOjE4JEmd/H9V\npfa3H13wnAAAAABJRU5ErkJggg==\n", 709 | "text/plain": [ 710 | "" 711 | ] 712 | }, 713 | "metadata": {}, 714 | "output_type": "display_data" 715 | } 716 | ], 717 | "source": [ 718 | "# e.g. calculate counts of reviews per business entity, and plot it\n", 719 | "df_joined['business_id'].value_counts().plot.hist(bins = 200)\n", 720 | "plt.show" 721 | ] 722 | }, 723 | { 724 | "cell_type": "markdown", 725 | "metadata": {}, 726 | "source": [ 727 | "## Save your preprocessed dataset to csv file\n", 728 | "\n", 729 | "* Respect your laptop's hard work! You don't want to make it run everything again." 730 | ] 731 | }, 732 | { 733 | "cell_type": "code", 734 | "execution_count": 31, 735 | "metadata": { 736 | "collapsed": true 737 | }, 738 | "outputs": [], 739 | "source": [ 740 | "# Save to ./data/last_2_years_restaurant_reviews.csv for your next task\n", 741 | "df_joined.to_csv('dataset/last_2_years_restaurant_reviews.csv', index = False)" 742 | ] 743 | }, 744 | { 745 | "cell_type": "code", 746 | "execution_count": null, 747 | "metadata": { 748 | "collapsed": true 749 | }, 750 | "outputs": [], 751 | "source": [] 752 | } 753 | ], 754 | "metadata": { 755 | "anaconda-cloud": {}, 756 | "kernelspec": { 757 | "display_name": "Python 3", 758 | "language": "python", 759 | "name": "python3" 760 | }, 761 | "language_info": { 762 | "codemirror_mode": { 763 | "name": "ipython", 764 | "version": 3 765 | }, 766 | "file_extension": ".py", 767 | "mimetype": "text/x-python", 768 | "name": "python", 769 | "nbconvert_exporter": "python", 770 | "pygments_lexer": "ipython3", 771 | "version": "3.5.5" 772 | } 773 | }, 774 | "nbformat": 4, 775 | "nbformat_minor": 1 776 | } 777 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/Yelp_Dataset_-_NLP-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Yelp Data Challenge - NLP" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 8, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import pandas as pd\n", 17 | "import numpy as np\n", 18 | "\n", 19 | "import matplotlib.pyplot as plt\n", 20 | "%matplotlib inline" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 3, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "df = pd.read_csv('../dataset/2016_restaurant_reviews.csv')" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 4, 35 | "metadata": {}, 36 | "outputs": [ 37 | { 38 | "data": { 39 | "text/html": [ 40 | "
\n", 41 | "\n", 54 | "\n", 55 | " \n", 56 | " \n", 57 | " \n", 58 | " \n", 59 | " \n", 60 | " \n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | "
business_idnamecategoriesavg_starscooldatefunnyreview_idstarstextusefuluser_id
0--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Cajun/Creole', 'Steakhouses', 'Restaurants']4.002016-03-3106SgvNWJltnZhW7duJgZ42w5This is mine and my fiancé's favorite steakhou...0oFyOUOeGTRZhFPF9uTqrTQ
1--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Cajun/Creole', 'Steakhouses', 'Restaurants']4.002016-02-100UxFpgng8dPMWOj99653k5Q5Truly Fantastic! Best Steak ever. Service was...0aVOGlN9fZ-BXcbtj6dbf0g
2--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Cajun/Creole', 'Steakhouses', 'Restaurants']4.002016-07-130GN7KnAaxJDrYgfJzgsvmkA5We decided to give Delmonico's a try because w...0C6kw0Rny7jZAGjTj0MWA3Q
3--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Cajun/Creole', 'Steakhouses', 'Restaurants']4.002016-04-010i7xD3FY-EaF9O08QL69l5w5Absolutely impressed with this restaurant. The...0tTifjrXlRrUme-4c0UW9Bw
4--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Cajun/Creole', 'Steakhouses', 'Restaurants']4.002016-10-150ZqMUHOJg9lGOmfqQ7RXj7A1This was supposed to be a very special dinner ...0kOll36Me-rM9NsqoDnejKA
\n", 150 | "
" 151 | ], 152 | "text/plain": [ 153 | " business_id name \\\n", 154 | "0 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 155 | "1 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 156 | "2 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 157 | "3 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 158 | "4 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 159 | "\n", 160 | " categories avg_stars cool \\\n", 161 | "0 ['Cajun/Creole', 'Steakhouses', 'Restaurants'] 4.0 0 \n", 162 | "1 ['Cajun/Creole', 'Steakhouses', 'Restaurants'] 4.0 0 \n", 163 | "2 ['Cajun/Creole', 'Steakhouses', 'Restaurants'] 4.0 0 \n", 164 | "3 ['Cajun/Creole', 'Steakhouses', 'Restaurants'] 4.0 0 \n", 165 | "4 ['Cajun/Creole', 'Steakhouses', 'Restaurants'] 4.0 0 \n", 166 | "\n", 167 | " date funny review_id stars \\\n", 168 | "0 2016-03-31 0 6SgvNWJltnZhW7duJgZ42w 5 \n", 169 | "1 2016-02-10 0 UxFpgng8dPMWOj99653k5Q 5 \n", 170 | "2 2016-07-13 0 GN7KnAaxJDrYgfJzgsvmkA 5 \n", 171 | "3 2016-04-01 0 i7xD3FY-EaF9O08QL69l5w 5 \n", 172 | "4 2016-10-15 0 ZqMUHOJg9lGOmfqQ7RXj7A 1 \n", 173 | "\n", 174 | " text useful \\\n", 175 | "0 This is mine and my fiancé's favorite steakhou... 0 \n", 176 | "1 Truly Fantastic! Best Steak ever. Service was... 0 \n", 177 | "2 We decided to give Delmonico's a try because w... 0 \n", 178 | "3 Absolutely impressed with this restaurant. The... 0 \n", 179 | "4 This was supposed to be a very special dinner ... 0 \n", 180 | "\n", 181 | " user_id \n", 182 | "0 oFyOUOeGTRZhFPF9uTqrTQ \n", 183 | "1 aVOGlN9fZ-BXcbtj6dbf0g \n", 184 | "2 C6kw0Rny7jZAGjTj0MWA3Q \n", 185 | "3 tTifjrXlRrUme-4c0UW9Bw \n", 186 | "4 kOll36Me-rM9NsqoDnejKA " 187 | ] 188 | }, 189 | "execution_count": 4, 190 | "metadata": {}, 191 | "output_type": "execute_result" 192 | } 193 | ], 194 | "source": [ 195 | "df.head()" 196 | ] 197 | }, 198 | { 199 | "cell_type": "markdown", 200 | "metadata": {}, 201 | "source": [ 202 | "### Define feature variables, here is the text of the review" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": 5, 208 | "metadata": {}, 209 | "outputs": [], 210 | "source": [ 211 | "# Take the values of the column that contains review text data, save to a variable named \"documents\"\n", 212 | "documents = [t for i,t in enumerate(df.text)]" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": 6, 218 | "metadata": {}, 219 | "outputs": [ 220 | { 221 | "data": { 222 | "text/plain": [ 223 | "178080" 224 | ] 225 | }, 226 | "execution_count": 6, 227 | "metadata": {}, 228 | "output_type": "execute_result" 229 | } 230 | ], 231 | "source": [ 232 | "# inspect your documents, e.g. check the size, take a peek at elements of the numpy array\n", 233 | "len(documents)" 234 | ] 235 | }, 236 | { 237 | "cell_type": "markdown", 238 | "metadata": {}, 239 | "source": [ 240 | "### Define the target variable (any categorical variable that may be meaningful)" 241 | ] 242 | }, 243 | { 244 | "cell_type": "markdown", 245 | "metadata": {}, 246 | "source": [ 247 | "#### For example, I am interested in perfect (5 stars) and imperfect (1-4 stars) rating" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": 9, 253 | "metadata": {}, 254 | "outputs": [ 255 | { 256 | "data": { 257 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAF1dJREFUeJzt3X+MXeWd3/H3JzYk5AcxhIFS26mpYqVLUEPAIt4iRWlIjSGrGKlBMmqDg7xyS8lu0lbaOvtHrSVBIlK1bGkTVt7gYtIkQNmkuImJ14VEq0qBMAQWQgj1LGFhaopnYyBsaZI6++0f9/Hmas4dz52xPXeC3y/p6p7zfZ5zznOPGT5zftw5qSokSer3ulEPQJK0+BgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUsHfUA5uuMM86oVatWjXoYkvQr4+GHH/7Lqhobpu+vbDisWrWK8fHxUQ9Dkn5lJPmLYft6WkmS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktTxK/sNaUkapVVbvzGS7T5z44cWZDseOUiSOoYKhyT/MskTSb6f5CtJ3pDknCQPJtmX5M4kJ7e+r2/zE619Vd96PtXqTyW5tK++vtUmkmw91h9SkjQ3s4ZDkuXAbwNrquo8YAmwEfgscFNVrQZeBDa3RTYDL1bVO4CbWj+SnNuWexewHvh8kiVJlgCfAy4DzgWuan0lSSMy7GmlpcApSZYCbwSeBz4A3N3adwJXtOkNbZ7WfkmStPodVfWzqvoRMAFc1F4TVfV0Vf0cuKP1lSSNyKzhUFX/C/h3wLP0QuFl4GHgpao61LpNAsvb9HLgubbsodb/bf31acvMVJckjcgwp5VOo/eb/DnA3wbeRO8U0HR1eJEZ2uZaHzSWLUnGk4xPTU3NNnRJ0jwNc1rpg8CPqmqqqv4f8FXgHwDL2mkmgBXA/jY9CawEaO1vBQ7216ctM1O9o6q2V9WaqlozNjbUw4wkSfMwTDg8C6xN8sZ27eAS4AfAt4CPtD6bgHva9K42T2u/v6qq1Te2u5nOAVYD3wUeAla3u59OpnfRetfRfzRJ0nzN+iW4qnowyd3A94BDwCPAduAbwB1JPtNqt7ZFbgW+mGSC3hHDxraeJ5LcRS9YDgHXVdUvAJJ8HNhD706oHVX1xLH7iJKkuRrqG9JVtQ3YNq38NL07jab3/Slw5QzruQG4YUB9N7B7mLFIko4/vyEtSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVLHrOGQ5J1JHu17/STJJ5OcnmRvkn3t/bTWP0luTjKR5LEkF/Sta1Prvy/Jpr76hUkeb8vc3B5HKkkakVnDoaqeqqrzq+p84ELgVeBrwFbgvqpaDdzX5gEuo/d86NXAFuAWgCSn03ua3HvpPUFu2+FAaX229C23/ph8OknSvMz1tNIlwJ9X1V8AG4Cdrb4TuKJNbwBur54HgGVJzgYuBfZW1cGqehHYC6xvbadW1XeqqoDb+9YlSRqBuYbDRuArbfqsqnoeoL2f2erLgef6lplstSPVJwfUJUkjMnQ4JDkZ+DDwX2brOqBW86gPGsOWJONJxqempmYZhiRpvuZy5HAZ8L2qeqHNv9BOCdHeD7T6JLCyb7kVwP5Z6isG1DuqantVramqNWNjY3MYuiRpLuYSDlfxy1NKALuAw3ccbQLu6atf3e5aWgu83E477QHWJTmtXYheB+xpba8kWdvuUrq6b12SpBFYOkynJG8E/hHwz/rKNwJ3JdkMPAtc2eq7gcuBCXp3Nl0DUFUHk3waeKj1u76qDrbpa4HbgFOAe9tLkjQiQ4VDVb0KvG1a7cf07l6a3reA62ZYzw5gx4D6OHDeMGORJB1/fkNaktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqSOocIhybIkdyf5YZInk/x6ktOT7E2yr72f1vomyc1JJpI8luSCvvVsav33JdnUV78wyeNtmZvb40IlSSMy7JHDvwe+WVV/D3g38CSwFbivqlYD97V5gMuA1e21BbgFIMnpwDbgvcBFwLbDgdL6bOlbbv3RfSxJ0tGYNRySnAq8D7gVoKp+XlUvARuAna3bTuCKNr0BuL16HgCWJTkbuBTYW1UHq+pFYC+wvrWdWlXfaY8Yvb1vXZKkERjmyOHvAlPAf0rySJIvJHkTcFZVPQ/Q3s9s/ZcDz/UtP9lqR6pPDqhLkkZkmHBYClwA3FJV7wH+D788hTTIoOsFNY96d8XJliTjScanpqaOPGpJ0rwNEw6TwGRVPdjm76YXFi+0U0K09wN9/Vf2Lb8C2D9LfcWAekdVba+qNVW1ZmxsbIihS5LmY9ZwqKr/DTyX5J2tdAnwA2AXcPiOo03APW16F3B1u2tpLfByO+20B1iX5LR2IXodsKe1vZJkbbtL6eq+dUmSRmDpkP1+C/hSkpOBp4Fr6AXLXUk2A88CV7a+u4HLgQng1daXqjqY5NPAQ63f9VV1sE1fC9wGnALc216SpBEZKhyq6lFgzYCmSwb0LeC6GdazA9gxoD4OnDfMWCRJx5/fkJYkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1DBUOSZ5J8niSR5OMt9rpSfYm2dfeT2v1JLk5yUSSx5Jc0LeeTa3/viSb+uoXtvVPtGVzrD+oJGl4czly+IdVdX5VHX5c6FbgvqpaDdzX5gEuA1a31xbgFuiFCbANeC9wEbDtcKC0Plv6lls/708kSTpqR3NaaQOws03vBK7oq99ePQ8Ay5KcDVwK7K2qg1X1IrAXWN/aTq2q77TnT9/ety5J0ggMGw4F/EmSh5NsabWzqup5gPZ+ZqsvB57rW3ay1Y5UnxxQ70iyJcl4kvGpqakhhy5JmqulQ/a7uKr2JzkT2Jvkh0foO+h6Qc2j3i1WbQe2A6xZs2ZgH0nS0RvqyKGq9rf3A8DX6F0zeKGdEqK9H2jdJ4GVfYuvAPbPUl8xoC5JGpFZwyHJm5K85fA0sA74PrALOHzH0Sbgnja9C7i63bW0Fni5nXbaA6xLclq7EL0O2NPaXkmytt2ldHXfuiRJIzDMaaWzgK+1u0uXAl+uqm8meQi4K8lm4FngytZ/N3A5MAG8ClwDUFUHk3waeKj1u76qDrbpa4HbgFOAe9tLkjQis4ZDVT0NvHtA/cfAJQPqBVw3w7p2ADsG1MeB84YYryRpAfgNaUlSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOoYOhyRLkjyS5Ott/pwkDybZl+TOJCe3+uvb/ERrX9W3jk+1+lNJLu2rr2+1iSRbj93HkyTNx1yOHD4BPNk3/1ngpqpaDbwIbG71zcCLVfUO4KbWjyTnAhuBdwHrgc+3wFkCfA64DDgXuKr1lSSNyFDhkGQF8CHgC20+wAeAu1uXncAVbXpDm6e1X9L6bwDuqKqfVdWP6D1j+qL2mqiqp6vq58Adra8kaUSGPXL4A+B3gL9u828DXqqqQ21+EljeppcDzwG09pdb/7+pT1tmpnpHki1JxpOMT01NDTl0SdJczRoOSX4DOFBVD/eXB3StWdrmWu8Wq7ZX1ZqqWjM2NnaEUUuSjsbSIfpcDHw4yeXAG4BT6R1JLEuytB0drAD2t/6TwEpgMslS4K3Awb76Yf3LzFSXJI3ArEcOVfWpqlpRVavoXVC+v6r+CfAt4COt2ybgnja9q83T2u+vqmr1je1upnOA1cB3gYeA1e3up5PbNnYdk08nSZqXYY4cZvJvgDuSfAZ4BLi11W8Fvphkgt4Rw0aAqnoiyV3AD4BDwHVV9QuAJB8H9gBLgB1V9cRRjEuSdJTmFA5V9W3g2236aXp3Gk3v81PgyhmWvwG4YUB9N7B7LmORJB0/R3PkIEkArNr6jZFs95kbPzSS7Z4I/PMZkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqWPWcEjyhiTfTfJnSZ5I8nutfk6SB5PsS3Jne8Qn7TGgdyaZaO2r+tb1qVZ/KsmlffX1rTaRZOux/5iSpLkY5sjhZ8AHqurdwPnA+iRrgc8CN1XVauBFYHPrvxl4sareAdzU+pHkXHqPDH0XsB74fJIlSZYAnwMuA84Frmp9JUkjMms4VM9ftdmT2quADwB3t/pO4Io2vaHN09ovSZJWv6OqflZVPwIm6D1m9CJgoqqerqqfA3e0vpKkERnqmkP7Df9R4ACwF/hz4KWqOtS6TALL2/Ry4DmA1v4y8Lb++rRlZqoPGseWJONJxqempoYZuiRpHoYKh6r6RVWdD6yg95v+rw3q1t4zQ9tc64PGsb2q1lTVmrGxsdkHLkmalzndrVRVLwHfBtYCy5IsbU0rgP1tehJYCdDa3woc7K9PW2amuiRpRIa5W2ksybI2fQrwQeBJ4FvAR1q3TcA9bXpXm6e1319V1eob291M5wCrge8CDwGr291PJ9O7aL3rWHw4SdL8LJ29C2cDO9tdRa8D7qqqryf5AXBHks8AjwC3tv63Al9MMkHviGEjQFU9keQu4AfAIeC6qvoFQJKPA3uAJcCOqnrimH1CSdKczRoOVfUY8J4B9afpXX+YXv8pcOUM67oBuGFAfTewe4jxSpIWgN+QliR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6hvmrrK85q7Z+YyTbfebGD41ku5I0Vx45SJI6DAdJUofhIEnqGOYxoSuTfCvJk0meSPKJVj89yd4k+9r7aa2eJDcnmUjyWJIL+ta1qfXfl2RTX/3CJI+3ZW5OkuPxYSVJwxnmyOEQ8K+r6teAtcB1Sc4FtgL3VdVq4L42D3AZvedDrwa2ALdAL0yAbcB76T1BbtvhQGl9tvQtt/7oP5okab5mDYeqer6qvtemXwGeBJYDG4CdrdtO4Io2vQG4vXoeAJYlORu4FNhbVQer6kVgL7C+tZ1aVd+pqgJu71uXJGkE5nTNIckqes+TfhA4q6qeh16AAGe2bsuB5/oWm2y1I9UnB9QlSSMy9PcckrwZ+GPgk1X1kyNcFhjUUPOoDxrDFnqnn3j7298+25ClkfB7NHotGOrIIclJ9ILhS1X11VZ+oZ0Sor0faPVJYGXf4iuA/bPUVwyod1TV9qpaU1VrxsbGhhm6JGkehrlbKcCtwJNV9ft9TbuAw3ccbQLu6atf3e5aWgu83E477QHWJTmtXYheB+xpba8kWdu2dXXfuiRJIzDMaaWLgY8Cjyd5tNV+F7gRuCvJZuBZ4MrWthu4HJgAXgWuAaiqg0k+DTzU+l1fVQfb9LXAbcApwL3tJUkakVnDoar+B4OvCwBcMqB/AdfNsK4dwI4B9XHgvNnGIklaGH5DWpLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdZyQz5A+Efn3fiTNhUcOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUM85jQHUkOJPl+X+30JHuT7Gvvp7V6ktycZCLJY0ku6FtmU+u/L8mmvvqFSR5vy9zcHhUqSRqhYY4cbgPWT6ttBe6rqtXAfW0e4DJgdXttAW6BXpgA24D3AhcB2w4HSuuzpW+56duSJC2wWcOhqv4UODitvAHY2aZ3Alf01W+vngeAZUnOBi4F9lbVwap6EdgLrG9tp1bVd9rjRW/vW5ckaUTme83hrKp6HqC9n9nqy4Hn+vpNttqR6pMD6pKkETrWF6QHXS+oedQHrzzZkmQ8yfjU1NQ8hyhJms18w+GFdkqI9n6g1SeBlX39VgD7Z6mvGFAfqKq2V9WaqlozNjY2z6FLkmYz33DYBRy+42gTcE9f/ep219Ja4OV22mkPsC7Jae1C9DpgT2t7JcnadpfS1X3rkiSNyKzPc0jyFeD9wBlJJunddXQjcFeSzcCzwJWt+27gcmACeBW4BqCqDib5NPBQ63d9VR2+yH0tvTuiTgHubS9J0gjNGg5VddUMTZcM6FvAdTOsZwewY0B9HDhvtnFIkhaO35CWJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKlj0YRDkvVJnkoykWTrqMcjSSeyRREOSZYAnwMuA84Frkpy7mhHJUknrkURDsBFwERVPV1VPwfuADaMeEySdMJaLOGwHHiub36y1SRJI5CqGvUYSHIlcGlV/Wab/yhwUVX91rR+W4AtbfadwFPz3OQZwF/Oc9njyXHNjeOaG8c1N6/Fcf2dqhobpuPSeW7gWJsEVvbNrwD2T+9UVduB7Ue7sSTjVbXmaNdzrDmuuXFcc+O45uZEH9diOa30ELA6yTlJTgY2ArtGPCZJOmEtiiOHqjqU5OPAHmAJsKOqnhjxsCTphLUowgGgqnYDuxdoc0d9auo4cVxz47jmxnHNzQk9rkVxQVqStLgslmsOkqRF5DUbDkl2JDmQ5PsztCfJze3PdTyW5IJFMq73J3k5yaPt9W8XaFwrk3wryZNJnkjyiQF9FnyfDTmuBd9nSd6Q5LtJ/qyN6/cG9Hl9kjvb/nowyapFMq6PJZnq21+/ebzH1bftJUkeSfL1AW0Lvr+GHNdI9leSZ5I83rY5PqD9+P48VtVr8gW8D7gA+P4M7ZcD9wIB1gIPLpJxvR/4+gj219nABW36LcD/BM4d9T4bclwLvs/aPnhzmz4JeBBYO63PvwD+sE1vBO5cJOP6GPAfF/q/sbbtfwV8edC/1yj215DjGsn+Ap4BzjhC+3H9eXzNHjlU1Z8CB4/QZQNwe/U8ACxLcvYiGNdIVNXzVfW9Nv0K8CTdb6kv+D4bclwLru2Dv2qzJ7XX9At4G4Cdbfpu4JIkWQTjGokkK4APAV+YocuC768hx7VYHdefx9dsOAxhMf/Jjl9vpwXuTfKuhd54O5x/D73fOvuNdJ8dYVwwgn3WTkU8ChwA9lbVjPurqg4BLwNvWwTjAvjH7VTE3UlWDmg/Hv4A+B3gr2doH8n+GmJcMJr9VcCfJHk4vb8OMd1x/Xk8kcNh0G8ki+E3rO/R+4r7u4H/APzXhdx4kjcDfwx8sqp+Mr15wCILss9mGddI9llV/aKqzqf3jf6Lkpw3rctI9tcQ4/pvwKqq+vvAf+eXv60fN0l+AzhQVQ8fqduA2nHdX0OOa8H3V3NxVV1A769VX5fkfdPaj+v+OpHDYag/2bHQquonh08LVO+7HyclOWMhtp3kJHr/A/5SVX11QJeR7LPZxjXKfda2+RLwbWD9tKa/2V9JlgJvZQFPKc40rqr6cVX9rM3+EXDhAgznYuDDSZ6h91eXP5DkP0/rM4r9Neu4RrS/qKr97f0A8DV6f72633H9eTyRw2EXcHW74r8WeLmqnh/1oJL8rcPnWZNcRO/f6McLsN0AtwJPVtXvz9BtwffZMOMaxT5LMpZkWZs+Bfgg8MNp3XYBm9r0R4D7q11JHOW4pp2X/jC96zjHVVV9qqpWVNUqeheb76+qfzqt24Lvr2HGNYr9leRNSd5yeBpYB0y/w/G4/jwumm9IH2tJvkLvLpYzkkwC2+hdnKOq/pDet7EvByaAV4FrFsm4PgJcm+QQ8H+Bjcf7B6S5GPgo8Hg7Xw3wu8Db+8Y2in02zLhGsc/OBnam96Cq1wF3VdXXk1wPjFfVLnqh9sUkE/R+A954nMc07Lh+O8mHgUNtXB9bgHENtAj21zDjGsX+Ogv4WvudZynw5ar6ZpJ/Dgvz8+g3pCVJHSfyaSVJ0gwMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1PH/ASVn8WEMV9FYAAAAAElFTkSuQmCC\n", 258 | "text/plain": [ 259 | "
" 260 | ] 261 | }, 262 | "metadata": {}, 263 | "output_type": "display_data" 264 | } 265 | ], 266 | "source": [ 267 | "# Make a column and take the values, save to a variable named \"target\"\n", 268 | "df.target = df.stars\n", 269 | "\n", 270 | "plt.hist(df.target)\n", 271 | "plt.show()" 272 | ] 273 | }, 274 | { 275 | "cell_type": "markdown", 276 | "metadata": {}, 277 | "source": [ 278 | "## Let's create training dataset and test dataset" 279 | ] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": 10, 284 | "metadata": {}, 285 | "outputs": [ 286 | { 287 | "data": { 288 | "text/plain": [ 289 | "count 178080.000000\n", 290 | "mean 0.462809\n", 291 | "std 0.498616\n", 292 | "min 0.000000\n", 293 | "25% 0.000000\n", 294 | "50% 0.000000\n", 295 | "75% 1.000000\n", 296 | "max 1.000000\n", 297 | "Name: stars, dtype: float64" 298 | ] 299 | }, 300 | "execution_count": 10, 301 | "metadata": {}, 302 | "output_type": "execute_result" 303 | } 304 | ], 305 | "source": [ 306 | "# df.target = df.stars.apply(lambda x: 1 if x > 4 else 0)\n", 307 | "# df.target.describe()" 308 | ] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "execution_count": 26, 313 | "metadata": {}, 314 | "outputs": [ 315 | { 316 | "name": "stderr", 317 | "output_type": "stream", 318 | "text": [ 319 | "/Users/jessie/anaconda/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n", 320 | " \"This module will be removed in 0.20.\", DeprecationWarning)\n" 321 | ] 322 | } 323 | ], 324 | "source": [ 325 | "from sklearn.cross_validation import train_test_split" 326 | ] 327 | }, 328 | { 329 | "cell_type": "code", 330 | "execution_count": 27, 331 | "metadata": { 332 | "collapsed": true 333 | }, 334 | "outputs": [], 335 | "source": [ 336 | "# Split to documents_train, documents_test, target_train, target_test\n", 337 | "documents_train, documents_test, target_train, target_test = train_test_split(\n", 338 | " documents, df.stars, test_size=0.33, random_state=42)" 339 | ] 340 | }, 341 | { 342 | "cell_type": "markdown", 343 | "metadata": {}, 344 | "source": [ 345 | "## Let's get NLP representation of the documents" 346 | ] 347 | }, 348 | { 349 | "cell_type": "code", 350 | "execution_count": 29, 351 | "metadata": { 352 | "collapsed": true 353 | }, 354 | "outputs": [], 355 | "source": [ 356 | "from sklearn.feature_extraction.text import TfidfVectorizer" 357 | ] 358 | }, 359 | { 360 | "cell_type": "code", 361 | "execution_count": 30, 362 | "metadata": {}, 363 | "outputs": [], 364 | "source": [ 365 | "# Create TfidfVectorizer, and name it vectorizer\n", 366 | "vectorizer = TfidfVectorizer(stop_words = 'english', max_features=5000)" 367 | ] 368 | }, 369 | { 370 | "cell_type": "code", 371 | "execution_count": 37, 372 | "metadata": {}, 373 | "outputs": [], 374 | "source": [ 375 | "# Train the model with your training data\n", 376 | "vectors_train = vectorizer.fit_transform(documents_train)" 377 | ] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "execution_count": 38, 382 | "metadata": {}, 383 | "outputs": [], 384 | "source": [ 385 | "# Get the vocab of your tfidf\n", 386 | "words = vectorizer.get_feature_names()" 387 | ] 388 | }, 389 | { 390 | "cell_type": "code", 391 | "execution_count": 34, 392 | "metadata": {}, 393 | "outputs": [], 394 | "source": [ 395 | "# Use the trained model to transform your test data\n", 396 | "vectors_test = vectorizer.transform(documents_test).toarray()" 397 | ] 398 | }, 399 | { 400 | "cell_type": "markdown", 401 | "metadata": {}, 402 | "source": [ 403 | "## Similar review search engine" 404 | ] 405 | }, 406 | { 407 | "cell_type": "code", 408 | "execution_count": 88, 409 | "metadata": {}, 410 | "outputs": [], 411 | "source": [ 412 | "import numpy as np\n", 413 | "\n", 414 | "# We will need these helper methods pretty soon\n", 415 | "\n", 416 | "def get_top_values(lst, n, labels):\n", 417 | " '''\n", 418 | " INPUT: LIST, INTEGER, LIST\n", 419 | " OUTPUT: LIST\n", 420 | "\n", 421 | " Given a list of values, find the indices with the highest n values.\n", 422 | " Return the labels for each of these indices.\n", 423 | "\n", 424 | " e.g.\n", 425 | " lst = [7, 3, 2, 4, 1]\n", 426 | " n = 2\n", 427 | " labels = [\"cat\", \"dog\", \"mouse\", \"pig\", \"rabbit\"]\n", 428 | " output: [\"cat\", \"pig\"]\n", 429 | " '''\n", 430 | " return [labels[i] for i in np.argsort(lst)[::-1][:n]] # np.argsort by default sorts values in ascending order\n", 431 | "\n", 432 | "def get_bottom_values(lst, n, labels):\n", 433 | " '''\n", 434 | " INPUT: LIST, INTEGER, LIST\n", 435 | " OUTPUT: LIST\n", 436 | "\n", 437 | " Given a list of values, find the indices with the lowest n values.\n", 438 | " Return the labels for each of these indices.\n", 439 | "\n", 440 | " e.g.\n", 441 | " lst = [7, 3, 2, 4, 1]\n", 442 | " n = 2\n", 443 | " labels = [\"cat\", \"dog\", \"mouse\", \"pig\", \"rabbit\"]\n", 444 | " output: [\"mouse\", \"rabbit\"]\n", 445 | " '''\n", 446 | " return [labels[i] for i in np.argsort(lst)[::1][:n]] \n" 447 | ] 448 | }, 449 | { 450 | "cell_type": "code", 451 | "execution_count": 36, 452 | "metadata": { 453 | "collapsed": true 454 | }, 455 | "outputs": [], 456 | "source": [ 457 | "# Let's use cosine similarity\n", 458 | "from sklearn.metrics.pairwise import cosine_similarity" 459 | ] 460 | }, 461 | { 462 | "cell_type": "code", 463 | "execution_count": 43, 464 | "metadata": {}, 465 | "outputs": [ 466 | { 467 | "name": "stderr", 468 | "output_type": "stream", 469 | "text": [ 470 | "/Users/jessie/anaconda/lib/python3.6/site-packages/ipykernel/__main__.py:2: VisibleDeprecationWarning: converting an array with ndim > 0 to an index will result in an error in the future\n", 471 | " from ipykernel import kernelapp as app\n" 472 | ] 473 | } 474 | ], 475 | "source": [ 476 | "# Draw an arbitrary review from test (unseen in training) documents\n", 477 | "doc_test = documents_test[np.random.randint(len(documents_test), size=1)]" 478 | ] 479 | }, 480 | { 481 | "cell_type": "code", 482 | "execution_count": 50, 483 | "metadata": {}, 484 | "outputs": [], 485 | "source": [ 486 | "# Transform the drawn review(s) to vector(s)\n", 487 | "doc_test_vector = vectorizer.transform([doc_test]).toarray()" 488 | ] 489 | }, 490 | { 491 | "cell_type": "code", 492 | "execution_count": 51, 493 | "metadata": {}, 494 | "outputs": [], 495 | "source": [ 496 | "# Calculate the similarity score(s) between vector(s) and training vectors\n", 497 | "similarity_scores = cosine_similarity(doc_test_vector, vectors_train.toarray())" 498 | ] 499 | }, 500 | { 501 | "cell_type": "code", 502 | "execution_count": 61, 503 | "metadata": {}, 504 | "outputs": [], 505 | "source": [ 506 | "# Let's find top 5 similar reviews\n", 507 | "n = 2\n", 508 | "#np.shape(similarity_scores[0])\n", 509 | "searched_result = get_top_values(similarity_scores[0], n, documents_train)" 510 | ] 511 | }, 512 | { 513 | "cell_type": "code", 514 | "execution_count": 63, 515 | "metadata": {}, 516 | "outputs": [ 517 | { 518 | "name": "stdout", 519 | "output_type": "stream", 520 | "text": [ 521 | "Our search query:\n", 522 | "Had to come check this place out, I'd never had a fresh White Castle burger and I was not disappointed. I got the 4 cheeseburgers with fries and a drink, it tasted fresh and was nice and hot. The onions were a tasty addition and even though I could only finish three of the burgers and half the fries, it was really tasty! Would definitely go back.\n" 523 | ] 524 | } 525 | ], 526 | "source": [ 527 | "print ('Our search query:')\n", 528 | "print (doc_test)" 529 | ] 530 | }, 531 | { 532 | "cell_type": "code", 533 | "execution_count": 64, 534 | "metadata": {}, 535 | "outputs": [ 536 | { 537 | "name": "stdout", 538 | "output_type": "stream", 539 | "text": [ 540 | "Most 5 similar reviews:\n", 541 | "[\"After walking up and down the Strip twice I was craving some White Castle almost as much as Roldy and Kumar!\\n\\nFrom the outside it looked like a tiny standalone but actually the location is inside the Casino Royale.\\n\\nGot the #7 Castle Pack, which was 10 sliders, two regular fries and two drinks.\\n\\nService was good as they got my order out quickly in spite of the place being packed but for my first time eating White Castle burgers at a White Castle I was rather disappointed. \\n\\nThese certainly did not hit the spot! I know sliders don't have much meat but these were practically meatless! And the onions could've tasted a whole lot better. The Fries were meh too as they were a bit soggy.\\n\\nNot gonna lie, I enjoyed the microwave White Castle burgers you get from the grocery better...\", 'After a night of dancing, my friends and I went to White Castle. No regrets.\\n\\nI have had White Castle burgers in the past. From the freezer section at the supermarket. I have always wonder if they are really that, uh, good, in real life. After Tao, we stopped at White Castle and bought the 20 piece combo with fries and chicken rings. Yes, chicken rings. \\n\\nThe verdict: White Castle burgers taste just like the burgers you get out of the freezer aisle. They were pretty damn satisfying and tasty. The fries were good and the chicken rings were definitely questionable. I would definitely go back!', \"Been wanting to venture out to White Castle for about a decade now. Ever since Harold and Kumar made their voyage, I've been anticipating my own. Had a few days off from work and figured it was time to try these burgers fresh, instead of the ones from the frozen food section that I have once in a while.\\n\\nWalking down the strip towards White Castle was awesome and exciting but when I finally pulled out my first slider my excitement faded a bit (not much, it's Vegas!). The bread was very soggy and they tasted about the same as the frozen ones to me. Though these had more cheese. Was expecting a fresh White Castle patty to be tastier. Maybe this just isn't a great location. I'm definitely trying White Castle again if I ever encounter another chain.\\n\\nIf you happen to stop by a White Castle I would firstly advise you not to settle for this and turn away. Vegas has so many better options! What? Sorry! If you're on some quest like me or just dgaf than I'd recommend asking for a toasted bun if you choose to eat here. It is a cool experience to have if you are a fan of the Harold and Kumar movies, but beyond that these burgers are kinda average (both frozen ones and these).\\n\\nWhile the burger itself wasn't what i was hoping for, I was still glad I made the drive to White Castle. It was an experience I knew I wanted to have and though the burgers weren't satisfying, the overall experience was fun. Oh yeah and the fries were meh too. Though crispy. Throw some Cajun Seasoning on them sumbitches or something! Thank God for Ketchup. \\n\\nPros:\\n+ Staff was nice\\n\\nCons:\\n- Bun was very soggy\\n- Fries were bland\\n- Location could have been cleaner (several dirty tables)\", \"For over 30 years I've been hearing about White Castle sliders from those that lived in, or traveled to parts of the country that offered them. Then, after Harold and Kumar made their movie, I truly hoped that one day I would travel to a place in the mid west or eastern parts that had a White Castle Burger joint, or even better, I wished that White Castle would open a store here in Southern California. Sadly, no luck either way. Then voila! I saw them in the frozen section of my local super market. I bought a pack, took them home and immediately nuked a few in the microwave. Not bad I thought to myself. They had a unique flavor with kind of an onion tang. Nevertheless, these were frozen burgers. I knew the freshly made ones had to be better, so I withheld final judgement, and continued to hope for a real experience at a White Castle location.\\n\\nFast forward several years and I'm walking the strip in Las Vegas and the White Castle sign hit my like a ton of bricks! There it was! Boom! A real deal White Castle burger joint right here in Las Vegas, and right in front of me! I dragged the wife into the place and immediately placed an order for sliders, fries and a drink. So let me get this out of the way first, I don't like krinkle fries. And, the White Castle krinkle fries were no different than any other krinkle fries I've tried. So what did I think of the sliders? They were really tasty and much better than the frozen ones, as I expected them to be. These are truly unique sliders. They are steamed with the onion on the patties. I think that's part of what makes them distinctive.\\n\\nNow, if I'm only judging the slider, in the context of the fast food world of large or popular burger chains, I give it 5 stars considering the under $2 price. I don't smoke weed, but if I did, I could see myself eating a bag of these little morsels after smoking a bowl, just like Harold and Kumar did in the movie. \\n\\nThe slider is the only thing that stood out. Again, I didn't care for the fries, and everything else was just okay. Service was decent, drink fountain was nothing special. Over all, 4 stars for White Castle, mostly because of their famous sliders.\", '西元1921年在堪薩斯州成立的 White Castle ，可是在美國無人不知無人不曉，距今將近一個世紀歷史的 速食 Fast Food 品牌，在近年來也到 賭城 Las Vegas 的賭城大道上開設分店，以全天候二十四小時的開業時間來提供給來自四面八方慕名而來的顧客。比起一般漢堡尺寸小了許多的特色小漢堡（又稱做Slider）不僅是陪伴著許多美國人成長的風味也是美國文化的縮影，在以其為主題的電影 Harold and Kumar Go to White Castle 中就可以感受其風潮與熱度，也是全美第一個以冷凍方式銷售到外州的漢堡。現在不用舟車勞頓前往東岸、也不用屈就於冷凍食物，趁著來到賭城的機會就可以品嚐到這個全美國資歷最深的經典速食風味。\\n\\n作為一個經典的速食品牌，相關的特色與紀念產品相信是許多粉絲們不可錯過的，而料理區就在點餐臺的後方，可以看到這個小漢堡特殊的料理方式，並不是採用一般香煎或火烤方式來料理漢堡肉，而是一種結合蒸煮同時又帶有煎的方式，並讓熱度與蒸氣把洋蔥的香甜味傳達到肉片上。\\n\\n一個小漢堡對一個成年男性來說，不用幾口就可以經鬆解決，特殊的烹煮方式讓品質尚可的肉質吃起來還挺有水分的，加上融化的起司帶來香濃的奶香與洋蔥的甜味，吃起來確實是挺有特色也是居住在加州的居民很少有機會品嚐到的口味。單吃小漢堡說起來並不難吃，但在當前眾多速食漢堡店都強調著食材品質和創新口味的衝擊下，相比起來食感薄弱了不少，但依靠著各種醬料的搭配來調整口位到也是挺有趣的。\\n\\n下次來到賭城，無論是早起的鳥兒、飢腸轆轆的食客、宵夜時間睡不著的夜貓子，都可以來到這間二十四小時經營的經典漢堡店，依照自己的食量來享用這個吃起來感覺不會太有負擔、卻是在美國速食業具有重高地位的可愛小漢堡。']\n" 542 | ] 543 | } 544 | ], 545 | "source": [ 546 | "print ('Most %s similar reviews:' % n)\n", 547 | "print (searched_result)" 548 | ] 549 | }, 550 | { 551 | "cell_type": "markdown", 552 | "metadata": {}, 553 | "source": [ 554 | "## Classifying positive/negative review" 555 | ] 556 | }, 557 | { 558 | "cell_type": "markdown", 559 | "metadata": {}, 560 | "source": [ 561 | "#### Naive-Bayes Classifier" 562 | ] 563 | }, 564 | { 565 | "cell_type": "code", 566 | "execution_count": 65, 567 | "metadata": {}, 568 | "outputs": [ 569 | { 570 | "data": { 571 | "text/plain": [ 572 | "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)" 573 | ] 574 | }, 575 | "execution_count": 65, 576 | "metadata": {}, 577 | "output_type": "execute_result" 578 | } 579 | ], 580 | "source": [ 581 | "# Build a Naive-Bayes Classifier\n", 582 | "\n", 583 | "from sklearn.naive_bayes import MultinomialNB\n", 584 | "\n", 585 | "nbf = MultinomialNB()\n", 586 | "nbf.fit(vectors_train.toarray(),target_train)" 587 | ] 588 | }, 589 | { 590 | "cell_type": "code", 591 | "execution_count": 67, 592 | "metadata": {}, 593 | "outputs": [ 594 | { 595 | "data": { 596 | "text/plain": [ 597 | "0.81235194038850844" 598 | ] 599 | }, 600 | "execution_count": 67, 601 | "metadata": {}, 602 | "output_type": "execute_result" 603 | } 604 | ], 605 | "source": [ 606 | "# Get score for training set\n", 607 | "nbf.score(vectors_train.toarray(),target_train)" 608 | ] 609 | }, 610 | { 611 | "cell_type": "code", 612 | "execution_count": 68, 613 | "metadata": {}, 614 | "outputs": [ 615 | { 616 | "data": { 617 | "text/plain": [ 618 | "0.80715542778439708" 619 | ] 620 | }, 621 | "execution_count": 68, 622 | "metadata": {}, 623 | "output_type": "execute_result" 624 | } 625 | ], 626 | "source": [ 627 | "# Get score for test set\n", 628 | "nbf.score(vectors_test,target_test)" 629 | ] 630 | }, 631 | { 632 | "cell_type": "markdown", 633 | "metadata": {}, 634 | "source": [ 635 | "#### Logistic Regression Classifier" 636 | ] 637 | }, 638 | { 639 | "cell_type": "code", 640 | "execution_count": 69, 641 | "metadata": {}, 642 | "outputs": [ 643 | { 644 | "data": { 645 | "text/plain": [ 646 | "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n", 647 | " intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n", 648 | " penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n", 649 | " verbose=0, warm_start=False)" 650 | ] 651 | }, 652 | "execution_count": 69, 653 | "metadata": {}, 654 | "output_type": "execute_result" 655 | } 656 | ], 657 | "source": [ 658 | "# Build a Logistic Regression Classifier\n", 659 | "from sklearn.linear_model import LogisticRegression\n", 660 | "vectors_train = vectors_train.toarray()\n", 661 | "\n", 662 | "lg = LogisticRegression()\n", 663 | "lg.fit(vectors_train,target_train)" 664 | ] 665 | }, 666 | { 667 | "cell_type": "code", 668 | "execution_count": 70, 669 | "metadata": {}, 670 | "outputs": [ 671 | { 672 | "data": { 673 | "text/plain": [ 674 | "0.84339061894301592" 675 | ] 676 | }, 677 | "execution_count": 70, 678 | "metadata": {}, 679 | "output_type": "execute_result" 680 | } 681 | ], 682 | "source": [ 683 | "# Get score for training set\n", 684 | "lg.score(vectors_train,target_train)" 685 | ] 686 | }, 687 | { 688 | "cell_type": "code", 689 | "execution_count": 71, 690 | "metadata": { 691 | "scrolled": true 692 | }, 693 | "outputs": [ 694 | { 695 | "data": { 696 | "text/plain": [ 697 | "0.83342260310658811" 698 | ] 699 | }, 700 | "execution_count": 71, 701 | "metadata": {}, 702 | "output_type": "execute_result" 703 | } 704 | ], 705 | "source": [ 706 | "# Get score for test set\n", 707 | "lg.score(vectors_test,target_test)" 708 | ] 709 | }, 710 | { 711 | "cell_type": "markdown", 712 | "metadata": {}, 713 | "source": [ 714 | "#### Q: What are the key features(words) that make the positive prediction?" 715 | ] 716 | }, 717 | { 718 | "cell_type": "code", 719 | "execution_count": 86, 720 | "metadata": {}, 721 | "outputs": [ 722 | { 723 | "data": { 724 | "text/plain": [ 725 | "['worst',\n", 726 | " 'horrible',\n", 727 | " 'ok',\n", 728 | " 'bland',\n", 729 | " 'amazing',\n", 730 | " 'mediocre',\n", 731 | " 'disappointing',\n", 732 | " 'rude',\n", 733 | " 'best',\n", 734 | " 'terrible',\n", 735 | " 'okay',\n", 736 | " 'slow',\n", 737 | " 'lacked',\n", 738 | " 'average',\n", 739 | " 'poor',\n", 740 | " 'lacking',\n", 741 | " 'overpriced',\n", 742 | " 'meh',\n", 743 | " 'poisoning',\n", 744 | " 'tasteless']" 745 | ] 746 | }, 747 | "execution_count": 86, 748 | "metadata": {}, 749 | "output_type": "execute_result" 750 | } 751 | ], 752 | "source": [ 753 | "# Let's find it out by ranking\n", 754 | "n = 20\n", 755 | "get_top_values(lg.coef_[0],n, words)" 756 | ] 757 | }, 758 | { 759 | "cell_type": "markdown", 760 | "metadata": {}, 761 | "source": [ 762 | "A: (insert your comments here)" 763 | ] 764 | }, 765 | { 766 | "cell_type": "markdown", 767 | "metadata": {}, 768 | "source": [ 769 | "#### Q: What are the key features(words) that make the negative prediction?" 770 | ] 771 | }, 772 | { 773 | "cell_type": "code", 774 | "execution_count": 90, 775 | "metadata": {}, 776 | "outputs": [ 777 | { 778 | "data": { 779 | "text/plain": [ 780 | "['worst',\n", 781 | " 'horrible',\n", 782 | " 'ok',\n", 783 | " 'bland',\n", 784 | " 'mediocre',\n", 785 | " 'disappointing',\n", 786 | " 'rude',\n", 787 | " 'terrible',\n", 788 | " 'okay',\n", 789 | " 'slow',\n", 790 | " 'lacked',\n", 791 | " 'average',\n", 792 | " 'poor',\n", 793 | " 'lacking',\n", 794 | " 'overpriced',\n", 795 | " 'meh',\n", 796 | " 'poisoning',\n", 797 | " 'tasteless',\n", 798 | " 'alright',\n", 799 | " 'unfortunately']" 800 | ] 801 | }, 802 | "execution_count": 90, 803 | "metadata": {}, 804 | "output_type": "execute_result" 805 | } 806 | ], 807 | "source": [ 808 | "# Let's find it out by ranking\n", 809 | "n = 20\n", 810 | "get_bottom_values(lg.coef_[0], n, words)" 811 | ] 812 | }, 813 | { 814 | "cell_type": "markdown", 815 | "metadata": {}, 816 | "source": [ 817 | "A: (insert your comments here)" 818 | ] 819 | }, 820 | { 821 | "cell_type": "markdown", 822 | "metadata": {}, 823 | "source": [ 824 | "#### Random Forest Classifier" 825 | ] 826 | }, 827 | { 828 | "cell_type": "code", 829 | "execution_count": 91, 830 | "metadata": {}, 831 | "outputs": [ 832 | { 833 | "data": { 834 | "text/plain": [ 835 | "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n", 836 | " max_depth=None, max_features='auto', max_leaf_nodes=None,\n", 837 | " min_impurity_decrease=0.0, min_impurity_split=None,\n", 838 | " min_samples_leaf=20, min_samples_split=2,\n", 839 | " min_weight_fraction_leaf=0.0, n_estimators=5, n_jobs=-1,\n", 840 | " oob_score=False, random_state=None, verbose=0,\n", 841 | " warm_start=False)" 842 | ] 843 | }, 844 | "execution_count": 91, 845 | "metadata": {}, 846 | "output_type": "execute_result" 847 | } 848 | ], 849 | "source": [ 850 | "# Build a Random Forest Classifier\n", 851 | "\n", 852 | "from sklearn.ensemble import RandomForestClassifier\n", 853 | "\n", 854 | "rfc = RandomForestClassifier(n_estimators = 5, min_samples_leaf = 20, n_jobs = -1)\n", 855 | "rfc.fit(vectors_train,target_train)" 856 | ] 857 | }, 858 | { 859 | "cell_type": "code", 860 | "execution_count": 92, 861 | "metadata": {}, 862 | "outputs": [ 863 | { 864 | "data": { 865 | "text/plain": [ 866 | "0.80161627255890078" 867 | ] 868 | }, 869 | "execution_count": 92, 870 | "metadata": {}, 871 | "output_type": "execute_result" 872 | } 873 | ], 874 | "source": [ 875 | "# Get score for training set\n", 876 | "rfc.score(vectors_train,target_train)" 877 | ] 878 | }, 879 | { 880 | "cell_type": "code", 881 | "execution_count": 93, 882 | "metadata": {}, 883 | "outputs": [ 884 | { 885 | "data": { 886 | "text/plain": [ 887 | "0.77830853819835377" 888 | ] 889 | }, 890 | "execution_count": 93, 891 | "metadata": {}, 892 | "output_type": "execute_result" 893 | } 894 | ], 895 | "source": [ 896 | "# Get score for test set\n", 897 | "rfc.score(vectors_test,target_test)" 898 | ] 899 | }, 900 | { 901 | "cell_type": "markdown", 902 | "metadata": {}, 903 | "source": [ 904 | "#### Q: What do you see from the training score and the test score?" 905 | ] 906 | }, 907 | { 908 | "cell_type": "markdown", 909 | "metadata": {}, 910 | "source": [ 911 | "A: The number of estimators is not enough and the prediction score is even less than logistic regression. If we increase the number of estimators, the training time may be much longer but the result is expected to be improved. The test score is lower than the training score, this means that our model doesn't have overfitting problem yet." 912 | ] 913 | }, 914 | { 915 | "cell_type": "markdown", 916 | "metadata": {}, 917 | "source": [ 918 | "### Important features (words) by inspecting the RFC model" 919 | ] 920 | }, 921 | { 922 | "cell_type": "code", 923 | "execution_count": 94, 924 | "metadata": {}, 925 | "outputs": [ 926 | { 927 | "data": { 928 | "text/plain": [ 929 | "['amazing',\n", 930 | " 'delicious',\n", 931 | " 'like',\n", 932 | " 'best',\n", 933 | " 'great',\n", 934 | " 'asked',\n", 935 | " 'awesome',\n", 936 | " 'bad',\n", 937 | " 'vegas',\n", 938 | " 'decent',\n", 939 | " 'didn',\n", 940 | " 'worst',\n", 941 | " 'okay',\n", 942 | " 'love',\n", 943 | " 'ok',\n", 944 | " 'order',\n", 945 | " 'bland',\n", 946 | " 'rude',\n", 947 | " 'definitely',\n", 948 | " 'dry']" 949 | ] 950 | }, 951 | "execution_count": 94, 952 | "metadata": {}, 953 | "output_type": "execute_result" 954 | } 955 | ], 956 | "source": [ 957 | "n = 20\n", 958 | "get_top_values(rfc.feature_importances_,n,words)" 959 | ] 960 | }, 961 | { 962 | "cell_type": "markdown", 963 | "metadata": {}, 964 | "source": [ 965 | "## TODO: Use cross validation to evaluate your classifiers\n", 966 | "\n", 967 | "[sklearn cross validation](http://scikit-learn.org/stable/modules/cross_validation.html)" 968 | ] 969 | }, 970 | { 971 | "cell_type": "code", 972 | "execution_count": 11, 973 | "metadata": {}, 974 | "outputs": [ 975 | { 976 | "ename": "NameError", 977 | "evalue": "name 'lg' is not defined", 978 | "output_type": "error", 979 | "traceback": [ 980 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 981 | "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", 982 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_selection\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcross_val_score\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m cv_scores = cross_val_score(lg,\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mvectors_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mtarget_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 983 | "\u001b[0;31mNameError\u001b[0m: name 'lg' is not defined" 984 | ] 985 | } 986 | ], 987 | "source": [ 988 | "from sklearn.model_selection import cross_val_score\n", 989 | "\n", 990 | "cv_scores = cross_val_score(lg,\n", 991 | " vectors_train,\n", 992 | " target_train,\n", 993 | " cv = 5,\n", 994 | " scoring=\"accuracy\")\n", 995 | "cv_scores" 996 | ] 997 | }, 998 | { 999 | "cell_type": "markdown", 1000 | "metadata": {}, 1001 | "source": [ 1002 | "## TODO: Use grid search to find best predictable classifier\n", 1003 | "\n", 1004 | "\n", 1005 | "[sklearn grid search tutorial (with cross validation)](http://scikit-learn.org/stable/modules/grid_search.html#grid-search)\n", 1006 | "\n", 1007 | "[sklearn grid search documentation (with cross validation)](http://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html#sklearn.model_selection.GridSearchCV)" 1008 | ] 1009 | }, 1010 | { 1011 | "cell_type": "code", 1012 | "execution_count": null, 1013 | "metadata": { 1014 | "collapsed": true 1015 | }, 1016 | "outputs": [], 1017 | "source": [ 1018 | "# To be implemented\n" 1019 | ] 1020 | }, 1021 | { 1022 | "cell_type": "code", 1023 | "execution_count": null, 1024 | "metadata": { 1025 | "collapsed": true 1026 | }, 1027 | "outputs": [], 1028 | "source": [] 1029 | } 1030 | ], 1031 | "metadata": { 1032 | "anaconda-cloud": {}, 1033 | "kernelspec": { 1034 | "display_name": "Python 3", 1035 | "language": "python", 1036 | "name": "python3" 1037 | }, 1038 | "language_info": { 1039 | "codemirror_mode": { 1040 | "name": "ipython", 1041 | "version": 3 1042 | }, 1043 | "file_extension": ".py", 1044 | "mimetype": "text/x-python", 1045 | "name": "python", 1046 | "nbconvert_exporter": "python", 1047 | "pygments_lexer": "ipython3", 1048 | "version": "3.5.5" 1049 | } 1050 | }, 1051 | "nbformat": 4, 1052 | "nbformat_minor": 1 1053 | } 1054 | -------------------------------------------------------------------------------- /Yelp_Dataset_-_NLP.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Yelp Data Challenge - NLP" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import pandas as pd\n", 17 | "import numpy as np\n", 18 | "\n", 19 | "import matplotlib.pyplot as plt\n", 20 | "%matplotlib inline" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 2, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "df = pd.read_csv('../dataset/2016_restaurant_reviews.csv')" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 3, 35 | "metadata": {}, 36 | "outputs": [ 37 | { 38 | "data": { 39 | "text/html": [ 40 | "
\n", 41 | "\n", 54 | "\n", 55 | " \n", 56 | " \n", 57 | " \n", 58 | " \n", 59 | " \n", 60 | " \n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | "
business_idnamecategoriesavg_starscooldatefunnyreview_idstarstextusefuluser_id
0--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Cajun/Creole', 'Steakhouses', 'Restaurants']4.002016-03-3106SgvNWJltnZhW7duJgZ42w5This is mine and my fiancé's favorite steakhou...0oFyOUOeGTRZhFPF9uTqrTQ
1--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Cajun/Creole', 'Steakhouses', 'Restaurants']4.002016-02-100UxFpgng8dPMWOj99653k5Q5Truly Fantastic! Best Steak ever. Service was...0aVOGlN9fZ-BXcbtj6dbf0g
2--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Cajun/Creole', 'Steakhouses', 'Restaurants']4.002016-07-130GN7KnAaxJDrYgfJzgsvmkA5We decided to give Delmonico's a try because w...0C6kw0Rny7jZAGjTj0MWA3Q
3--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Cajun/Creole', 'Steakhouses', 'Restaurants']4.002016-04-010i7xD3FY-EaF9O08QL69l5w5Absolutely impressed with this restaurant. The...0tTifjrXlRrUme-4c0UW9Bw
4--9e1ONYQuAa-CB_Rrw7TwDelmonico Steakhouse['Cajun/Creole', 'Steakhouses', 'Restaurants']4.002016-10-150ZqMUHOJg9lGOmfqQ7RXj7A1This was supposed to be a very special dinner ...0kOll36Me-rM9NsqoDnejKA
\n", 150 | "
" 151 | ], 152 | "text/plain": [ 153 | " business_id name \\\n", 154 | "0 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 155 | "1 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 156 | "2 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 157 | "3 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 158 | "4 --9e1ONYQuAa-CB_Rrw7Tw Delmonico Steakhouse \n", 159 | "\n", 160 | " categories avg_stars cool \\\n", 161 | "0 ['Cajun/Creole', 'Steakhouses', 'Restaurants'] 4.0 0 \n", 162 | "1 ['Cajun/Creole', 'Steakhouses', 'Restaurants'] 4.0 0 \n", 163 | "2 ['Cajun/Creole', 'Steakhouses', 'Restaurants'] 4.0 0 \n", 164 | "3 ['Cajun/Creole', 'Steakhouses', 'Restaurants'] 4.0 0 \n", 165 | "4 ['Cajun/Creole', 'Steakhouses', 'Restaurants'] 4.0 0 \n", 166 | "\n", 167 | " date funny review_id stars \\\n", 168 | "0 2016-03-31 0 6SgvNWJltnZhW7duJgZ42w 5 \n", 169 | "1 2016-02-10 0 UxFpgng8dPMWOj99653k5Q 5 \n", 170 | "2 2016-07-13 0 GN7KnAaxJDrYgfJzgsvmkA 5 \n", 171 | "3 2016-04-01 0 i7xD3FY-EaF9O08QL69l5w 5 \n", 172 | "4 2016-10-15 0 ZqMUHOJg9lGOmfqQ7RXj7A 1 \n", 173 | "\n", 174 | " text useful \\\n", 175 | "0 This is mine and my fiancé's favorite steakhou... 0 \n", 176 | "1 Truly Fantastic! Best Steak ever. Service was... 0 \n", 177 | "2 We decided to give Delmonico's a try because w... 0 \n", 178 | "3 Absolutely impressed with this restaurant. The... 0 \n", 179 | "4 This was supposed to be a very special dinner ... 0 \n", 180 | "\n", 181 | " user_id \n", 182 | "0 oFyOUOeGTRZhFPF9uTqrTQ \n", 183 | "1 aVOGlN9fZ-BXcbtj6dbf0g \n", 184 | "2 C6kw0Rny7jZAGjTj0MWA3Q \n", 185 | "3 tTifjrXlRrUme-4c0UW9Bw \n", 186 | "4 kOll36Me-rM9NsqoDnejKA " 187 | ] 188 | }, 189 | "execution_count": 3, 190 | "metadata": {}, 191 | "output_type": "execute_result" 192 | } 193 | ], 194 | "source": [ 195 | "df.head()" 196 | ] 197 | }, 198 | { 199 | "cell_type": "markdown", 200 | "metadata": {}, 201 | "source": [ 202 | "### Define feature variables, here is the text of the review" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": 4, 208 | "metadata": {}, 209 | "outputs": [], 210 | "source": [ 211 | "# Take the values of the column that contains review text data, save to a variable named \"documents\"\n", 212 | "documents = [t for i,t in enumerate(df.text)]" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": 5, 218 | "metadata": {}, 219 | "outputs": [ 220 | { 221 | "data": { 222 | "text/plain": [ 223 | "178080" 224 | ] 225 | }, 226 | "execution_count": 5, 227 | "metadata": {}, 228 | "output_type": "execute_result" 229 | } 230 | ], 231 | "source": [ 232 | "# inspect your documents, e.g. check the size, take a peek at elements of the numpy array\n", 233 | "len(documents)" 234 | ] 235 | }, 236 | { 237 | "cell_type": "markdown", 238 | "metadata": {}, 239 | "source": [ 240 | "### Define the target variable (any categorical variable that may be meaningful)" 241 | ] 242 | }, 243 | { 244 | "cell_type": "markdown", 245 | "metadata": {}, 246 | "source": [ 247 | "#### For example, I am interested in perfect (5 stars) and imperfect (1-4 stars) rating" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": 6, 253 | "metadata": {}, 254 | "outputs": [ 255 | { 256 | "name": "stderr", 257 | "output_type": "stream", 258 | "text": [ 259 | "/Users/jessie/anaconda3/lib/python3.5/site-packages/ipykernel_launcher.py:2: UserWarning: Pandas doesn't allow columns to be created via a new attribute name - see https://pandas.pydata.org/pandas-docs/stable/indexing.html#attribute-access\n", 260 | " \n" 261 | ] 262 | }, 263 | { 264 | "data": { 265 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAF1dJREFUeJzt3X+MXeWd3/H3JzYk5AcxhIFS26mpYqVLUEPAIt4iRWlIjSGrGKlBMmqDg7xyS8lu0lbaOvtHrSVBIlK1bGkTVt7gYtIkQNmkuImJ14VEq0qBMAQWQgj1LGFhaopnYyBsaZI6++0f9/Hmas4dz52xPXeC3y/p6p7zfZ5zznOPGT5zftw5qSokSer3ulEPQJK0+BgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUsHfUA5uuMM86oVatWjXoYkvQr4+GHH/7Lqhobpu+vbDisWrWK8fHxUQ9Dkn5lJPmLYft6WkmS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktTxK/sNaUkapVVbvzGS7T5z44cWZDseOUiSOoYKhyT/MskTSb6f5CtJ3pDknCQPJtmX5M4kJ7e+r2/zE619Vd96PtXqTyW5tK++vtUmkmw91h9SkjQ3s4ZDkuXAbwNrquo8YAmwEfgscFNVrQZeBDa3RTYDL1bVO4CbWj+SnNuWexewHvh8kiVJlgCfAy4DzgWuan0lSSMy7GmlpcApSZYCbwSeBz4A3N3adwJXtOkNbZ7WfkmStPodVfWzqvoRMAFc1F4TVfV0Vf0cuKP1lSSNyKzhUFX/C/h3wLP0QuFl4GHgpao61LpNAsvb9HLgubbsodb/bf31acvMVJckjcgwp5VOo/eb/DnA3wbeRO8U0HR1eJEZ2uZaHzSWLUnGk4xPTU3NNnRJ0jwNc1rpg8CPqmqqqv4f8FXgHwDL2mkmgBXA/jY9CawEaO1vBQ7216ctM1O9o6q2V9WaqlozNjbUw4wkSfMwTDg8C6xN8sZ27eAS4AfAt4CPtD6bgHva9K42T2u/v6qq1Te2u5nOAVYD3wUeAla3u59OpnfRetfRfzRJ0nzN+iW4qnowyd3A94BDwCPAduAbwB1JPtNqt7ZFbgW+mGSC3hHDxraeJ5LcRS9YDgHXVdUvAJJ8HNhD706oHVX1xLH7iJKkuRrqG9JVtQ3YNq38NL07jab3/Slw5QzruQG4YUB9N7B7mLFIko4/vyEtSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVLHrOGQ5J1JHu17/STJJ5OcnmRvkn3t/bTWP0luTjKR5LEkF/Sta1Prvy/Jpr76hUkeb8vc3B5HKkkakVnDoaqeqqrzq+p84ELgVeBrwFbgvqpaDdzX5gEuo/d86NXAFuAWgCSn03ua3HvpPUFu2+FAaX229C23/ph8OknSvMz1tNIlwJ9X1V8AG4Cdrb4TuKJNbwBur54HgGVJzgYuBfZW1cGqehHYC6xvbadW1XeqqoDb+9YlSRqBuYbDRuArbfqsqnoeoL2f2erLgef6lplstSPVJwfUJUkjMnQ4JDkZ+DDwX2brOqBW86gPGsOWJONJxqempmYZhiRpvuZy5HAZ8L2qeqHNv9BOCdHeD7T6JLCyb7kVwP5Z6isG1DuqantVramqNWNjY3MYuiRpLuYSDlfxy1NKALuAw3ccbQLu6atf3e5aWgu83E477QHWJTmtXYheB+xpba8kWdvuUrq6b12SpBFYOkynJG8E/hHwz/rKNwJ3JdkMPAtc2eq7gcuBCXp3Nl0DUFUHk3waeKj1u76qDrbpa4HbgFOAe9tLkjQiQ4VDVb0KvG1a7cf07l6a3reA62ZYzw5gx4D6OHDeMGORJB1/fkNaktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqSOocIhybIkdyf5YZInk/x6ktOT7E2yr72f1vomyc1JJpI8luSCvvVsav33JdnUV78wyeNtmZvb40IlSSMy7JHDvwe+WVV/D3g38CSwFbivqlYD97V5gMuA1e21BbgFIMnpwDbgvcBFwLbDgdL6bOlbbv3RfSxJ0tGYNRySnAq8D7gVoKp+XlUvARuAna3bTuCKNr0BuL16HgCWJTkbuBTYW1UHq+pFYC+wvrWdWlXfaY8Yvb1vXZKkERjmyOHvAlPAf0rySJIvJHkTcFZVPQ/Q3s9s/ZcDz/UtP9lqR6pPDqhLkkZkmHBYClwA3FJV7wH+D788hTTIoOsFNY96d8XJliTjScanpqaOPGpJ0rwNEw6TwGRVPdjm76YXFi+0U0K09wN9/Vf2Lb8C2D9LfcWAekdVba+qNVW1ZmxsbIihS5LmY9ZwqKr/DTyX5J2tdAnwA2AXcPiOo03APW16F3B1u2tpLfByO+20B1iX5LR2IXodsKe1vZJkbbtL6eq+dUmSRmDpkP1+C/hSkpOBp4Fr6AXLXUk2A88CV7a+u4HLgQng1daXqjqY5NPAQ63f9VV1sE1fC9wGnALc216SpBEZKhyq6lFgzYCmSwb0LeC6GdazA9gxoD4OnDfMWCRJx5/fkJYkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1DBUOSZ5J8niSR5OMt9rpSfYm2dfeT2v1JLk5yUSSx5Jc0LeeTa3/viSb+uoXtvVPtGVzrD+oJGl4czly+IdVdX5VHX5c6FbgvqpaDdzX5gEuA1a31xbgFuiFCbANeC9wEbDtcKC0Plv6lls/708kSTpqR3NaaQOws03vBK7oq99ePQ8Ay5KcDVwK7K2qg1X1IrAXWN/aTq2q77TnT9/ety5J0ggMGw4F/EmSh5NsabWzqup5gPZ+ZqsvB57rW3ay1Y5UnxxQ70iyJcl4kvGpqakhhy5JmqulQ/a7uKr2JzkT2Jvkh0foO+h6Qc2j3i1WbQe2A6xZs2ZgH0nS0RvqyKGq9rf3A8DX6F0zeKGdEqK9H2jdJ4GVfYuvAPbPUl8xoC5JGpFZwyHJm5K85fA0sA74PrALOHzH0Sbgnja9C7i63bW0Fni5nXbaA6xLclq7EL0O2NPaXkmytt2ldHXfuiRJIzDMaaWzgK+1u0uXAl+uqm8meQi4K8lm4FngytZ/N3A5MAG8ClwDUFUHk3waeKj1u76qDrbpa4HbgFOAe9tLkjQis4ZDVT0NvHtA/cfAJQPqBVw3w7p2ADsG1MeB84YYryRpAfgNaUlSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOoYOhyRLkjyS5Ott/pwkDybZl+TOJCe3+uvb/ERrX9W3jk+1+lNJLu2rr2+1iSRbj93HkyTNx1yOHD4BPNk3/1ngpqpaDbwIbG71zcCLVfUO4KbWjyTnAhuBdwHrgc+3wFkCfA64DDgXuKr1lSSNyFDhkGQF8CHgC20+wAeAu1uXncAVbXpDm6e1X9L6bwDuqKqfVdWP6D1j+qL2mqiqp6vq58Adra8kaUSGPXL4A+B3gL9u828DXqqqQ21+EljeppcDzwG09pdb/7+pT1tmpnpHki1JxpOMT01NDTl0SdJczRoOSX4DOFBVD/eXB3StWdrmWu8Wq7ZX1ZqqWjM2NnaEUUuSjsbSIfpcDHw4yeXAG4BT6R1JLEuytB0drAD2t/6TwEpgMslS4K3Awb76Yf3LzFSXJI3ArEcOVfWpqlpRVavoXVC+v6r+CfAt4COt2ybgnja9q83T2u+vqmr1je1upnOA1cB3gYeA1e3up5PbNnYdk08nSZqXYY4cZvJvgDuSfAZ4BLi11W8Fvphkgt4Rw0aAqnoiyV3AD4BDwHVV9QuAJB8H9gBLgB1V9cRRjEuSdJTmFA5V9W3g2236aXp3Gk3v81PgyhmWvwG4YUB9N7B7LmORJB0/R3PkIEkArNr6jZFs95kbPzSS7Z4I/PMZkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqWPWcEjyhiTfTfJnSZ5I8nutfk6SB5PsS3Jne8Qn7TGgdyaZaO2r+tb1qVZ/KsmlffX1rTaRZOux/5iSpLkY5sjhZ8AHqurdwPnA+iRrgc8CN1XVauBFYHPrvxl4sareAdzU+pHkXHqPDH0XsB74fJIlSZYAnwMuA84Frmp9JUkjMms4VM9ftdmT2quADwB3t/pO4Io2vaHN09ovSZJWv6OqflZVPwIm6D1m9CJgoqqerqqfA3e0vpKkERnqmkP7Df9R4ACwF/hz4KWqOtS6TALL2/Ry4DmA1v4y8Lb++rRlZqoPGseWJONJxqempoYZuiRpHoYKh6r6RVWdD6yg95v+rw3q1t4zQ9tc64PGsb2q1lTVmrGxsdkHLkmalzndrVRVLwHfBtYCy5IsbU0rgP1tehJYCdDa3woc7K9PW2amuiRpRIa5W2ksybI2fQrwQeBJ4FvAR1q3TcA9bXpXm6e1319V1eob291M5wCrge8CDwGr291PJ9O7aL3rWHw4SdL8LJ29C2cDO9tdRa8D7qqqryf5AXBHks8AjwC3tv63Al9MMkHviGEjQFU9keQu4AfAIeC6qvoFQJKPA3uAJcCOqnrimH1CSdKczRoOVfUY8J4B9afpXX+YXv8pcOUM67oBuGFAfTewe4jxSpIWgN+QliR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6hvmrrK85q7Z+YyTbfebGD41ku5I0Vx45SJI6DAdJUofhIEnqGOYxoSuTfCvJk0meSPKJVj89yd4k+9r7aa2eJDcnmUjyWJIL+ta1qfXfl2RTX/3CJI+3ZW5OkuPxYSVJwxnmyOEQ8K+r6teAtcB1Sc4FtgL3VdVq4L42D3AZvedDrwa2ALdAL0yAbcB76T1BbtvhQGl9tvQtt/7oP5okab5mDYeqer6qvtemXwGeBJYDG4CdrdtO4Io2vQG4vXoeAJYlORu4FNhbVQer6kVgL7C+tZ1aVd+pqgJu71uXJGkE5nTNIckqes+TfhA4q6qeh16AAGe2bsuB5/oWm2y1I9UnB9QlSSMy9PcckrwZ+GPgk1X1kyNcFhjUUPOoDxrDFnqnn3j7298+25ClkfB7NHotGOrIIclJ9ILhS1X11VZ+oZ0Sor0faPVJYGXf4iuA/bPUVwyod1TV9qpaU1VrxsbGhhm6JGkehrlbKcCtwJNV9ft9TbuAw3ccbQLu6atf3e5aWgu83E477QHWJTmtXYheB+xpba8kWdu2dXXfuiRJIzDMaaWLgY8Cjyd5tNV+F7gRuCvJZuBZ4MrWthu4HJgAXgWuAaiqg0k+DTzU+l1fVQfb9LXAbcApwL3tJUkakVnDoar+B4OvCwBcMqB/AdfNsK4dwI4B9XHgvNnGIklaGH5DWpLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdZyQz5A+Efn3fiTNhUcOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUM85jQHUkOJPl+X+30JHuT7Gvvp7V6ktycZCLJY0ku6FtmU+u/L8mmvvqFSR5vy9zcHhUqSRqhYY4cbgPWT6ttBe6rqtXAfW0e4DJgdXttAW6BXpgA24D3AhcB2w4HSuuzpW+56duSJC2wWcOhqv4UODitvAHY2aZ3Alf01W+vngeAZUnOBi4F9lbVwap6EdgLrG9tp1bVd9rjRW/vW5ckaUTme83hrKp6HqC9n9nqy4Hn+vpNttqR6pMD6pKkETrWF6QHXS+oedQHrzzZkmQ8yfjU1NQ8hyhJms18w+GFdkqI9n6g1SeBlX39VgD7Z6mvGFAfqKq2V9WaqlozNjY2z6FLkmYz33DYBRy+42gTcE9f/ep219Ja4OV22mkPsC7Jae1C9DpgT2t7JcnadpfS1X3rkiSNyKzPc0jyFeD9wBlJJunddXQjcFeSzcCzwJWt+27gcmACeBW4BqCqDib5NPBQ63d9VR2+yH0tvTuiTgHubS9J0gjNGg5VddUMTZcM6FvAdTOsZwewY0B9HDhvtnFIkhaO35CWJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKlj0YRDkvVJnkoykWTrqMcjSSeyRREOSZYAnwMuA84Frkpy7mhHJUknrkURDsBFwERVPV1VPwfuADaMeEySdMJaLOGwHHiub36y1SRJI5CqGvUYSHIlcGlV/Wab/yhwUVX91rR+W4AtbfadwFPz3OQZwF/Oc9njyXHNjeOaG8c1N6/Fcf2dqhobpuPSeW7gWJsEVvbNrwD2T+9UVduB7Ue7sSTjVbXmaNdzrDmuuXFcc+O45uZEH9diOa30ELA6yTlJTgY2ArtGPCZJOmEtiiOHqjqU5OPAHmAJsKOqnhjxsCTphLUowgGgqnYDuxdoc0d9auo4cVxz47jmxnHNzQk9rkVxQVqStLgslmsOkqRF5DUbDkl2JDmQ5PsztCfJze3PdTyW5IJFMq73J3k5yaPt9W8XaFwrk3wryZNJnkjyiQF9FnyfDTmuBd9nSd6Q5LtJ/qyN6/cG9Hl9kjvb/nowyapFMq6PJZnq21+/ebzH1bftJUkeSfL1AW0Lvr+GHNdI9leSZ5I83rY5PqD9+P48VtVr8gW8D7gA+P4M7ZcD9wIB1gIPLpJxvR/4+gj219nABW36LcD/BM4d9T4bclwLvs/aPnhzmz4JeBBYO63PvwD+sE1vBO5cJOP6GPAfF/q/sbbtfwV8edC/1yj215DjGsn+Ap4BzjhC+3H9eXzNHjlU1Z8CB4/QZQNwe/U8ACxLcvYiGNdIVNXzVfW9Nv0K8CTdb6kv+D4bclwLru2Dv2qzJ7XX9At4G4Cdbfpu4JIkWQTjGokkK4APAV+YocuC768hx7VYHdefx9dsOAxhMf/Jjl9vpwXuTfKuhd54O5x/D73fOvuNdJ8dYVwwgn3WTkU8ChwA9lbVjPurqg4BLwNvWwTjAvjH7VTE3UlWDmg/Hv4A+B3gr2doH8n+GmJcMJr9VcCfJHk4vb8OMd1x/Xk8kcNh0G8ki+E3rO/R+4r7u4H/APzXhdx4kjcDfwx8sqp+Mr15wCILss9mGddI9llV/aKqzqf3jf6Lkpw3rctI9tcQ4/pvwKqq+vvAf+eXv60fN0l+AzhQVQ8fqduA2nHdX0OOa8H3V3NxVV1A769VX5fkfdPaj+v+OpHDYag/2bHQquonh08LVO+7HyclOWMhtp3kJHr/A/5SVX11QJeR7LPZxjXKfda2+RLwbWD9tKa/2V9JlgJvZQFPKc40rqr6cVX9rM3+EXDhAgznYuDDSZ6h91eXP5DkP0/rM4r9Neu4RrS/qKr97f0A8DV6f72633H9eTyRw2EXcHW74r8WeLmqnh/1oJL8rcPnWZNcRO/f6McLsN0AtwJPVtXvz9BtwffZMOMaxT5LMpZkWZs+Bfgg8MNp3XYBm9r0R4D7q11JHOW4pp2X/jC96zjHVVV9qqpWVNUqeheb76+qfzqt24Lvr2HGNYr9leRNSd5yeBpYB0y/w/G4/jwumm9IH2tJvkLvLpYzkkwC2+hdnKOq/pDet7EvByaAV4FrFsm4PgJcm+QQ8H+Bjcf7B6S5GPgo8Hg7Xw3wu8Db+8Y2in02zLhGsc/OBnam96Cq1wF3VdXXk1wPjFfVLnqh9sUkE/R+A954nMc07Lh+O8mHgUNtXB9bgHENtAj21zDjGsX+Ogv4WvudZynw5ar6ZpJ/Dgvz8+g3pCVJHSfyaSVJ0gwMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1PH/ASVn8WEMV9FYAAAAAElFTkSuQmCC\n", 266 | "text/plain": [ 267 | "
" 268 | ] 269 | }, 270 | "metadata": {}, 271 | "output_type": "display_data" 272 | } 273 | ], 274 | "source": [ 275 | "# Make a column and take the values, save to a variable named \"target\"\n", 276 | "df.target = df.stars\n", 277 | "\n", 278 | "plt.hist(df.target)\n", 279 | "plt.show()" 280 | ] 281 | }, 282 | { 283 | "cell_type": "markdown", 284 | "metadata": {}, 285 | "source": [ 286 | "## Let's create training dataset and test dataset" 287 | ] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "execution_count": 7, 292 | "metadata": {}, 293 | "outputs": [], 294 | "source": [ 295 | "# df.target = df.stars.apply(lambda x: 1 if x > 4 else 0)\n", 296 | "# df.target.describe()" 297 | ] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "execution_count": 8, 302 | "metadata": {}, 303 | "outputs": [ 304 | { 305 | "name": "stderr", 306 | "output_type": "stream", 307 | "text": [ 308 | "/Users/jessie/anaconda3/lib/python3.5/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n", 309 | " \"This module will be removed in 0.20.\", DeprecationWarning)\n" 310 | ] 311 | } 312 | ], 313 | "source": [ 314 | "from sklearn.cross_validation import train_test_split" 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "execution_count": 9, 320 | "metadata": {}, 321 | "outputs": [], 322 | "source": [ 323 | "# Split to documents_train, documents_test, target_train, target_test\n", 324 | "documents_train, documents_test, target_train, target_test = train_test_split(\n", 325 | " documents, df.stars, test_size=0.33, random_state=42)" 326 | ] 327 | }, 328 | { 329 | "cell_type": "markdown", 330 | "metadata": {}, 331 | "source": [ 332 | "## Let's get NLP representation of the documents" 333 | ] 334 | }, 335 | { 336 | "cell_type": "code", 337 | "execution_count": 10, 338 | "metadata": {}, 339 | "outputs": [], 340 | "source": [ 341 | "from sklearn.feature_extraction.text import TfidfVectorizer" 342 | ] 343 | }, 344 | { 345 | "cell_type": "code", 346 | "execution_count": 11, 347 | "metadata": {}, 348 | "outputs": [], 349 | "source": [ 350 | "# Create TfidfVectorizer, and name it vectorizer\n", 351 | "vectorizer = TfidfVectorizer(stop_words = 'english', max_features=5000)" 352 | ] 353 | }, 354 | { 355 | "cell_type": "code", 356 | "execution_count": 12, 357 | "metadata": {}, 358 | "outputs": [], 359 | "source": [ 360 | "# Train the model with your training data\n", 361 | "vectors_train = vectorizer.fit_transform(documents_train)" 362 | ] 363 | }, 364 | { 365 | "cell_type": "code", 366 | "execution_count": 13, 367 | "metadata": {}, 368 | "outputs": [], 369 | "source": [ 370 | "# Get the vocab of your tfidf\n", 371 | "words = vectorizer.get_feature_names()" 372 | ] 373 | }, 374 | { 375 | "cell_type": "code", 376 | "execution_count": 14, 377 | "metadata": {}, 378 | "outputs": [], 379 | "source": [ 380 | "# Use the trained model to transform your test data\n", 381 | "vectors_test = vectorizer.transform(documents_test).toarray()" 382 | ] 383 | }, 384 | { 385 | "cell_type": "markdown", 386 | "metadata": {}, 387 | "source": [ 388 | "## Similar review search engine" 389 | ] 390 | }, 391 | { 392 | "cell_type": "code", 393 | "execution_count": 15, 394 | "metadata": {}, 395 | "outputs": [], 396 | "source": [ 397 | "import numpy as np\n", 398 | "\n", 399 | "# We will need these helper methods pretty soon\n", 400 | "\n", 401 | "def get_top_values(lst, n, labels):\n", 402 | " '''\n", 403 | " INPUT: LIST, INTEGER, LIST\n", 404 | " OUTPUT: LIST\n", 405 | "\n", 406 | " Given a list of values, find the indices with the highest n values.\n", 407 | " Return the labels for each of these indices.\n", 408 | "\n", 409 | " e.g.\n", 410 | " lst = [7, 3, 2, 4, 1]\n", 411 | " n = 2\n", 412 | " labels = [\"cat\", \"dog\", \"mouse\", \"pig\", \"rabbit\"]\n", 413 | " output: [\"cat\", \"pig\"]\n", 414 | " '''\n", 415 | " return [labels[i] for i in np.argsort(lst)[::-1][:n]] # np.argsort by default sorts values in ascending order\n", 416 | "\n", 417 | "def get_bottom_values(lst, n, labels):\n", 418 | " '''\n", 419 | " INPUT: LIST, INTEGER, LIST\n", 420 | " OUTPUT: LIST\n", 421 | "\n", 422 | " Given a list of values, find the indices with the lowest n values.\n", 423 | " Return the labels for each of these indices.\n", 424 | "\n", 425 | " e.g.\n", 426 | " lst = [7, 3, 2, 4, 1]\n", 427 | " n = 2\n", 428 | " labels = [\"cat\", \"dog\", \"mouse\", \"pig\", \"rabbit\"]\n", 429 | " output: [\"mouse\", \"rabbit\"]\n", 430 | " '''\n", 431 | " return [labels[i] for i in np.argsort(lst)[::1][:n]] \n" 432 | ] 433 | }, 434 | { 435 | "cell_type": "code", 436 | "execution_count": 16, 437 | "metadata": {}, 438 | "outputs": [], 439 | "source": [ 440 | "# Let's use cosine similarity\n", 441 | "from sklearn.metrics.pairwise import cosine_similarity" 442 | ] 443 | }, 444 | { 445 | "cell_type": "code", 446 | "execution_count": 21, 447 | "metadata": {}, 448 | "outputs": [], 449 | "source": [ 450 | "# Draw an arbitrary review from test (unseen in training) documents\n", 451 | "doc_test = documents_test[np.random.randint(len(documents_test))]" 452 | ] 453 | }, 454 | { 455 | "cell_type": "code", 456 | "execution_count": 22, 457 | "metadata": {}, 458 | "outputs": [], 459 | "source": [ 460 | "# Transform the drawn review(s) to vector(s)\n", 461 | "doc_test_vector = vectorizer.transform([doc_test]).toarray()" 462 | ] 463 | }, 464 | { 465 | "cell_type": "code", 466 | "execution_count": 23, 467 | "metadata": {}, 468 | "outputs": [], 469 | "source": [ 470 | "# Calculate the similarity score(s) between vector(s) and training vectors\n", 471 | "similarity_scores = cosine_similarity(doc_test_vector, vectors_train.toarray())" 472 | ] 473 | }, 474 | { 475 | "cell_type": "code", 476 | "execution_count": 24, 477 | "metadata": {}, 478 | "outputs": [], 479 | "source": [ 480 | "# Let's find top 5 similar reviews\n", 481 | "n = 2\n", 482 | "#np.shape(similarity_scores[0])\n", 483 | "searched_result = get_top_values(similarity_scores[0], n, documents_train)" 484 | ] 485 | }, 486 | { 487 | "cell_type": "code", 488 | "execution_count": 25, 489 | "metadata": {}, 490 | "outputs": [ 491 | { 492 | "name": "stdout", 493 | "output_type": "stream", 494 | "text": [ 495 | "Our search query:\n", 496 | "I was prepared to not like Margaritaville. I figured this was going to a formulaic theme restaurant with mediocre burgers. You typically see Margaritaville's in resort areas. This time we were at the Flamingo Hotel in Las Vegas waiting to see an 8:00 show. Everybody was hungry and thirsty and our alternative was the Food Court.\n", 497 | "\n", 498 | "We started off with drinks. While a margarita would have was the natural choice, I lighted on the specialty cocktails. Most were rum-based with fruit juices. I had my party face on so I went for the \"It's Five O'clock Somewhere\" cocktail. One aspect of the sweet party drinks is they go down easily. If you don't pace yourself, you'll find yourself face down again in Margaritaville.\n", 499 | "\n", 500 | "Fortunately dinner came before that could happen! I had the salmon with bourbon sauce. It came with rice and a generous amount of green beans. This was a good meal. The salmon was a bit dry as they left it cooking to get the bourbon sauce to caramelize but they offset it by including a nice amount of tartare sauce. The combination worked perfectly.\n", 501 | "\n", 502 | "By now I was both sated and buzzed. At this point, the floor show started. The show consisted of two people sort of dancing with stilts. It was cheesy but fun. We lustily sang along with the chorus then had a piece of key lime pie. Everyone left with a big grin.\n" 503 | ] 504 | } 505 | ], 506 | "source": [ 507 | "print ('Our search query:')\n", 508 | "print (doc_test)" 509 | ] 510 | }, 511 | { 512 | "cell_type": "code", 513 | "execution_count": 26, 514 | "metadata": {}, 515 | "outputs": [ 516 | { 517 | "name": "stdout", 518 | "output_type": "stream", 519 | "text": [ 520 | "Most 2 similar reviews:\n", 521 | "[\"Order the cocktail called 'the viale', it is one of the best bourbon drinks I've ever had.\", 'Staff was on point, food was good, cocktails were terrible. I got some tropical cocktail with rum it only tasted like rum.']\n" 522 | ] 523 | } 524 | ], 525 | "source": [ 526 | "print ('Most %s similar reviews:' % n)\n", 527 | "print (searched_result)" 528 | ] 529 | }, 530 | { 531 | "cell_type": "markdown", 532 | "metadata": {}, 533 | "source": [ 534 | "## Classifying positive/negative review" 535 | ] 536 | }, 537 | { 538 | "cell_type": "markdown", 539 | "metadata": {}, 540 | "source": [ 541 | "#### Naive-Bayes Classifier" 542 | ] 543 | }, 544 | { 545 | "cell_type": "code", 546 | "execution_count": 27, 547 | "metadata": {}, 548 | "outputs": [ 549 | { 550 | "data": { 551 | "text/plain": [ 552 | "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)" 553 | ] 554 | }, 555 | "execution_count": 27, 556 | "metadata": {}, 557 | "output_type": "execute_result" 558 | } 559 | ], 560 | "source": [ 561 | "# Build a Naive-Bayes Classifier\n", 562 | "\n", 563 | "from sklearn.naive_bayes import MultinomialNB\n", 564 | "\n", 565 | "nbf = MultinomialNB()\n", 566 | "nbf.fit(vectors_train.toarray(),target_train)" 567 | ] 568 | }, 569 | { 570 | "cell_type": "code", 571 | "execution_count": 28, 572 | "metadata": {}, 573 | "outputs": [], 574 | "source": [ 575 | "nbf_predict_test = nbf.predict(vectors_test)" 576 | ] 577 | }, 578 | { 579 | "cell_type": "code", 580 | "execution_count": 41, 581 | "metadata": {}, 582 | "outputs": [], 583 | "source": [ 584 | "def make_confusion_matrix_relative(confusion_matrix):\n", 585 | " star_category_classes = [1, 2, 3, 4, 5]\n", 586 | " N = list(map(lambda clazz : sum(target_test == clazz), star_category_classes))\n", 587 | " relative_confusion_matrix = np.empty((len(star_category_classes), len(star_category_classes)))\n", 588 | " \n", 589 | " for j in range(0, len(star_category_classes)):\n", 590 | " if N[j] > 0:\n", 591 | " relative_frequency = confusion_matrix[j, :] / float(N[j])\n", 592 | " relative_confusion_matrix[j, :] = relative_frequency\n", 593 | " \n", 594 | " return relative_confusion_matrix\n", 595 | "\n", 596 | "# http://www.wenda.io/questions/4330313/heatmap-with-text-in-each-cell-with-matplotlibs-pyplot.html\n", 597 | "# http://stackoverflow.com/questions/20520246/create-heatmap-using-pandas-timeseries\n", 598 | "# http://sebastianraschka.com/Articles/heatmaps_in_r.html\n", 599 | "# http://code.activestate.com/recipes/578175-hierarchical-clustering-heatmap-python/\n", 600 | "def plot_confusion_matrix(confusion_matrix=[[]], title='CM', savefilename=''):\n", 601 | " rcm = make_confusion_matrix_relative(confusion_matrix)\n", 602 | " #plt.imshow(rcm, vmin=0, vmax=1, interpolation='nearest')\n", 603 | " c = plt.pcolor(rcm, edgecolors='k', linewidths=4, cmap='jet', vmin=0.0, vmax=1.0)\n", 604 | " plt.title(title)\n", 605 | " plt.colorbar()\n", 606 | " plt.ylabel('Actual Label')\n", 607 | " plt.xlabel('Predicted Label')\n", 608 | " plt.xticks(0.5 + np.arange(5), np.arange(1,6))\n", 609 | " plt.yticks(0.5 + np.arange(5), np.arange(1,6))\n", 610 | "\n", 611 | " def show_values(pc, fmt=\"%.2f\", **kw):\n", 612 | " pc.update_scalarmappable()\n", 613 | " ax = pc.axes\n", 614 | " for p, color, value in zip(pc.get_paths(), pc.get_facecolors(), pc.get_array()):\n", 615 | " x, y = p.vertices[:-2, :].mean(0)\n", 616 | " if sum(color[:2] > 0.3) >= 2:\n", 617 | " color = (0.0, 0.0, 0.0)\n", 618 | " else:\n", 619 | " color = (1.0, 1.0, 1.0)\n", 620 | " ax.text(x, y, fmt % value, ha=\"center\", va=\"center\", color=color, **kw)\n", 621 | " \n", 622 | " show_values(c)\n", 623 | "\n", 624 | " if savefilename:\n", 625 | " plt.savefig(savefilename, bbox_inches='tight')\n", 626 | " \n", 627 | " return plt.show()" 628 | ] 629 | }, 630 | { 631 | "cell_type": "code", 632 | "execution_count": 30, 633 | "metadata": {}, 634 | "outputs": [ 635 | { 636 | "data": { 637 | "text/plain": [ 638 | "0.6258245119978544" 639 | ] 640 | }, 641 | "execution_count": 30, 642 | "metadata": {}, 643 | "output_type": "execute_result" 644 | } 645 | ], 646 | "source": [ 647 | "# Get score for training set\n", 648 | "nbf.score(vectors_train.toarray(),target_train)" 649 | ] 650 | }, 651 | { 652 | "cell_type": "code", 653 | "execution_count": 42, 654 | "metadata": {}, 655 | "outputs": [ 656 | { 657 | "name": "stdout", 658 | "output_type": "stream", 659 | "text": [ 660 | "[[7.72460555e-01 4.48559134e-02 3.18164037e-02 3.40331204e-02\n", 661 | " 1.16834007e-01]\n", 662 | " [3.87141999e-01 9.29301021e-02 1.51011416e-01 1.42399359e-01\n", 663 | " 2.26517124e-01]\n", 664 | " [1.43563591e-01 2.70437336e-02 1.35682275e-01 3.05671457e-01\n", 665 | " 3.88038943e-01]\n", 666 | " [2.63616213e-02 1.34578847e-03 1.24287524e-02 2.32504750e-01\n", 667 | " 7.27359088e-01]\n", 668 | " [9.25857344e-03 1.85171469e-04 9.25857344e-04 3.72564995e-02\n", 669 | " 9.52373898e-01]]\n" 670 | ] 671 | }, 672 | { 673 | "data": { 674 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEWCAYAAABLzQ1kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xt8FNX5+PHPk5Bwv6MgSSAgEbl4h1hv1YqIaCVasQZriy0/W7+I/bbVFm2rNdqLWP3S1lKrLVSk1lSx1qRyqYpWi1ITBARDMCkBsgQECXcIJJvn98dMks1mk2xgLxN43q/XvtiZOTvzHHbz7JkzZ+eIqmKMMcZ7EuIdgDHGmNAsQRtjjEdZgjbGGI+yBG2MMR5lCdoYYzzKErQxxnjUSZegRURFZFgL2z8WkStiGFKbj9taHY4zjq+IyD+jse+TmYgMF5FVIrJfRL59HPv5vYg8EMnY4kFEDojI0HjH4XXtJkGLyCYROSoi/YLWr3YTVvox7PNZEflp4DpVHaWqbx9XsMcgUscVkbdFpEpE0gLWXSUim8KM43lVvfp44wgR17Pu+3fATVIrReTySB/neIhIsog8JCIlInLQ/czNO5bPVgg/AN5W1e6q+ptj3Ymq3qmqj0Qgnkbcemvwl4eIfMdd/1CY+3lbRP5fa+VUtZuqbjzGcE8a7SZBu8qAKXULInIW0Dl+4XjWQcCLrazHVLUb0BN4CvibiCTGOaZAC4FJwK04MZ4DrATGRWDfg4GPI7CfaPoEmBq07mvu+ogQkQ6R2tfJoL0l6AU4H5g6U4HnAgsEf4OLyO0i8u/gHYnIN4GvAD9wW3X57vpNInKV+/whEXlRRJ5zW30fi8iYgH2McI+3x902KWDbsyLyOxFZ7O5/uYgMEJFfichuESkWkfMCygceN1NE3nf3u01EfisiyW34f/oNMKW5bhARuU9E/uvWqUhEbgz1/+WeTj8e9NpXReR77vOBIvKyiOwUkbJwT91VtRb4C9AH6O/u63QRWSYiu0TkMxF5XkR6udu+LyIvB8XxpIj8yn3eU0Tmuv9XW0Xkp3WJX0SGici/RGSvu9+/NvN/chUwHshS1QJVrVHVvao6R1XnBtQ3T0QqRaRURO4IeH2znxURWQZ8Afit+1k4o6XPqThmi8gON+6PRGS0u63RWZ+I3OHGUunGNjBgm4rIneKcEewWkTkiIi28NQVAFxEZ5b5+FE4DqCBgn71F5B/ue77bfZ7qbvsZcFlAPX8bEMddIlIClASsGybOWctqEbnbXZ/o/q082EKcJ432lqBXAD3cxJgI3AL8+Vh2pKrPAM/jtupU9fpmik4CcoFeQB5Q96FLAvKBfwKnAncDz4vI8IDXfhn4MdAPOAK8D3zoLi8E/q+ZY/qB77rlLsJpwU1vQ/W2An8AHmpm+39x/pB6AjnAn0XktBDl/gLcUvdHLSK9gauBXBFJwKn/GiDFjfE7IjKhteDc9+5rOGdEn9atBn4BDARGAGkB8f8ZuCYgYXfAee8XuNvnAzXAMOA8N8a65PcIznvUG0gFnmwmrKuAD1S1vIXQXwB8boyTgZ+LSGDrOuRnRVWvBN4FZriftdZapFcDnwfOcPd1C7AruJCIXInzf/Zl4DRgs3v8QF8ExuKcDXwZaO39CWwENWkA4eSMP+GcEQwCDgfU80dB9ZwR8LobgAuBkYE7U9WjwG3AwyIyArgPSAR+1kqcJ4X2lqCh4QM0HijGSUbR9G9VXaSqfvfY57jrPwd0Ax5V1aOqugz4BwFdMMArqrpSVauAV4AqVX3O3ddfcZJJE+5rVrituE3A00Bb+2t/AVxf1xoK2v9LqlqhqrWq+lecVk1miH28CyhOMgcnKb2vqhU4f/SnqOrDbv034nwpZLcQ070isgenC+ZXwAPu/wWqWqqqr6vqEVXdifPldbm7bRvwDnCzu59rgM9UdaWI9AcmAt9R1YOqugOYHRBHNU4yGaiqVara5GzK1RfY1lzg4vTpXwrMdPezGvgj8NWAYs19VtqqGugOnAmIqq53/w+CfQWYp6ofquoR4H7gImncZ/6oqu5R1S3AW8C5rRz7zzhnX0k4/4eNGkCquktVX1bVQ6q6HyeRhvPZ/IWqVqrq4eANqroO+CnO38i9wFfrPhcnu/aaoG8Fbqfpt3s0bA94fgjo5LbgBgLl7ul6nc04rck6nwY8PxxiuVuoA7qnwP8Qke0isg/4OU5rOmxukvst8HCI/X/NPa3c4ybM0aH2r86dtHJp+NK5FeesA9ykV7cPdz8/xO2yaMbjqtoL57R5DPBLEZnoxnSqiOS6XRT7cBJDYEzzcVpauP/WtZ4HA0nAtoA4nsY5qwHn4pwAH7jdDt9oJrZdOK3Q5gwEKt2kVCf4/W7us9Im7pf9b4E5wKci8oyI9Ggmps0BrzuAU4+WYgr5mQvYxxagFOczVxJ8RiEiXUTkaRHZ7L5P7wC9pPVrCS2dmYDz/qYDi1S1pJWyJ412l6BVdTPOqfG1wN9CFDkIdAlYHtDS7o4jlAogzT3VrzOIyLTon8I5O8hQ1R44ia+lvsPm/BKn7/OCuhUiMhinpTsD6OsmzHUt7P8FYLL7uguBur7gcqBMVXsFPLqr6rWtBaWOdcBy4Dp39S9w3o+z3TrfFhTT34Gz3b7YL9LwRVGO033ULyCOHqo6yj3WdlW9Q1UHAt8Cfieh++bfADLr+lNDqAD6iEj3gHXH8363+DlV1d+o6gXAKJyuju83E9PgugUR6YpzJnC8n8HngHsI3QC6BxgOXOi+T5+vO3xd6M3ss7W/td/hnIFOEJFL2xbuiavdJWjXNOBKVT0YYttq4EvuN/0wt2xzPgWOdSzmf3D+yH4gIknijGG+nqZ9gMeiO7APOCAiZwL/cyw7UdU9wBM4rcg6XXH+WHYCiMjXcVrQze1jlVv2j8BSd58AHwD7RGSmiHR2L+6MFpGx4cTm1utSGkY2dAcOAHtEJIWghOR2Ey3E6Rf/wG3p1XV//BN4QkR6iEiCOBccL3ePc3NA0t3t1r3J6bOqvgG8DrwiIheISAcR6e5eZPuG25J8D/iFiHQSkbNxPlvPB+8rTM1+TkVkrIhc6HYzHASqQsXs/l98XUTOFZGOOK3e/7jdYsfjrzj94C+G2NYd5+xvj4j0AX4StL3Nf1Mi8lWcRsTtwLeB+SLSYkv/ZNEuE7Sq/ldVC5vZPBs4ivNBmU/Lf0BzgZHuqfHf2xjDUZyLQhOBz3BaAF9T1eK27KcZ9+J0J+zHae2GHHkQpl8T8MetqkU4Sft9nP+js3Basi15Aeci2l8C9uPH+UI6F+eM5jOcJN6zhf3UjZg5iJNU/4TTHQHOxcrzgb3Aa4Q+O5rvxrsgaP3XgGSgCCcJL6Shu2Is8B8ROYBz4e5/VbWsmfgmA4tw/r/34pxZjMFpXYPT1ZOO03J9BfiJqr7eQn1b0tLntAfO+74bpwtjF/B48A5U9U2c4ZQv4/Sfn07L1wDCoqqHVfWNUP3FONcOOuO83yuAJUHbf41zxrVbRFod7y0ig9x9fk1VD6jqX4BCnP+fk56o3bDftBPuH3MxMEBV98U7HmOirV22oM3Jx+3r/x6Qa8nZnCwsQRvPcy9+7cMZWhnc52lM3IlzS4AdIrKume0iIr8R50dFH4nI+eHs1xK08Tx3fHM3de5X0tpwLWPi4Vmc8fnNmQhkuI9v4ozUapUlaGOMOU6q+g5Q2UKRLOA5d4jpCpyx4y2NuwfAUzcuERG7YmmMCcdnqnrK8exgmIgeCrPsNmc4aFXAqmfUuV1EuFJo/GMdn7uu2V+vgscStDHGhGlz60Vadgjnl0vheMi5TcOYVgs2L9QPwVptkHowQQ8j4Idv7diLNP7/H08rv7JtB9bjjHKr0x+4OE6xRNJrOEOS61yP8yPB9mwLzr2sHL1p+Rdb7cXrOHfnigQhpgnQh3MDsDqpOOPpW+TBBH0qzu8V2ruXaJygz8T5FW57VknjBN2dY78fkJcE/9ZiNO2/XqsJTNBdgCvjFkvkrCdyCTqBmN5MPg+YISK5OLdM2NvMDbAa8WCCNsaY6BOcu2xFZF8iLwBXAP1ExIczHDQJQFV/j/ML1WtxbkR1CPh6OPu1BG2MOSlFsotDVae0sl2Bu9q6X0vQxpiTUiRb0NFiCdoYc1KK8UXCY+L1+IwxJiqsBW2MMR4V41Ecx8QStDHmpGQtaGOM8TCvJ0Cvx2eMMVFhLWhjjPEoG8VhjDEeZRcJjTHGo6yLwxhjPMq6OIwxxqPaQwv6hJvyasKEIRQX/z9KSu5g5swLm2xPTk4kN3cSJSV3sGLFbQwe3AOAPn06sWxZNvv3f4cnn7wq1mG3asKE0ykuvouSkruZOfOSJtudet1EScndrFgxjcGDe9Zvu+++SykpuZvi4ru4+urTYxl2i07EOgFMmDCA4uKJlJRcy8yZZzbZnpycQG7uRZSUXMuKFVcxeHCXRtvT0rqwf/+XuOee4bEKuVX9JkzgsuJiPl9SwtCZM5ts7zRoEGPfeINL1qwh86236JSSUr/tmpoaLlm1iktWreL8V1+NZdgtqmtBh/OIl6gmaBHZJCJrRWS1iBRG81gACQnCnDlXMXHiS4wcOZcpU0YwYkTjezBPm3YWu3dXkZHxB2bPLmTWrCsAqKry88AD73LvvW9HO8w2c+p1LRMnPs/IkXOYMmU0I0b0a1Rm2rTz3Ho9yezZK5g1y/mSGTGiH9nZoxg16ndcc83z/O5315KQEGpyh9g6EesEdfW6gIkT32HkyCVMmTKYESN6NCozbdpQdu8+SkbGImbP3sCsWY3vPT179rksXrw9lmG3LCGBUXPmUDhxIu+OHMlpU6bQbcSIRkXOfPxxKp57juXnnEPpww9zxi9+Ub/Nf/gwy887j+XnnceHWVmxjr5ZdS3ocB7xEosW9BdU9dzjnC4mLJmZp1Fauoeysr1UV9eSm7uerKxhjcpkZWUwf74zM/rChRsYN86ZOePQoWqWL99KVVVNtMNss8zMFEpLKykr2+PW62Oyshq3zLKyhjN/vnMr84ULixg3bqi7/kxycz/m6FE/mzbtobS0kszMlCbHiLUTsU4AmZl9KC3dT1nZQbdeW8jKahxbVtZA5s/fBMDChT7GjesfsC2FjRsP8vHHe2MZdot6ZWZysLSUw2VlaHU123JzOTUo0XYbOZJdb74JQOVbb9HfQ4m4OYIziiOcR7ycUF0cKSndKC/fX7/s8+0nJaV7iDL7APD7lb17j9C3r7cH26SkdK+PGcDn2xeiXj0oL3f+qJ16VdG3b2f3tXsDXtv0/yQeTsQ6AaSkdKa8/HD9ss93iJSUzkFlulBe7kxX6tSrmr59k+nSJZGZM88kJ+fjmMbcmk4pKVSVN8x3WuXzNerCANi/Zg39b7oJgP433kiHHj1I6tMHgIROnbi4oICL3n+/SWKPJwGSOoT3iJdoH1qBf7qzdT8dahZcEfkm8M1IHEyk6Wmuc5/stpXxmtAxB5dp+jrV8F4bDydinaD5mMMpk5MzmtmzP+HgQY+dxYXxN1N8772M/O1vSb39dirfeYcqnw+tcerx9qBBHNm2jc5DhpC5bBkH1q7l0MaNMQm9JSLQIdwMGKe3JNoJ+hJVrRCRU4HXRaRYVd8JLOAm7WcA3ER+zHy+/aSlNbSkUlO7U1FxIESZHmzdeoDERKFnz45UVlYF78pTfL59pKU19GOmpvagomJ/iDI92bp1v1uvTlRWHq5f3/Da7k1eGw8nYp0AfL7DpKU1tJhTU7tQUXE4qMwh0tK6sHXrYbdeSVRWHuXCC/syeXIajz12Dr16JVFbq1RV+ZkzpzTW1WikyuejU1rDfKedUlM5UtF4vtMj27axym1BJ3btyoCbbqJm3776bQCHy8qofPttepx3nmcSdFJivKNoWVS7OFS1wv13B/AKkBnN4xUUbCMjozfp6T1JSkogO3sEeXmNP9x5eaVMnToagMmTh7Ns2ZZohhQRBQVbycjoS3p6L7deo8jL29CoTF7eJ0yd6lxsmjx5JMuWlbnrN5CdPYrk5ETS03uRkdGXDz7YGvM6BDsR6wRQUFBJRkZ30tO7uvUaRF5e49jy8iqYOjUdgMmTU1m27FMAPv/5ZQwZ8g+GDPkHv/rVJ/z85+vjnpwB9hYU0DUjg87p6UhSEqdlZ7MjL69RmaS+fetb2kPvvx/fvHkAdOjVi4Tk5PoyvS+5hANFRbGtQDPqWtDhPOIlaocWka5Agqrud59fDTwcreOB0583Y8YbLF16M4mJwrx5aykq2kVOzqUUFm4nP7+UuXM/YsGC6ygpuYPKyiqysxs+aGVl36JHj2SSkxO54YYMrr76Rdav3xXNkMPi1GsRS5fe5tZrNUVFO8nJuYLCwgry8z9h7twPWbDgRkpK7qay8jDZ2QsBKCrayYsvFlFUNJ2amlruumsRtbXx7w84EesEdfX6kKVLL3frtZGion3k5IymsLCS/PwK5s7dyIIFn6Ok5FoqK4+Snf1+vMNukfr9FM2YwdilS5HERHzz5nGgqIiMnBz2FhayIz+fvldc4YzcUKXynXcousuZfq/biBGMfvpptLYWSUhg46OPcmD9+jjXyCECSR3jHUXLJFr9ryIyFKfVDM4XwV9U9WetvEbhYsA7FxKO3f1AbcDy3UDfZsq2F28B/wpYHgbcFqdYImkWENgNcT9wTjNl24vVwKP1SynAH+IWS+TMAV5znq483pFhY5JFC08Jr6xUHP/xjkXUWtCqupH2/yk3xpyo2sFvvT0enjHGRJHHM6DHwzPGmCgRwOOjOCxBG2NOTtbFYYwxHiWAx0dxWII2xpycrAVtjDEeZQnaGGM8zC4SGmOMB1kL2hhjPMoStDHGeJSN4jDGGI+yFrQxxniUJWhjjPEo+6n3sTgM7Ix3EFGwG2cGsPbsUNDyUeCzeAQSYcHvyy6gIlTBdqTx+1ID+OITSEQdaL1I+CLcghaRa4Bf46T9P6rqo0HbBwHzgV5umftUdVGL+/TSfHzHO+WVMeakcfz3g+4nWnh9eGXl2ZaPJyKJwCfAeJzvwgJgiqoWBZR5Blilqk+JyEhgkaqmt3TcE2pWb2OMCVtdCzqcR+sygVJV3aiqR4Fcms48okDdRJw9CeM0zYNdHMYYEwNt6+LoJyKFAcvPuBNe10kBygOWfcCFQft4CPiniNwNdAWuau2gHkzQwonRsPcHLZ8IdVKa9td6/CpLWILeK+lQPwFqu6UKWtNolddnsA6HvxYiOv1k+Bnws1a6VEJ9YIIjnQI8q6pPiMhFwAIRGa2qtSFe28bwYuYK4NZ4BxEBd9L4D/8uTrw5Cc8E7ohTLJH0AI0ugF7/Dxg8IW7RRMSmxZB/bf3iGafBht/EMZ4Imf4HeOqfEdpZZEdx+IC0gOVUmnZhTAOuAVDV90WkE9AP2NHcTk+EZp0xxrRdZPugC4AMERkiIslANpAXVGYLMA5AREYAnWhlyJoHW9DGGBMDEfypt6rWiMgMYClOu3yeqn4sIg8DhaqaB9wD/EFEvovT/XG7tjKMzhK0MebkFOFx0O6Y5kVB6x4MeF4EXNKWfVqCNsacnOyn3sYY41GWoI0xxsM8PvTQErQx5uRkLWhjjPEou2G/McZ4lLWgjTHGoyxBG2OMR9kN+40xxqOsBW2MMR4lOHfD8DBL0MaYk1M76OI44e5mN2FCKsXFkykpuZmZM89usj05OYHc3CspKbmZFSsmMXhwNwDGjj2FVatuZNWqG1m9+kZuuGFwrENv0YQJp1NcfBclJXczc2bTn/MnJyeSm3sTJSV3s2LFNAYP7lm/7b77LqWk5G6Ki+/i6qtPj2XYLZowIZ3i4tspKfkGM2eObbLdqdN1lJR8gxUrpjB4sDMZRZ8+nVi27Gb275/Bk09eGeuwWzVhBBT/CEoegJkhbsn+3S/Axz+ENTPhjbtgUG9n/aDeUPh9WPUDWHc/fKtNd22IriWrYPi3YdgMePSVptu/+yyce6/zOOPb0Guqs37zTrjgB876Ud+F30fqVqGRENm72UVF1A/tztVVCGxV1S9G81gJCcKcORczfvxifL6DFBRkkZe3hfXr99SXmTZtOLt3HyEj4yVuuWUos2Zlkp29jHXrKhkz5u/4/cqAAZ1Zs+ZL5Odvwe+P/zSJTr2uZfz4Bfh8+ygouIO8vA2sX98wMei0aeexe3cVGRlPcssto5g16yqys19mxIh+ZGePYtSo3zFwYHfeeOOrnHHGb6mN6F3P286p05WMH/8yPt9+Cgq+Ql7ef1m/vrK+zLRpo906zeOWW4Yza9ZlZGe/RlVVDQ88sJzRo/sxenS/ONaiqQSBOTfD+Dng2wMF90LeOli/vaHMKh+M+SUcroY7L4XHsiD7Wdi2Dy6eDUdroGuyk6Tz1jrr48nvh7vmwusPQGofGHs/TBoDIwPufjz79obnTy6GVWXO89N6wXs/g45JcOAwjL7Hee3APjGtQvM83ocQixb0/wLrY3AcMjNPobR0H2Vl+6muriU3dyNZWY1bwllZg5k/vwSAhQvLGDduIACHD/vrk3GnTol4aC5dMjNTKC2tpKxsj1uvj8nKOrNRmays4cyfvwaAhQuLGDduqLv+THJzP+boUT+bNu2htLSSzMyUmNchWGbmAEpL91BWttetUzFZWY1b91lZpzN/vjPn5sKFnzBu3CAADh2qYfnyCqqqaprsN94yB0PpTijbBdV+yP0Qss5qXObtEic5A6zYBKm9nOfVfic5A3Ts4CR7L/igFIYNgKH9ITkJsi+BVwubL//Cv2GK2/pPTnKSM8CRGqhtdu6QOKjr4gjnESdRTdAikgpcB/wxmsepk5LShfLyg/XLPt9BUlK6hCjjTN7u9yt79x6lb1/n50SZmaewbt1NrF17E3fe+W9PtJ4BUlK6U17e0Izy+faRktI9qEwPysv3AnX1qqJv387ua/cGvHZ/k9fGQ0pKN8rL99cv+3wHQtSpoYxTpyP07evtqzopvaC84YQN3x5I6dl8+Wmfg8VFDcupvZyuj/KHYdab8W89A2ythLSAyYBS+8DWXaHLbt4JZTvgytEN68o/g7PvgbQ7YeYNHmo9t4Mujmi3oH8F/ACIyfemhJhHLrgl3FKZDz7YyejRLzN27Kvcf/85dOzojSsI4dWr6etUw3ttPISOV8MoE6WAIiTkxHTNxPyVMTBmEPxyWcM63x44ZxYMeximZsKp8f8ubTKxHjQ/ZWPucpj8OUgM+NNJ6wcfPQGlT8L8t+HTPaFfG3N1P/UO5xEnUUvQIvJFYIeqrmyl3DdFpDBoxtxj4vMdJC2ta/1yampXKioOhSjjXBhMTBR69kymsvJIozLFxXs4eLCG0aN7H29IEeHz7SMtrUf9cmpqDyoq9oco4zTVnHp1orLycKP1zmu7N3ltPPh8B0hLa8g+qandqKg40GwZp04dqaysimmcbeXbA2m9GpZTe0FFiFbwuDPgR1fDpGcaujUCbdsHH2+DyzxwTTe1D5QHtJh9lc23gnOXw5RLQ28b2AdGpcG7MenwDMNJ3oK+BJgkIpuAXOBKEflzcCFVfUZVx7QyY25YCgp2kpHRg/T0biQlJZCdPZS8vM2NyuTlbWbq1AwAJk8ewrJlzryO6endSEx0mgWDBnVj+PCebNoU/0QGUFCwlYyMvqSn93LrNYq8vA2NyuTlfcLUqecAMHnySJYtK3PXbyA7exTJyYmkp/ciI6MvH3ywNeZ1CFZQsJ2MjF6kp/dw63QmeXkbG5XJy/svU6eOBGDy5DNYtmxLPEJtk4ItkHEKpPdxZtHOPt+50Bfo3FR4Ohsm/QF2BnwnpfSCTm5/ba/OcMlQ2PBp7GJvzthhULINyj6Fo9VOEp4U4q91w1bYfRAuOqNhnW8XHHbbP7sPwPINMHxgbOJuVTtI0FE7tKreD9wPICJXAPeq6m3ROh44/ZQzZrzH0qUTSUwU5s37hKKiPeTknE9h4Wfk529h7txPWLDgckpKbqay8gjZ2W8BcOmlA7jvvnOorq6ltlaZPv09du060soRY8Op1yKWLr3Nrddqiop2kpNzBYWFFeTnf8LcuR+yYMGNlJTcTWXlYbKzFwJQVLSTF18soqhoOjU1tdx116K4j+CAujq9xdKlN7l1WkdR0S5yci6msHA7+fkbmTt3HQsWTKSk5BtUVlaRnf1a/evLyqbRo0dHkpMTuOGG07n66pcbjQCJF38tzFgIS6dDYgLMWwFF2yHnWijcAvnr4JdZ0C0ZXvq685otuyHrDzCiPzxxg9OlIMDjy2DdtnjWxtEhEX47DSb8zKnfN77gtIQfzIUxp8Mkd4TkC8sh++LG3R/rfXDPc846Vbj3ejjLKyNY28EvCaWVOQsjc5CGBN3iMDsRUfgCcGvUY4q+OwF/wPLdQN9myrYXbwH/Clg+E7gjTrFE0gNAQFfYpCUweELcoomITYsh/9r6xTNOgw2/iWM8ETL9D/CUM5Z65fGedY8ZJVr41/DKylnHf7xjEZPvD1V9G3g7FscyxpiwtIMWtMfDM8aYKLEb9htjjEdZC9oYYzzKErQxxniUJWhjjPEu9caPhZtlCdoYc1LSBDjq7Vu7WII2xpycVKAmMdwfU8fnNnyWoI0xJyUVwd8h3BR4NKqxNMcStDHmpOVP9HYntCVoY8xJSRH8Hp+UsNkELSL5hL4VLACqOikqERljTAwoQk17TdDA4zGLwhhjYkwRjnr8t97NJmhVrb9tmYh0Bgap6obmyhtjTHvSHro4Wh1jIiLXA6uBJe7yuSKSF+3AjDEm2vwkhvUIh4hcIyIbRKRURO5rpsyXRaRIRD4Wkb+0ts9wLhI+BGTi3i5UVVeLSHpYER+TdcDvo7f7mAkeN7kYSI5HIBG0M2h5K/BcPAKJsKCJGQp/BkVz4xNKpBxsfKf/it3w5f+LUywRtHJj62XCFck+aBFJBOYA4wEfUCAieapaFFAmA2cSk0tUdbeInNrafsNJ0DWqujfU5KPRsZOmieBEUBrvAKJgP7Am3kFEXsW78Y4g4g5UwUvvxzsKb3G6OCI2kC0TKFXVjQAikgtkAQFztnMHMEdVdwOo6o7WdhpOdOtE5FYg0f0G+DbwXhuDN8YYT3EuEoZ5Xn8VAAAgAElEQVR9VtsvaGLrZ1T1mYDlFKA8YNkHXBi0jzMARGQ5kAg8pKpLWjpoOAn6buBHOOeBLwBLgUfCeJ0xxniWQlu6OD5rZcqrUF0MwcOUOwAZwBVAKvCuiIxW1T3N7bTVBK2qh4AficgsZ1GjO9V1wjmQ9IWoHiImjvyaxu/PRUCXOAUTKRuBsoDlwZB4Y7yCiRz/UzTqh77oBuifHq9oIuPTTfD+3+sXpWd3un29/b9XVW+uoHrtJxHaW0S7OHxAWsByKlARoswKVa0GykRkA07CLmhup61GJyJjgXlAd3d5L/ANVV3ZpvDDlXQZdJkdlV3H1JE5QHXAiuuA/nEKJlLyaZSgE0ZC8gnwXh1eQKME/cU7YUw7nzS2YHGjBJ3Yvy99Zt8fx4AiY9f0nIgl6AgPsysAMkRkCM7V82yazn79d2AK8KyI9MPp8mjxsmc4Xx9zgemq+i6AiFwK/Ak4u03hG2OMx0QqQatqjYjMwOkCTgTmqerHIvIwUKiqee62q0WkCPAD31fVXS3tN5wEvb8uObuB/FtEotvNYYwxURbpH6qo6iJgUdC6BwOeK/A99xGWlu7Fcb779AMReRrnAqECt+COiTbGmPZKEY601596A08ELf8k4HmzN1Eyxpj2oD381Lule3GcAEMpjDEmtHadoAOJyHXAKKB+Bi9VfThaQRljTCy059uNAiAiv8cZwPsF4I/AZOCDKMdljDFRFeGfekdFODMmXqyqXwN2q2oOzi8u0lp5jTHGeFpdF0ek7mYXDeF8fRx2/z0kIgOBXcCQ6IVkjDHR54zi8PYdJsNJ0P8QkV7AL4EPcUZw/CGqURljTJS1hy6OcO7FUXdjpJdF5B84FwrPjGpUxhgTAyfEKI46qnoEOCIiLwGDohOSMcZE3wkzzC6EWN293xhjouJETtD2S0JjTLvWHn7q3ewwOxHJF5G8EI98oG8MY2yTCeOg+D9QUggz/7fp9ssugpVvQfUOuGlS0+3du4NvHTw5K/qxtsWECakUF0+mpORmZs5seiPB5OQEcnOvpKTkZlasmMTgwd0AGDv2FFatupFVq25k9eobueGGwbEOvVkTxkPxaihZCzPvabr9sktg5XtQvQ9uuqHp9u7dwVcKT3psrr0JfaH4Eii5FGamN93+rVT46CJY9Tl4dyyM6Oqs75MEy8bA/ivhSY9d5bmcrrzJEN5mKP9Dnybbv0IvlpDOItJ5iUEMc0dHJAG/ZABLSGcx6XzOQ/dEb+/D7B4/xm0AiEgn4B2go3uchar6k5ZfdXwSEmDOYzD+S+CrgII3IW8JrN/QUGaLD26/C+6dEXofj/wQ/uWxCb0SEoQ5cy5m/PjF+HwHKSjIIi9vC+vXN0zEMG3acHbvPkJGxkvccstQZs3KJDt7GevWVTJmzN/x+5UBAzqzZs2XyM/fgt8f35OghASYMxvGfxF8W6HgXch7DdYXN5TZUg63fxPuDfFFC/DIg/Avj00fmADMGQHjV4KvCgo+B3k7Yf3BhjJ/2QZP+5zn158C/zccJn4IVbXwQCmM7uY8vCIBeJj+3EY526kmj3Re5wClHK0v8yr7eB7n83gV3XiAU5mKj2x6AXANm+hLIs+SxiQ2eeYU3OtdHM22oFX1Xy09wtj3EeBKVT0HOBe4RkQ+F6nAQ8m8AErLoGwzVFdD7t8ga2LjMpvLYW0R1AZPug2cfw70PwX++VY0o2y7zMxTKC3dR1nZfqqra8nN3UhWVuOWcFbWYObPLwFg4cIyxo0bCMDhw/76ZNypUyLqkb+MzDFQ+l8o2+S+Vwsh64uNy2zeAmvXNfNenQf9T4V/vhmTcMOW2RNKD0HZYahWyN0OWUFzN+/3NzzvmtjQX3jID8v3OInaS86lE5s5SjnVVAP57ONqGn+DHAiYxb4LUl+nDDqynEMA7MLPPvyc3XDHiLiqm9U7nEe8hPNLwmOijgPuYpL7iGp6SDkNyrc2LPsqnHXhEIEnHoHvR7WNf2xSUrpQXt7QBPP5DpKS0iVEGee/2+9X9u49St++Tv9aZuYprFt3E2vX3sSdd/477q1ngJSBQe/VVmddOETgiV/A938YndiOR0onKK9qWPZVQUqIbs7paVB6KTx2Bny7uOl2L+lPEhXU1C9vo4b+JDUp91V68S+Gch+n8hCfArCeKsbTjUQglSTOohOnhXhtPNSNgw7nES9RS9AAIpIoIquBHcDrqvqfEGW+KSKFQTPmHuPxmq4Lt8U4fRoset1JFF4jISoWXK+WynzwwU5Gj36ZsWNf5f77z6Fjx/if1h3Xe/UtWLTUo+9ViHWhqvW7chj2b5j5Cfx4aLSjOj7h1mkBe7icjTzKTu6mHwAvspft1JBPOj/hVFZyGL9nOjho133Qx01V/cC57i8RX3FnsF0XVOYZ4BkAETmud85XAWkpDcupA6Fie3ivvWiscwFx+jTo1hWSk+HAQbjfA/fs8/kOkpbWtX45NbUrFRWHQpTpxtath0hMFHr2TKay8kijMsXFezh4sIbRo3uzcuVnMYm9Ob6tQe9VClRsC++1F2U6FxCnfzPgvToA9z/Y+mujzVcFaQFn8KmdoOJI8+Vzt8NTI6If1/HYTjUDA1LFaXRgR6P5NhvLZx8/deff9AOPsKN+28sMoiyg7zqeFOFoe/2ptztao9mEqaohxkA0W3aPiLwNXAOsa6X4MSv4EDKGQvog2LoNsr8Et34zvNfe9q2G51OnwJhzvZGcAQoKdpKR0YP0dCcBZ2cP5dZbG3eU5+VtZurUDFas2MHkyUNYtsyZUDg9vRvl5Qfx+5VBg7oxfHhPNm2K/4xlBSshYxikD4atFZA9GW79enivve0bDc+n3gZjzvdGcgYo2AcZXSC9M2ytguwBcOtHjcsM6+L0UwNcdwqUHGq6Hy9ZQxXpJJNKEp9SzfX04NtBE1ank8QmN2lfSTc2uUm4E4IAh1EupQs10OjiYjzV9UF72bGO4miViJwCVLvJuTNwFRDVwWt+P8z4ASxdCImJMO95KCqGnPuhcBXkL4Ex58ErC6B3T7j+Gsi5D0ZfHM2ojp/fr8yY8R5Ll04kMVGYN+8Tior2kJNzPoWFn5Gfv4W5cz9hwYLLKSm5mcrKI2RnOwn80ksHcN9951BdXUttrTJ9+nvs2tVCky5G/H6Y8T1Ymue+V89B0XrIeQAKP4T812DMBfBKLvTuBddfCzk/htFj4h15y/wKM4ph6fmQKDBvKxQdhJzToXAf5O+EGWlwVV+oroXdNTA1oMlSdhn06ADJAjecClevbDwCJB78wIN8ynOkkYjTbVHCUb5LP9ZSxRscYCq9uYSu1KDsxc89OKdD/ejAfFJRYDs1fC8oscdTe7gXh2iULuuLyNnAfJwZbhOAF1u7yb+IKB1nQJcnoxJTTO1OhkangT8F97Sv/coH8hoWEyZCx0XNlm43DvfDuUmj62dLYMyEuIUTEQWL4cfX1i92OCOdlA2L4xhQZOyansOBp3IBVqrqcX1d9x+TqrcUNjOGM8iT8oPjPt6xCOeG/RnAL4CRNJ5RpcVLG6r6EXDe8QZojDHRcKL81PtPOBPGzsaZVeXr2L04jDHtXHvogw5nmF1nVX0Tpztks6o+BFwZ3bCMMSa6nFEcHcN6xEs4LegqEUkASkRkBrAVOLWV1xhjjKedKF0c38GZNPbbwCM4reep0QzKGGNiod0naFUtcJ8ewOl/NsaYdq899EGHM4rjLUL8YEVVrR/aGNNutYdx0OFEd2/A807ATRBw5xRjjGmH2vVPveuo6sqgVctFJJzbjRpjjGedKF0cgdMnJAAXAAOiFpExxsTIidDFsRKnD1pwujbKgGnRDMoYY6LtRBlmN0JVqwJXiIi3Z1o0xphWtIcEHc4vCUPN0Pd+pAMxxphY8/qUVy3dD3oAkAJ0FpHzaLj/Rg/w0NS8xhhzDGpJiOvPuMPRUhfHBOB2IBV4goYEvQ+I3mxwtZ+Bf23Udh87wUPHt0MLs1C0D3uDlvdB7YnwXgWNGt1eBmXtvF7byxot6pGjHF37SZyCiZzaXXtaL9QGkeziEJFrgF/j3GL5j6r6aDPlJgMvAWNVtcWp/lq9H7SI3KSqLx9byG1zvFNeGWNOGsd9f+YuY0bo8MI/hVV2tVzU4vFEJBH4BBgP+IACYIqqFgWV6w68BiQDM1pL0OH0QV/gzilYd4DeIvLTMF5njDGepUS0DzoTKFXVjap6FMgFskKUewR4DKgKsa2JcBL0RFWtP69Q1d3AtS2UN8aYdsD5qXc4D6CfiBQGPIJnO00BygOWfe66hqM51/LSVPUf4UYYzjC7RBHpqKpH3IN0hij2rCcmQAdvDx4Py5HgiTGTCO/70MtqcGaoqyPg8Z/Khid4jsYT4b2qpck1jwRvXxALi1aD1kZmV20bZvdZK10qoSYxqe+ydW/ZPBvnul7YwsmEfwbeFJE/uQf8BvBcWw7SFt3/Zwp9nvxxtHYfM5uTz4LqwItP/wJanCWsHXicRnMJd7wG+pwAcxJu7wcaMCdht3xIaudzEh5dDAcDTnR7ngE3b4hfPJGyfDqsfyoiu1KEI5FrYPiAtIDlVGg0Q253YDTwtoiA82vsPBGZ1FI/dDj34nhMRD7CmZVbgEdUdWnb4zfGGO+I8N3sCoAMERmCM6lJNnBr/bFU9wL96pZF5G3g3tYuEoYVnaouAZa4O75EROao6l1trYExxnhJpIbZqWqNO+PUUpxhdvNU9WMReRgoVNW8Y9lvWAlaRM4FpgC34NyL42/HcjBjjPGKSP/UW1UXAYuC1j3YTNkrwtlnS78kPAOnmT4F2AX8FWfc9BfCjNcYYzxLEfy13r4XR0st6GLgXeB6VS0FEJHvxiQqY4yJMq0VjlR5e2RLSwn6JpwW9FsisgRn4HWooSTGGNPuqAr+mnbaglbVV4BXRKQrcAPwXaC/iDwFvKKq/4xRjMYYE3mK5xN0q6PxVfWgqj6vql/EGdu3Grgv6pEZY0wUqQo11YlhPeKlTYMAVbUSeNp9GGNMOybU+r39q2VvR2eMMdGigMe7OCxBG2NOTrUCVd5Ogd6Ozhhjoqmm9SLxZAnaGHNycm4I7WmWoI0xJ6d2kKDb+01vm7icrrzJEN5mKP9Dnybbv0IvlpDOItJ5iUEMc283mAT8kgEsIZ3FpPM5D82LO2FCMsXFfSkp6cvMmU3jSk6G3NyelJT0ZcWKPgwe7LytSUkwb14PPvqoD6tX9+Hyy5NiHXqLJlwBxe9CyXKYOaPp9ssuhJVLoXoL3HRd42015bDqdefx6rOxiDZ8E8ZB8X+gpBBm/m/T7ZddBCvfguodcNOkptu7dwffOnhyVvRjDdeEUVD8CJT8DGZe03T7d8fDxzmw5ifwxvdgkPunN6gPFP4YVj0I63LgW5fHNu4WKc4ts8N5xEnUWtAikoZz3+gBOHcPf0ZVfx2t44HzbfMw/bmNcrZTTR7pvM4BSmm4ef6r7ON5nAlirqIbD3AqU/GRjTOr1zVsoi+JPEsak9jUZOrXWEtIgDlzujN+/B58Pj8FBX3IyzvC+vUNN86fNq0zu3fXkpGxi1tu6cisWd3Jzt7LHXd0BuDssys55RRh8eLejB1bSSvTUMZEQgLM+TmMzwbfNihYBHlLYX1JQ5ktW+H278C9dzZ9/eEqOG987OINV0ICzHkMxn8JfBVQ8CbkLYH1Abdi3uKD2++Ce0N8KQE88kP413uxiTccCQJzboXxs8G3Gwp+BHlrYP22hjKrtsCYn8Hho3Dn5fDYZMh+BrbthYsfhaM10LUjrHsI8lY76+NOaTpXg8dEswVdA9yjqiOAzwF3icjIKB6Pc+nEZo5STjXVQD77uJpujcocoGE2hi5IfQLOoCPLOQTALvzsw8/ZdIpmuGHJzEyitNRPWZmf6mrIza0iK6vx/QOysjoyf74zxdnChUcYN845Kxg5sgNvvul8Oe3cqezZU8uYMd7o1co8D0o3QdkWnHq9CllB98jf7IO166E2MhNoxETmBVBaBmWb3Xr9DbImNi6zuRzWFoWu1/nnQP9T4J9vxSbecGQOgdKdUPYZVPshtwCyzm1c5u0NTnIGWLERUns7z6v9TnIG6NjBSfaeUdfFEc4jTqKWoFV1m6p+6D7fD6wnaI6uSOtPEhUB/5vbqKE/TU/rv0ov/sVQ7uNUHuJTANZTxXi6kQikksRZdOK0EK+NtZSUBMrLG/6Sfb5aUlISg8okUl7utKj9fti7t5a+fYU1a2rIyupIYiKkpydwwQVJpKV5Y9xnygAoD5hvwrcNUk4L//WdOkLBYng/H7JCnHLHS8ppUL61YdlXEX69ROCJR+D7P4lObMcqpReUVzYs+3Y765oz7VJYvK5hObW30/VRPgtmLfFI6xnaRYKOSXNKRNKB84D/RPU4IdaFOptfwB4WsIdJ9OBu+nEP23iRvQyjI/mks5VqVnIYf9w7OJw/2mDBXRTNlZk37zAjRiRSWNiHzZv9vPdeNTUeuSgSTr1aMmgsbPsUhgyCZS85Le2NmyMX37E6nnpNnwaLXgff1tbLxlLIOjVT9isXwph0uPyXDet8u+GcHDitJ/z9Lli4Enbsj0akbdQOLhJGPUGLSDfgZeA7qrovxPZvAsEz5B6T7VQzMKBKp9GBHS308Oezj5/SH3CmQn2EHfXbXmYQZQRP/Bp7Pl8taWkNJzqpqQlUVPiDyvhJS0tk69ZaEhOhZ88EKiudP6Hvfe9Afbnly3tTUuKNT6RvG6QNbFhOPQ0qtof/+m3OiQ9lW+Dt9+C80d5I0L4KSAs4T0wdGH69LhrrXECcPg26dXUu/h44CPc/HJ1Yw+XbDWkB19tTe0PFnqblxo2AH13nJOejIT5m2/bCxxVwWQa8/GH04g1bO0jQUR3FISJJOMn5eVUNOQuLqj6jqmNamTE3LGuoIp1kUkkiCbieHrzOgUZl0gO6La6kG5vcJNwJobPbBr+ULtRAo4uL8VJQUE1GRiLp6QkkJUF2dify8hpf2cjLO8LUqU5/+eTJHVm2zIm7c2fo4g76uOqqZGpqaHRxMZ4KVkPGEEhPc0abZGdBXpj3R+zV00leAH37wCVjoeiT6MXaFgUfQsZQSB/k1utLzkXCcNz2LRh8Ngw5F+59EJ7LjX9yBijYBBmnQno/SEqE7LHORcJA56bB07fBpN/CzoDWcUpv6OT+yfXqApcMgw2fxiz01p2sXRziTF07F1ivqv8XreME8gMP8inPkUYi8CJ7KeEo36Ufa6niDQ4wld5cQldqUPbi5x6cS9H96MB8UlFgOzV8r9GEvPHj98OMGftZurQ3iYkwb14VRUV+cnK6UlhYQ37+EebOPcyCBc4wu8pKJTvb6eQ79dQEli7tTW0tbN3q56tf9Urnn1uvH8HSv+DUK9dJsjnfh8I1kP9PGHMOvDIXeveC68dDzr0w+gswIgOenuVcZEtIgEfnNB79EU9+P8z4ASxd6NbreSgqhpz7oXAV5C+BMefBKwugd0+4/hrIuQ9GXxzvyJvnr4UZf4Gl34FEgXnLoagCciZB4WbIXwO/nAzdOsFL7oibLbsgaw6MGABPfNnp5hGBx5fCOq904dQCVfEOomWiURpzJSKX4szIshbqh0780J23q7nXaPcZX6HPkz+OSkyxtDn5LKgO/Op9Dxgar3Ai5HH34eo4Efo0+3a2H9v7ge5qWO62BJImNF++PTi6GA5e27Dc8wy4eUPz5duL5dNh/VMAK4/3rFuGjFFyWpxUu8FUOe7jHYuotaBV9d/YDCzGGK9qB33Q3hgUa4wxsWYJ2hhjPMwStDHGeJC1oI0xxqNqgcPxDqJllqCNMScnxRmb62GWoI0xJy/r4jDGGA+yPmhjjPEoS9DGGONR7eCn3pagjTEnL2tBG2OMB1kXhzHGeFTdpLEeZgnaGHNyagfjoKN6w35jjPGsCM9JKCLXiMgGESkVkftCbP+eiBSJyEci8qaIDG5tn55rQR9+7V/sKG/D3EdeVRP81TwT6BqPSCIo6K741Suh8ob4hBJJwTOxHX4IjjwVl1Aipjbob+jgVnj9BHivdq1pvUy4lIj91FtEEoE5wHjABxSISJ6qFgUUWwWMUdVDIvI/wGPALS3t13MJuqbMR02ZL95hRMG78Q4g8mp3wJFX4x1F5PlXeP7Ut81qDsLmE/C9Oh6R7eLIBEpVdSOAiOQCWUB9glbVtwLKrwBua22nnkvQxhgTE20bxdFPRAKnX3lGVZ8JWE4BygOWfcCFLexvGrC4tYNagjbGnJzalqA/a2XKq1CzR4WcT1BEbgPGAJe3dlDPJejzT4Orh8U7iuP32LsNEzE6JgDd4hNMxBQB6wOWT6HlRkJ7sZTG460uxKlbe7YD+CBguTfwrTjFEklLgNWR2VVkh9n5gLSA5VRoOvO0iFwF/Ai4XFWPtLZTzyXoiwfBL8bHO4rj98RyZ9bpBjcCA+MUTaS8QOMEPRDIjlMskfQ2jf9SLwfOjk8oEbOaxgn6FOAXcYolkvYSsQQNkeyDLgAyRGQIsBXnD+PWwAIich7wNHCNqu4IZ6eeS9DGGBMTEbwXh6rWiMgMnNOxRGCeqn4sIg8DhaqaB/wS5zT6JREB2KKqk1raryVoY8zJKcK/JFTVRcCioHUPBjy/qq37tARtjDk5tYNfElqCNsacvOxmScYY40F2NztjjPEou2G/McZ4lLWgjTHGwyxBG2OMB9kN+40xxqNsmJ0xxniU9UEbY4xH1RKxG/ZHiyVoY8zJy7o4jDHGo0Lesdk7TrhJY5eUwPBfw7BfwaPvNN3+3cVw7u+cxxm/hl4/d9a/tbFh/bm/g04Pw9/XN319vEyYcArFxV+gpORKZs5sesPs5OQEcnPPp6TkSlasuJTBgzs32p6W1pn9+ydyzz1DYxVyqyZMOI3i4uspKZnEzJkjm2x36nQpJSWTWLFiAoMHO3M6jh3bl1WrJrJq1URWr76WG25IjXXoLToR6zVhAhQXQ0kJzJzZdPtll8HKlVBdDTfd1LD+nHPgvfdg3TpYswa+/OXYxXwiiFoLWkTmAV8Edqjq6GgdJ5C/Fu76B7w+FVJ7wNinYdKZMPLUhjKzJzY8f3IFrNrmPP/CUFg93XleeQiG/RquPj0WUbcuIQHmzDmL8eNX4PMdpqDgMvLytrN+/YH6MtOmpbF7dzUZGcu45ZaBzJo1guzsD+u3z549isWLw7oFbUwkJAhz5oxl/Phl+HyHKCi4hrw8H+vXN0zgOm3a6ezefZSMjDxuuWUws2adR3b2v1m3bg9jxizB71cGDOjEmjXXkZ+/Fb8//s2hE7FezucPxo8Hnw8KCiAvD9YHNGC2bIHbb4d772382kOH4Gtfg9JSOO00J4kvXQp798a0Cu1WNFvQzwLXRHH/TXzgg2F9YGgfSO4A2WfBq8XNl39hLUw5q+n6hUUwMQO6JEcv1rbIzOxNaelBysoOUV2t5OZWkJU1oFGZrKwBzJ/vTLa7cOE2xo07pdG2jRsP8vHH+2Mad0syM/tSWrqfsrIDVFfXkpu7maystEZlsrJSmT9/IwALF25h3Lj+ABw+7K9PWp06JaIa/8Rc50SsV2amk2DLypwWcm4uZGU1LrN5M6xdGzxJhdPiLi11nm/bBjt2wCntfbKaGIpaglbVd4DKaO0/lK37Ia1nw3JqD9i6L3TZzXugbDdcGeKMP7eZxB0vKSmdKC9vuNzs81WRktKp2TJ+v7J3bzV9+ybTpUsiM2eeTk7OJzGNuTUpKZ0pLz9Uv+zzHSIlpXNQmS6Ulx8EAuvUEXAS4bp117F27XXceecHcW9l1jkR65WSAuUB06H6fM66tho7FpKT4b//jVxsx6duGEc4j/iIex+0iHxTRAqDZsw9JqEaHBJqKkecJDx5FCQG/Q9s2w9rP4UJHpoXMVQdgusqIQqpKjk5w5k9eyMHD3rrcnXoeIPLNH1dXavygw92MXr0a4wdu4T77x9Fx45x/ygDJ2a9wvn8tWbAAFiwAL7+9ba/NnrqfkoYziM+4v7uq+ozqjqmlRlzw5LaA8oD+rZ8+2Bg99Blm2slv7gObhwBSYnHG03k+HxVpKU1tMJSUztRUVEVVOZwfZnERKFnzyQqK6u58MJePPbYSMrKxvGd7wzlhz/M4K670mMZfkg+3yHS0rrUL6emdqGi4nCIMs4FtIY6HW1Uprh4HwcP1jB6dK/oBx2GE7FePh+kBfTSpKZCRZPpUJvXvTu89hr8+Mfwn/9EPr5jV/dLlXAe8RH3BB1JY1OgpNLpujha4yThSWc2LbfhM9hdBRelNd3WXL90PBUU7CEjoyvp6Z1JShKysweSl7e9UZm8vE+ZOtW56j958mksW/YZAJ///HsMGfImQ4a8ya9+tZGf/7yEOXM2xboKTRQU7CIjozvp6V1JSkogO3sweXm+RmXy8rYydarTBzV58iCWLfsUgPT0riQmOs26QYO6Mnx4DzZtOhjbCjTjRKxXQQFkZEB6OiQlQXa2c5EwHElJ8Mor8NxzsHBhVMM8Bt5vQZ9Q46A7JMJvr4MJzzkjOr5xPow6FR58E8akNCTrFz6C7NFNT9027XZa4Jenxzz0Fvn9yowZ61i69HMkJgrz5pVTVHSAnJzhFBbuIT//U+bO3cKCBedRUnIllZVHG43g8CKnToUsXXqlW6f/UlS0l5ycsyks3EV+/lbmzi1lwYKLKSmZRGXlEbKzlwNw6aWnct99I6murqW2FqZPL2DXrlZnsI+JE7Fefj/MmOGMvkhMhHnzoKgIcnKgsBDy82HMGCcR9+4N11/vbBs92hlW9/nPQ9++zigPcP5dsyaeNarj/bslSbSuFIvIC8AVQD/gU+Anqjq3ldfojAvhyeuiElJMJT8E1Y2uaP8eGBifYCLmBfdR5xzg+3GKJZLuBA4ELP8AODtOsUTKauDxgOUzgA1xiiWSpgNPAaw83m5RkbMU/hZm6TOO+/GuKPoAAAeRSURBVHjHImotaFWdEq19G2PM8fP+3ZJOqC4OY4wJn/e7OCxBG2NOUtaCNsYYj7IWtDHGeJS1oI0xxqO8f8d+S9DGmJOUdXEYY4yHWReHMcZ4kLWgjTHGoyxBG2OMR9koDmOM8SgbxWGMMR5lXRzGGONR1sXRZjsOwso2zNbgVU1v4roZONS0YLsSPMXkIaAsHoFEWPB0YNuBZqbiaTc+DVquAlbGI5AIi+TM9JFtQYvINcCvgUTgj6r6aND2jsBzwAXALuAWVd3U4j69MnMwOPeDjncMxph2IQL3g05V+HaYpWe2eDwRSQQ+AcYDPqAAmKKqRQFlpgNnq/7/9u4/5M6yDOD495s/2tymC1223GgybSMDN2dWjsRMhpa6Fkazn6IQ9suJaBgJmf2RIEVECK0mVJaKPxYjYdP8ucWWztfN7WVGlkovCStJc9OQ5OqP537X8bj3fXfec9xz3ue9PnDY8zy7z/1c92Fcu899nud64jJ1JbAiIj4z2lkb9cirlFI6cD19qvdpwNMR8deIeA24DVje1mY58IuyfSfwMff3lOEW/bbE8U+qtYC32jHlXE3TxHE1cUyQ4+rWe7rv4vkNcN0xB9h4irq1ZX91RKxu2T8O+FvL/hDwwbY+9rWJiP+qLwFHM8rn1VcJOiJmHYzzqFvreHzNW62J42rimCDH1Q8i4pwedre/mXD7ku2BtHmDXOJIKaXuDQFzW/bnAO2XO+xrox4KHMWbf3l/g0zQKaXUvceAE9Xj1cOBlcC6tjbrgC+V7QuBB2KMqzT6aonjIFo9dpMJqYnjauKYIMfVKGVN+evABqrL7G6OiEH1emBrRKwD1gC/Up+mmjmvHKvfvrrMLqWU0v/lEkdKKfWpTNAppdSnJlWCVm9Wd6s7646lV9S56oPqLnVQXVV3TL2gTlEfVbeXcX237ph6RT1EfUL9Xd2x9Ir6rLpD3dZ2vXDqwqRag1bPAPYAv4yI99cdTy+os4HZETGgzqAquPDJ1ltMJ6Jyh9W0iNijHgZsAlZFxJaaQ+uaeiVwKnBkRJxXdzy9oD4LnBoRTbz5pjaTagYdEY8wxnWHE01EPB8RA2X7ZWAX1R1LE1pU9pTdw8prws8m1DnAJ4Cf1x1L6n+TKkE3nToPWAz8sd5IeqMsBWyjKmF2X0Q0YVw/Ar5JVQiiSQK4V31c/XLdwTRFJuiGUKcDdwFXRMS/646nFyLi9YhYRHVX1mnqhF6WUs8DdkdEE+p+tlsaEacA5wJfK8uJqUuZoBugrNHeBfw6Iu6uO55ei4gXgYeAXtZOqMNS4IKyXnsbcJZ6S70h9UZE/L38uRtYS1XdLXUpE/QEV35MWwPsiogf1h1Pr6iz1JlleypwNvBUvVF1JyK+FRFzImIe1V1kD0TE52sOq2vqtPIDNeo0YBnQmCul6jSpErR6K7AZWKAOqZfWHVMPLAW+QDUb21ZeH687qB6YDTyoPklV5+C+iGjMZWkNcyywSd0OPArcExHra46pESbVZXYppTSRTKoZdEopTSSZoFNKqU9lgk4ppT6VCTqllPpUJuiUUupTmaDTPurr5TK9neod6hFd9HXmcLU29QL1mlHazlS/Oo5zXKdedaDHR+lnz9itxt9/SuOVCTq1ejUiFpVKf68Bl7X+pZWO/81ExLqIuGGUJjOBjhN0Sk2XCTqNZCNwgjqv1Jq+CRgA5qrL1M3qQJlpTwdQz1GfUjcBnxruSL1Y/UnZPlZdW+o8b1dPB24A5pfZ+42l3dXqY+qTrbWg1W+rf1J/DyzoZEDqb0sxn8H2gj7qD8p47ldnlWPz1fXlPRvVheP4HFMat0zQ6U3KI+HPBXaUQwuoamgvBvYC1wJnl+I4W4Er1SnAz4DzgY8A7xqh+x8DD0fEycApwCBwDfCXMnu/Wl0GnEhVz2ERsEQ9Q11CdYv0Yqr/AD7Q4dAuiYglVLWYL1ePLsenAQNlPA8D3ynHVwPfKO+5Cripw/Ol1JXJ+lTvtH9TS3lPqGbQa4B3A8+1FMr/EPA+4A9VGRAOp7p9fiHwTET8GaAUAdpf2cmzgC9CVa0OeEl9R1ubZeX1RNmfTpWwZwBrI+KVco72x9qP5XJ1RdmeW/p8gar05+3l+C3A3eVbwenAHWWcAG/v8HwpdSUTdGr1ainvuU9JTntbD1HVxbiord0ieldQX+D7EfHTtnNcMd5zqGdSFVz6cES8oj4ETBmheVB9u3yx/fNI6WDKJY7UqS3AUvUEAPUI9b1UleaOV+eXdheN8P77ga+U9x6iHgm8TDU7HrYBuKRlbfs49Z3AI8AKdWqpnnZ+B3EfBfyrJOeFVN8Ehr0NuLBsfxbYVGpqP6N+usSgenIH50upa5mgU0ci4h/AxcCtpdLcFmBhRPyHaknjnvIj4XMjdLEK+Ki6g+r5iSdFxAtUSyY71Rsj4l7gN8Dm0u5OYEZ5tNftwDaq+tcbRwn12lKxcEgdAtYDh5aYv1fiHrYXOEl9nGoJ5vpy/HPApaVK2yCw/EA/p5R6IavZpZRSn8oZdEop9alM0Cml1KcyQaeUUp/KBJ1SSn0qE3RKKfWpTNAppdSnMkGnlFKf+h/lacAHhRwdrgAAAABJRU5ErkJggg==\n", 675 | "text/plain": [ 676 | "
" 677 | ] 678 | }, 679 | "metadata": {}, 680 | "output_type": "display_data" 681 | } 682 | ], 683 | "source": [ 684 | "# Get score for test set\n", 685 | "from sklearn import metrics\n", 686 | "from sklearn.metrics import confusion_matrix, classification_report, precision_score, recall_score, roc_auc_score\n", 687 | "multinomial_confusion_matrix = confusion_matrix(target_test, nbf_predict_test)\n", 688 | "print(make_confusion_matrix_relative(multinomial_confusion_matrix))\n", 689 | "plot_confusion_matrix(multinomial_confusion_matrix, 'Multinomial Naive Bayes Confusion Matrix', savefilename='MultinomialCM.png')" 690 | ] 691 | }, 692 | { 693 | "cell_type": "markdown", 694 | "metadata": {}, 695 | "source": [ 696 | "#### Logistic Regression Classifier" 697 | ] 698 | }, 699 | { 700 | "cell_type": "code", 701 | "execution_count": 43, 702 | "metadata": {}, 703 | "outputs": [ 704 | { 705 | "data": { 706 | "text/plain": [ 707 | "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n", 708 | " intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n", 709 | " penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n", 710 | " verbose=0, warm_start=False)" 711 | ] 712 | }, 713 | "execution_count": 43, 714 | "metadata": {}, 715 | "output_type": "execute_result" 716 | } 717 | ], 718 | "source": [ 719 | "# Build a Logistic Regression Classifier\n", 720 | "from sklearn.linear_model import LogisticRegression\n", 721 | "vectors_train = vectors_train.toarray()\n", 722 | "\n", 723 | "lg = LogisticRegression()\n", 724 | "lg.fit(vectors_train,target_train)" 725 | ] 726 | }, 727 | { 728 | "cell_type": "code", 729 | "execution_count": 44, 730 | "metadata": {}, 731 | "outputs": [ 732 | { 733 | "data": { 734 | "text/plain": [ 735 | "0.6988341588929957" 736 | ] 737 | }, 738 | "execution_count": 44, 739 | "metadata": {}, 740 | "output_type": "execute_result" 741 | } 742 | ], 743 | "source": [ 744 | "# Get score for training set\n", 745 | "lg.score(vectors_train,target_train)" 746 | ] 747 | }, 748 | { 749 | "cell_type": "code", 750 | "execution_count": 45, 751 | "metadata": { 752 | "scrolled": true 753 | }, 754 | "outputs": [ 755 | { 756 | "data": { 757 | "text/plain": [ 758 | "0.6602174689876972" 759 | ] 760 | }, 761 | "execution_count": 45, 762 | "metadata": {}, 763 | "output_type": "execute_result" 764 | } 765 | ], 766 | "source": [ 767 | "# Get score for test set\n", 768 | "lg.score(vectors_test,target_test)" 769 | ] 770 | }, 771 | { 772 | "cell_type": "markdown", 773 | "metadata": {}, 774 | "source": [ 775 | "#### Q: What are the key features(words) that make the positive prediction?" 776 | ] 777 | }, 778 | { 779 | "cell_type": "code", 780 | "execution_count": 46, 781 | "metadata": {}, 782 | "outputs": [ 783 | { 784 | "data": { 785 | "text/plain": [ 786 | "['worst',\n", 787 | " 'horrible',\n", 788 | " 'terrible',\n", 789 | " 'poisoning',\n", 790 | " 'zero',\n", 791 | " 'disgusting',\n", 792 | " 'awful',\n", 793 | " 'waste',\n", 794 | " 'joke',\n", 795 | " 'sick',\n", 796 | " 'rude',\n", 797 | " 'worse',\n", 798 | " 'sucks',\n", 799 | " 'money',\n", 800 | " 'nasty',\n", 801 | " 'garbage',\n", 802 | " 'poor',\n", 803 | " 'disappointment',\n", 804 | " 'rip',\n", 805 | " 'shitty']" 806 | ] 807 | }, 808 | "execution_count": 46, 809 | "metadata": {}, 810 | "output_type": "execute_result" 811 | } 812 | ], 813 | "source": [ 814 | "# Let's find it out by ranking\n", 815 | "n = 20\n", 816 | "get_top_values(lg.coef_[0],n, words)" 817 | ] 818 | }, 819 | { 820 | "cell_type": "markdown", 821 | "metadata": {}, 822 | "source": [ 823 | "#### Q: What are the key features(words) that make the negative prediction?" 824 | ] 825 | }, 826 | { 827 | "cell_type": "code", 828 | "execution_count": 47, 829 | "metadata": {}, 830 | "outputs": [ 831 | { 832 | "data": { 833 | "text/plain": [ 834 | "['delicious',\n", 835 | " 'amazing',\n", 836 | " 'great',\n", 837 | " 'good',\n", 838 | " 'excellent',\n", 839 | " 'best',\n", 840 | " 'awesome',\n", 841 | " 'friendly',\n", 842 | " 'loved',\n", 843 | " 'love',\n", 844 | " 'perfect',\n", 845 | " 'stars',\n", 846 | " 'favorite',\n", 847 | " 'fantastic',\n", 848 | " 'nice',\n", 849 | " 'tasty',\n", 850 | " 'enjoyed',\n", 851 | " 'yummy',\n", 852 | " 'attentive',\n", 853 | " 'bit']" 854 | ] 855 | }, 856 | "execution_count": 47, 857 | "metadata": {}, 858 | "output_type": "execute_result" 859 | } 860 | ], 861 | "source": [ 862 | "# Let's find it out by ranking\n", 863 | "n = 20\n", 864 | "get_bottom_values(lg.coef_[0], n, words)" 865 | ] 866 | }, 867 | { 868 | "cell_type": "markdown", 869 | "metadata": {}, 870 | "source": [ 871 | "#### Random Forest Classifier" 872 | ] 873 | }, 874 | { 875 | "cell_type": "code", 876 | "execution_count": 48, 877 | "metadata": {}, 878 | "outputs": [ 879 | { 880 | "data": { 881 | "text/plain": [ 882 | "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n", 883 | " max_depth=None, max_features='auto', max_leaf_nodes=None,\n", 884 | " min_impurity_decrease=0.0, min_impurity_split=None,\n", 885 | " min_samples_leaf=20, min_samples_split=2,\n", 886 | " min_weight_fraction_leaf=0.0, n_estimators=5, n_jobs=-1,\n", 887 | " oob_score=False, random_state=None, verbose=0,\n", 888 | " warm_start=False)" 889 | ] 890 | }, 891 | "execution_count": 48, 892 | "metadata": {}, 893 | "output_type": "execute_result" 894 | } 895 | ], 896 | "source": [ 897 | "# Build a Random Forest Classifier\n", 898 | "\n", 899 | "from sklearn.ensemble import RandomForestClassifier\n", 900 | "\n", 901 | "rfc = RandomForestClassifier(n_estimators = 5, min_samples_leaf = 20, n_jobs = -1)\n", 902 | "rfc.fit(vectors_train,target_train)" 903 | ] 904 | }, 905 | { 906 | "cell_type": "code", 907 | "execution_count": 49, 908 | "metadata": {}, 909 | "outputs": [ 910 | { 911 | "data": { 912 | "text/plain": [ 913 | "0.5952410885653701" 914 | ] 915 | }, 916 | "execution_count": 49, 917 | "metadata": {}, 918 | "output_type": "execute_result" 919 | } 920 | ], 921 | "source": [ 922 | "# Get score for training set\n", 923 | "rfc.score(vectors_train,target_train)" 924 | ] 925 | }, 926 | { 927 | "cell_type": "code", 928 | "execution_count": 50, 929 | "metadata": {}, 930 | "outputs": [ 931 | { 932 | "name": "stdout", 933 | "output_type": "stream", 934 | "text": [ 935 | "[[6.25635676e-01 8.73647151e-03 2.69917851e-02 4.31607772e-02\n", 936 | " 2.95475290e-01]\n", 937 | " [3.43280593e-01 2.18305628e-02 9.11275786e-02 1.39995994e-01\n", 938 | " 4.03765271e-01]\n", 939 | " [1.29964457e-01 1.35991346e-02 1.01684438e-01 2.37366713e-01\n", 940 | " 5.17385257e-01]\n", 941 | " [3.49905003e-02 2.21659278e-03 2.43825206e-02 1.78989867e-01\n", 942 | " 7.59420519e-01]\n", 943 | " [1.30731057e-02 2.96274350e-04 3.66639508e-03 4.03673802e-02\n", 944 | " 9.42596845e-01]]\n" 945 | ] 946 | }, 947 | { 948 | "data": { 949 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEWCAYAAABLzQ1kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xt4VNW5+PHvmyEJ9wSCokwCQQmYgHeItWq9cAloJbTgMbRWrBxrD2Dbc6pF29oa7a8WW0tbS21t4Yi0mioe20kV4wW8VEtNEBQIgYkEyBBQIAn3kGSyfn/sSTKTDMmEzGUH3s/zzJPZe6+997syM2vWrL32WmKMQSmllP3ExToApZRSwWkBrZRSNqUFtFJK2ZQW0EopZVNaQCullE1pAa2UUjalBbRNiMh1IuKJdRxnChH5iYjsF5G93TjGcBE5IiKOcMYWbSLyfRH5U6zjUO1pAd0BEdkhIsd9H8K9IvK0iPSPdVzdJSJGRI768nVERGqjfP6QvoxEJFtEXhGRWhGpFpEPROTrYTh/GvBdIMsYc86pHscYs8sY098Y4+1uTG35XqNPRaSX37peIvKZiIR080Ko/2djzE+NMf/ZnXhVZGgB3bmbjTH9gUuAS4EHYhxPuFzsK1z6G2OSu7qzf8ERCSJyJbAaeBsYBaQA/wVMC8PhRwAHjDGfheFYkVRLYH5vBGrCeYJIv46qe7SADpExZi9QhFVQAyAiN4nIehE5JCKVIvKQ37Z0Xy1ojojs8v2c/oHf9j6+GnmNiJQCE/zPJyKZIvKWr/a4WUSm+217WkR+JyKrfDXg90TkHBH5le94ZSJy6ankU0TuEpFyX43VJSLD/LYZEZkvIm7A7Vt3gYi87ku/VUT+wy/9jSJSKiKHRWS3iNwrIv2AVcAwvxr8sHaBwM+B5caYRcaY/cayzhjjf/zOYv2miLh9/5MlYpkEvO53/qeD1TR9v54m+Z5ni0iJ73X+VER+6Vvf/Br38i0P88VR7YvrLr/jPSQiz4vIM77/x2YRGd/Jy7ECuN1v+XbgmTZxfl1EtviOuV1E7vatD/p/9sWxUkT+LCKHgDt86/7s2+9W33EG+panifXr8axOYlWRYIzRx0kewA5gku95KrAR+LXf9uuAC7G+6C4CPgVm+LalAwb4I9AHuBg4AWT6tv8MeBcYDKQBmwCPb1s8UA58H0gAbgAOA2N8258G9gOXA72xapoVWB9gB/ATYE0H+TLAqCDrb/Ad9zIgEXgCeKfNfq/7Yu4D9AMqga8DvXz77QfG+tLvAa7xPR8EXOb3f/N0EF9fwAtc30GaUGL9B5AMDAf2AVODnT9YPG1e+38BX/M97w98rs1r3Mu3/DbwO99rconvnBN92x4C6rBqwQ7gUWBtJ6/ROKz3VLLv8alvnfFLdxNwPiDAtcCxjv7PvjgagBlY79s+vnV/9kvzF6z3WApQBXwx1p/FM/UR8wDs/PB9SI9gFY4GeBNI7iD9r4DFvufNH95Uv+0fAHm+59ubCwzf8jdoLaCvAfYCcX7bnwMe8j1/Gvij37Z7gC1+yxcCtR3EaYBDWD+ha4Hf+NYvBR7zS9ff92FO99vvBr/ttwLvtjn2H4Af+57vAu4GBrZJ067gaLPd6TvXBR2kCSXWq/22Pw/cH+z8JynIdtBaQL8D5AND2qRpfo17YX3JeoEBftsfBZ72PX8IeMNvWxZwvJPXaBTwJ9//8JtYX/aj8Cugg+z3N+DbHeTrIfy+yPzW+RfQyb7XbiPwh1h/Ds/khzZxdG6GMWYA1pv9AmBI8wYRuUJE1ojIPhE5iPUhGtJmf/9eAsewChKAYVi1z2Y7/Z4PAyqNMU1ttjv9lj/1e348yHJnFzMvM8Yk+x7f8jtvSxzGmCPAgTbn9Y95BHCFrxmmVqyLjV8Fmi+8zcSqMe4Ukbd97cqhqAGagHM7SBNKrCf733fVXGA0UCYixSLyxZPEU22MOey3ru1r1jae3iG0AT+D9cuoXfMGtDRBrPU1q9Ri/b/bvgfbquxoozGmFngBq7b+eCfHUhGkBXSIjDFvY9Vcf+G3+lnABaQZY5KA32P91AzFHqxaV7Phfs+rgDQRiWuzfXcXw+6qKqxCF2hpx0xpc17/HgSVwNt+BX2ysS46/heAMabYGJMLnI1Vs3s+yDHaMcYcw2pWmNnNWEN1FKtZpflYDqClzdUY4zbGzMbKxyJgpe98beMZLCID/NaF4zV7F+uLaijwT/8NIpIIvIj1nhxqrIu9r9D6HjzZ/7nD/7+IXALcifWr7TenHLnqNi2gu+ZXwGTfGxhgAFatqU5EsoGvdOFYzwMPiMggEUnFaqZo9m+sQuN7IhIvItcBNwMF3c5Bx54Fvi4il/g+/D8F/m2M2XGS9P8ARovI13xxxovIBLEucCaIyFdFJMkY04DVpNLcHe1TIEVEkjqI5XtYF7DuE5EUABG5WESa/wddjbUj27BqszeJSDzwQ6x2bXznvU1EzvL9omnukhjQtc4YUwm8DzwqIr1F5CKsmvdfTiEe/+MarNd+uu+5vwRfnPuARhGZBkzx2x7K/zmAiPQG/ox1/ePrgFNE5nUjC6obtIDuAmPMPqyfmQ/6Vs0DHhaRw8CPaK0hhiIf6ydwBfAa1hX75vPUA9Oxuljtx7rwdLsxpqy7eeiIMeZNrLy9iFXDPx/I6yD9YawCIQ+rBrkXq4bZXLh9Ddjh6y3wTeA2335lWLWz7b6mkXa9OIwx72NdCLzBl64aeAqrhtjlWDvJ90Gs1/JPWDXeo4B/r46pwGYROQL8Gus6Ql2QQ83GapeuAl7Caot//VRiahPfZmPM5iDrDwPfwnrf1WBVEFx+2zv9PwfxKFa79ZPGmBNYr9lPRCSju/lQXSftv5SVUkrZgdaglVLKprSAVkqpbhKRZWLdhr/pJNtFRH7ju4HpYxG5LJTjagGtlFLd9zTWtYqTmQZk+B7fAJ4M5aBaQCulVDcZY94BqjtIkgs8YyxrgWQR6aifP2DdAWUbEuIoXUqpM95+Y0y3xgcZJWKOhZh2D2zGulW/2VPGmKe6cDongTcIeXzr9nS0k60KaKWUCtHOzpN07BjWPfSheAjqjDGdDW7VkWA3sHVaIbVhAZ1Bm4HdeqjnCPz/T+bU7zS2iy2Af1fsocDnYxRLOL0M1Pst5+J3k2IPtQO/LtEMwhqrtadbBawP07GEqBaAHgLvHE7F6i/fIRsW0GdjDdLW0xUQWEBfgHUnck9WTWABPQBrkL6e7tU2yxdiDf3dkw3Av4Duj1VF6Ok2Eb4CunkovyhxAQt8d8JeARw0xnTYvAG2LKCVUiryBGtc37AcS+Q5rAHVhog1tviPmw9vjPk91h2wN2INI3wM6zb6TmkBrZQ6I4WzicM3mFZH2w0wv6vH1QJaKXVGCmcNOlK0gFZKnZGifJHwlNg9PqWUigitQSullE1FuRfHKdECWil1RtIatFJK2ZjdC0C7x6eUUhGhNWillLIp7cWhlFI2pRcJlVLKprSJQymlbEqbOJRSyqZ6Qg36tJvyKicnnbKyO3C772ThwvbjSickOCgouAm3+07Wrp3NiBEDARg8uDerV9/C4cMLeOKJG6Iddqdycs6nrGw+bvc9LFx4VbvtVr5m4nbfw9q1cxkxIqll2/33X43bfQ9lZfOZMuX8aIbdodMxTwA5OUMpK5uM2z2FhQtHt9uekBBHQUE2bvcU1q69jhEj+gZsT0vrw+HD0/nudzOiFXKnBufkkF1WxhVuN8MXLmy3PXH4cC5+4w3Gf/QRl6xZQ6LTGbDdMWAAV3o8ZDzxRLRC7lRzDTqUR6xEtIAWkR0islFENohISSTPBRAXJyxZcgPTpr1EVtbTzJ59AZmZgwPSzJ07jpqaOjIylrF48YcsWnQNAHV1jTz44Hvce+87kQ6zy6x83ci0aX8hK2sJs2ePIzNzSECauXMv9eXrCRYvXsuiRZMAyMwcQl7eWMaO/R1Tp/6F3/3uRuLigk3uEF2nY54A4uJgyZKLmTbtPbKyXmf27FQyMwcEpJk7N52amnoyMl5j8eJyFi0aF7B98eKLWLVqbzTD7lhcHBlLlvDxtGl8kJXF2bNn0zczMyDJ+b/4BXufeYaSiy9mx8MPM/LRRwO2j3zkEWrffjuaUXequQYdyiNWolGDvt4Yc0k3p4sJSXb2OZSX11JRcZCGhiYKCsrIzQ2sXeXmns/y5aUArFy5jYkThwNw7Fgj771XRV1dY6TD7LLsbCfl5dVUVNT68rWZ3NwLAtLk5o5h+fKPAFi5spSJE8/zrb+AgoLN1Nd72bGjlvLyarKzne3OEW2nY54AsrMHU15+lIqKYzQ0GAoKPOTmBs4Nmpt7LsuX7wJg5crdTJx4VsC27duPsnnz4ajG3ZGB2dkcLy+nrqIC09DAZwUFDMnNDUjTLyuL2jffBKB2zZqA7f0vu4yEoUOpee21qMbdGcHqxRHKI1ZOqyYOp7M/lZWtb2yP5whO54CTpvF6DQcPniAlpXdU4+wqp3MAlZWHWpY9nkNB8jWQysqDQHO+6khJ6ePb96Dfvofb7RsLp2OeAJzO3lRWHm9Z9niO43T2OWkaK18NpKQk0Levg4ULR5OfvyWqMXcm0enkRGXrfKcnPJ52TRhHPvqIITNnAjDkS1+i18CB9Bo8GEQY9fjjfHLffVGNORQCxPcK7RErkS6gDfCaiKwTkW8ESyAi3xCRknA0gUiQX7nWONmdpenumSNLggTdNuaT5SuUfWPhdMwThJqv4Gny8zNZvLico0e9kQrv1ASNNzBTn9x7L8nXXsvlH35I8rXXcsLjwTQ24pw3jwOvvMIJjyda0YZMBHr1Cu0RK5E+9VXGmCoRORt4XUTKjDEBjby+qcufAhCRbn3MPJ4jpKW11qRSU/tTVXUkaJrdu4/gcAhJSYlUV9e1PZSteDyHSEsb2LKcmjqQqqrDQdIksXv3YV++elNdfbxlfeu+A9rtGwunY57AqjGnpbXWmFNT+1BVdTxomt27j/vyFU91dT1XXDGYWbOcPPbYOJKT42lqgro6L0uWbI92NgKc8HhITGud7zQxNZX6qsD5Tuv37GGzrwbt6NePs2bOxHvoEAOvvJKka67BOW8ejv79kYQEvEeOsP2BB6Kah2BEIN4R6yg6FtEatDGmyvf3M+AlIDuS5ysu3ktGRjLp6QOJj48jL+8CXK7AN7fL9Qlz5mQBMGvWaFav3hXJkMKiuHg3GRkppKcn+/I1Fpdra0Aal2sbc+ZYE7jOmpXF6tUVvvVbycsbS0KCg/T0ZDIyUvjgg91Rz0Nbp2OeAIqLa8jI6E96el/i44W8vFRcrsC5QV2uPcyZY137mDXLyerV+wD4whfeYeTIIkaOLOJXv/qEn/50a8wLZ4DDxcX0ycigd3o6Eh/P2Xl57He5AtLEp6S01LSHP/AAe5YtA2DLbbexdsQI1o4cySf33sunzzxji8IZzvAatIj0A+KMMYd9z6cAD0fqfGC15y1YsIaiopk4HMKyZZsoLT1Afv7nKSnZS2HhdpYu3cSKFdNwu++kurqOvLyXW/avqJjLwIGJJCTEMWPG+UyZ8iJbtlRHMuSQWPl6haKi23z52kBp6T7y86+jpKSKwsJtLF36IStWfAm3+x6qq4+Tl7cSgNLSfTz/fCmlpfNobGxi/vxXaGqKfXvA6ZgnaM7XBoqKrvLlayelpYfJz8+kpKSWwsI9LF26gxUrxuN2T6G6up68vA9iHXaHjNeLe8ECLioqQhwO9ixbxrHSUtLz8zlcUsKBwkKSr7vO6rlhDAffeYdt87s8/V7UiUB8Yqyj6Ji0bUsK24FFzsOqNYP1RfCsMeb/dbKPgauAL0ckpui6D2jyW74HSIlRLOGyBvDvKjUKuC1GsYTTIsC/GeKHwKUxiiVcPgRaP25pwDMxiyV8FgO+uvu67vYMG58gpuSsztMBSFX3z3cqIlaDNsZsBy6O1PGVUqpbesC93jYPTymlIsjmJaDNw1NKqQgRwOa9OLSAVkqdmbSJQymlbEoAm/fi0AJaKXVm0hq0UkrZlBbQSillY3qRUCmlbEhr0EopZVNaQCullE1pLw6llLIprUErpZRNaQGtlFI2pbd6n4rjwGexDiICarBmAOvJjrVZrgf2xyKQMGv7uhwA7DEBwKk7ELDUANh/aorOHek8SejCXIMWkanAr7GK/T8ZY37WZvtwYDmQ7EtzvzHmlQ6PGanxoE9Fd6e8UkqdMbo/HvQQMSU3h5ZWnu74fCLiALYBkwEPUAzMNsaU+qV5ClhvjHlSRLKAV4wx6R2d97Sa1VsppULWXIMO5dG5bKDcGLPdGFMPFAC5bdIYoHkiziSgik7YsIlDKaWioGtNHENEpMRv+SnfhNfNnECl37IHuKLNMR4CXhORe4B+wKTOTmrDAlo4PSr23jbLp0OeDO3ba21+lSUkbV+rXi0ToPZYxgCNAavsPoN1KLxNENbpJ0MvAfd30qQS7A3TNtLZwNPGmMdF5EpghYiMM8Y0Bdm3i+FFzSRgTqyDCIM5BH7w53P6zUk4Dvh2jGIJp+8AR1sXM/4BSTkxiyYsDq4C940ti6PPgq33xzCeMJn3Ijz5fpgOFt5eHB6sqR+bpdK+CWMuMBXAGPMvEekNDKGDXhGnQ7VOKaW6Lrxt0MVAhoiMFJEEII+W+W1b7AImAohIJtAb2NfRQW1Yg1ZKqSgI463exphGEVkAFGHVy5cZYzaLyMNAiTHGBXwX+KOI/DdW88cdppNudFpAK6XOTGHuB+3r0/xKm3U/8nteClzVlWNqAa2UOjPprd5KKWVTWkArpZSN2bzroRbQSqkzk9aglVLKpnTAfqWUsimtQSullE1pAa2UUjalA/YrpZRNaQ1aKaVsSrBGw7AxLaCVUmemHtDEcdqNZpeTcy5lZTfjdk9n4cKsdtsTEuIoKLgat3s6a9fmMGJEPwAmTEhh/fpprF8/jQ0bbmTGjNRoh96hnJzzKSubj9t9DwsXtr+dPyHBQUHBTNzue1i7di4jRiS1bLv//qtxu++hrGw+U6acH82wO5STk0pZ2Szc7ltYuPCidtut1+oG3O5bWLt2OiNG9Adg0iQnJSUz+PjjL1NSMoPrrz832qF3KOdKKFsJ7v+DhUFGzr3mUli3Ahr+BTNvCNy26B7Y9FcofR5+/d3oxBuKV8tgzM9g1E/hZ2+23/7ff4dLHrceox+F5B+0bttVA1P+AJmLIOsx2FEdvbg7FN7R7CIi4qf2zdVVAuw2xnwxkueKixOWLJnA5Mmr8XiOUVw8FZfLw5Yth1rSzJ17PjU19WRkuLj11hEsWnQpeXn/ZNOmWsaPfxWv13DOOb356KObKCzcjdcb+2kSrXzdyOTJK/B4DlFcfBcu11a2bGmdsHXu3EupqakjI+MJbr11LIsWTSIv70UyM4eQlzeWsWN/x7BhA3jjja8xevRvaQrrqOddZ+Xp80yevAqP5yjFxbm4XLvYsqW2Jc3cuWOoqTlBRsYL3HrreSxalE1e3mr276/j5ptfY8+eY4wdO4iioqmkpj4Xw9y0iouDJd+DyQvA8ykULwfXO7ClojXNrr1wRz7ce1vgvldeBFddDBfNtpb/+Ue49jJ4+8PoxR+Mtwnm/x+8fjekJsGEX8H0sZB1TmuaxX6TOz3xLqz3m3P39ufgBxNh8hg4cgLi7DQXgs3bEKJRg/42sCUK5yE7O4Xy8sNUVByhoaGJgoKd5OamBaTJzU1l+fLtAKxcuYuJE4cCcPy4t6Uw7t3bgZ0m083OdlJeXk1FRa0vX5vJzb0gIE1u7hiWL/8IgJUrS5k48Tzf+gsoKNhMfb2XHTtqKS+vJjvbGfU8tJWdfRbl5YeoqDjsy9N2cnNHBKTJzR3B8uVuAFaurGDixGEAbNhwgD17rBnGN2+uoXdvBwkJ9vgxmD0WyiuhYjc0NELB65B7bWCanXtgY3n7mUGMgd4JkBAPifEQ3ws+tUFt84NdMCoFzkuBhF6Qdyn8ffPJ0z+3HmZfaj0v3QuNXqtwBuifCH0TIh9zSJqbOEJ5xEhE39UikgrcBPwpkudp5nT2obLyWMuyx3MMp7NPmzR9qay0Zs/weg0HDzaQkmLdTpSdncKmTTexceNNfPObH9ii9gzgdA6gsrL1V4DHcwinc0CbNAOprDwINOerjpSUPr59D/rte7jdvrHg/zoAeDxHcTr7BklzBGjOU33La9Vs5sx01q8/QH39SWcNiirnWVD5aeuy51NrXSjWboQ162DPKtjzKhSthbIdEQmzS3YfhLTk1uXUJGtdMDuroaIabsiwlrftg+Q+8OWn4dLH4b5Cq0ZuCz2giSPS1Y5fAd8DovKSSJB55NpWhINNNddcW/7ggwOMG/cyEya8ygMPjCUx0R61slPPV2j7xkJoeeo4TVZWMosWZXP33f8Md3in7GSvQyjOT4XMdEi9CZw3wg3jrfbqWAsW/slaKQo2wKyLwOH76DQ2wbsV8Iubofg7sP0APF0cqUi7qPlW71AeMRKxEkhEvgh8ZoxZ10m6b4hISZsZc0+Jx3OMtLTWWlhqal+qqo4HSWNdGHQ4hKSkeKqr6wPSlJUd4ujRRsaNS8YOPJ5DpKUNbFlOTR1IVdXhIGmsC4NWvnpTXX08YL2174B2+8aCx3O05XUASE3tR1XVsSBprAuDVp4SqK4+AVi165demsztt7/N9u2xz08zz2eQNrR1OXUoVO0/eXp/X7oO1m6Co8etx6p/wefGRSTMLklNgsrWSwN4DsKwpOBpC/yaNwBSk+FSp9U80ssBM8bBh57IxhuyM7wGfRUwXUR2AAXADSLy57aJjDFPGWPGdzJjbkiKiw+QkTGA9PR+xMfHkZc3Apcr8N3gcu1mzhyrfXbWrOGsXm39Hk1P74fDYdULhg/vx5gxA9mx4yh2UFy8m4yMFNLTk335GovLtTUgjcu1jTlzLgZg1qwsVq+u8K3fSl7eWBISHKSnJ5ORkcIHH+xud45oKy7eR0bGQNLT+/vydB4u186ANC7XTubMsX4rz5o1ktWrrTk4k5ISePnlHB54oJj33/+03bFjqbgUMoZD+jCrDTlvsnWRMBS7PrUuCjocVmF27WWwZUdEww3JhDRw74eKA1DfaBXC08e2T7f1M6g5DlemB+5bcwz2WS1VrC6HrKHt942JHlBAR+zUxpgHgAcAROQ64F5jzG0d7tRNXq9hwYISiopuwOEQli37hNLSg+TnX0RJyQEKC3ezdGk5K1Z8Hrd7OtXVJ8jLew+Aq68+m/vvz6KhoYmmJpg3r5gDB05EMtyQWfl6haKi23z52kBp6T7y86+jpKSKwsJtLF36IStWfAm3+x6qq4+Tl7cSgNLSfTz/fCmlpfNobGxi/vxXYt6DA5rz9D5FRdN8edpGaWkt+fmXUVKyn8LCXSxduo0VK67F7b7F91qtAWDBgixGjRrIgw9eyoMPWtW1KVNWsW9fXSyzBIDXCwseg6LfWAXtMheUbof8u6FkCxS+A+Oz4KXHYNBAuPlqa9u4W2Hlm1azxsbnrGaRV/8F/3g31jmyvix++2XIeQq8Bu7MhrHnwI9ehfGpMN1Xy39uPeRdEtjM44izmjcm/t7K0+WpcNfnYpOPdnrAnYQSjd4KfgV0h93sRMTAZCBI59EeZw7g9Vu+B0iJUSzhsgZ42295HFYnnZ7uO4Dfr6WMVyEpJ2bRhMXBVeC+sWVx9Fmw9f4YxhMm816EJ98HYF13f3WPHyum5K+hpZULu3++UxGV7w9jzFvAW9E4l1JKhaQH1KBtHp5SSkWIDtivlFI2pTVopZSyKS2glVLKprSAVkop+zI2H25UC2il1BnJxEG9DtivlFL2YwQaHaHeTB2bEZ60gFZKnZGMCN5eoRaB9Z0niQAtoJVSZyyvw96N0FpAK6XOSAbBa/NJCU9aQItIIcGHggXAGDM9IhEppVQUGITGnlpAA7+IWhRKKRVlBqHe5vd6n7SANsa0DFsmIn2A4caYrSdLr5RSPUlPaOLotI+JiNwMbABe9S1fIiKuSAemlFKR5sUR0iMUIjJVRLaKSLmIBB3cVUT+Q0RKRWSziDzb2TFDuUj4EJCNb7hQY8wGEUkPKeJT8hHwm8gdPmra9ptcBdhlOuNTta/N8i7g97EIJMzaDPS/5//B/qWxCSVc6vcELFYdgv94JkaxhNG6ME6XFc42aBFxAEuwBrT3AMUi4jLGlPqlycCaxOQqY0yNiJzd2XFDKaAbjTEHg03gGRmf+R6nm/JYBxABh4AOp5zsmY7YYBqTMDtyAl74KNZR2IvVxBG2jmzZQLkxZjuAiBQAuUCpX5q7gCXGmBoAY0ynBV0o0W0Ska8ADt83wLeA97sYvFJK2Yp1kTDkX7VD2kxs/ZQx5im/ZSdQ6bfsAa5oc4zRACLyHuAAHjLGvNrRSUMpoO8BfgCcAJ4DioBHQthPKaVsy0BXmjj2dzLlVbAmhrbdlHsBGcB1QCrwroiMM8bUtt3Rf4cOGWOOAT8QkUXWoonwHPcXg1wf2VNEg/k1ga/PeKBPjIIJl51Y7c7NhgOnQ3f4P2LVP3zOnwED02MVTHgc2gGf/K1lsXdSAuO/nhm7eMKk/E0PezceCNPRwtrE4QHS/JZTgaogadYaYxqAChHZilVgF5/soJ1GJyITgGXAAN/yQeBOY0xkGh/lGnAsjsiho6pxCdDgt2IScFaMggmX1wgsoC8AHotRLOH0LAEF9EXfhPQePmlsxaqAArr/0L5MX/yFGAYUHi/NeytsBXSYu9kVAxkiMhLYDeQBX2mT5m/AbOBpERmC1eSxvaODhvL1sRSYZ4x5F0BErgb+F7ioS+ErpZTNhKuANsY0isgCrCZgB7DMGLNZRB4GSowxLt+2KSJSCniB+4wxHX7bhFJAH24unH2B/FNEItzMoZRSkRXuG1WMMa8Ar7RZ9yO/5wb4H98jJB2NxXGZ7+kHIvIHrAuEBrgVX59opZTqqQzCiZ56qzfweJvlH/s9P+kgSkop1RP0hFu9OxqL4zToSqGUUsH16ALan4jcBIwFWmbwMsY8HKmglFIqGnrycKMAiMjvgb7A9cCfgFnABxGOSymlIirMt3pHRCgzJn7eGHM7UGOMyQeuJLBQAQbCAAAgAElEQVRDtlJK9TjNTRzhGs0uEkL5+jju+3tMRIYBB4CRkQtJKaUiz+rFYe8RJkMpoP8hIsnAz4EPsXpw/DGiUSmlVIT1hCaOUMbiaB4Y6UUR+QfWhcILIhqVUkpFwWnRi6OZMeYEcEJEXsAaKUcppXqk06abXRDRGr1fKaUi4nQuoPVOQqVUj9YTbvU+aTc7ESkUEVeQRyGQEsUYuyRnCpRtAvcWWHhf++3XXA3rPoCG4zDzy63rhw+Hkn/D+hLYtAHu/kb0Yu5MTk46ZWV34HbfycKFE9ptT0hwUFBwE273naxdO5sRIwYCMHhwb1avvoXDhxfwxBM3RDvsTuXkxFFWlojbncjChe3rCtdcE8e6dYk0NPRm5szAt+rttzvYti2RbdsSuf12e9WCci6AsvvB/X1YGOTf/t/XwubvwUf3whvfhOGDArcPSATPj+GJL7ffN1a2vrqTx8asYNGoZ1jzs5J220ue3kL+WX9k8SXPsfiS5/j3nzYDULVhH7+98gUeH/sXfnnRs2z467Zoh35SPb2b3S9OcRsAItIbeAdI9J1npTHmxx3v1T1xcbDkNzB5Gng8ULwWXP+ALVta0+yqhDvmwr1txpPaswc+fw3U10O/flYh7Sq01sdSXJywZMkNTJ78Ih7PYYqLv4rL9QlbtlS3pJk7dxw1NXVkZCzj1lvHsGjRNeTlvUxdXSMPPvge48YNYdy4ITHMRXtxcbBkSTyTJ9fj8RiKixNxubxs2dL642zXLsMdd9Rz772Bb9NBg+DHP+7F+PEnMAbWrbP2rT3pvBTREyew5Msw+ffgOQjF/w2uzbDl09Y063fD+MVwvAG++Xl47IuQt6J1+yPT4O1Poh/7yTR5m3hp/lvc9foMklL788SEv5I1/TyGZg0OSHfxrRnM+O11Aevi+/bi1mcmc1ZGMgerjvCby//KmJwR9Em2R83V7k0cJ61BG2Pe7ugRwrFPADcYYy4GLgGmisjnwhV4MNnZUP4JVFRAQwMU/BVybw5Ms3MnbNwITW0m3W5osApngMREqwCxg+zscygvr6Wi4iANDU0UFJSRm3t+QJrc3PNZvtyam3Llym1MnGhdvz12rJH33quirq4x6nF3Jjs7jvJyQ0WFsV6rAi+5uYEflp07DRs3mnavVU6Og9dfb6KmBmpr4fXXm5g61R4ftOzhUL4fKqqhwQsF6yF3XGCat8qtwhlg7U5ITW7ddlkqDB0Ar22NXsydqfzgU4aMSiblvCR6JTi4OG80m//e4TjzLc4aPYizMqwMJg3rT/+z+3Bk3/FO9oqO5lm9Q3nESsSKIWM54luM9z0i2nbtHAaVftOye3aD0xn6/qmp8NGHUFkBi34R+9ozgNPZn8rK1uG3PZ4jOJ0DTprG6zUcPHiClJTe2JnTCZWVrW8Hj8fgdIZ27dnplFPeN9KcSVDpV5P31FrrTmbuFbDK9wtPBB6fDvcVRjbGrjq4+yhJaf1blpNS+3No95F26Ta++Am/vOhZVsx6hdrK9kPG7/pgL976JlLO7+AfEkXN/aBDecRKROuJIuIQkQ3AZ8Drxph/B0nzDREpaTNj7imer/0604WvBI8HLr4MRl0Ac74GZ5/d3Yi6L3ieTAhpIhRQmHQnZjvntyuxffVyGJ8GP19jLc+7Cl7ZYhXqthIsA20ymnlzOg/suIP/+fgrjJqUxl/nvBGw/dCeoxR87XVu+d9JxMXZ48sUsH0bdEQLaGOM1xhzCdYEitkiMi5ImqeMMeM7mTE3JJ7dkJbaupzqhKq20zaGYM8e2FxqXVCMNY/nCGlprTXm1NT+VFUdOWkah0NISkqkurouqnF2lccDaWmtH9TUVKGqKrRS1uMxp7xvpHlqIc2vySI1GaoOtU83MQN+MAmmL4V6r7XuyhGw4Gqo+CH84ma4fTw8elN04u5IUmp/Dla2vucOeo4wcFi/gDT9UvrQK9EqyK64ayy7133Wsq3uUD3Lbipk6k8+x4jPnROdoENgEOpJCOkRK6fSi8MlIq6unMQ3rfhbwNRuxtuh4mLIGAXp6RAfD3m3WhcJQ+F0Qm9fq0ByMlx1JWy1wQXn4uK9ZGQkk54+kPj4OPLyLsDlCmz/c7k+Yc6cLABmzRrN6tW7gh3KVoqLm8jIENLTxXqt8hy4XN6Q9i0q8jJlShzJydZrNWVKHEVFoe0bacWVkHEWpA+GeAfkXQquTYFpLnHCH26xCud9ft+1t/0FRjwCI38C9xbCMyXwwMvRjT+Y1AlD2e+upbriII31Xj4q2EbW9MDheA7tOdryvNRVwdmZVteUxnovz3zpZS6//QIuuiUjqnF3pie0QZ9qL45OichZQIMxplZE+mBNa72oO8fsjNcLC74NRS+DwwHLnobSUsj/MZSsg8J/wPjx8NILVk+Am2+C/B/BuEsg8wJ4/OfWrzkR+MVi2LSp01NGnNdrWLBgDUVFM3E4hGXLNlFaeoD8/M9TUrKXwsLtLF26iRUrpuF230l1dR15ea2f6oqKuQwcmEhCQhwzZpzPlCkvBvQAiRWvFxYsaKCoKMF6rZZ5KS015Of3oqSkicLCJsaPF156KdF6rW52kJ9vGDfuBDU18MgjjRQXWz0BHn64kZqaGGfIx9sEC/4Pir4BjjhY9gGUfgr5U6GkEgo3w89vhv6J8MIca59dNZC7LLZxd8TRK47c317Ln3JcNHmbmHBnFueMTaHoR2tJHX82Y6efx3u/+YhSVwVxvYQ+g3vzH09PAuDj591sf6eKowfqKHnaamy/9elJDLsk9jPc94SxOKRte2bYDixyEbAca4bbOOD5zgb5FxGDLADHExGJKaoaE4AGvxX3A7F/U3bPa1gTEzebgjWTfE+XCvh9aX3pVUjPiVk0YVGxCv52Y8vikNHJfG/r12IYUHi8NO8t/vXkRoB13W0WHTo+1dxa8u2Q0j4h3+v2+U5FKAP2ZwCPAlkEzqhyXkf7GWM+Bi7tboBKKRUJp8ut3v+LNWHsYqxZVb6OjsWhlOrhmtug7SyUXhx9jDFvYjWH7DTGPATY775hpZTqAqsXR2JIj1gJpQZdJyJxgFtEFgC7ARv0EFZKqVN3ujRxfAdr0thvAY9g1Z7nRDIopZSKhh5fQBtjin1Pj2C1PyulVI/XE9qgQ+nFsYYgY2gYY7QdWinVY/WEftChRHev3/PewEzAfsOjKaVUFzTf6m1noTRxrGuz6j0RCWW4UaWUsq3TpYnDf1TuOOBywD4jniil1Ck6HZo41mG1QQtW00YFMDeSQSmlVKSdLt3sMo0xAWNXiog95qtRSqlT1BMK6FDuJHw/yLp/hTsQpZSKth473KiInAM4gT4icimt428MxLpxRSmleqwm4mJ6G3coOmriyAHuwBqL8XFaC+hDwPcjF9J+MBsjd/ioadt1fB89v3di23nmDgM2GDS729oM9n+wAvb38PfgoYqARe8JL3s27o9RMOFz9EB4J5wNZxOHiEwFfo01xPKfjDE/O0m6WcALwARjTIdT/XU6HrSIzDTGvHhqIXeNiNhj3iKllN11e3zmvuMzzZiS/w0p7Qa5ssPziYgD2AZMBjxAMTDbGFPaJt0A4GUgAVjQWQEdShv05SLSMsuaiAwSkZ+EsJ9SStmWIaxt0NlAuTFmuzGmHigAcoOkewR4DAhp0tBQCuhpvjkFATDG1AA3dpBeKaV6AOtW71AewBARKfF7fKPNwZxApd+yx7eu9WzWtbw0Y0yIM6WG1s3OISKJxpgTvpP0gQi2rMc5oJe9O4+HpP5EmxUOev48B02+RzMBm98qG5q2r1U8EZ7wPgqaCJxyDSSx579WpqERmpo6TxjKsbrWzW5/J00qwT7cLU22viGbF2Nd1wtZKCXhn4E3ReR/fSe8E3imKyfpkq9+Ex46DeYkHJMIjf4fkAfp+cNov+x7NLseeDZGsYRTJuA362yfQujVw+ckbFwFx1t/6CaMHs7orStjGFB4VM1bRPWT4bkkZhBOhK+C4QHS/JZTgSq/5QHAOOAtEQHrbmyXiEzvqB06lLE4HhORj7Fm5RbgEWNMUSe7KaWUrYV5NLtiIENERmJNapIHfKXlXMYcBIY0L4vIW8C9nV0kDCk6Y8yrwKu+A18lIkuMMfO7mgOllLKTcHWzM8Y0+macKsJqz1xmjNksIg8DJcYY16kcN6QCWkQuAWYDt2KNxfF/p3IypZSyi3Df6m2MeQV4pc26H50k7XWhHLOjOwlHY1XTZwMHgL9i9Zu+PsR4lVLKtgyCt8neY3F0VIMuA94FbjbGlAOIyH9HJSqllIow0yScqOu5t3rPxKpBrxGRV7E6Xvf0fmJKKQWAMYK3sYfWoI0xLwEviUg/YAbw38BQEXkSeMkY81qUYlRKqfAz2L6A7rQ3vjHmqDHmL8aYL2L17dsA3B/xyJRSKoKMERobHCE9YqVLnQCNMdXAH3wPpZTqwYQmr73vWrZ3dEopFSkGsHkThxbQSqkzU5NAnb2LQHtHp5RSkWTzOTS0gFZKnZmsAaFtTQtopdSZqQcU0D190Nt2cvpAWSq402BhUvvtdw+Aj1NhvRPeHQaZ8YHb0xxwOB2+G2TfWMnJSaOsbDZu91dYuPDSdtsTEuIoKJiM2/0V1q79MiNGDABg0qRUSkpm8fHH/0FJySyuv97Zbt9YyslJoKwsBbc7hYUL289DnJAABQVJuN0prF07mBEjrLdrfDwsWzaQjz8ezIYNg7n22vh2+8ZSziQoWwfuDbAwyL2313we1r0DDdUwM8icGwMGgKcMnvhF5GMN1VUMxMU4XuZC5nLOSdNNZhAbmUCW37zSczmXl7kQF+P4PAOjEW5oDNaQ2aE8YiRiBbSIpInIGhHZIiKbReTbkTpXszhgyRCYtheyKmF2//YF8LNH4CIPXLobHquFX6YEbl88BFYdi3SkoYuLE5YsuYZp0/5BVlYBs2ePIjNzUECauXMzqak5QUbGsyxe/DGLFn0OgP3767j55le46KLnmTNnNStW3BCLLAQVFwdLlgxg2rRasrIOMHt2bzIzA6+oz53bh5qaJjIyDrB48VEWLbK+eO66qw8AF11UzeTJNTz++ADEJve4xsXBksdh2kzImgCzZ0HmmMA0uzxwx3/Bsy8EP8YjP4S334t8rKGKA37ACObhJpdNTCOF8+jdLl1f4vgqQ/mIIy3rzqM30xjMDDbxX2zjh4ywT63QYM3VEMojRiL5v2oEvmuMyQQ+B8wXkawIno/sRChvgIpG60uv4Cjk9gtMc9hvWtp+Ejj3dm5f2N4Am2P4jdlWdvbZlJcfpKLiMA0NTRQUlJObmx6QJjc3neXLtwKwcuUnTJxo1ZQ3bNjPnj3Wt83mzdX07t2LhAR7fDyys+MpL/dSUeGloQEKCurIzQ0cFyE3N5Hly62p21auPMHEidbg6llZvXjzzXoA9u0z1NY2MX68PVrrssdD+Xao2IGVrxch96bANDt3wcbNwScGuewSGHo2vPZmVMINyYX0Yxcn8HCCRgyrqOZ6BrVLtwAny9hDvd+sO9cziFVU04BhN/Xs4gQX0q/dvjHR3MQRyiNGIvZpNcbsMcZ86Ht+GNhCmzm6ws3ZCyr9/pmeRnAG6eY4byCUp8FjKfAt30z0fQUWJkN+Tfv0seR09qOy8mjLssdzFKezX5s0/amstGotXq/h4MF6UlICazgzZ57H+vX7qa8Pz3RB3eV0xlFZ2RqLx9OEs82L5XQ6qKz0AuD1wsGDTaSkCB991EhubiIOB6Snx3H55fGkpdmjP6vzXKj0tC57qsA5LLR9ReDx/wf3/TAysZ2qs0lgL/Uty59Sz1ACf5peQF/OIYF3OBiwfijxfNpm37PtMk1aDyigo1LtEJF04FLg3xE9T5B1Jsi63x2yHrP7wQ8HwR37IH8QLD4IR4PtEEPBfrobE0qa1kRZWYNYtOhzTJkS8lyVEXfq+YJly46TmemgpGQwO3d6ef/9BhptcrEnlHydzLy74JXXwLM7vDF1V2efKwG+Rxo/pKLL+8ZUD7hIGPECWkT6Ay8C3zHGHAqy/RtA2xlyT4mnEdL8cpTaC6q8J09fcBSePAvYB1ckwqx+8NhgSI6zptysM7CkXcTR5fEcJS2ttcacmtqPqqqjbdIcIS2tP7t3H8XhEJKSEqiuthrOnM5+vPTSVG6/fTXbt8c4M348nibS0lp/wKWmxlHV5sXyeLykpTnYvbsJhwOSkuKorrY+3v/zP63tnO+9Nwi32x6fNE8VpKW2LqcOg6o9oe17ZTZccyXM+0/o3x8S4uHIEXjgoYiEGrJPqeccv1rvUBL4zO/KWT8cjKIPy7gAgCHE8wQZ3IObvTQwtM2++/xq1DHVAwroiDZIikg8VuH8F2NM0FlYjDFPGWPGdzJjbkiKT0BGPKT3suZlzusHrsCyjFF+BfhNfcHte599YQ+MrLQevzoEP62NfeEMUFz8GRkZyaSnDyA+Po68vFG4XDsC0rhcO5gzx7oSNWvW+axebVXBkpISePnlG3nggX/z/vt7ox16h4qLG8jIcJCeHkd8POTl9cblCrwa43KdYM4cq6lm1qxEVq+2Pth9+kBfXyeBSZMSaGyELVs6+CaOouJ1kHEepI+wepvkzQTXK53vB3Dbf8KIsTDyQrj3B/BMQewLZ4BNHGUEiThJoBfCNAbzlt8ku0fw8gU2MJWPmcrHfMwR7sFNKcd4ixqmMZh4BCcJjCCRjRzt4GxRdqY2cYg1de1SYIsx5peROo8/L7BgPxSdAw6BZYehtMFqvig5AYXHYEESTOoDDQZqmmDOZ9GI7NR5vYYFC96lqOiLOBzCsmVllJbWkJ8/gZKSfRQW7mDp0jJWrJiI2/0VqqvryMt7HYAFC8YxalQSDz54OQ8+eDkAU6b8g337jscyS4DVprxgwWGKigbhcMCyZXWUlnrJz+9HSUkjhYUnWLr0OCtWWN3sqqsNeXlW++bZZ8dRVDSIpibYvdvL1752sJOzRY/XCwvug6KXsPK1AkrLIP8HUPIhFK6C8ZfBS3+BQclw8zTI/z6MuyLWkZ+cF/gpu/g9Y3AAL7GfT6hjPsPYzDHeovak+35CHUVU83fG0Yjh/7ELe1wFwfczOdZBdExMqA1kXT2wyNVYM7JshJbX5Pu+ebtOto/ha/PhoSciElNUjUmERv/uIA8BZ8comHB52fdodgPwbIxiCadM8KsR0udV6JUTs2jConEVHL+xZTFh9HBGb10Zw4DCo2reIqqffBFgXXd/dcvI8Yb8DifVbjVHun2+UxGxGrQx5p/oDCxKKbvqAW3Q9ug8qpRS0aYFtFJK2ZgW0EopZUNag1ZKKZtqAmLfoalDWkArpc5MBqsPoY1pAa2UOnNpE4dSStmQtkErpZRNaQGtlFI21QNu9dYCWil15tIatFJK2ZA2cSillE01TxprY1pAK6XOTD2gH7Q9ZhBVSqloC/OchCIyVUS2iki5iNwfZPv/iEipiHwsIm+KyIjOjmm/GvRbr8Ddns7T2Z237av6HJAYLGUP0nZWlo+BO2IQR7gdCVw88RA0PBmTSMKmKfC1aty9j50z7o1RMOFT95E7fAczhO1WbxFxAEuAyYAHKBYRlzGm1C/ZemC8MeaYiPwX8Bhwa0fHtV8BXVlhPU47W2MdQATsB16NdRDh17QW+0z7ER5NR49z+O/vxDoMewlvE0c2UG6M2Q4gIgVALtBSQBtj1vilXwvc1tlB7VdAK6VUNHStF8cQEfGffuUpY8xTfstOoNJv2QN0NJHZXGBVZyfVAlopdWbqWgG9v5Mpr4LNHhV0PkERuQ0YD1zb2UltV0CPu8zBtVPiYx1Gtz35WB1NAT+Ts4G+MYomXCqAnX7LQ7DeZz3dGwR+Uq8GhsYolnDZC7zXuthvEEy5O2bRhM36V2HHhvAcK7zd7DxAmt9yKlDVNpGITAJ+AFxrjDnRdntbtiugx38+ngce7RfrMLrtD4+3LaCvxSrQerLXCSygzwFmxiiWcHqXwAJ6EnBZjGIJl3UEFNBJZ8Ftj8YsmrA5djB8BTSEsw26GMgQkZHAbiAP+Ip/AhG5FPgDMNUY81koB7VdAa2UUlERxrE4jDGNIrIAKAIcwDJjzGYReRgoMca4gJ8D/YEXRARglzFmekfH1QJaKXVmCvOdhMaYV4BX2qz7kd/zSV09phbQSqkzUw+4k1ALaKXUmUsHS1JKKRvS0eyUUsqmdMB+pZSyKa1BK6WUjWkBrZRSNqQD9iullE1pNzullLIpbYNWSimbaiJsA/ZHihbQSqkzlzZxKKWUTQUdsdk+TrtJY9e8Ws8XxtRw1agafvuz4L9fCp8/wfVZtdwwtpb5XzkMgGenl2mX1zLlEmv9it/bpwd7Ts5Iysr+E7f7LhYubD9JQ0KCg4KC6bjdd7F27W2MGDEQgMGDe7N6dR6HD3+HJ57o8jgtEZeTk0pZ2Szc7ltYuPCidtsTEuIoKLgBt/sW1q6dzogR/QGYMOEs1q//EuvXf4kNG77EjBmdzr0ZVTk5Z1FWdj1u9w0sXDiq3XYrX5fhdt/A2rVXM2JEn4DtaWl9OHx4Gt/97nnRCrlTOUOhbDK4p8DC0e233z0SPp4I62+Ad78AmQNat90/2tqvbDJMOTt6MZ8OIlZAi8gyEflMRDZF6hxteb2GH84/yopVA1lTmszfnzvBttLAqwDb3V5+++hxXnpvIKs3J5P/K2vs6bPPjeNv7yfx2oZkCv+dxJKfHWdvVewnpouLE5YsmcS0aS+QlbWU2bMzycxMCUgzd+6F1NTUkZHxRxYvLmHRousAqKvz8uCD73LvvW9FP/BOWPn6PNOmFZGV9SKzZ59PZmZyQJq5c8dQU3OCjIwXWLx4E4sWZQOwaVM148f/jUsvfYmpU1/lD3+4Gocj2IQW0RcXB0uWXMi0af8mK2sNs2cPIzOzf0CauXPTqKlpICNjNYsXb2fRosyA7YsXj2XVqpCGC46KOGDJxTDtPch6HWanBhbAAM9WwkVvwqWr4TE3/NL3fZs5APJSYewbMPU9+N0lp2GtMIIi+b96GpgaweO3s+GDRtJHORhxnoOEBCE3L5HX/h7Y0fHZP9YxZ35vkgdZWR9ytvU3IUFITLQ+5PUnTJvB9mMnO/tcystrqag4SENDEwUFW8jNDayV5eZmsHy59T24cuVWJk4cDsCxYw28995u6ursd6k6O/ssyssPUVFx2Jev7eTmBtaEc3NHsHy5NYvzypUVTJw4DIDjx714vdZv0969HRgb/UzNzh5EeflRKiqO0dBgKCioIjf3nIA0ubnnsHy5NXP9ypV7mDjxrIBt27cfZfPmw1GNuyPZg6H8KFQcgwYDBR7IPTcwzWG/t1g/By2vSe65Vvr6JthxzDpO9uDoxd7TRayANsa8A1RH6vjB7NndxLlprVk6JzWOPbsDrwJUbPOyfZuXGVcd5ObPHWTNq/Ut26oqvUy6qJYJaTXMW9iHc4bF/rve6exPZWXrh9XjOYzTOSBImkOA9Svi4METpKQE/my2G6ezL5WVR1uWPZ6jOJ19g6Q5AjTnq56UlETAKuA3bZrJxo0z+eY3/9lSYMea09mbysrWpjWPpw6ns/dJ01j5aiAlJYG+fR0sXHg++fnbohpzZ5y9wS9LeI6DM8jba955UD4FHhsH3/rIt2+fIPv2br9vbDR34wjlERsxL4FE5BsiUtJmxtxTE+Qz6pu5oEVjI1S4vbzw1kCWPNef+/7zKAdrrerysDQHb3yczD/LB/HC8jr2fRr7anTb+AFMmypjKGnsJnjMoaf54IN9jBv3IhMm/J0HHriYxERHJMLssiAhh5gvQ37+GBYv3s7Ro/bqWhA0T0HS/W47jHoNFm6CH17g2zdIOvu8M5tvJQzlERsxL6CNMU8ZY8Z3MmNuSM5NjWNPZWuhutfT1K4WfG5qHDm5CcTHC8NHOjh/TBwV7sCC+JxhcYwZ24t/vxv7+0A9nsOkpbXWmFNTB1BVdSRIGuvCoMMhJCUlUl1tn4ucwXg8R0lLa517MjW1H1VVx4KksdpvrXwlUF0dOM9mWVktR482Mm7coMgHHQKPp460tNbqZWpqb6qq6tqkOd6SxspXPNXVDVxxRTKPPZZFRcVEvvOd8/j+9zOYPz89muEH5TkOflkitQ9UdVCpLPDAjGEd7Gubt2bznSqhPGIj5gV0OF08oRcVbi+7KrzU1xv+XnCCydMDZwjPmZHA+2usf3j1/ia2b2tixHlxVHm8HD9ufbfX1jRR/F4D54+Jfa2suHgPGRmDSE9PIj4+jry8TFyu8oA0Llc5c+aMA2DWrDGsXr0rFqF2SXHxPjIyBpKe3t+Xr/NwuXYGpHG5djJnTgYAs2aNZPVqa5Lk9PT+LRcFhw/vz5gxSezYYY822+LiWjIy+pGe3of4eCEvbxgu196ANC7Xp8yZkwrArFnnsnr1fgC+8IX3GTnyTUaOfJNf/Wo7P/2pmyVLdkQ7C+0U10BGf0jvC/FiXfRz7QlMM8pvnuebzgG3rw7h2mOlT4iz9s/oDx9EteGzI/avQZ9W/aB79RIe+W0/vppziCYv3HpnImPG9uLnPzrGxeN7MWV6AtflxPPOaw1cn1VLnAN++PO+DEqJY+Pr9Tz83cOIWD9J7763D5kXxv7f4/UaFix4g6KiW3A4hGXLNlJaeoD8/KspKdlLYWE5S5d+zIoVN+F230V1dR15ea6W/Ssq7mbgwAQSEhzMmJHBlCnPs2XLgRjmyGLl632Kiqb58rWN0tJa8vMvo6RkP4WFu1i6dBsrVlyL230L1dUnyMtbA8DVV5/D/fdfTENDE01Nhnnz3ufAgU5nsI8KK1+bKCr6nC9flZSWHiE/fwwlJbUUFn7K0qW7WLHiUtzuG6iuricv78NYh90hr4EFG6DoKnAILNsJpYchPxNKaqFwDyw4HyadDQ1NUNMAc3wNlqWH4fndUDoJGg3M32C1/NqD/ZgQ4wIAAAf9SURBVEdLkki1VYrIc8B1wBDgU+DHxpilnexj7ljQm5880a+jZD1CesIBGgNe+/uw/hU92evAG37L44BvxyiWcPoOcNRv+SHgstiEEjbrgPzWxWGj4YmtMYsmbJ6aB0VPAqzrbrOoyIUG/i/E1KO7fb5TEbEqojFmdqSOrZRS3Wf/0ZJi/xteKaViwv5NHFpAK6XOUFqDVkopm9IatFJK2ZTWoJVSyqbsP2K/FtBKqTOUNnEopZSNaROHUkrZkNaglVLKprSAVkopm9JeHEopZVPai0MppWxKmziUUsqmtImjyw581sTH6+z9TwtF+1Fc9wK2mUriFB1qs3yc/9/evcdYUZ5xHP/+xAvItbFoqRAxaCG1qSDWWkmN9ULwSmlshF6NJkZtC8Zoq6lJrf1DEmPTGmNSKiS1eIsXGlIT0HqFBiqwgrDiXUg3mKCkEkENqfn1j3kXj4fds3v2TJk5s88n2TAz+56Z5z0hz77nPTPPC9t7athm6isU7wBGFRFIjnZ8fnffJ/DWhmJCydPuPFc7z3cELWkW8EdgCHCv7YV1vz8CuA+YDuwCLrO9reE5y7R2naTyBBNCKLMc6kGPN8zvZ+tfNbyepCHA68B5QBewDphn+5WaNtcCX7d9taS5wBzblzW6aqWWvAohhP7LdVXv04A3bb9tex/wEDC7rs1s4C9p+1HgHPW0gnCNsk1xvM/B+cz8xXStqqliv6rYJ4h+teq41k/x7kq4tb/LHA2VtL5mf5HtRTX7xwL/rtnvAr5Zd479bWz/V9Ju4CgavF+lStC2xx6M60haX8TyNf9vVexXFfsE0a8ysD0rx9P1NBKun7LtT5vPiSmOEEJoXRcwoWZ/PAd8U/tZG0mHAqOBhmucR4IOIYTWrQNOlHS8pMOBucDyujbLgZ+m7UuBZ9zHXRqlmuI4iBb13aQtVbFfVewTRL8qJc0p/xxYSXab3RLbnZJuA9bbXg4sBv4q6U2ykfPcvs5bqtvsQgghfCamOEIIoaQiQYcQQkkNqgQtaYmknZK2FB1LXiRNkPSspK2SOiUtKDqmPEgaKulFSZtSv35bdEx5kTRE0kuS/l50LHmRtE3SZkkb6+4XDi0YVHPQks4E9gD32f5a0fHkQdI4YJztDkkjgQ3Ad2sfMW1H6Qmr4bb3SDoMWA0ssL224NBaJul64FRglO2Lio4nD5K2AafaruLDN4UZVCNo2y/Qx32H7cb2u7Y70vaHwFayJ5bamjN70u5h6aftRxOSxgMXAvcWHUsov0GVoKtO0kRgGvCvYiPJR5oK2AjsBJ6yXYV+/QH4JQeW0Gt3Bp6UtEHSVUUHUxWRoCtC0gjgMeA62/V1QduS7U9tTyV7Kus0SW09LSXpImCn7QrU/TzADNunAOcDP0vTiaFFkaArIM3RPgbcb/vxouPJm+0PgOeAPGsnFGEGcEmar30IOFvS0mJDyoftHenfncAysupuoUWRoNtc+jJtMbDV9u+LjicvksZKGpO2hwHnAq8WG1VrbN9se7ztiWRPkT1j+0cFh9UyScPTF9RIGg7MBCpzp1SRBlWClvQgsAaYLKlL0pVFx5SDGcCPyUZjG9PPBUUHlYNxwLOSXiarc/CU7crcllYxxwCrJW0CXgSesL2i4JgqYVDdZhdCCO1kUI2gQwihnUSCDiGEkooEHUIIJRUJOoQQSioSdAghlFQk6LCfpE/TbXpbJD0i6cgWznVWd7U2SZdIuqlB2zGSrh3ANW6VdEN/jzc4z56+Ww38/CEMVCToUOtj21NTpb99wNW1v1Sm6f8ztpfbXtigyRig6QQdQtVFgg69WQWcIGliqjV9D9ABTJA0U9IaSR1ppD0CQNIsSa9KWg18r/tEki6XdHfaPkbSslTneZOkM4CFwKQ0er8jtbtR0jpJL9fWgpb0a0mvSfoHMLmZDkn6Wyrm01lf0EfSnak/T0sam45NkrQivWaVpCkDeB9DGLBI0OEAaUn484HN6dBkshra04C9wC3Auak4znrgeklDgT8DFwPfBr7Uy+nvAp63fTJwCtAJ3AS8lUbvN0qaCZxIVs9hKjBd0pmSppM9Ij2N7A/AN5rs2hW2p5PVYp4v6ah0fDjQkfrzPPCbdHwR8Iv0mhuAe5q8XggtGayreoeeDUvlPSEbQS8GvgxsrymUfzrwVeCfWRkQDid7fH4K8I7tNwBSEaCeyk6eDfwEsmp1wG5JX6hrMzP9vJT2R5Al7JHAMtsfpWvUL2vfl/mS5qTtCemcu8hKfz6cji8FHk+fCs4AHkn9BDiiyeuF0JJI0KHWx6m8534pOe2tPURWF2NeXbup5FdQX8Dttv9Ud43rBnoNSWeRFVz6lu2PJD0HDO2luck+XX5Q/36EcDDFFEdo1lpghqQTACQdKekrZJXmjpc0KbWb18vrnwauSa8dImkU8CHZ6LjbSuCKmrntYyUdDbwAzJE0LFVPu7iJuEcD/0nJeQrZJ4FuhwCXpu0fAKtTTe13JH0/xSBJJzdxvRBaFgk6NMX2e8DlwIOp0txaYIrtT8imNJ5IXxJu7+UUC4DvSNpMtn7iSbZ3kU2ZbJF0h+0ngQeANando8DItLTXw8BGsvrXqxqEekuqWNglqQtYARyaYv5dirvbXuAkSRvIpmBuS8d/CFyZqrR1ArP7+z6FkIeoZhdCCCUVI+gQQiipSNAhhFBSkaBDCKGkIkGHEEJJRYIOIYSSigQdQgglFQk6hBBK6n8m85xYfhrtwgAAAABJRU5ErkJggg==\n", 950 | "text/plain": [ 951 | "
" 952 | ] 953 | }, 954 | "metadata": {}, 955 | "output_type": "display_data" 956 | } 957 | ], 958 | "source": [ 959 | "# Get score for test set\n", 960 | "rfc_predict_test = rfc.predict(vectors_test)\n", 961 | "rfc_confusion_matrix = confusion_matrix(target_test, rfc_predict_test)\n", 962 | "print(make_confusion_matrix_relative(rfc_confusion_matrix))\n", 963 | "plot_confusion_matrix(rfc_confusion_matrix, 'Random Forest Confusion Matrix', savefilename='rfcCM.png')" 964 | ] 965 | }, 966 | { 967 | "cell_type": "markdown", 968 | "metadata": {}, 969 | "source": [ 970 | "__Imbalaned Data__\n", 971 | "\n", 972 | "1. The number of estimators may not be enough. \n", 973 | "2. We can see that the classifier tend to classify samples to the majority classes. We can try to downsample 5 and 1 star reviews or oversample 2,3,4 star reviews" 974 | ] 975 | }, 976 | { 977 | "cell_type": "markdown", 978 | "metadata": {}, 979 | "source": [ 980 | "### Important features (words) by inspecting the RFC model" 981 | ] 982 | }, 983 | { 984 | "cell_type": "code", 985 | "execution_count": 51, 986 | "metadata": {}, 987 | "outputs": [ 988 | { 989 | "data": { 990 | "text/plain": [ 991 | "['amazing',\n", 992 | " 'good',\n", 993 | " 'best',\n", 994 | " 'great',\n", 995 | " 'worst',\n", 996 | " 'delicious',\n", 997 | " 'terrible',\n", 998 | " 'told',\n", 999 | " 'bad',\n", 1000 | " 'love',\n", 1001 | " 'ok',\n", 1002 | " 'awesome',\n", 1003 | " 'said',\n", 1004 | " 'rude',\n", 1005 | " 'wasn',\n", 1006 | " 'slow',\n", 1007 | " 'average',\n", 1008 | " 'asked',\n", 1009 | " 'perfect',\n", 1010 | " 'little']" 1011 | ] 1012 | }, 1013 | "execution_count": 51, 1014 | "metadata": {}, 1015 | "output_type": "execute_result" 1016 | } 1017 | ], 1018 | "source": [ 1019 | "n = 20\n", 1020 | "get_top_values(rfc.feature_importances_,n,words)" 1021 | ] 1022 | }, 1023 | { 1024 | "cell_type": "markdown", 1025 | "metadata": {}, 1026 | "source": [ 1027 | "## Use cross validation to evaluate classifiers\n", 1028 | "\n", 1029 | "[sklearn cross validation](http://scikit-learn.org/stable/modules/cross_validation.html)" 1030 | ] 1031 | }, 1032 | { 1033 | "cell_type": "code", 1034 | "execution_count": 52, 1035 | "metadata": {}, 1036 | "outputs": [ 1037 | { 1038 | "data": { 1039 | "text/plain": [ 1040 | "array([0.56541234, 0.56958471, 0.56939904, 0.57342218, 0.57308692])" 1041 | ] 1042 | }, 1043 | "execution_count": 52, 1044 | "metadata": {}, 1045 | "output_type": "execute_result" 1046 | } 1047 | ], 1048 | "source": [ 1049 | "from sklearn.model_selection import cross_val_score\n", 1050 | "\n", 1051 | "cv_scores = cross_val_score(rfc,\n", 1052 | " vectors_train,\n", 1053 | " target_train,\n", 1054 | " cv = 5,\n", 1055 | " scoring=\"accuracy\")\n", 1056 | "cv_scores" 1057 | ] 1058 | }, 1059 | { 1060 | "cell_type": "markdown", 1061 | "metadata": {}, 1062 | "source": [ 1063 | "## TODO: Use grid search to find best predictable classifier\n", 1064 | "\n", 1065 | "\n", 1066 | "[sklearn grid search tutorial (with cross validation)](http://scikit-learn.org/stable/modules/grid_search.html#grid-search)\n", 1067 | "\n", 1068 | "[sklearn grid search documentation (with cross validation)](http://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html#sklearn.model_selection.GridSearchCV)" 1069 | ] 1070 | }, 1071 | { 1072 | "cell_type": "code", 1073 | "execution_count": null, 1074 | "metadata": {}, 1075 | "outputs": [], 1076 | "source": [ 1077 | "# To be implemented\n" 1078 | ] 1079 | }, 1080 | { 1081 | "cell_type": "code", 1082 | "execution_count": null, 1083 | "metadata": {}, 1084 | "outputs": [], 1085 | "source": [] 1086 | } 1087 | ], 1088 | "metadata": { 1089 | "anaconda-cloud": {}, 1090 | "kernelspec": { 1091 | "display_name": "Python 3", 1092 | "language": "python", 1093 | "name": "python3" 1094 | }, 1095 | "language_info": { 1096 | "codemirror_mode": { 1097 | "name": "ipython", 1098 | "version": 3 1099 | }, 1100 | "file_extension": ".py", 1101 | "mimetype": "text/x-python", 1102 | "name": "python", 1103 | "nbconvert_exporter": "python", 1104 | "pygments_lexer": "ipython3", 1105 | "version": "3.5.5" 1106 | } 1107 | }, 1108 | "nbformat": 4, 1109 | "nbformat_minor": 1 1110 | } 1111 | --------------------------------------------------------------------------------