├── .gitignore ├── README.md ├── book-crossing-eda.ipynb ├── book-crossing-preprocessing.ipynb ├── collaborative-filtering-memory-based.ipynb ├── collaborative-filtering-model-based.ipynb ├── functions.py └── img ├── books_header.jpg ├── test_actual.jpg ├── test_pred.jpg └── train_actual.jpg /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/python,pycharm,windows,jupyternotebooks 3 | # Edit at https://www.gitignore.io/?templates=python,pycharm,windows,jupyternotebooks 4 | 5 | ### JupyterNotebooks ### 6 | # gitignore template for Jupyter Notebooks 7 | # website: http://jupyter.org/ 8 | 9 | .ipynb_checkpoints 10 | */.ipynb_checkpoints/* 11 | 12 | # IPython 13 | profile_default/ 14 | ipython_config.py 15 | 16 | # Remove previous ipynb_checkpoints 17 | # git rm -r .ipynb_checkpoints/ 18 | 19 | ### PyCharm ### 20 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 21 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 22 | 23 | # User-specific stuff 24 | .idea/**/workspace.xml 25 | .idea/**/tasks.xml 26 | .idea/**/usage.statistics.xml 27 | .idea/**/dictionaries 28 | .idea/**/shelf 29 | 30 | # Generated files 31 | .idea/**/contentModel.xml 32 | 33 | # Sensitive or high-churn files 34 | .idea/**/dataSources/ 35 | .idea/**/dataSources.ids 36 | .idea/**/dataSources.local.xml 37 | .idea/**/sqlDataSources.xml 38 | .idea/**/dynamic.xml 39 | .idea/**/uiDesigner.xml 40 | .idea/**/dbnavigator.xml 41 | 42 | # Gradle 43 | .idea/**/gradle.xml 44 | .idea/**/libraries 45 | 46 | # Gradle and Maven with auto-import 47 | # When using Gradle or Maven with auto-import, you should exclude module files, 48 | # since they will be recreated, and may cause churn. Uncomment if using 49 | # auto-import. 50 | # .idea/modules.xml 51 | # .idea/*.iml 52 | # .idea/modules 53 | # *.iml 54 | # *.ipr 55 | 56 | # CMake 57 | cmake-build-*/ 58 | 59 | # Mongo Explorer plugin 60 | .idea/**/mongoSettings.xml 61 | 62 | # File-based project format 63 | *.iws 64 | 65 | # IntelliJ 66 | out/ 67 | 68 | # mpeltonen/sbt-idea plugin 69 | .idea_modules/ 70 | 71 | # JIRA plugin 72 | atlassian-ide-plugin.xml 73 | 74 | # Cursive Clojure plugin 75 | .idea/replstate.xml 76 | 77 | # Crashlytics plugin (for Android Studio and IntelliJ) 78 | com_crashlytics_export_strings.xml 79 | crashlytics.properties 80 | crashlytics-build.properties 81 | fabric.properties 82 | 83 | # Editor-based Rest Client 84 | .idea/httpRequests 85 | 86 | # Android studio 3.1+ serialized cache file 87 | .idea/caches/build_file_checksums.ser 88 | 89 | ### PyCharm Patch ### 90 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 91 | 92 | # *.iml 93 | # modules.xml 94 | # .idea/misc.xml 95 | # *.ipr 96 | 97 | # Sonarlint plugin 98 | .idea/**/sonarlint/ 99 | 100 | # SonarQube Plugin 101 | .idea/**/sonarIssues.xml 102 | 103 | # Markdown Navigator plugin 104 | .idea/**/markdown-navigator.xml 105 | .idea/**/markdown-navigator/ 106 | 107 | ### Python ### 108 | # Byte-compiled / optimized / DLL files 109 | __pycache__/ 110 | *.py[cod] 111 | *$py.class 112 | 113 | # C extensions 114 | *.so 115 | 116 | # Distribution / packaging 117 | .Python 118 | build/ 119 | develop-eggs/ 120 | dist/ 121 | downloads/ 122 | eggs/ 123 | .eggs/ 124 | lib/ 125 | lib64/ 126 | parts/ 127 | sdist/ 128 | var/ 129 | wheels/ 130 | pip-wheel-metadata/ 131 | share/python-wheels/ 132 | *.egg-info/ 133 | .installed.cfg 134 | *.egg 135 | MANIFEST 136 | 137 | # PyInstaller 138 | # Usually these files are written by a python script from a template 139 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 140 | *.manifest 141 | *.spec 142 | 143 | # Installer logs 144 | pip-log.txt 145 | pip-delete-this-directory.txt 146 | 147 | # Unit test / coverage reports 148 | htmlcov/ 149 | .tox/ 150 | .nox/ 151 | .coverage 152 | .coverage.* 153 | .cache 154 | nosetests.xml 155 | coverage.xml 156 | *.cover 157 | .hypothesis/ 158 | .pytest_cache/ 159 | 160 | # Translations 161 | *.mo 162 | *.pot 163 | 164 | # Scrapy stuff: 165 | .scrapy 166 | 167 | # Sphinx documentation 168 | docs/_build/ 169 | 170 | # PyBuilder 171 | target/ 172 | 173 | # pyenv 174 | .python-version 175 | 176 | # pipenv 177 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 178 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 179 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 180 | # install all needed dependencies. 181 | #Pipfile.lock 182 | 183 | # celery beat schedule file 184 | celerybeat-schedule 185 | 186 | # SageMath parsed files 187 | *.sage.py 188 | 189 | # Spyder project settings 190 | .spyderproject 191 | .spyproject 192 | 193 | # Rope project settings 194 | .ropeproject 195 | 196 | # Mr Developer 197 | .mr.developer.cfg 198 | .project 199 | .pydevproject 200 | 201 | # mkdocs documentation 202 | /site 203 | 204 | # mypy 205 | .mypy_cache/ 206 | .dmypy.json 207 | dmypy.json 208 | 209 | # Pyre type checker 210 | .pyre/ 211 | 212 | ### Windows ### 213 | # Windows thumbnail cache files 214 | Thumbs.db 215 | Thumbs.db:encryptable 216 | ehthumbs.db 217 | ehthumbs_vista.db 218 | 219 | # Dump file 220 | *.stackdump 221 | 222 | # Folder config file 223 | [Dd]esktop.ini 224 | 225 | # Recycle Bin used on file shares 226 | $RECYCLE.BIN/ 227 | 228 | # Windows Installer files 229 | *.cab 230 | *.msi 231 | *.msix 232 | *.msm 233 | *.msp 234 | 235 | # Windows shortcuts 236 | *.lnk 237 | 238 | # Custom 239 | /data/ 240 | .idea/ 241 | 242 | # End of https://www.gitignore.io/api/python,pycharm,windows,jupyternotebooks -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Collaborative Filtering Recommender System with Python 2 | 3 | ![books recommendations](img/books_header.jpg) 4 | 5 | 6 | 7 | **Collaborative filtering** is a technique commonly used to build personalized recommendations in online products. Among companies using the collaborative filtering technology we can find some popular websites like: Amazon, Netflix, IMDB. In collaborative filtering, algorithms are used to make automatic predictions about a user's interests by compiling preferences from several users. 8 | 9 | The main focus of this repository is to build collaborative filtering recommender systems for a **Book-Crossing dataset**. It contains data about book ratings collected in a 4-week crawl in 2004 as well as detailed information about books and users. Further details on the dataset are given in this publication: 10 | 11 | > [Improving Recommendation Lists Through Topic Diversification](http://www2.informatik.uni-freiburg.de/~dbis/Publications/05/WWW05.html), 12 | > 13 | > Cai-Nicolas Ziegler, Sean M. McNee, Joseph A. Konstan, Georg Lausen; *Proceedings of the 14th International World Wide Web Conference (WWW '05),* May 10-14, 2005, Chiba, Japan. *To appear.* 14 | 15 | 16 | 17 | ------ 18 | 19 | **Contents:** 20 | 21 | 1. [**Preprocessing of Book-Crossing dataset**](book-crossing-preprocessing.ipynb) - the script includes loading data in the correct format, filtering out incorrect rows and reducing dimensionality of the dataset. 22 | 2. [**Exploratory Data Analysis of Book-Crossing dataset**](book-crossing-eda.ipynb) - the analysis provides insights about distribution of ratings, most popular readings and characteristics of users giving the scores. 23 | 3. [**Memory-based approach to Collaborative Filtering**](collaborative-filtering-memory-based.ipynb) - memory based algorithms apply statistical techniques to the entire dataset to calculate the predictions. In this notebook two methods are compared (user-user and user-item) and the model is optimized to provide the best predictions. 24 | 4. [**Model-based approach to Collaborative Filtering**](collaborative-filtering-model-based.ipynb) - model based approach involves building machine learning algorithms to predict user's ratings. In this notebook SVD and NMF methods are compared and the model is optimized to provide the best predictions. 25 | 26 | ------ 27 | 28 | **Reference:** 29 | 30 | 1. https://surprise.readthedocs.io/en/stable/getting_started.html#getting-started 31 | 2. https://realpython.com/build-recommendation-engine-collaborative-filtering/ 32 | 3. https://towardsdatascience.com/various-implementations-of-collaborative-filtering-100385c6dfe0 33 | 4. https://towardsdatascience.com/building-and-testing-recommender-systems-with-surprise-step-by-step-d4ba702ef80b -------------------------------------------------------------------------------- /book-crossing-preprocessing.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Preprocessing of Book-Crossing Dataset\n", 8 | "\n", 9 | "The [Book-Crossing dataset](http://www2.informatik.uni-freiburg.de/~cziegler/BX/) contains data about book ratings, books and users collected by Cai-Nicolas Ziegler in a 4-week crawl (August / September 2004)." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 1, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "import pandas as pd\n", 19 | "import numpy as np\n", 20 | "\n", 21 | "import functions as f" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 2, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "path = 'data/original/'\n", 31 | "\n", 32 | "df_ratings = pd.read_csv(path + 'BX-Book-Ratings.csv', sep=';', encoding='ansi')\n", 33 | "df_books = pd.read_csv(path + 'BX-Books.csv', sep=';', encoding='ansi', escapechar='\\\\')\n", 34 | "df_users = pd.read_csv(path + 'BX-Users.csv', sep=';', encoding='ansi')" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 3, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "for df in [df_ratings, df_books, df_users]:\n", 44 | " df.columns = [f.colname_fix(col) for col in df.columns]" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 4, 50 | "metadata": {}, 51 | "outputs": [ 52 | { 53 | "name": "stdout", 54 | "output_type": "stream", 55 | "text": [ 56 | "Ratings:\n", 57 | "Number of ratings: 1149780\n", 58 | "Number of books: 340556\n", 59 | "Number of users: 105283\n", 60 | "\n", 61 | "Number of books: 271379\n", 62 | "\n", 63 | "Number of users: 278858\n" 64 | ] 65 | } 66 | ], 67 | "source": [ 68 | "print('Ratings:\\nNumber of ratings: %d\\nNumber of books: %d\\nNumber of users: %d' % (len(df_ratings),\n", 69 | " len(df_ratings['isbn'].unique()),\n", 70 | " len(df_ratings['user_id'].unique())))\n", 71 | "print('\\nNumber of books: %d' % len(df_books))\n", 72 | "print('\\nNumber of users: %d' % len(df_users))" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 5, 78 | "metadata": {}, 79 | "outputs": [ 80 | { 81 | "name": "stdout", 82 | "output_type": "stream", 83 | "text": [ 84 | "items with non-ascii characters in user_id: 0\n", 85 | "items with non-ascii characters in isbn: 55\n", 86 | "items with non-ascii characters in book_rating: 0\n", 87 | "\n", 88 | "items with non-ascii characters in isbn: 0\n", 89 | "items with non-ascii characters in book_title: 365\n", 90 | "items with non-ascii characters in book_author: 21\n", 91 | "items with non-ascii characters in year_of_publication: 0\n", 92 | "items with non-ascii characters in publisher: 33\n", 93 | "items with non-ascii characters in image_url_s: 0\n", 94 | "items with non-ascii characters in image_url_m: 0\n", 95 | "items with non-ascii characters in image_url_l: 0\n", 96 | "\n", 97 | "items with non-ascii characters in user_id: 0\n", 98 | "items with non-ascii characters in location: 560\n", 99 | "items with non-ascii characters in age: 0\n", 100 | "\n" 101 | ] 102 | } 103 | ], 104 | "source": [ 105 | "f.ascii_check_bulk(df_ratings)\n", 106 | "f.ascii_check_bulk(df_books)\n", 107 | "f.ascii_check_bulk(df_users)" 108 | ] 109 | }, 110 | { 111 | "cell_type": "markdown", 112 | "metadata": {}, 113 | "source": [ 114 | "## Filtering observations\n", 115 | "* Remove (incorrect) ISBN with non-ascii characters\n", 116 | "* Use only country instead of whole 'location' data\n", 117 | "* Remove images' urls\n", 118 | "* Separate explicit (1-10) and implicit (0) ratings" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": 6, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "df_ratings['isbn_check'] = df_ratings['isbn'].apply(f.ascii_check)\n", 128 | "df_ratings = df_ratings[df_ratings['isbn_check']==0]" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 7, 134 | "metadata": {}, 135 | "outputs": [], 136 | "source": [ 137 | "df_users['country'] = df_users['location'].apply(lambda x: x.split(', ')[-1].title())\n", 138 | "df_users['country_check'] = df_users['country'].apply(f.ascii_check)\n", 139 | "df_users.loc[df_users['country_check']==1, 'country'] = np.nan" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": 8, 145 | "metadata": {}, 146 | "outputs": [], 147 | "source": [ 148 | "df_ratings.drop(['isbn_check'], axis=1, inplace=True)\n", 149 | "df_books.drop(['image_url_s', 'image_url_m', 'image_url_l'], axis=1, inplace=True)\n", 150 | "df_users.drop(['country_check'], axis=1, inplace=True)" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": 9, 156 | "metadata": {}, 157 | "outputs": [ 158 | { 159 | "name": "stdout", 160 | "output_type": "stream", 161 | "text": [ 162 | "Explicit ratings: 433642\n", 163 | "Implicit ratings: 716083\n" 164 | ] 165 | } 166 | ], 167 | "source": [ 168 | "df_ratings_explicit = df_ratings[df_ratings['book_rating']!=0]\n", 169 | "df_ratings_implicit = df_ratings[df_ratings['book_rating']==0]\n", 170 | "\n", 171 | "print('Explicit ratings: %d\\nImplicit ratings: %d' % (len(df_ratings_explicit), len(df_ratings_implicit)))" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": 10, 177 | "metadata": {}, 178 | "outputs": [], 179 | "source": [ 180 | "df_ratings_explicit.to_csv('data/ratings_explicit.csv', encoding='utf-8', index=False)\n", 181 | "df_ratings_implicit.to_csv('data/ratings_implicit.csv', encoding='utf-8', index=False)\n", 182 | "df_books.to_csv('data/books.csv', encoding='utf-8', index=False)\n", 183 | "df_users.to_csv('data/users.csv', encoding='utf-8', index=False)" 184 | ] 185 | }, 186 | { 187 | "cell_type": "markdown", 188 | "metadata": {}, 189 | "source": [ 190 | "## Reducing the dimensionality\n", 191 | "To reduce the dimensionality of the dataset and avoid running into memory error it will focus on users with at least 3 ratings and top 10% most frequently rated books. It consists of 176,594 records." 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": 11, 197 | "metadata": {}, 198 | "outputs": [ 199 | { 200 | "name": "stdout", 201 | "output_type": "stream", 202 | "text": [ 203 | "Filter: users with at least 3 ratings\n", 204 | "Number of records: 368563\n" 205 | ] 206 | } 207 | ], 208 | "source": [ 209 | "user_ratings_threshold = 3\n", 210 | "\n", 211 | "filter_users = df_ratings_explicit['user_id'].value_counts()\n", 212 | "filter_users_list = filter_users[filter_users >= user_ratings_threshold].index.to_list()\n", 213 | "\n", 214 | "df_ratings_top = df_ratings_explicit[df_ratings_explicit['user_id'].isin(filter_users_list)]\n", 215 | "\n", 216 | "print('Filter: users with at least %d ratings\\nNumber of records: %d' % (user_ratings_threshold, len(df_ratings_top)))" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": 12, 222 | "metadata": {}, 223 | "outputs": [ 224 | { 225 | "name": "stdout", 226 | "output_type": "stream", 227 | "text": [ 228 | "Filter: top 10% most frequently rated books\n", 229 | "Number of records: 176594\n" 230 | ] 231 | } 232 | ], 233 | "source": [ 234 | "book_ratings_threshold_perc = 0.1\n", 235 | "book_ratings_threshold = len(df_ratings_top['isbn'].unique()) * book_ratings_threshold_perc\n", 236 | "\n", 237 | "filter_books_list = df_ratings_top['isbn'].value_counts().head(int(book_ratings_threshold)).index.to_list()\n", 238 | "df_ratings_top = df_ratings_top[df_ratings_top['isbn'].isin(filter_books_list)]\n", 239 | "\n", 240 | "print('Filter: top %d%% most frequently rated books\\nNumber of records: %d' % (book_ratings_threshold_perc*100, len(df_ratings_top)))" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": 13, 246 | "metadata": {}, 247 | "outputs": [], 248 | "source": [ 249 | "df_ratings_top.to_csv('data/ratings_top.csv', encoding='utf-8', index=False)" 250 | ] 251 | } 252 | ], 253 | "metadata": { 254 | "kernelspec": { 255 | "display_name": "master", 256 | "language": "python", 257 | "name": "master" 258 | }, 259 | "language_info": { 260 | "codemirror_mode": { 261 | "name": "ipython", 262 | "version": 3 263 | }, 264 | "file_extension": ".py", 265 | "mimetype": "text/x-python", 266 | "name": "python", 267 | "nbconvert_exporter": "python", 268 | "pygments_lexer": "ipython3", 269 | "version": "3.7.6" 270 | } 271 | }, 272 | "nbformat": 4, 273 | "nbformat_minor": 2 274 | } 275 | -------------------------------------------------------------------------------- /collaborative-filtering-model-based.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Model Based Collaborative Filtering Recommender\n", 8 | "\n", 9 | "The goal of the **recommender system** is to predict user preference for a set of items based on the past experience. Two the most popular approaches are Content-Based and Collaborative Filtering.\n", 10 | "\n", 11 | "**Collaborative filtering** is a technique used by websites like Amazon, YouTube, and Netflix. It filters out items that a user might like on the basis of reactions of similar users. There are two categories of collaborative filtering algorithms: memory based and model based.\n", 12 | "\n", 13 | "**Model based approach** involves building machine learning algorithms to predict user's ratings. They involve dimensionality reduction methods that reduce high dimensional matrix containing abundant number of missing values with a much smaller matrix in lower-dimensional space.\n", 14 | "\n", 15 | "The goal of this exercise is to compare SVD and NMF algorithms, try different configurations of parameters and explore obtained results." 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 1, 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "import pandas as pd\n", 25 | "import numpy as np\n", 26 | "import seaborn as sns\n", 27 | "import matplotlib.pyplot as plt\n", 28 | "\n", 29 | "from surprise import Dataset, Reader\n", 30 | "from surprise import SVD, NMF\n", 31 | "from surprise.model_selection import cross_validate, train_test_split, GridSearchCV\n", 32 | "\n", 33 | "import functions as f" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "This analysis will focus on book recommendations based on [Book-Crossing dataset](http://www2.informatik.uni-freiburg.de/~cziegler/BX/). To reduce the dimensionality of the dataset and avoid running into memory error it will focus on users with at least 3 ratings and top 10% most frequently rated books. It consists of 176,594 records.\n", 41 | "\n", 42 | "The recommender systems will be built using [surprise package](https://surprise.readthedocs.io/en/stable/getting_started.html) (Matrix Factorization - based models)." 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 2, 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "df = pd.read_csv('data/ratings_top.csv')\n", 52 | "\n", 53 | "reader = Reader(rating_scale=(1, 10))\n", 54 | "data = Dataset.load_from_df(df[['user_id', 'isbn', 'book_rating']], reader)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 3, 60 | "metadata": {}, 61 | "outputs": [ 62 | { 63 | "name": "stdout", 64 | "output_type": "stream", 65 | "text": [ 66 | "Number of ratings: 176594\n", 67 | "Number of books: 16766\n", 68 | "Number of users: 20149\n" 69 | ] 70 | } 71 | ], 72 | "source": [ 73 | "print('Number of ratings: %d\\nNumber of books: %d\\nNumber of users: %d' % (len(df), len(df['isbn'].unique()), len(df['user_id'].unique())))" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "## SVD and NMF models comparison\n", 81 | "\n", 82 | "Singular Value Decomposition (SVD) and Non-negative Matrix Factorization (NMF) are matrix factorization techniques used for dimensionality reduction. Surprise package provides implementation of those algorithms.\n", 83 | "\n", 84 | "It's clear that for the given dataset much better results can be obtained with SVD approach - both in terms of accuracy and training / testing time." 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 15, 90 | "metadata": {}, 91 | "outputs": [ 92 | { 93 | "data": { 94 | "text/plain": [ 95 | "test_rmse 1.606926\n", 96 | "test_mae 1.242338\n", 97 | "fit_time 18.130412\n", 98 | "test_time 1.120190\n", 99 | "dtype: float64" 100 | ] 101 | }, 102 | "execution_count": 15, 103 | "metadata": {}, 104 | "output_type": "execute_result" 105 | } 106 | ], 107 | "source": [ 108 | "model_svd = SVD()\n", 109 | "cv_results_svd = cross_validate(model_svd, data, cv=3)\n", 110 | "pd.DataFrame(cv_results_svd).mean()" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 16, 116 | "metadata": {}, 117 | "outputs": [ 118 | { 119 | "data": { 120 | "text/plain": [ 121 | "test_rmse 2.640803\n", 122 | "test_mae 2.255504\n", 123 | "fit_time 22.795353\n", 124 | "test_time 1.005285\n", 125 | "dtype: float64" 126 | ] 127 | }, 128 | "execution_count": 16, 129 | "metadata": {}, 130 | "output_type": "execute_result" 131 | } 132 | ], 133 | "source": [ 134 | "model_nmf = NMF()\n", 135 | "cv_results_nmf = cross_validate(model_nmf, data, cv=3)\n", 136 | "pd.DataFrame(cv_results_nmf).mean()" 137 | ] 138 | }, 139 | { 140 | "cell_type": "markdown", 141 | "metadata": {}, 142 | "source": [ 143 | "## Optimisation of SVD algorithm\n", 144 | "\n", 145 | "Grid Search Cross Validation computes accuracy metrics for an algorithm on various combinations of parameters, over a cross-validation procedure. It's useful for finding the best configuration of parameters.\n", 146 | "\n", 147 | "It is used to find the best setting of parameters:\n", 148 | "* n_factors - the number of factors\n", 149 | "* n_epochs - the number of iteration of the SGD procedure\n", 150 | "* lr_all - the learning rate for all parameters\n", 151 | "* reg_all - the regularization term for all parameters\n", 152 | "\n", 153 | "As a result, regarding the majority of parameters, the default setting is the most optimal one. The improvement obtained with Grid Search is very small." 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": 17, 159 | "metadata": {}, 160 | "outputs": [ 161 | { 162 | "name": "stdout", 163 | "output_type": "stream", 164 | "text": [ 165 | "1.5981785240945765\n", 166 | "{'n_factors': 80, 'n_epochs': 20, 'lr_all': 0.005, 'reg_all': 0.2}\n" 167 | ] 168 | } 169 | ], 170 | "source": [ 171 | "param_grid = {'n_factors': [80,100,120],\n", 172 | " 'n_epochs': [5, 10, 20],\n", 173 | " 'lr_all': [0.002, 0.005],\n", 174 | " 'reg_all': [0.2, 0.4, 0.6]}\n", 175 | "\n", 176 | "gs = GridSearchCV(SVD, param_grid, measures=['rmse', 'mae'], cv=3)\n", 177 | "gs.fit(data)\n", 178 | "\n", 179 | "print(gs.best_score['rmse'])\n", 180 | "print(gs.best_params['rmse'])\n", 181 | "\n", 182 | "#1.5981785240945765\n", 183 | "#{'n_factors': 80, 'n_epochs': 20, 'lr_all': 0.005, 'reg_all': 0.2}" 184 | ] 185 | }, 186 | { 187 | "cell_type": "markdown", 188 | "metadata": {}, 189 | "source": [ 190 | "## Analysis of Collaborative Filtering model results\n", 191 | "\n", 192 | "In this part, let's examine in detail the results obtained by the SVD model that provided the best RMSE score." 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": 4, 198 | "metadata": {}, 199 | "outputs": [], 200 | "source": [ 201 | "trainset, testset = train_test_split(data, test_size=0.2)\n", 202 | "\n", 203 | "model = SVD(n_factors=80, n_epochs=20, lr_all=0.005, reg_all=0.2)\n", 204 | "model.fit(trainset)\n", 205 | "predictions = model.test(testset)" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": 5, 211 | "metadata": {}, 212 | "outputs": [ 213 | { 214 | "data": { 215 | "text/html": [ 216 | "
\n", 217 | "\n", 230 | "\n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | "
user_idisbnactual_ratingpred_ratingimpossiblepred_rating_roundabs_err
611824299903453745685.07.246858False7.02.246858
1718567840155166951X10.08.513183False9.01.486817
2131378553045140432710.09.083398False9.00.916602
2342310778403730314675.05.890978False6.00.890978
98999525003757256019.08.035049False8.00.964951
\n", 296 | "
" 297 | ], 298 | "text/plain": [ 299 | " user_id isbn actual_rating pred_rating impossible \\\n", 300 | "6118 242999 0345374568 5.0 7.246858 False \n", 301 | "17185 67840 155166951X 10.0 8.513183 False \n", 302 | "21313 78553 0451404327 10.0 9.083398 False \n", 303 | "23423 107784 0373031467 5.0 5.890978 False \n", 304 | "9899 95250 0375725601 9.0 8.035049 False \n", 305 | "\n", 306 | " pred_rating_round abs_err \n", 307 | "6118 7.0 2.246858 \n", 308 | "17185 9.0 1.486817 \n", 309 | "21313 9.0 0.916602 \n", 310 | "23423 6.0 0.890978 \n", 311 | "9899 8.0 0.964951 " 312 | ] 313 | }, 314 | "execution_count": 5, 315 | "metadata": {}, 316 | "output_type": "execute_result" 317 | } 318 | ], 319 | "source": [ 320 | "df_pred = pd.DataFrame(predictions, columns=['user_id', 'isbn', 'actual_rating', 'pred_rating', 'details'])\n", 321 | "\n", 322 | "df_pred['impossible'] = df_pred['details'].apply(lambda x: x['was_impossible'])\n", 323 | "df_pred['pred_rating_round'] = df_pred['pred_rating'].round()\n", 324 | "df_pred['abs_err'] = abs(df_pred['pred_rating'] - df_pred['actual_rating'])\n", 325 | "df_pred.drop(['details'], axis=1, inplace=True)\n", 326 | "\n", 327 | "df_pred.sample(5)" 328 | ] 329 | }, 330 | { 331 | "cell_type": "markdown", 332 | "metadata": {}, 333 | "source": [ 334 | "### Distribution of actual and predicted ratings in the test set\n", 335 | "\n", 336 | "According to the distribution of actual ratings of books in the test set, the biggest part of users give positive scores - between 7 and 10. The mode equals 8 but count of ratings 7, 9, 10 is also noticeable. The distribution of predicted ratings in the test set is visibly different. One more time, 8 is a mode but scores 7, 9 and 10 are clearly less frequent.\n", 337 | "\n", 338 | "It shows that the recommender system is not perfect and it cannot reflect the real distribution of book ratings." 339 | ] 340 | }, 341 | { 342 | "cell_type": "code", 343 | "execution_count": 6, 344 | "metadata": {}, 345 | "outputs": [ 346 | { 347 | "data": { 348 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0kAAAEXCAYAAAB4Yg/uAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZglZX33//dHFpFNQEaCAwgaJEGSIM4jJG4kKCIxookaeFwQSYgJJBp9fhGyPBCVBPOYGE0Ug4KAUUDFZWJQRBSJRpYBkUU0jIAyMMLgsLmh4Pf3R90NNT2ne7qnt9PT79d1navr3LV9q+qcuvt76q67UlVIkiRJkjqPmOsAJEmSJGmYmCRJkiRJUo9JkiRJkiT1mCRJkiRJUo9JkiRJkiT1mCRJkiRJUo9J0jRK8t4kfzNNy9olyQ+SbNTeX5TkD6Zj2W15n0ly+HQtbxLrfWuSO5N8b7bXPVFJXp3kyzO8juuS7D+T65iMJDskuTjJfUn+ccD405O8dQbWe0KSf5/iMp6Z5FvTGNOMbKs0k6x/JrTeOa1/klSSX2zD03a81rFO67O1x1ufaUJMkiYoyc1Jfty+dHcn+e8kr03y0D6sqtdW1VsmuKznjDdNVX23qrasqgenIfa1vrhV9fyqOmOqy55kHDsDbwT2rKpfmMbl7p9kxXQtb7oNOklV1ZOr6qI5CmmQo4A7ga2r6o1zHcxkVNV/VdUe6zPvTP8DMV3/XM7kZ3wi5yPNLeufqZup+md9TeJ4TWuCOlXWZzPL+mzKy57W+swkaXJ+p6q2Ah4PnAS8CTh1uleSZOPpXuaQeDzw/aq6Y64DmS4b0LF6PPCN8unS0rCy/pmaaa1/Rq6ybUg2oGNvfabpUVW+JvACbgaeM6rsacDPgb3a+9OBt7bh7YFPA3cDq4H/oktKP9jm+THwA+AvgF2BAo4Evgtc3CvbuC3vIuDvgcuAe4BPAdu1cfsDKwbFCxwE/BT4WVvf13vL+4M2/Ajgr4HvAHcAZwKPbuNG4ji8xXYn8Ffj7KdHt/lXteX9dVv+c9o2/7zFcfqAebdt+2wVcFcb3qk3fjvgA8BtbfwngS1GLfcHwOP6x2LQPgKOBb4N3Ad8A3hxb9yrgS+PsX1rHatW/lHge+3YXAw8uZUf1fb9T1ts/zH68wScAHyk7bf7gOuAJb117gN8rY37KHAO6/icjRH7bwCXtxgvB36j97ntx/icAfOeDrwXuKDF8SXg8etadhv3OGBpi2858Ie9cScA/96GNwHOAs4FNqX7fi0D7gVuB/5pjO0afWxvBv4PcHWL5xxgswHz/TLwE+DBtt1397b13cB/tm29FHhib75favthNfAt4GVjxHViW/ZP2vL/dV3zAwfTfR7vA25t2zHwMz5gfWvN2xv3AuCq9jn5b+BXW/la56O5Ptf6GvhZunn09xLrn0H7aSr1z/7ACuAv23puBl7eG386cDJwHvDDtsxHAm9vsd1Od458VG+e/w9YSVdnvaZtyy+OPl7t/SF039F76eqmg1i/c8hj6M6397bj9Rasz0bPezrWZyPzWZ+Nd+6d7pP5hvpiQCXVyr8L/HHvwzjyZf/79iXcpL2eCWTQsnj4RHVm+wA9isGV1K3AXm2ac3tfxjW+VKPXQe+L2xt/EQ9XUq+h+7I/AdgS+DjwwVGxva/F9WvA/cAvj7GfzqSrQLdq8/4PcORYcY6a9zHA7wGbt/k/CnyyN/4/6U4Q27Z9+uxxtv+hYzFoGuCldCe7RwC/T1fp7djGvZp1VyoPHavePtyKrtL8Z+CqsWIZ4/j8hO6ksBHdZ+eSNm5Tusr+dW2bf5fu5L/Oz9mo9W1Hl1i+EtgYOKy9f8xYMQ7Yn/cBz2rb+M6RfTSBZX8JeA+wGbA33T8wB/Q/m3Sfrf9s69mojfsq8Mo2vCWw3xixjT62N9P9c/C4Ftv1wGvHmHetY91iWE1XqW0MfAg4u43bArgFOKKN24fuH6onj7H8i2jfs4nMT/cP1TPb8LbAPhP57qxj3n3o/vncl+7zdXjbR48c79zma3heYx0jrH9G74+p1D/7Aw8A/0R3jns2Xb2wR2//3gM8na7e2IzuXL+U7jyzFfAfwN+36Q+i+2d4ZJ99mDGSJLpzzT3Ac9uyFwO/NHpftffrOoecTZekbNHWfSvWZ6PnPx3rM7A+W+fL5nZTdxvdB3e0nwE70v068bPq2pnWOpZ1QlX9sKp+PMb4D1bVtVX1Q+BvgJdN0yX/l9P9qnFjVf0AOA44dNSl97+tqh9X1deBr9NVVmtosfw+cFxV3VdVNwP/SHeyWaeq+n5VnVtVP6qq++h+uXh2W/aOwPPpTg53tX36pfXd4Kr6aFXdVlU/r6pzgBvoTiITtcaxqqrT2jbfT3ei/LUkj57E8r5cVedVdw/AB3l4/+5Hd/J5V9vmj9OdMEdM9HP228ANVfXBqnqgqs4Cvgn8ziRi/M+qurht418Bv97a+Y+57Db+GcCbquonVXUV8H7W/ExsDXyW7tfTI+rh+yB+Bvxiku2r6gdVdckkYn1XO76r6f5x2XsS8wJ8vKouq6oH6CqVkflfANxcVR9o23ol3T+ML5ngctc1/8+APZNs3T7nV04i5rHm/UPg36rq0qp6sLp7Qe6n+2xpfrP+aaZa//T8TVXd3+qX/wRe1hv3qar6SlX9nO479IfAn1fV6lZn/R1waJv2ZcAHevvshHHWeSRwWlVd0OqkW6vqm2NMO+Y5pO2D3wP+bzuW1wITuffL+sz6zPpsAJOkqVtMl6WP9v/ofh37XJIbkxw7gWXdMonx36H7pWX7CUU5vse15fWXvTGwQ6+s3xvQj+h+CRltex7+pai/rMUTCSLJ5kn+Lcl3ktxLd5l/m3bi3xlYXVV3TWRZE1jXq5Jc1W6CvpvuF7fJ7MuHjkWSjZKclOTbLe6b26jJLG/0/t2s/ZPwOODWURVF/3Mw0c/Z6GMMkzg2o9fb/plZ3ZY73rIfR3fc7htnvfsBvwqcNGo7jwSeBHwzyeVJXjCJWCfyeV2f+R8P7DvyuWmfnZcDE70RfF3z/x7dL7DfSfKlJL8+iZjHmvfxwBtHrXNnumOj+c3652FTqn+au1pC05+//z3p74NFdK0eruh9rz7bymnzjd5nY9mZ7p/qiRjvHLKIbt9NdL0jrM+sz6zPBjBJmoIk/4vuy7FWbyLtV5g3VtUT6H7deEOSA0ZGj7HIdf3St3NveBe6TPtOuiYBm/fi2oiHT9QTWe5tdB+8/rIfoGsqMBl3tphGL+vWCc7/RmAPYN+q2pruUjhA6E5o2yXZZsB8g7ZvjX1C70uf5PF0zTeOobuEvg1wbVvPRPXX+b/p2pM/h65N/K69uMeKb6JWAouT9GN76HOwjs9Z3+hjDJM7NmusN8mWdL9g37aOZd9Gd9y2Gme9n6NrZnFhkof+MaqqG6rqMOCxwNuAjyXZYhLxTsRkj80twJeqapvea8uq+uMJLn/c+avq8qo6hG6bP0nXbGZCcY4z7y3AiaPWuXn7hXRCy9bwsf5Zy1TrH4BtR51jdmnxjehvy5109z48ufe9enRVjfwDupK199lYbgGeOMa4yZxDVtHtu4mud9A6rM+sz6zPGpOk9ZBk6/YrwNl0ba2vGTDNC5L8YjsZ3Et3w9vIZdfb6dpfT9YrkuyZZHPgzcDH2qXc/6H7pea3k2xCd7PqI3vz3Q7sml53saOcBfx5kt3ayeLvgHPapdkJa7F8BDgxyVYtGXkDXRvdidiKrtK5O8l2wPG9Za8EPgO8J8m2STZJMpJE3Q48ZlRzgKuAg5Nsl+QXgNf3xm1B90VaBZDkCLorSetrK7rLvd+n+2fh70aNX9/jDV075geBY5JsnOQQes0C1/E56zsPeFKS/92W8/vAnnQ3yU7UwUmekWRTupuBL62qW8Zbdhv/38DfJ9ksya/S/aL2of6Cq+of6NrsX5hk+7Ztr0iyqLqmLXe3SafcJfEotwM7tW2aiE/Tbesr22dwkyT/K8kvj7P8J0xk/iSbJnl5kkdX1c94+HiOLGf0Z/wh65j3fcBrk+ybzhbtXLFVb9nr+/nULLP+GWwa6p8Rf9u+T8+ka0700THW93O679Y7kjwWIMniJM9rk3wEeHVvnx0/aDnNqcARSQ5I8oi2nF9q4yZ8Dmn74OPACelaZuxJd8/GZFifWZ9ZnzUmSZPzH0nuo8tk/4ruBs8jxph2d+DzdD1sfBV4Tz38HIG/B/463aXC/zOJ9X+Q7ia879HdNPhnAFV1D/AndG1jb6X7Za/fB/3ISf77SQa1CT2tLfti4Ca6my7/dBJx9f1pW/+NdL9wfrgtfyL+me6GxzuBS+iaLvS9ku6Xwm/S3bj3eoDq2m6fBdzY9unj2vZ8na6pwOfoOnygTf8NurbqX6X7Qv0K8JXJbeYazqS75H4rXW8so9san0rXtvbuJJ+czIKr6qd0N7ceSXdifQXdien+Nsl4n7P+cr5PV+G/ka7y+wvgBVV15yTC+TBdRb8aeCrdZfWJLPswul8jbwM+ARxfVRcMiPEtdL8YfT5dknwQcF2SH9DdWHtoVf1kEvFOxBfoel/6XpJ17ovWzOJAuvsObqP7Lr6NNf8p7Hsn3b0CdyV51wTmfyVwc7pmLq+lO95jfcZHG2veZXTtuP+V7gbk5XQ3+I5Y3/ORZpf1z7pNpf6BbtvuovtufojuHtix7g2Crhv25cAl7Xv3ebrWEFTVZ+jqtC+0ab4w1kKq6jK6Y/kOug4cvsTDVzMmew45hq451ffojtcHJr75gPWZ9Zn12UNGeruRNE8kuRR4b1VNtvKTJA2QZH+6K3M7zXUsC4n1mYaZV5KkIZfk2Ul+oV36P5zuptDRV9kkSRpq1meaTzaUpytLG7I96Nq3b0nXA9JL2j1akiTNJ9ZnmjdsbidJkiRJPTa3kyRJkqSeDbK53fbbb1+77rrrXIchSQvaFVdccWdVLVr3lAuP9ZQkDYex6qoNMknaddddWbZs2VyHIUkLWpLRT65XYz0lScNhrLrK5naSpA1akp2TfDHJ9UmuS/K6Vr5dkguS3ND+btvKk+RdSZYnuTrJPr1lHd6mv6H1zjVS/tQk17R53tUeiClJmqdMkiRJG7oHgDdW1S8D+wFHJ9kTOBa4sKp2By5s7wGeT/dgy92Bo4CToUuq6B5AuS/wNOD4kcSqTXNUb76DZmG7JEkzxCRJkrRBq6qVVXVlG74PuB5YDBwCnNEmOwN4URs+BDizOpcA2yTZEXgecEFVra6qu4ALgIPauK2r6qvVdRl7Zm9ZkqR5yCRJkrRgJNkVeApwKbDDyDNa2t/HtskWA7f0ZlvRysYrXzGgfPS6j0qyLMmyVatWTcfmSJJmiEmSJGlBSLIlcC7w+qq6d7xJB5TVepSvWVB1SlUtqaolixbZ6Z8kDTOTJEnSBi/JJnQJ0oeq6uOt+PbWVI72945WvgLYuTf7TsBt6yjfaUC5JGmeMkmSJG3QWk9zpwLXV9U/9UYtBUZ6qDsc+FSv/FWtl7v9gHtac7zzgQOTbNs6bDgQOL+Nuy/Jfm1dr+otS5I0D22Qz0mSJKnn6cArgWuSXNXK/hI4CfhIkiOB7wIvbePOAw4GlgM/Ao4AqKrVSd4CXN6me3NVrW7DfwycDjwK+Ex7SZLmKZMkSdIGraq+zOD7hgAOGDB9AUePsazTgNMGlC8D9ppCmJKkIWKSJGle+/tPL5v1dR73giWzvk5JUucnd9461yGs02bbr9XBpeYZ70mSJEmSpB6TJEmSJEnqMUmSJEmSpB6TJEmSJEnqMUmSJEmSpB6TJEmSJEnqMUmSJEmSpB6TJEmSJEnqMUmSJEmSpB6TJEmSJEnqMUmSJEmSpB6TJEmSJEnqMUmSJEmSpB6TJEmSJEnqmdEkKcmfJ7kuybVJzkqyWZLdklya5IYk5yTZtE37yPZ+eRu/a285x7XybyV53kzGLEmSJGlhm7EkKcli4M+AJVW1F7ARcCjwNuAdVbU7cBdwZJvlSOCuqvpF4B1tOpLs2eZ7MnAQ8J4kG81U3JIkSZIWtplubrcx8KgkGwObAyuB3wI+1safAbyoDR/S3tPGH5Akrfzsqrq/qm4ClgNPm+G4JUkbiCSnJbkjybW9snOSXNVeNye5qpXvmuTHvXHv7c3z1CTXtJYN72p1FEm2S3JBayFxQZJtZ38rJUnTacaSpKq6FXg78F265Oge4Arg7qp6oE22AljchhcDt7R5H2jTP6ZfPmCehyQ5KsmyJMtWrVo1/RskSZqvTqdrifCQqvr9qtq7qvYGzgU+3hv97ZFxVfXaXvnJwFHA7u01ssxjgQtbC4kL23tJ0jw2k83ttqW7CrQb8DhgC+D5AyatkVnGGDdW+ZoFVadU1ZKqWrJo0aL1C1qStMGpqouB1YPGtatBLwPOGm8ZSXYEtq6qr1ZVAWcyuCVEv4WEJGmemsnmds8BbqqqVVX1M7pf6X4D2KY1vwPYCbitDa8AdgZo4x9NV6k9VD5gHkmSpuKZwO1VdUOvbLckX0vypSTPbGWL6eqjEf1WDTtU1UqA9vexMx20JGlmzWSS9F1gvySbt1/qDgC+AXwReEmb5nDgU214aXtPG/+F9mvdUuDQ1vvdbnRNHC6bwbglSQvHYax5FWklsEtVPQV4A/DhJFszwVYN47FZuCTNHzN5T9KldB0wXAlc09Z1CvAm4A1JltPdc3Rqm+VU4DGt/A20Nt1VdR3wEboE67PA0VX14EzFLUlaGFqrhd8Fzhkpa50Efb8NXwF8G3gS3ZWjnXqz91s13N6a4400y7tj0PpsFi5J88fG655k/VXV8cDxo4pvZEDvdFX1E+ClYyznRODEaQ9QkrSQPQf4ZlU91IwuySJgdVU9mOQJdK0Xbqyq1UnuS7IfcCnwKuBf2mwjLSFOYs0WEpKkeWqmuwCXJGlOJTkL+CqwR5IVSUaez3coa3fY8Czg6iRfp2sN8dqqGun04Y+B99M9iuLbwGda+UnAc5PcADy3vZckzWMzeiVJkqS5VlWHjVH+6gFl59J1CT5o+mXAXgPKv093360kaQPhlSRJkiRJ6jFJkiRJkqQekyRJkiRJ6jFJkiRJkqQekyRJkiRJ6jFJkiRJkqQekyRJkiRJ6jFJkiRJkqQekyRJkiRJ6jFJkiRJkqQekyRJkiRJ6jFJkiRJkqQekyRJkiRJ6jFJkiRJkqQekyRJkiRJ6jFJkiRJkqQekyRJkiRJ6jFJkiRJkqQekyRJ0gYtyWlJ7khyba/shCS3JrmqvQ7ujTsuyfIk30ryvF75Qa1seZJje+W7Jbk0yQ1Jzkmy6extnSRpJpgkSZI2dKcDBw0of0dV7d1e5wEk2RM4FHhym+c9STZKshHwbuD5wJ7AYW1agLe1Ze0O3AUcOaNbI0macSZJkqQNWlVdDKye4OSHAGdX1f1VdROwHHhaey2vqhur6qfA2cAhSQL8FvCxNv8ZwIumdQMkSbNu47kOQJI0M/Z+zbtnfZ1XnXb0rK9zCo5J8ipgGfDGqroLWAxc0ptmRSsDuGVU+b7AY4C7q+qBAdOvIclRwFEAu+yyy3RtgyRpBnglSZK0EJ0MPBHYG1gJ/GMrz4Bpaz3K1y6sOqWqllTVkkWLFk0+YknSrPFKkiRpwamq20eGk7wP+HR7uwLYuTfpTsBtbXhQ+Z3ANkk2bleT+tNLkuYpryRJkhacJDv23r4YGOn5bilwaJJHJtkN2B24DLgc2L31ZLcpXecOS6uqgC8CL2nzHw58aja2QZI0c7ySJEnaoCU5C9gf2D7JCuB4YP8ke9M1jbsZ+COAqrouyUeAbwAPAEdX1YNtOccA5wMbAadV1XVtFW8Czk7yVuBrwKmztGmSpBlikiRJ2qBV1WEDisdMZKrqRODEAeXnAecNKL+Rrvc7SdIGwuZ2kiRJktRjkiRJkiRJPSZJkiRJktRjkiRJkiRJPSZJkiRJktRjkiRJkiRJPSZJkiRJktRjkiRJkiRJPSZJkiRJktQzo0lSkm2SfCzJN5Ncn+TXk2yX5IIkN7S/27Zpk+RdSZYnuTrJPr3lHN6mvyHJ4TMZsyRJkqSFbeMZXv47gc9W1UuSbApsDvwlcGFVnZTkWOBY4E3A84Hd22tf4GRg3yTbAccDS4ACrkiytKrumuHYJWnSXnjSJ2Z1fUuPffGsrk+SpIVgxq4kJdkaeBZwKkBV/bSq7gYOAc5ok50BvKgNHwKcWZ1LgG2S7Ag8D7igqla3xOgC4KCZiluSJEnSwjaTze2eAKwCPpDka0nen2QLYIeqWgnQ/j62Tb8YuKU3/4pWNlb5GpIclWRZkmWrVq2a/q2RJEmStCDMZJK0MbAPcHJVPQX4IV3TurFkQFmNU75mQdUpVbWkqpYsWrRofeKVJEmSpBm9J2kFsKKqLm3vP0aXJN2eZMeqWtma093Rm37n3vw7Abe18v1HlV80g3FLWoeLv3XrrK7vWXusdfFYkiRpxszYlaSq+h5wS5I9WtEBwDeApcBID3WHA59qw0uBV7Ve7vYD7mnN8c4HDkyybesJ78BWJkmSJEnTbqZ7t/tT4EOtZ7sbgSPoErOPJDkS+C7w0jbtecDBwHLgR21aqmp1krcAl7fp3lxVq2c4bkmSJEkL1IwmSVV1FV3X3aMdMGDaAo4eYzmnAadNb3SSJEmStLYZfZisJElzLclpSe5Icm2v7P+1B51fneQTSbZp5bsm+XGSq9rrvb15nprkmvbQ83clSSsf+JB0SdL8ZZIkSdrQnc7az9e7ANirqn4V+B/guN64b1fV3u312l75ycBRPPzg85FlHkv3kPTdgQsZvydXSdI8YJIkSdqgVdXFwOpRZZ+rqgfa20voek4dU+uNdeuq+mprHn4maz4MfdBD0iVJ85RJkiRpoXsN8Jne+93aQ9C/lOSZrWwx3SMpRvQfbD7WQ9LX4EPPJWn+MEmSJC1YSf4KeAD4UCtaCezSHoL+BuDDSbZmgg82H48PPZek+WOmuwCXJGkoJTkceAFwQGtCR1XdD9zfhq9I8m3gSXRXjvpN8kYeeA5jPyRdkjRPeSVJkrTgJDkIeBPwwqr6Ua98UZKN2vAT6DpouLE1o7svyX6tV7tXsebD0Ac9JF2SNE95JUmStEFLchawP7B9khXA8XS92T0SuKD15H1J68nuWcCbkzwAPAi8tvcA8z+m6ynvUXT3MI3cx3QSgx+SLkmap0ySJEkbtKo6bEDxqWNMey5w7hjjlgF7DSj/PgMeki5Jmr9sbidJkiRJPSZJkiRJktRjkiRJkiRJPSZJkiRJktRjkiRJkiRJPSZJkiRJktRjkiRJkiRJPRNKkpJcOJEySZJminWRJGm2jPsw2SSbAZvTPaV8WyBt1NbA42Y4NkmSrIskSbNu3CQJ+CPg9XSV0BU8XDHdC7x7BuOSJGmEdZEkaVaNmyRV1TuBdyb506r6l1mKSZKkh1gXSZJm27quJAFQVf+S5DeAXfvzVNWZMxSXJElrsC6SJM2WCSVJST4IPBG4CniwFRdgxSRJmhXWRZKk2TKhJAlYAuxZVTWTwUiSNA7rIknSrJjoc5KuBX5hJgORJGkdrIskSbNioleStge+keQy4P6Rwqp64YxEJUnS2qyLJEmzYqJJ0gkzGYQkSRNwwlwHIElaGCbau92XZjoQSZLGs751UZLTgBcAd1TVXq1sO+Acup7ybgZeVlV3JQnwTuBg4EfAq6vqyjbP4cBft8W+tarOaOVPBU4HHgWcB7zO+6YkaX6b0D1JSe5Lcm97/STJg0nunengJEkaMYW66HTgoFFlxwIXVtXuwIXtPcDzgd3b6yjg5Lbu7YDjgX2BpwHHJ9m2zXNym3ZkvtHrkiTNMxO9krRV/32SF9FVEpIkzYr1rYuq6uIku44qPgTYvw2fAVwEvKmVn9muBF2SZJskO7ZpL6iq1W3dFwAHJbkI2LqqvtrKzwReBHxm0hsoSRoaE+3dbg1V9Ungt6Y5FkmSJmyKddEOVbWyLWcl8NhWvhi4pTfdilY2XvmKAeVrSXJUkmVJlq1atWo9w5YkzYaJPkz2d3tvH0H3rArbW0uSZs0s1UUZUFbrUb52YdUpwCkAS5YssQ6VpCE20d7tfqc3/ADdTa6HTHs0kiSNbTrrotuT7FhVK1tzujta+Qpg5950OwG3tfL9R5Vf1Mp3GjC9JGkem+g9SUfMdCCSJI1nmuuipcDhwEnt76d65cckOZuuk4Z7WiJ1PvB3vc4aDgSOq6rVrUOJ/YBLgVcB/zKNcUqS5sBEe7fbKcknktyR5PYk5ybZad1zSpI0Pda3LkpyFvBVYI8kK5IcSZccPTfJDcBz23vouvC+EVgOvA/4E4DWYcNbgMvb680jnTgAfwy8v83zbey0QZLmvYk2t/sA8GHgpe39K1rZc2ciKEmSBlivuqiqDhtj1AEDpi3g6DGWcxpw2oDyZcBe48UgSZpfJpokLaqqD/Ten57k9TMRkCRJY7AukqbgnmXnz3UI43r0kufNdQjSQybaBfidSV6RZKP2egXw/ZkMTJKkUayLJEmzYqJJ0muAlwHfA1YCLwHszEGSNJusiyRJs2KiSdJbgMOralFVPZauojphIjO2X/u+luTT7f1uSS5NckOSc5Js2sof2d4vb+N37S3juFb+rSRei5WkhWm96yJJkiZjoknSr1bVXSNvWo8+T5ngvK8Dru+9fxvwjqraHbgLOLKVHwncVVW/CLyjTUeSPYFDgScDBwHvSbLRBNctSdpwTKUukiRpwiaaJD2i92wIkmzHBDp9aF2z/jZd16gkCfBbwMfaJGcAL2rDh7T3tPEHtOkPAc6uqvur6ia6LlafNsG4JUkbjvWqiyRJmqyJVi7/CPx3ko8BRdcm/MQJzPfPwF8AW7X3jwHurqoH2vsVwOI2vBi4BaCqHkhyT5t+MXBJb5n9eSRJC8f61kWSJE3KhJKkqjozyTK6q0ABfreqvjHePEleANxRVVck2X+keNDi1zFuvHn66zsKOApgl112GS80SdI8tD51kSRJ62PCzRRaRTSZyujpwAuTHAxsBmxNd2VpmyQbt6tJOwG3telXADsDK5JsDDwaWN0rH9Gfpx/fKcApAEuWLFkriZIkzanJG5QAABXWSURBVH/rURdJkjRpE70nadKq6riq2qmqdqXreOELVfVy4It03bYCHA58qg0vbe9p47/Qnny+FDi09X63G7A7cNlMxS1JkiRpYZuLG17fBJyd5K3A14BTW/mpwAeTLKe7gnQoQFVdl+QjdL8cPgAcXVUPzn7YkiRJkhaCWUmSquoi4KI2fCMDeqerqp8ALx1j/hPx5lxJkiRJs2DGmttJkiRJ0nxkkiRJkiRJPSZJkiRJktRjkiRJkiRJPXPRu500L/zkzltndX2bbb94VtcnSZKkwbySJEmSJEk9JkmSpAUpyR5Jruq97k3y+iQnJLm1V35wb57jkixP8q0kz+uVH9TKlic5dm62SJI0XWxuJ0lakKrqW8DeAEk2Am4FPgEcAbyjqt7enz7JnnQPOn8y8Djg80me1Ea/G3gusAK4PMnSqvrGrGyIJGnamSRJkgQHAN+uqu8kGWuaQ4Czq+p+4KYky3n44ejL28PSSXJ2m9YkSZLmKZvbSZLUXSE6q/f+mCRXJzktybatbDFwS2+aFa1srPI1JDkqybIky1atWjW90UuSppVJkiRpQUuyKfBC4KOt6GTgiXRN8VYC/zgy6YDZa5zyNQuqTqmqJVW1ZNGiRVOOW5I0c2xuJ0la6J4PXFlVtwOM/AVI8j7g0+3tCmDn3nw7Abe14bHKJUnzkFeSJEkL3WH0mtol2bE37sXAtW14KXBokkcm2Q3YHbgMuBzYPclu7arUoW1aSdI85ZUkSdKClWRzul7p/qhX/A9J9qZrMnfzyLiqui7JR+g6ZHgAOLqqHmzLOQY4H9gIOK2qrpu1jZAkTTuTJEnSglVVPwIeM6rsleNMfyJw4oDy84Dzpj1ASdKcsLmdJEmSJPWYJEmSJElSj0mSJEmSJPWYJEmSJElSj0mSJEmSJPWYJEmSJElSj0mSJEmSJPWYJEmSJElSjw+TleaB2++6b9bXucO2W836OiVJkoaBV5IkSZIkqcckSZIkSZJ6TJIkSZIkqcckSZIkSZJ6TJIkSZIkqcckSZIkSZJ6TJIkSZIkqcckSZIkSZJ6TJIkSQtWkpuTXJPkqiTLWtl2SS5IckP7u20rT5J3JVme5Ook+/SWc3ib/oYkh8/V9kiSpodJkiRpofvNqtq7qpa098cCF1bV7sCF7T3A84Hd2+so4GTokirgeGBf4GnA8SOJlSRpfjJJkiRpTYcAZ7ThM4AX9crPrM4lwDZJdgSeB1xQVaur6i7gAuCg2Q5akjR9TJIkSQtZAZ9LckWSo1rZDlW1EqD9fWwrXwzc0pt3RSsbq1ySNE9tPNcBSJI0h55eVbcleSxwQZJvjjNtBpTVOOVrztwlYUcB7LLLLusTqyRplnglSZK0YFXVbe3vHcAn6O4pur01o6P9vaNNvgLYuTf7TsBt45SPXtcpVbWkqpYsWrRoujdFkjSNZuxKUpKdgTOBXwB+DpxSVe9sN7ieA+wK3Ay8rKruShLgncDBwI+AV1fVlW1ZhwN/3Rb91qo6A0mSpiDJFsAjquq+Nnwg8GZgKXA4cFL7+6k2y1LgmCRn03XScE9VrUxyPvB3vc4aDgSOm8VNkTRP3X7XfXMdwjrtsO1Wcx3CnJjJ5nYPAG+sqiuTbAVckeQC4NV0vQadlORYul6D3sSavQbtS9dr0L69XoOW0DVfuCLJ0nZzrCRJ62sH4BPdb3RsDHy4qj6b5HLgI0mOBL4LvLRNfx7dD3nL6X7MOwKgqlYneQtweZvuzVW1evY2Q5I03WYsSWo3u47c+HpfkuvpbmQ9BNi/TXYGcBFdkvRQr0HAJUlGeg3an9ZrEEBLtA4Czpqp2CVJG76quhH4tQHl3wcOGFBewNFjLOs04LTpjlGSNDdm5Z6kJLsCTwEuxV6DJEmSJA2xGU+SkmwJnAu8vqruHW/SAWWT6jUoybIky1atWrV+wUqSJEla8GY0SUqyCV2C9KGq+ngrttcgSZIkSUNrxpKk1lvdqcD1VfVPvVEjvQbB2r0GvSqd/Wi9BgHnAwcm2bb1HHRgK5MkSZKkaTeTvds9HXglcE2Sq1rZX9J1qWqvQZIkSZKG0kz2bvdlBt9PBPYaJEmSJGlIzUrvdpIkSZI0X5gkSZIkSVKPSZIkSZIk9ZgkSZIkSVKPSZIkSZIk9ZgkSZIkSVKPSZIkSZIk9ZgkSZIkSVKPSZIkSZIk9ZgkSZIkSVKPSZIkSZIk9ZgkSZIkSVKPSZIkaUFKsnOSLya5Psl1SV7Xyk9IcmuSq9rr4N48xyVZnuRbSZ7XKz+olS1PcuxcbI8kafpsPNcBSJI0Rx4A3lhVVybZCrgiyQVt3Duq6u39iZPsCRwKPBl4HPD5JE9qo98NPBdYAVyeZGlVfWNWtkKSNO1MkiRJC1JVrQRWtuH7klwPLB5nlkOAs6vqfuCmJMuBp7Vxy6vqRoAkZ7dpTZIkaZ6yuZ0kacFLsivwFODSVnRMkquTnJZk21a2GLilN9uKVjZW+eh1HJVkWZJlq1atmuYtkCRNJ5MkSdKClmRL4Fzg9VV1L3Ay8ERgb7orTf84MumA2Wuc8jULqk6pqiVVtWTRokXTErskaWbY3E6StGAl2YQuQfpQVX0coKpu741/H/Dp9nYFsHNv9p2A29rwWOWSpHnIK0mSpAUpSYBTgeur6p965Tv2JnsxcG0bXgocmuSRSXYDdgcuAy4Hdk+yW5JN6Tp3WDob2yBJmhleSZIkLVRPB14JXJPkqlb2l8BhSfamazJ3M/BHAFV1XZKP0HXI8ABwdFU9CJDkGOB8YCPgtKq6bjY3RJI0vUySJEkLUlV9mcH3E503zjwnAicOKD9vvPkkSfOLze0kSZIkqcckSZIkSZJ6TJIkSZIkqcckSZIkSZJ6TJIkSZIkqcckSZIkSZJ67AJcQ+OeZefP+jofveR5s75OSZIkDTevJEmSJElSj1eSJEmSxvDtf3vLXIewTk/8o7+Z6xCkDY5XkiRJkiSpxytJC9xs/0Lmr12SJEkadl5JkiRJkqQekyRJkiRJ6jFJkiRJkqQekyRJkiRJ6jFJkiRJkqQee7ebZV884tBZX+dvfuDsWV+nJEmSNF95JUmSJEmSeubNlaQkBwHvBDYC3l9VJ0103n/75f1mLK5B/uj6S2Z1fZKkuTeVekqSNFzmRZKUZCPg3cBzgRXA5UmWVtU35jYySZKsp0abi6blk2VTdGl6XfytW+c6hHE9a4/Fk5p+vjS3exqwvKpurKqfAmcDh8xxTJIkjbCekqQNSKpqrmNYpyQvAQ6qqj9o718J7FtVx/SmOQo4qr3dA/jWNKx6e+DOaVjOdBmmeIYpFhiueIYpFhiueIYpFhiueIYpFpieeB5fVYumI5hhN4f11LoM2+dqKjaUbdlQtgPclmG0oWwHzN62DKyr5kVzOyADytbI7qrqFOCUaV1psqyqlkznMqdimOIZplhguOIZplhguOIZplhguOIZplhg+OKZB+aknlqXDek4bijbsqFsB7gtw2hD2Q6Y+22ZL83tVgA7997vBNw2R7FIkjSa9ZQkbUDmS5J0ObB7kt2SbAocCiyd45gkSRphPSVJG5B50dyuqh5IcgxwPl3XqqdV1XWzsOpZbRYxAcMUzzDFAsMVzzDFAsMVzzDFAsMVzzDFAsMXz1Cbw3pqXTak47ihbMuGsh3gtgyjDWU7YI63ZV503CBJkiRJs2W+NLeTJEmSpFlhkiRJkiRJPQs+SUpyWpI7klw7xvgkeVeS5UmuTrLPDMayc5IvJrk+yXVJXjfH8WyW5LIkX2/x/O2AaR6Z5JwWz6VJdp2peNr6NkrytSSfHoJYbk5yTZKrkiwbMH42j9U2ST6W5Jvt8/PrcxjLHm2fjLzuTfL6OYznz9vn99okZyXZbNT42f7cvK7Fct3o/dLGz+i+GXTOS7JdkguS3ND+bjvGvIe3aW5Icvh0xqWpGaZz41QM03l1qobpvDwVw3ZOn4phqw+mYq7rkqmYN/VQVS3oF/AsYB/g2jHGHwx8hu4ZGPsBl85gLDsC+7ThrYD/Afacw3gCbNmGNwEuBfYbNc2fAO9tw4cC58zw8XoD8GHg0wPGzXYsNwPbjzN+No/VGcAftOFNgW3mKpZR690I+B7dg9pmPR5gMXAT8Kj2/iPAq+fqcwPsBVwLbE7Xcc7ngd1nc98MOucB/wAc24aPBd42YL7tgBvb323b8Laz8TnyNaHjOjTnxilux9CcV6dhW4byvDzFbZrTc/oUYx+q+mCK2zLndckU458X9dCCv5JUVRcDq8eZ5BDgzOpcAmyTZMcZimVlVV3Zhu8Drqf7Us9VPFVVP2hvN2mv0T19HEJXEQB8DDggyaCHKk5Zkp2A3wbeP8YksxbLBM3KsUqyNd0J51SAqvppVd09F7EMcADw7ar6zhzGszHwqCQb01Uoo59dM5ufm18GLqmqH1XVA8CXgBcPiGfG9s0Y57z+PjgDeNGAWZ8HXFBVq6vqLuAC4KDpikvrbx6eG6dirs5lkzLk5+WpGIZz+lQMU30wFXNel0zFfKmHFnySNAGLgVt671ewduIy7dol3qfQXb2Zs3haE46rgDvoPphjxtO+qPcAj5mhcP4Z+Avg52OMn81YoEsYP5fkiiRHjRdPM1PH6gnAKuADrbnN+5NsMUexjHYocNaA8lmJp6puBd4OfBdYCdxTVZ8bK5ZZ+NxcCzwryWOSbE73S9/Oo6aZi2O1Q1WthO7HGuCxA6aZq8+Q1m3Yzo1TMSzn1aka5vPyVMzpOX0qhrA+mIphrUumYujqIZOkdRv0C8KM9pueZEvgXOD1VXXvXMZTVQ9W1d50T49/WpK95iKeJC8A7qiqK8abbDZi6Xl6Ve0DPB84Osmz5iiejekuW59cVU8Bfkh3qXouYnl4hd0DNV8IfHTQ6NmIp7VpPgTYDXgcsEWSV8xFLABVdT3wNrpfvz4LfB14YK7imaRhjWtBG9Jz41QMy3l1qobyvDwVw3BOn4phqw+mYp7XJVMxq9tkkrRuK1gzO9+JtS/PTpskm9AlSB+qqo/PdTwjWjOBi1j7suZD8bTL149m/OaL6+vpwAuT3AycDfxWkn+fo1gAqKrb2t87gE8ATxsrnmamjtUKYEXvKt/H6CrnuYil7/nAlVV1+4BxsxXPc4CbqmpVVf0M+DjwG2PFMkufm1Orap+qelZbzw1jxdPMxrG6faQZRvt7x4Bp5uTco3UaunPjVAzReXWqhvW8PBXDcE6fiqGrD6ZiSOuSqRi6esgkad2WAq9qvYTsR3d5duVMrKi1ez0VuL6q/mkI4lmUZJs2/Ci6E8w3B8Qz0rvIS4AvVNW0Z/VVdVxV7VRVu9Jd7v9CVY3+BWhWYgFIskWSrUaGgQPpLn+PjmfGj1VVfQ+4JckeregA4BtzEcsohzG4WcZsxvNdYL8km7fv1wF09/qNjmVWPjcASR7b/u4C/C5r76O5OFb9fXA48KkB05wPHJhk2/aL7IGtTHNo2M6NUzFM59WpGuLz8lQMwzl9KoauPpiKIa1LpmL46qEagl4u5vJF96FaCfyMLkM9Engt8No2PsC7gW8D1wBLZjCWZ9BdNrwauKq9Dp7DeH4V+FqL51rg/7byNwMvbMOb0V16Xw5cBjxhFo7Z/rQenOYqFrr25l9vr+uAv2rlc3Ws9gaWtWP1SbpeX+Yklra+zYHvA4/ulc3VvvlbuuT+WuCDwCPn8jMM/BfdP0tfBw6Y7X0zxjnvMcCFdL9EXghs16ZdAry/N+9r2n5aDhwxk/vJ13od2zk/N04x/qE6r07D9gzVeXmK2zI05/QpbsdQ1QdT3JY5rUumGPu8qIfSVihJkiRJwuZ2kiRJkrQGkyRJkiRJ6jFJkiRJkqQekyRJkiRJ6jFJkiRJkqQekyRJkiRJ6jFJkqZBkv2TjH5y92SX8YNpiuUvR73/7+lYriRp/rBOWdt07RMtDD4nSZoGSU4AflBVb5/CMn5QVVtOYLqNqurBqS5HkjS/rOv8P2raeVenJNm4qh6YweVbP2rCvJIkjSPJJ5NckeS6JEe1soOSXJnk60kuTLIr3VOu/zzJVUmemeT0JC/pLecH7e+WbZ4rk1yT5JAJxrF/ki8m+TDdk7PHiu0k4FEtjg+NWvf+SS5K8rEk30zyoSRp4w5uZV9O8q4kn56ePShJmogku7bz8BlJrm7n6s2T3Jzk/yb5MvDSJE9M8tl2/v+vJL/U5t8tyVeTXJ7kLetY19DUKUlOSHJKks8BZybZLMkHWh35tSS/2aZ7dZJ/7c336ST7j8SU5MRWL1+SZIfJ7hNptI3nOgBpyL2mqlYneRRweZJPAe8DnlVVNyXZro1/L70rSUmOHGN5PwFeXFX3JtkeuCTJ0prYJd2nAXtV1U1jxHZuVR2b5Jiq2nuMZTwFeDJwG/AV4OlJlgH/1tumsyYQiyRp+u0BHFlVX0lyGvAnrfwnVfUMgCQXAq+tqhuS7Au8B/gt4J3AyVV1ZpKjJ7CuYapTngo8o6p+nOSNAFX1Ky0B/FySJ61j/i2AS6rqr5L8A/CHwFuZ/D6RHuKVJGl8f5bk68AlwM7AUcDFI5VKVa2e5PIC/F2Sq4HPA4uBHSY472W9ymxQbLtPcBkrqurnwFXArsAvATf2lm2SJElz45aq+kob/nfgGW34HOhaIwC/AXw0yVV0yciObZqn8/D5+4MTWNcw1SlLq+rHbfgZI/FX1TeB7wDrSpJ+CoxcrbqixQGT3yfSQ7ySJI2hXcZ/DvDrVfWjJBcBX6f7pW9dHqD9CNGaH2zayl8OLAKeWlU/S3IzsNkEQ/rhOmKbyHLu7w0/SHcOyATXL0maWaNbFYy8Hzn/PwK4e5wrO5O50XyY6pQf9obHmv+herXpx/ezXouMkThGePO91otXkqSxPRq4q1UYvwTsBzwSeHaS3QCSbNemvQ/YqjfvzXTNBwAOATbpLfOOliD9JvD4aYxtxM+SbDLGfIN8E3hCunurAH5/PWOSJE3NLkl+vQ0fBny5P7Kq7gVuSvJS6H6ES/JrbfRXgEPb8Msnud5hqlMupsXfmtntAnyLrl7dO8kjkuxM11xwXaayT7TAmSRJY/sssHFrGvcWuiYIq+ia3H28NUs4p037H8CL282tz6S7b+nZSS4D9uXhX8k+BCxpbbZfTleZTFdsI04Brh65yXZdWhOHPwE+224Mvh24Zz3jkiStv+uBw9u5fTvg5AHTvBw4stVB19H9EAfwOuDoJJfTJT2TMUx1ynuAjZJcQ1fHvrqq7qdLeG6i62ji7cCVE1jWVPaJFji7AJdEki2r6getaeC7gRuq6h1zHZckLRTtysunq2qvOQ5lyqxTtCHwSpIkgD9sNwFfR/dr27/NcTySpPnLOkXznleSpCGS5FdYuwee+6tq37mIR5I0fw1TnZLkCLrmb31fqSq75tZQMkmSJEmSpB6b20mSJElSj0mSJEmSJPWYJEmSJElSj0mSJEmSJPX8/94LCyZSph3sAAAAAElFTkSuQmCC\n", 349 | "text/plain": [ 350 | "
" 351 | ] 352 | }, 353 | "metadata": { 354 | "needs_background": "light" 355 | }, 356 | "output_type": "display_data" 357 | } 358 | ], 359 | "source": [ 360 | "palette = sns.color_palette(\"RdBu\", 10)\n", 361 | "fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(14, 4))\n", 362 | "\n", 363 | "sns.countplot(x='actual_rating', data=df_pred, palette=palette, ax=ax1)\n", 364 | "ax1.set_title('Distribution of actual ratings of books in the test set')\n", 365 | "\n", 366 | "sns.countplot(x='pred_rating_round', data=df_pred, palette=palette, ax=ax2)\n", 367 | "ax2.set_title('Distribution of predicted ratings of books in the test set')\n", 368 | "\n", 369 | "plt.show()" 370 | ] 371 | }, 372 | { 373 | "cell_type": "markdown", 374 | "metadata": {}, 375 | "source": [ 376 | "### Absolute error of predicted ratings\n", 377 | "\n", 378 | "The distribution of absolute errors is right-skewed, showing that the majority of errors is small: between 0 and 1. There is a long tail that indicates that there are several observations for which the absolute error was close to 10.\n", 379 | "\n", 380 | "How good/bad the model is with predicting certain scores? As expected from the above charts, the model deals very well with predicting score = 8 (the most frequent value). The further the rating from score = 8, the higher the absolute error. The biggest errors happen to observations with scores 1 or 2 which indicates that probably the model is predicting high ratings for those observations." 381 | ] 382 | }, 383 | { 384 | "cell_type": "code", 385 | "execution_count": 7, 386 | "metadata": {}, 387 | "outputs": [ 388 | { 389 | "data": { 390 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzIAAAEXCAYAAAB2y3GBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeZwcdZ3/8dene47MTGZyTu7JfZADCBACGMQAKiAKuqKiK+quiq736u6KPy+8XXVddZdV8AIRjBiugMihEC4hJIQECEkghNyBTM5J5j4+vz+qJnSGnpnOpLurp/v9fDz6MX3U8e7qmqr+9PdbVebuiIiIiIiI9CexqAOIiIiIiIgcLRUyIiIiIiLS76iQERERERGRfkeFjIiIiIiI9DsqZEREREREpN9RISMiIiIiIv2OCpkcYWa/MLOvpmla483skJnFw8dLzewj6Zh2OL2/mNkH0zW9o5jvt81st5m93Idx07oMwmleaWa/T+c0c124Xk2OOoeIFBZtw9PDzBaY2QvhtvztUecBMLN/NLN7MzTtNWa2MBPTltygQiYLzGyTmTWa2UEz229mfzezj5vZ4eXv7h9392+lOK039jSMu29x94Hu3p6G7K/Z0Lv7Be5+3bFO+yhz1ABfAGa5+6hszjsd8mWHGa5XG492PDObaGZuZkXHmsHMrjWzbx/rdJJMd6GZbUv3dEVyUbgvaTGz4V2eXxX+r06MJlluypdtOPBN4H/Dbflt2Z55sn2Bu9/g7m/OxPzcfba7L+3LuGHOqceaIVPrTjr3q/2ZCpnseZu7VwITgO8DXwR+ne6Z5PEKPQHY4+67og6S65KtA0e7XuTxeiQir3oJeG/nAzM7HiiLLo5AxrfhE4A16cqVZJh4X6Yt0mfurluGb8Am4I1dnpsPdABzwsfXAt8O7w8H7gT2A3uBhwmKzuvDcRqBQ8B/ABMBBz4MbAEeSniuKJzeUuB7wBPAAeB2YGj42kJgW7K8wPlAC9Aazm91wvQ+Et6PAV8BNgO7gN8Bg8LXOnN8MMy2G/hyD8tpUDh+bTi9r4TTf2P4njvCHNcmGXdIuMxqgX3h/XEJr/e0DAYAvwf2hMt8OTAyfG0MsCT8HDYAH02Y5pXA749hOQ4iKGZ3AtuBbwPxbpZNDLgCeDHMeVNC/p7WgcPPhcNeRLAT2x8uk5ld8n4ReBpoJlx/uuRwYGrCOnsV8GfgILAMmNJN/i3huIfC2xnh8/8MrA0/s3uACeHzBvw3wTp1IMw0B7g8XI4t4XTuSDKvpOOGr5UCPwrzvAL8guCLWwVHrmOHgDFRbzt00y1Tt/D//SvA8oTnfgR8OfxfnRg+l/R/Jnwtle3ut4BHw23EvcDwbvJoG57hbXg478TvEKUpLJ/F4bKtI9zvd5nmtcDPgbuA+nB5XQg8FY6zFbgyYfjX7AuADwGPJAzjwMeBF8J14SrAwtfiwH8RfJ94CfgUCd93ulnP35jwfm4i+J5xMFyO87oZ76FwuvVhzveEz78VWBUu/78DJySM88VwPTgIrAfOpZt1J8n8XjNuCutN0v1qod0iD1AIN5IUMuHzW4B/Ce9fy6uFzPcIdhbF4e31Cf/ER0yLVzd2vyP4MlZG8kJmO8EXwQrgZlLYeIf3r+wcNuH1pbxayPwzwcZvMjAQuAW4vku2X4a5TiTYuM7sZjn9jmDnVBmO+zzw4e5ydhl3GPBOoDwc/0/AbV0yd7cMPgbcEY4bB04BqsLXHgT+j2BHOZdgJ3tu12XTx+V4G3B1mGcEwQ76Y928v88BjwPjCHY+VwN/SGEdSHxuOsFG+U0E69V/hJ9dSULeVUAN4ReVJDm6FjJ7CYryIuAGYFE343XmKUp47u3h/GeG438F+Hv42nnAk8BggsJkJjC66/9KN/PqadyfEOy0h4bryR3A91JZx3TTLZ9uvPolfX34PxIn+NI5gSMLmZ7+Z1LZ7r4YbnvKwsff7yaPtuHZ2YYffk8pLp9Wgm11LNk0CbbHB4AF4TADwmV5fPj4BIIC+O1d3mvivuBDvLaQuZNgGz4+zHR++NrHgefC5TgE+GvX6fXyGTYBbyFYT74HPN7D/8jh/V34+GSCH8hOC8f/YDj9UmAGwf/PmIT3OaW7dafLfHoaN5X1Jul7L5SbupZFawfBzqGrVmA0wa/Tre7+sIdrbQ+udPd6d2/s5vXr3f1Zd68Hvgq8O01NwP8I/NjdN7r7IeBLwKVdmqC/4e6N7r4aWE1Q0BwhzPIe4EvuftDdNxH86nJZKiHcfY+73+zuDe5+EPgO8IYug3W3DFoJdqJT3b3d3Z9097rwuJwzgS+6e5O7rwJ+lWqmnpjZSOAC4HPh57aLoBXh0m5G+RhBa9Y2d28m2DBe0mU5J1sHEp97D/Bnd7/P3VsJfmUtA16XMPzP3H1rD+tRV7e4+xPu3kZQyMxNcbzO9/Q9d18bjv9dYK6ZTSD4TCqB4wiK+LXuvjPF6SYd18wM+Cjwr+6+N1xPvkv3y1ykEFwPfIDgy/E6gmIBgN7+Z1Lc7v7W3Z8Ptyk30c02QttwIMvb8BSXz2Pufpu7d/Qwzdvd/dFwmCZ3X+ruz4SPnwb+wGs/y9583933u/sW4AFeXW/eDfw0XI77CLrqH41H3P0uD44hvp4k30d68FHgandfFq5n1xH8OHs60E5QaMwys2J33+TuL6Y43Z7GTWW9KWgqZKI1luAX7a5+SPAry71mttHMrkhhWluP4vXNBL/mDO9m2KMxJpxe4rSLgJEJzyWeZayBoOWmq+FASZJpjU0lhJmVm9nVZrbZzOoImoUHdynWulsG1xN0a1pkZjvM7AdmVhy+t86d91Fn6sWEcP47wxNA7Cf4pWVED8PfmjDsWoKNX+JyTrYOJD53xGfl7h3h62O7GT4VqXy23ZkA/DThPe0laEEZ6+73A/9L0KXgFTO7xsyqUploD+NWE/xi+2TCPO8OnxcpVNcD7yP4Vfx3XV7r8X8mxe1uStsIbcOB7G/DU1k+qUzviGHM7DQze8DMas3sAEErytF+3+huvRnTZX7Hus8acBRFwQTgC52fYfg51hC0pGwgaD25EthlZovMbEwqE+1l3FTWm4KmQiYiZnYqwcbika6vhS0SX3D3ycDbgM+b2bmdL3czyd5abGoS7o8n+AVrN0EzdXlCrjhHfrHrbbo7CP7REqfdRtCUfDR2h5m6Tmt78sFf4wsEzbOnuXsVcFb4vCUMk3QZhK1e33D3WQS/bL2V4BfKHcBQM6tMIdPRLsetBL/kDHf3weGtyt1nd/P+tgIXJAw72N0HuHtilmSfVeJzR3xW4a+tNV3eT2+fd18lm+5Wgm4Yie+pzN3/DuDuP3P3U4DZBF0q/j3VjN2Mu5ugb/jshPkNcvfOHWSm3rtIznL3zQTHGryFoGtwot7+Z1LZ7qZK2/Dsb8NTWT6pTK/rMDcSdEescfdBBF3lrZthj9ZOgm5WnWq6GzADtgLf6fIZlrv7HwDc/UZ3P5NXu2f+ZzheKvus7sbtab3RPgsVMllnZlVm9lZgEUGfyWeSDPNWM5sabqTqCKrvzlMpv0JwPMrRer+ZzTKzcoLTLy4Om1afJ/hF4sLwF6yvEDRxdnoFmGgJp4ru4g/Av5rZJDMbSNDt4I8edBVKWZjlJuA7ZlYZdi/6PMFBhqmoJNjh7jezocDXkwyTdBmY2dlmdny446oj2Dm2u/tWgoP5vmdmA8zsBIIDL29IMu2jWo4edJO6F/ivcJ2ImdkUM+uu+f0X4bKZAGBm1WZ2cYrLptNNwIVmdm6Y8QsEO+K/H+V0+qKW4CDTxHX3F8CXzGw2gJkNMrN3hfdPDX/VKyb4gtFEiv8D3Y0b/nr5S+C/zWxEOOxYMzsvYbrDzGxQet6ySL/xYeCcsMvWYSn8z6Sy3U2VtuG9S+s2/CiXz9GoJGjpaTKz+QQtfp2S7QuOxk3AZ8P1cDDBQfKZ0nVf80vg4+H+xcysIlxfKs1shpmdY2alBPucRo7cZ3X7PaqXcXtab451WeYFFTLZc4eZHSSorr8M/Bj4p26GnUZwANsh4DHg//zV86B/D/hK2Mz4b0cx/+sJDsp7meBgvM8AuPsB4BME/WK3E3zxS7yWxp/Cv3vMbGWS6f4mnPZDBL/qNQGfPopciT4dzn8jQUvVjeH0U/ETgr7CuwkOjLs7yTBJlwEwiuDMLHUEzbYP8moB9V6CA+p2ALcCX3f3+7pOuI/L8QME3emeIzgzy2KCY6OS+SnBL1z3huvR4wQHHKbM3dcD7wf+h2A5vY3gtOAtRzOdvnD3BoI+74+G6+7p7n4rwa9OiyzoSvIsQZ9zgCqCncY+gq4Oewj6g0NwlqBZ4XSSXQehp3G/SNBt8/Fwnn8l+BUYd19HUJhvDKedUrcAkf7O3V909xXdvNzt/wypbXdTpW14LzK0DU9p+RylTwDfDN/n1wiKDyD5vuAop/1LggLyaYIzo91F0AvkmK+bl8SVwHVhzneH/yMfJei6vI/g/+JD4bClBMfr7CZYP0cA/y98rbfvUT2N2+16k4ZlmRc6z4QlIiIiItJvmNkFwC/cfUKvA0teUouMiIiIiOQ8Myszs7eYWZGZjSXognhr1LkkOmqREREREZGcFx4f9SDB6fUbCS7I/Fl3r4s0mERGhYyIiIiIiPQ76lomIiIiIiL9TmRXBh0+fLhPnDgxqtmLiAjw5JNP7nZ3XRQ0Ce2nRESi19N+KrJCZuLEiaxY0d3ZHkVEJBvMbHPvQxUm7adERKLX035KXctERERERKTfUSEjIiIiIiL9jgoZERERERHpd1TIiIiIiIhIv6NCRkRERERE+h0VMiIiUjDMbLCZLTazdWa21szOiDqTiIj0TWSnXxYREYnAT4G73f0SMysByqMOJCIifaNCRkRECoKZVQFnAR8CcPcWoCXKTCIi0nfqWiYiIoViMlAL/NbMnjKzX5lZRdShRESkbwquRWbx0jVJn79k4ewsJxERkSwrAk4GPu3uy8zsp8AVwFc7BzCzy4HLAcaPH3/EyFfPPD17SYGPrX08q/MTEelv1CIjIiKFYhuwzd2XhY8XExQ2h7n7Ne4+z93nVVdXZz2giIikruBaZLqjlhoRkfzm7i+b2VYzm+Hu64FzgeeiziUiIn2jFplutLa1s+qFnbyy91DUUUREJH0+DdxgZk8Dc4HvRpxHRET6SC0ySby0cx93Pf48+w428sBTG/ncu17HO98wm1jMoo4mIiLHwN1XAfOiziEiIsdOLTJdLHtuKzfctxpw3vH6mcyZNJLvXP8g//LjJTQ2t0YdT0REREREUIvMa6xYv52aEYN43xtPoLgozqyJIxg6qJy/PP48H/jOzbxr4WxisZiOnRERERERiVBKLTJmdr6ZrTezDWZ2RZLXP2RmtWa2Krx9JP1RM29vXQP7DjYxe+IIioviAJgZp0wfwwWnTeOFbXv482PP4+4RJxURERERKWy9tsiYWRy4CngTwakrl5vZEnfveqaXP7r7pzKQMWte3LEXgMljh77mtVNmjOVQYwsPP72ZyvJS3nX2nGzHExERERGRUCotMvOBDe6+0d1bgEXAxZmNFY0Xt+9laGUZQyvLkr5+1okTmTt1FI88s5k7Hl2X5XQiIiIiItIplUJmLLA14fG28Lmu3mlmT5vZYjOrSUu6LGprb2fTy/uZkqQ1ppOZccHp05k4ajDfuPYBVqzfnsWEIiIiIiLSKZVCJtk5h7seJHIHMNHdTwD+ClyXdEJml5vZCjNbUVtbe3RJM2zLKwdoa+9gypjuCxmAeCzGO98wm5oRg/j8//6FzS/vz1JCERERERHplEohsw1IbGEZB+xIHMDd97h7c/jwl8ApySbk7te4+zx3n1ddXd2XvBnz4o69xGPG+JGDex22rLSYt5w+nba2Dj70vVv43d2rWLx0TRZSioiIiIgIpFbILAemmdkkMysBLgWWJA5gZqMTHl4ErE1fxOx4cftexo8cTElxPKXhh1SW8a6z51BX38Sflj5LW3tHhhOKiIiIiEinXgsZd28DPgXcQ1Cg3OTua8zsm2Z2UTjYZ8xsjZmtBj4DfChTgTPhQH0Tuw809Hh8TDI1IwbxtgXHsXXXAf6yTKdlFhERERHJlpQuiOnudwF3dXnuawn3vwR8Kb3Rsmf3/gYAxgyrPOpx50waye79DTzyzGZueeg53vkGXShTRERERCTTUrogZr6ra2gCoKpiQJ/GP+vEiUwePYTv3/AQz770SjqjiYiIiIhIEipkgAP1zZhBZXlJn8aPxYy3v34WwwdV8O//dzd19U1pTigiIiIiIolUyAB19c0MLCshHuv74igfUMwPP3Eeu/bV88NFj6YxnYiIiIiIdKVCBqirb2JQH7uVJZozaSQfvvAU7nh0HQ+ueikNyUREREREJBkVMgQtMlXlpWmZ1kffNo/p44bxreuWcuCQupiJiIiIiGRCwRcy7k5dQ3OfD/RPtHjpGm5/ZB1nnTiRvXWNfPqnd+pCmSIiIiIiGVDwhUxDcytt7R1UVaSnRQZg1LBKTp9dw9MvvsK22gNpm66IiIiIiAQKvpCpq28GYFAaCxmAM48fT2VZCfc88QIdHbpQpoiIiIhIOqmQqT+2a8h0p6S4iHNPmcLOPYe47ZG1aZ22iIiIiEihK/hC5kDYIpPOrmWdZk8awbjqKv5n8WPUNTSnffoiIiIiIoWq4AuZuvomiuIxykuL0z5tM+O8+dPYd6iJ3939VNqnLyIiIiJSqFTIhKdeNrOMTH/0sErOmz+VG+5bzZ4DDRmZh4iIiIhIoSn4QuZAfXNGupUl+sQ7TqOltZ1f3bkio/MRERERESkUBV/I1DU0pf1A/64mjBzMxWfO5E9L17B9d11G5yUiIiIiUggKupBp7+jgYENL2k+9nMzHLjqVmBlX37484/MSEREREcl3RVEHiNLBhhYg/ade7mrx0jUAzJ02ijv+vo6aEYMYUlnGJQtnZ3S+IiJyJDPbBBwE2oE2d58XbSIREemrgm6RefUaMplvkQE4ffZ4YmY8tmZLVuYnIiJJne3uc1XEiIj0bwXdIlMXXkMmG13LAKrKSzlx6mhWb9jJmSdMzMo8RUQk/zzwT5dmfZ5n/3ZR1ucpItKTgm6ROdDZIlOe2a5liV43p4YOdx5fszVr8xQRkcMcuNfMnjSzy7u+aGaXm9kKM1tRW1sbQTwREUlVQRcydQ3NlJUUUVIcz9o8Bw8s4/jJo1j5/A721um6MiIiWbbA3U8GLgA+aWZnJb7o7te4+zx3n1ddXR1NQhERSUlhFzJZuIZMMgvmjKetvYNFf3sm6/MWESlk7r4j/LsLuBWYH20iERHpq4IuZOqbWqgYUJL1+Q4bVM70mmH88f5naGxuzfr8RUQKkZlVmFll533gzcCz0aYSEZG+KuhCpqm5jQGlxZHM+4zZ4zlQ38ySR9dFMn8RkQI0EnjEzFYDTwB/dve7I84kIiJ9VNBnLWtsaaWsNJpFMK66ihOmjOT3967ikoWziccKuqYUEck4d98InBh1DhERSY+C/fbs7jS1tDGgJJpCxsz4wHknsXVXHQ+sfCmSDCIiIiIi/VXBFjLNre24Q1lEXcsAzj55EjUjqrju7qdw98hyiIiIiIj0NwVbyDS1BAfZR9UiAxCPxXj/m+fyzMZXWPXCzshyiIiIiIj0NwVbyDQ2twFQVhJdiwzARQuOY/DAAVx3z6pIc4iIiIiI9CcFW8g0tQSFzICIDvbvVFZazLvPmcODq15i0859kWYREREREekvCvasZZ3XbymLsGvZ4qVrAKgoLSFmxjeufYALz5jBJQtnR5ZJRERERKQ/UItMhAf7d6ooK+GEKaN4+sWXOdTYEnUcEREREZGcl1IhY2bnm9l6M9tgZlf0MNwlZuZmNi99ETMjF1pkEp0+q4b2Dmfl8zuijiIiIiIikvN6LWTMLA5cBVwAzALea2azkgxXCXwGWJbukJnQ1NJGPGYUF8WjjgLAsEHlTB07lCfXb6e5tS3qOCIiIiIiOS2VFpn5wAZ33+juLcAi4OIkw30L+AHQlMZ8GdPY3BbpNWSSOW1WDfVNrdy97IWoo4iIiIiI5LRUCpmxwNaEx9vC5w4zs5OAGne/s6cJmdnlZrbCzFbU1tYeddh0amppjfQaMslMHDWY6sEV3HDfal0gU0RERESkB6kUMpbkucPfss0sBvw38IXeJuTu17j7PHefV11dnXrKDGhsyb0WGTNj/sxxPL91DyvW61gZEREREZHupFLIbANqEh6PAxK/ZVcCc4ClZrYJOB1YkusH/Dc1t+VciwzAnEkjGDJwADfcqwtkioiIiIh0J5VCZjkwzcwmmVkJcCmwpPNFdz/g7sPdfaK7TwQeBy5y9xUZSZwmjS2tlEV8McxkioviXLJwDg+u3sSWV/ZHHUdEREREJCf1Wsi4exvwKeAeYC1wk7uvMbNvmtlFmQ6YKU0tbQwoya2uZZ3efc4c4rEYf/jbM1FHERERERHJSSk1Sbj7XcBdXZ77WjfDLjz2WJnV3tFBS2t7TrbIAFQPruD8+dO4/eG1fOLt86ksL406koiIiIhITknpgpj5pqkluE5LLh4j0+l9bzqBhuZWbnt4bdRRRERERERyTu5+k8+gpuagkCnL0a5li5euAaBmxCB+decKSovjxGIxLlk4O+JkIiIiIiK5oSBbZBpbWgEYkKNdyzqdNmscB+qbWb91d9RRRERERERySkEWMrneItNp+rjhDKks49FntugCmSIiIiIiCXK7SSJDGjuPkcnxFplYzFgwZzx3PraeF7fvjTqOiIhIUi9e/a2szm/Kx76a1fmJSG4q0BaZoGtZWQ4f7N/p+Ckjqaoo5eGnN6tVRkREREQkVJCFTGM/OGtZp3gsxoI549m+u47l67ZHHUdEREREJCcUZCHT1NJ6+Exg/cGJU0cxsKyEa+5YoVYZEREREREKtJBpbG7rF60xnYricV43Zzwr1m3nodWboo4jIiIiIhK5gixkmlraKCvN7TOWdXXKjDFMHj2EHy16hJbW9qjjiIj0S2YWN7OnzOzOqLOIiMixKchCprG5tV+1yEBwrMy/v+/1bN1Vx+/vXRV1HBGR/uqzwNqoQ4iIyLEryEKmP7bIAJwxu4azT5rEL+9cwSv7DkUdR0SkXzGzccCFwK+iziIiIseuYAuZ/tYi0+kL71lAe7vzoz88EnUUEZH+5ifAfwAdUQcREZFjV3CFjLvT2NzaL64hk8y4EYP46Nvmcd+KF3lw1UtRxxER6RfM7K3ALnd/spfhLjezFWa2ora2NkvpRESkLwqukGlr76C9wxnQD7uWdfrQBScxZexQvnv9Q9Q3tkQdR0SkP1gAXGRmm4BFwDlm9vuuA7n7Ne4+z93nVVdXZzujiIgchYIrZBqbW4H+cTHMrhYvXcPipWu4/ZF1vP6ECbyy7xCf+5+7oo4lIpLz3P1L7j7O3ScClwL3u/v7I44lIiLHoOAKmaaWNgDKSvtfIZNoXPUg5s0Yw/J123lh256o44iIiIiIZFXBFTKNzUEhM6Ck/3Yt63TWiZMoLY5z1S2PRx1FRKTfcPel7v7WqHOIiMixKbhCprNFpj92LeuqfEAxZ8wez9JVm1i94eWo44iIiIiIZE3BFTLNrUEhU5oHhQzA/JljGVpVxs9ufgx3jzqOiIiIiEhW5Me3+aPQ0toOQGlxPOIk6VFSXMSpM8Zyz/IN/ODGR5gydujh1y5ZODvCZCIiIiIimVNwLTJNnS0yxflTw500fQxVFaX8/dktUUcREREREcmKgitkWlraiMeMonj+vPWieIxTpo9h8yv72b2/Puo4IiIiIiIZlz/f5lPU3NqeN8fHJJo7dTSxmPHk8zuijiIiIiIiknEFV8g0tbblzfExiSrKSpg5vpqnX3z58HFAIiIiIiL5quAKmZbW9rw6PibRKTPG0NzazppNu6KOIiIiIiKSUQVXyDS3tuVtIVMzYhDVg8tZ+fz2qKOIiGSEmcXM7Nmoc4iISPQKr5Bpyc+uZQBmxinTx7JzzyF27jkYdRwRkbRz9w5gtZmNjzqLiIhEKz+bJnrQ3NpOSR4e7N9p9qQR3LdiA89sfCXqKCIimTIaWGNmTwCHT9Xo7hdFF0lERLItf7/Rd6O5tY0BedoiA1BWWsy0ccNY89IrtLV35NVppkVEQt+IOoCIiEQvpW+5Zna+ma03sw1mdkWS1z9uZs+Y2Soze8TMZqU/6rFzd5pb2inJ02NkOh0/eST1Ta0se25r1FFERNLKzOLAV939wa63qLOJiEh29VrIhDuNq4ALgFnAe5MUKje6+/HuPhf4AfDjtCdNg+bWdjrc8/YYmU5Txw6jrKSIOx9bH3UUEZG0cvd2oMHMBkWdRUREopVK08R8YIO7bwQws0XAxcBznQO4e13C8BWApzNkutQ3tgAwII+PkQGIx2PMmjiCB1a+RH1jCxVlJVFHEhFJpybgGTO7jyOPkflMdJFERCTbUvlGPxZI7KO0DTit60Bm9kng80AJcE6yCZnZ5cDlAOPHZ/+EM4fCQibfu5YBHD9lJE8+v4O/rdzIRQuOizqOiEg6/Tm8iYhIAUvlGBlL8txrWlzc/Sp3nwJ8EfhKsgm5+zXuPs/d51VXVx9d0jToLGTyvWsZwNjhVdSMGMQdj66LOoqISFq5+3XATcDj7n5d5y3qXCIikl2pFDLbgJqEx+OAHT0Mvwh4+7GEypT6prCQyfOuZRBcU+aiBcexfN12tu06EHUcEZG0MbO3AauAu8PHc81sSbSpREQk21L5Rr8cmGZmk4DtwKXA+xIHMLNp7v5C+PBC4AVy0KGGwmmRAbhowXH8323LWPLoOj7xjtf0BhQR6a+uJDh+cymAu68K91EiHFhxT9bnOWjeeVmfp4ik0CLj7m3Ap4B7gLXATe6+xsy+aWadFx/7lJmtMbNVBMfJfDBjiY/Boc4WmQI4RgZg5NCBvG72eG5/ZB3tHR1RxxERSZc2d+/a1JyTJ5kREZHMSekbvbvfBdzV5bmvJdz/bJpzZUR9Y2EVMgAXv34m//Hze1j23DZeNyf7J1gQEcmAZ83sfUDczKYBnwH+HnEmERHJsoK67HshHezfaeHcSQweOHd7+lwAACAASURBVIDbHl4bdRQRkXT5NDAbaAZuBA4An4s0kYiIZF3BFTJF8RjxeOG87ZLiOG85fToPPLWRfQcbo44jInLM3L3B3b/s7qeGt6+4e1Pn62b2P1HmExGR7Cicb/QEZy0rpNaYTv9w1ixa2zq4+cE1UUcREcmGBVEHEBGRzCuoQuZQY0tBHR/Taeq4YZwxu4ZFf3uGltb2qOOIiIiIiByzwitkSgqvRQbgsvPmsvtAA39Z9nzUUUREREREjllBNU/UN7ZQUkAtMouXvtqVzN0ZMbiCq25ZxkULjsPMIkwmIpJRSTdwZjYAeAgoJdj/LXb3r2czmIiIpE9BtcjUN7YwoIAKmURmxmmza9i1v57H1myNOo6ISFqYWczMqro8/dNuBm8GznH3E4G5wPlmdnpGA4qISMYUVCFzsLGFkgI82L/TnIkjGFhWwnV/eSrqKCIifWZmN5pZlZlVAM8B683s3ztfd/drk43ngUPhw+Lwpgtpioj0UwVVyARnLSvMFhmAeDzG6bNrWLZ2Gyuf3xF1HBGRvprl7nXA2wku1jweuCyVEc0sbmargF3Afe6+rMvrl5vZCjNbUVtbm+7cIiKSRgVTyLg79QV8sH+nU6aPYfigcn5+2xNRRxER6atiMysmKGRud/dWUmxZcfd2d58LjAPmm9mcLq9f4+7z3H1edXV12oOLiEj6FEzzRFNLG+0dXtAtMgDFRXFOnj6Ge5dv4EeLHmHiqCGHX7tk4ewIk4mIpOxqYBOwGnjIzCYAdUczAXffb2ZLgfOBZ9MdUEREMq9gWmQONbYAFOQFMbs6efpoKstKeHDVJtzVPVxE+hd3/5m7j3X3t4THvWwGzu5tPDOrNrPB4f0y4I3AugzHFRGRDCmYQqa+s5ApKewWGYCieJwFx09g664DvLRzX9RxRESOipkNM7OfmdlKM3vSzH4KDEph1NHAA2b2NLCc4BiZOzMaVkREMqZgCpmDh1tkVMgAzJ02msryUh5arVYZEel3FgG1wDuBS8L7f+xtJHd/2t1PcvcT3H2Ou38zwzlFRCSDCqaQqVfXsiMUxWOcecIEttXWsXHH3qjjiIgcjaHu/i13fym8fRsYHHUoERHJrsIpZJrUtayruVNGMaiiVMfKiEh/84CZXRpeDDNmZu8G/hx1KBERya6CKWQONqhFpqt4PMaZJ0xkx56DbNi+J+o4IiI9MrODZlYHfAy4EWgOb4uAf40ym4iIZF/BFDKHW2R0jMwRTpgyksEDB/Dgqk10dKhVRkRyl7tXunuVu1cCw4EzCc48djbwtkjDiYhI1hVOIaNjZJKKx2KcdeJEXt57iPtWbIg6johIr8zsI8CDwN3AleHfr0WZSUREsq9gCplDjS0MKCkiFiuYt5yyOZNGUj24nKtuXUZrW3vUcUREevNZ4FRgs7ufDZwE7I42koiIZFvBfKs/1NjCwLKSqGPkpFjMOPukyWx55QBLHtW14UQk5zW5exOAmZW6+zpgRsSZREQkywqmkKlvUiHTk2njhnHClFH84vblNLW0RR1HRKQn28xsMHAbcJ+Z3Q7siDiTiIhkWcEUMocaW6hQIdMtM+Mzl5xO7f56/nj/M1HHERHplru/w933u/uVwFeBXwNvjzaViIhkW0EVMgMHqJDpybwZY1kwZzy/+fOTHGxojjqOiEiv3P1Bd1/i7i1RZxERkewqmEKmXi0yKfn0O0/nQH0zv7t7VdRRRERERES6VTAXVTnY0EJluQqZnixeugaAWROrufbulVSUlTCwrIRLFs6OOJmIiIiIyJEKpkXmYEMzleWlUcfoFxbOnURbewePPrM56igiIiIiIkkVRCHT1t5BQ3OrCpkUDa0qZ+7U0Tz5/A72HWyMOo6IiIiIyGsURCFzqDE4BrRSx8ik7PUnTiRmxkOrN0UdRURERETkNQqikOk8A5daZFJXVV7KqceN5ZmNr7Bh256o44iIiIiIHCGlQsbMzjez9Wa2wcyuSPL6583sOTN72sz+ZmYT0h+171TI9M3r5oyntDjO/966LOooIiIiIiJH6LWQMbM4cBVwATALeK+Zzeoy2FPAPHc/AVgM/CDdQY+FCpm+KSst5ow541n61Eus3vBy1HFERERERA5LpUVmPrDB3TeGFxxbBFycOIC7P+DuDeHDx4Fx6Y15bA42hMfI6PTLR23+ceMYUlnGL+9cEXUUEREREZHDUilkxgJbEx5vC5/rzoeBvyR7wcwuN7MVZraitrY29ZTH6FCjWmT6qqQ4zj++6QQeeXoz67fsjjqOiIiIiAiQWiFjSZ7zpAOavR+YB/ww2evufo27z3P3edXV1amnPEavtsiokOmLd59zPBUDivntXSujjiIiIiIiAkBRCsNsA2oSHo8DdnQdyMzeCHwZeIO7N6cnXnocbGgmZkZ5aXHUUfqle5/YwAlTRnHP8heYPGYIQ6vKAbhk4eyIk4mIiIhIoUqlkFkOTDOzScB24FLgfYkDmNlJwNXA+e6+K+0pj9HBhmYGlpcQiyVrXJJUzJ85jifWbuPx57byltNnRB1HRESkX2javT2r8xswvKfe/yL5pdeuZe7eBnwKuAdYC9zk7mvM7JtmdlE42A+BgcCfzGyVmS3JWOI+ONjQQmWZupUdi8ryUk6cOorVG14+fIFREREREZGopNIig7vfBdzV5bmvJdx/Y5pzpdXBxmadsSwNTptZw8rnd/Lk+u28Ye6kqOOIiIiISAFL6YKY/d3BhmYd6J8GwwaVM3XsUJ58fgdt7e1RxxERERGRAlYghUyLCpk0OW1WDQ1NrTy7MecOhRIR6ZGZ1ZjZA2a21szWmNlno84kIiJ9l1LXsv4uaJFR17J0mDhqMCMGV/DE2m24O2Y6gYKI9BttwBfcfaWZVQJPmtl97v5c1MFEROToFUiLjLqWpYuZMX/mOHbtr+eJtduijiMikjJ33+nuK8P7BwlOYKNTPImI9FN5X8i0tXdQ39SqQiaN5kweQcWAYn5/7+qoo4iI9ImZTQROApZ1ef5yM1thZitqa2ujiCYiIinK+0KmPjxVcGWZupalS1E8zsnTx/Dw05vZtHNf1HFERI6KmQ0EbgY+5+51ia+5+zXuPs/d51VXV0cTUEREUpL3hczBhmYAtcik2SkzxlJcFOOGvz4ddRQRkZSZWTFBEXODu98SdR4REem7/C9kOltkVMik1cCyEt5y+nTueHQdBw41RR1HRKRXFpyd5NfAWnf/cdR5RETk2OR/IaMWmYx5/5vn0tTSxs0Prok6iohIKhYAlwHnmNmq8PaWqEOJiEjf5P3pl18tZHSMTLpNGzeM02aOY9H9z3DZeXMpLopHHUlEpFvu/gigc8aLiOQJtcjIMbnsvLns2lfPnx9bH3UUERERESkgBVDI6BiZTFpw/HhmTajm139+krb2jqjjiIiIiEiBKIBCppmYGeWlxVFHyTuLl67h5gefY/akEWzdVce3rnsg6kgiIiIiUiAKopCpKCshFlO36EyZXjOcEYMreOSZLbR3qFVGRERERDKvAA72b9GB/hlmZpx5wgRueeg5/rriRc6bPy3qSCIiIiKSBnP/+aqsz3PVbz6Z0nB53yJzqLFZx8dkwXHjqxk+qJyf3/YErW3tUccRERERkTyX94VM0CKjQibTYjHjnJMns+nl/dzy0HNRxxERERGRPFcAhUwzlWXqWpYN08YNY95xY/nFbU8cPu21iIiIiEgm5H8ho65lWWNmfOE9C9h3qInf3rUy6jgiIiIiksfyv5BR17KsmjmhmreeMYPf37ua7bV1UccRERERkTyV14VMe0cHhxp11rJsWrx0DZPHDAHgsz/7M3964FkWL10TcSoRERERyTd5XcjUN7YAqEUmy6oqBvCGuRPZsH0vazfXRh1HRERERPJQXl9H5mCDCpmonHrcWJ7Z+Ar3Lt/A5DFDo44jIiJS8F7ZdzDr8xw5pDLr85TCkdctMp1nzlLXsuyLxWK85fTp1De1cP/KjVHHEREREZE8UyCFjFpkojBmeBXzjxvHyud38PiarVHHEREREZE8kueFTNi1rEyFTFQWnjSJYVVlXPnb+3VtGRERERFJm7wuZA7UNwFQVaFCJirFRXEuOnMmu/bV88M/PBJ1HBERERHJE3ldyOw+0ADAsKryiJMUtrHDq/jnC09myaPr+NuTL0YdR0RERETyQF4XMnvqGqiqKKWkOB51lIL3sYtOZdbEEVz52wfYuSf7Z00RERERkfyS14XM7gMNDFdrTE4oLorz/Y+9ifb2Dr78y/toa++IOpKIiIiI9GMpFTJmdr6ZrTezDWZ2RZLXzzKzlWbWZmaXpD9m3+w50MCwQSpkcsX4kYP58gfewMrnd/LLO1ZEHUdERERE+rFeL4hpZnHgKuBNwDZguZktcffnEgbbAnwI+LdMhOyr3QfqmTN5ZNQxBFi8dM3h+8dPHsk1dyynobmVL7xnQYSpRERERKS/SqVFZj6wwd03unsLsAi4OHEAd9/k7k8DOdVfSF3LctP5p01j8MAybnt4LfsPNUUdR0RERET6oVQKmbFA4tUMt4XP5bSGphYam9vUtSwHlRYX8Q9nzaK+qYVv/PZ+3D3qSCIiIiLSz6RSyFiS5/r0zdPMLjezFWa2ora2ti+TSFnnqZeHD6rI6Hykb0YPq+SckyfzwFMvcdMDz0YdR0RERET6mVQKmW1ATcLjccCOvszM3a9x93nuPq+6urovk0jZns5ryKhFJmedNnMcC44fz38tepQXtu2JOo6I5Dkz+42Z7TIz/XoiIpIHUilklgPTzGySmZUAlwJLMhvr2O2u62yRUSGTq8yMb334XKoqSvniz++hsbk16kgikt+uBc6POoSIiKRHr4WMu7cBnwLuAdYCN7n7GjP7ppldBGBmp5rZNuBdwNVmtqb7KWbH7v1hi4wO9s9p9698iTefOpWNO/fxiR/fweKla444w5mISLq4+0PA3qhziIhIevR6+mUAd78LuKvLc19LuL+coMtZzth9oIGieIzBAwdEHUV6MXnMUM6YXcNja7YybdwwptcMjzqSiBQoM7scuBxg/PjxEacRkUJ10fdvzer8llzxjqzOL11SuiBmf7TnQANDq8qIxZKdq0ByzcK5kxg5pII/P7ae+qaWqOOISIHK5rGcIiJybPK2kNldp2vI9CfxeIyLz5xJU0sbdz3+vE7JLCIiIiI9yttCZs+BBp2xrJ8ZMWQgC0+axPotu7n9kbVRxxERERGRHJa3hczuAw06Y1k/dNrMGiaMGsz3b3iYF7frmFwRSR8z+wPwGDDDzLaZ2YejziQiIn2Xl4VMe0cHe+vUItMfxWLG28+cSVlpMV/8hU7JLCLp4+7vdffR7l7s7uPc/ddRZxIRkb7Ly0LmwKEm2jtcLTL9VGV5Kd/56BvZsH0vP7jxYR0vIyIiIiKvkZeFzO4DnRfDrIg4ifTV6+aM58MXnsKtD6/l9/eujjqOiIiIiOSYlK4j0990FjLqWta/ffIdp7H55f381x8fZeTQgbz51KlRRxIRERGRHJGXhcyewy0yKmT6s1jM+PZH30jtgXq+8su/MrSqjHkzxkYdS0RERERyQF4WMrvrwhaZqrKIk0hfLV665vD9c0+ewrbaOj7+oyX8z+feyhmzayJMJiIiIiK5ID8Lmf0NlJcWUz6gJOookgblA4q57M1zufGvq/nMT+/kR584nzfMnRR1LBERETkGD63fntX5naVeHXknLw/218Uw88/AshIue/NcptcM5wtX3c09T7wQdSQRERERiVBeFjK76xqoHqxCJt+UlRZz9b9dzAlTRvKlq+/jtofXRh1JRERERCKSl13L9hxoYOrYoVHHkAy4e9kLvGneVPYfbOLK397PY89u4dSZ47hk4eyoo4mIiIhIFuVdi0x7Rwc79xxkxJCBUUeRDCkuivPuc45nRs1w7lm+gaVPvaSLZoqIiIgUmLwrZDa/vJ+mljZmjB8edRTJoKJ4jHe+YRZzp47ikWc2863rltLW3hF1LBERERHJkrzrWrZuy24AjlMhk/disRgXnjGDirISbnnoOXbuOch//st5VJWXRh1NRERERDIs71pk1m2ppaQozqTRQ6KOIllgZpx90mS+/qGzWb5uOx/8zs1s3XUg6lgiIiIikmH51yKzuZap44ZRXBSPOopk0TvOmsW4EYP4t6v+wnu/cRNf/eBCzps/LepYIiIi0g98784VWZ/nl946L+vzzDd51SLj7qzbslvdygrQ4qVr2Pzyfi47by6DBw7gi7+4lw9+92bqGpqjjiYiIiIiGZBXhcyOPQepq29m5oTqqKNIRAYPLOOy8+ay4PjxrN7wMhd/6QZufeg5Ojp0VjMRERGRfJJXhcz6zTrQXyAei3H2SZP58IWnMH7kIL5x7QO86+uLWLz0WRqaWqKOJyIiIiJpkFfHyKzbUkvMjKnjhkUdRXLA6GGVXPulf+DuZS9w7d1P8e3fPchP/vQYp8+u4fRZ45g7dTTjRw6mpFjHU4mIiIj0N3lVyKzdXMukMUMoKy2OOorkiJsffA6AS94wm221dazasJNlz23jryteBCBmxpjhlYweVsnIIQMZOXQgo4YOPHx/5NCBDKooxcyifBsiIiIi0kVeFTLrtuxm/syxUceQHGRm1IwYRM2IQbg7e+sa2bnnIHvqGg7fX791NwcbmvEuh9OUFsc5fsoo5kwawexJI5g9cQSjh1WquBERERGJUN4UMnsONFC7v57jxutAf+mZmTFsUDnDBpW/5rWODqe+qYW6+mbqGpqoq29mb10jDU2t/P7e1bS1dwAwtKqM4yePZM6kkZwwZSSzJ41kYFlJtt+KiIiISMHKm0Jm3ZZaAI7TGcvkGMRiRmV5KZXlpYyl6ojX2to72LXvEDt2H2THnjqefWkXD67aBIAZTBo9hBnjhzO9ZjjTxw1jes1whg8qV8uNiIiISAbkTSHzyNObiceMGTpjmWRIUTzGmOFVjBleBQRdGBubW9mx5yDba+vYsbuOR5/ewl8ef+HwOEMGDmDCqCFMGDWI8SMHM2HkYCaMGkzNiEEMKMmbfz8RERGRrMuLb1I7dtex+ME1XLTgOKrKS6OOIwWkrLSYKWOGMmXM0MPPNTa3smtfPbv2HWLX/nr21DWwYfseDjUeeernUUMHUjNiEKOGVTJ66MDg77BKRgyuYPDAAVRVlFJcpDOqiYiIiCSTF4XM1UuWYxgfu/jUqKOIUFZazIRRQctLoubWNvbWNbK3roG9BxvZE55kYN2W3RxqfO1JBgCKi2KUFhdRWhyn5PDfOANKihhQUsSgigEMqSxLuL36eFBFUAyVFsfVvU1ERETyTr8vZF7cvpc7Hl3PP77pBEYNrYw6jki3SouLGB22unTV3tHBwYZmDtQ3c6ihmcbmNhpbWmlt66CtvZ229o7g1hb8PdjQwt66RjZs30tjUyuNLW3dzre4KEZleSlV5aVUVQxgWFUZQ6vKGFZVztCq8sP3O5+vLNfppkVERCT3pVTImNn5wE+BOPArd/9+l9dLgd8BpwB7gPe4+6b0Rn2t5tY2fvKnv1NWWsQ/X3hKpmcnkjHxWIzBA8sYPLCsT+N3dHTQ0NxGQ1MLDc2tNITFTVNzK80tbTS1ttHU3EZdfRM79xw8PFx3rUBDK4MCZ2hVGVUVpQwsK6WyrISBZSVUhH8ry0opLysmHosRjxmxmBGPxYiZEY8ZZsFz7o47OOHMHBwOPw/ByRJKiuIUF8cpjgetTiVFMYqL4hTFYyqsJG1625+JiEj/0WshY2Zx4CrgTcA2YLmZLXH35xIG+zCwz92nmtmlwH8C78lEYIDWtnZue3gtv7rzSV7Zd4jPvusMhlT27QugSD6IxWIMDAuMVHV0eFj0tHCosYX6puB+fWMLh8L7G3fspbG5jebWNppb2ulIVvlkmBmHi5viojglRXFKioMi5/DjsAgqKYpTHBZAwXCvDlNcFDtiuCOGPTxuOM7h+cVenU78yHFjMRVX/U2K+zMREeknUmmRmQ9scPeNAGa2CLgYSNzwXwxcGd5fDPyvmZl7Zr71/ODGh/nT0jWcOHUU3/rIucyfOS4TsxHJa7GYHS5+RgzpfXh3p62943BR09zaRnNrUNx4eOvoCFtaCO/jBF/3jcRGleC+Ef7BHdrbO2jv6KCtw4P77R20dXTQ3u60d4SvtXvC8x20dzj1jS0cSBiuLXz+8PTCx53XAEqXonjQEhW8nyOLms6H1vnuOx+/Zrgur3d9vssEkw03a+IIfvbZC4/pvRSQVPZnIiLST1hvtYaZXQKc7+4fCR9fBpzm7p9KGObZcJht4eMXw2F2d5nW5cDl4cMZwPp0vZGjMBzY3etQouWUGi2n1Gg5pS7by2qCuxfEBbhS3J9lYj+VS+t/LmWB3MqTS1kgt/LkUhZQnp7kUhZIT55u91OptMgk6z/RtfpJZRjc/RrgmhTmmTFmtsLd50WZoT/QckqNllNqtJxSp2WVUb3uqzKxn8qlzzSXskBu5cmlLJBbeXIpCyhPT3IpC2Q+TyyFYbYBNQmPxwE7uhvGzIqAQcDedAQUERFJk1T2ZyIi0k+kUsgsB6aZ2SQzKwEuBZZ0GWYJ8MHw/iXA/Zk6PkZERKSPUtmfiYhIP9Fr1zJ3bzOzTwH3EJyu8jfuvsbMvgmscPclwK+B681sA0FLzKWZDH2MIu3a1o9oOaVGyyk1Wk6p07LKkO72Z1mYdS59prmUBXIrTy5lgdzKk0tZQHl6kktZIMN5ej3YX0REREREJNek0rVMREREREQkp6iQERERERGRfqdgChkzO9/M1pvZBjO7Iuo8ucrMaszsATNba2ZrzOyzUWfKVWYWN7OnzOzOqLPkMjMbbGaLzWxduF6dEXWmXGRm/xr+zz1rZn8wswFRZ5LUmdlvzGxXeF21ZK+bmf0s3Ac9bWYnZzBLr9vxLOcZYGZPmNnqMM83kgxTamZ/DPMsM7OJmcoTzq/b7XcEWTaZ2TNmtsrMViR5PZufVY/b6yxnmREuk85bnZl9LsI8PW6jI1hvPhtmWdN1uYSvZ3TZJNvmmdlQM7vPzF4I/ya93LaZfTAc5gUz+2CyYVLmCVflztcbwUGdLwKTgRJgNTAr6ly5eANGAyeH9yuB57Wsul1WnwduBO6MOksu34DrgI+E90uAwVFnyrUbMBZ4CSgLH98EfCjqXLod1Wd4FnAy8Gw3r78F+AvBtWxOB5ZlMEuv2/Es5zFgYHi/GFgGnN5lmE8AvwjvXwr8McOfV7fb7wiybAKG9/B6Nj+rHrfX2czSZb5x4GWCCyNmPU8q2+hsrjfAHOBZoJzgxF1/BaZlc9kk2+YBPwCuCO9fAfxnkvGGAhvDv0PC+0P6mqNQWmTmAxvcfaO7twCLgIsjzpST3H2nu68M7x8E1hL8A0sCMxsHXAj8KuosuczMqgg2dr8GcPcWd98fbaqcVQSUWXAtrnJ0fZN+xd0foufrp10M/M4DjwODzWx0hrKksh3PZh5390Phw+Lw1vVMQxcTfIkGWAyca2bJLmB6zFLYfmctS4qy8lmluL3O2nrTxbnAi+6+OcI8vW2js7nezAQed/cGd28DHgTekSRPxpZNN9u8xGVwHfD2JKOeB9zn7nvdfR9wH3B+X3MUSiEzFtia8Hgb+nLeq7BZ9CSCX8/kSD8B/gPoiDpIjpsM1AK/Dbtx/MrMKqIOlWvcfTvwI2ALsBM44O73RptK0iyS/VAP2/Gs5gm7cq0CdhF8iek2T/jF7AAwLENxett+ZzMLBEXdvWb2pJld3lOeUKY+q1S211F9n7oU+EOS57OSJ8VtdDbXm2eBs8xsmJmVE7S+1HQZJorPaqS774TgBxVgRJJh0pqrUAqZZBWxzjvdAzMbCNwMfM7d66LOk0vM7K3ALnd/Muos/UARQdPzz939JKCeoLlZEoT9iC8GJgFjgAoze3+0qSTNsr4f6mU7ntU87t7u7nOBccB8M5sTRZ4Ut9/Z/qwWuPvJwAXAJ83srIjypLK9jmI9LgEuAv6U7OVs5ElxG521ZePua4H/JGjNuJvgkIm2qPIcpbTmKpRCZhtHVqrjULeNbplZMcHO7wZ3vyXqPDloAXCRmW0i6KZ4jpn9PtpIOWsbsC3h19fFBDtKOdIbgZfcvdbdW4FbgNdFnEnSK6v7oRS245HsF8OuSkt5bVeSw3nCrjuD6LmrXl+lsv3OVhYA3H1H+HcXcCtBd/ikeUKZ+qxS2V5Hsd5cAKx091eSvJatPKlso7O93vza3U9297PC+bzQXZ5QNj6rVzq7r4V/dyUZJq25CqWQWQ5MM7NJYWV/KbAk4kw5KezP+Wtgrbv/OOo8ucjdv+Tu49x9IsG6dL+769fzJNz9ZWCrmc0InzoXeC7CSLlqC3C6mZWH/4PnEhzXIPljCfCB8ExCpxN0TdmZiRmluB3PZp5qMxsc3i8j+FK4LkmezrMXXUKwXU37r8cpbr+zkgXAzCrMrLLzPvBmgm5DXfNk/LNKcXudtfUmwXtJ3q0sm3lS2UZnbb0BMLMR4d/xwD/w2mUUxWeVuAw+CNyeZJh7gDeb2ZCwpevN4XN9UtTXEfsTd28zs08RLKg48Bt3XxNxrFy1ALgMeCbszwzw/9z9rggzSf/2aeCG8EeEjcA/RZwn57j7MjNbDKwk6B7wFHBNtKnkaJjZH4CFwHAz2wZ8neCgdtz9F8BdBP3YNwANZPb/IOl2HBgfUZ7RwHVmFif4AfUmd7/TzL4JrHD3JQSF1/VmtoHg1+VLM5jnNSLMMhK4NTwmvAi40d3vNrOPQySf1Wu21xFmITz+403AxxKey3qe7rbREa/DN5vZMKAV+KS778vmsvn/7d1dqO1zHsfx94dzGOPISYwLeSxRx0yJQnnYgySm0TSkcEGM5LG5IKmZhOSC1NRcoKScjRmaRkwZDznpHOl4Og9zTkcKF1KejqeNwgKKUwAABF5JREFUToyvi/VbWpa1WWdb27L2fr9u9v//+/9/v//3vy7Wd3//v99/71m+824B/pnkQjrF31nt3COBS6rqoqramuRGOpMMADdU1ZxnrjKPxaIkSZIkzYvFsrRMkiRJ0gJiISNJkiRp4ljISJIkSZo4FjKSJEmSJo6FjCRJkqSJYyEjSZIkaeJYyEg9ksyMOwZJ0uKUZCpJ/3+M394xRpLHklzXt//sKMaVRslCRvqJJVnSt7/juGKRJP2sTAE/qpAZ1hC551uFTFX9JHFJ28NCRotWkn8neTHJpiQX97TfluSlJE8l2au1XZlkc5INSR74njF3TXJ3kueTvJzkjNZ+fpIHkzwCPN6euj2d5D5g43zfqyRpfAblmySntlyzvuWbA4BLgD8nWZfkuCT3JDmzZ5yZ9nNZ6/NSko3dXDNEHN/JPbPEdguwS4tjuu/aU0lWJXkoyZYk00nSjp3W2lYn+VuSR0fzCUqDparGHYM0Fkn2qKqtSXYBngdOAN4Dzquq6SR/BX5VVZcneQs4sKq2JVleVR/OMubNwOaqWplkObAWOBw4C7gJ+E275hTwH+Cwqnp93m9WkjQ2A/LNScALwPFV9XrP8euBmaq6tfW7B3i0qh5q+zNVtazN7P+yqj5OsifwHHBwVVX3nFnimKIv9wzKhVX1fv84PdeeAh4GVgBvAWuAq9v9vNpzT/cDu1XV70b3SUrf5oyMFrMrk6ynkwD2BQ4GvgL+0Y6vBI5t2xuA6STnAV9+z5inANcmWQesAn4B7NeOPVFVW3vOXWsRI0mLQn++uRh4ppsD+nLDMALcnGQD8CSwD7D3kH37c8+gXDjMGG9W1VfAOuAA4FDgtZ6x7x8yHmnOlvzwKdLC054onQwcU1WfJVlFp+jo152yPB04Hvg98JckK6pqUEET4I9V9Urf9Y4CPu07t39fkrTAzJJv1gOHDNH9S9pD57Z8a6fWfi6wF3BEVX2R5A0G57BBvsk925EL+23r2f4/nd8nM+T1pZFxRkaL1e7AB+2L+1Dg6Na+A9Bdj3wOsDrJDsC+VfU0cA2wHBg4bQ/8F7iiZ73w4fN1A5KkiTAo3+wMnJDkQOgs72rnfgLs1tP3DeCItn0GsLRnzHdaEfNbYP8Rxtb1RZKls/QbZAtwUHvXB+DsOcYkDc1CRovVY8CSNi1/I50pdeg8qVqR5EXgROAGYEdgZZKNwMvA7bO9I9PGWgpsSPK/ti9JWrwG5Zt36Swv+1db1tVd0vwI8Ifuy/7AXXQKnrVA78z+NHBkkhfozM5sGWFsXXfSyWXTwwxUVZ8DlwKPJVkNvA18NMe4pKH4sr8kSZJ+tCTLqmqmrUr4O/BqVd0+7ri0cDkjI0mSpFH4U/tjN5voLFu7Y8zxaIFzRkaagyQXAFf1Na+pqsvGEY8kSV1Jfg3c29e8raqOGkc80nyxkJEkSZI0cVxaJkmSJGniWMhIkiRJmjgWMpIkSZImjoWMJEmSpInzNRMh18b3NxeTAAAAAElFTkSuQmCC\n", 391 | "text/plain": [ 392 | "
" 393 | ] 394 | }, 395 | "metadata": { 396 | "needs_background": "light" 397 | }, 398 | "output_type": "display_data" 399 | } 400 | ], 401 | "source": [ 402 | "df_pred_err = df_pred.groupby('actual_rating')['abs_err'].mean().reset_index()\n", 403 | "\n", 404 | "fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(14, 4))\n", 405 | "\n", 406 | "sns.distplot(df_pred['abs_err'], color='#2f6194', ax=ax1)\n", 407 | "ax1.set_title('Distribution of absolute error in test set')\n", 408 | "\n", 409 | "sns.barplot(x='actual_rating', y='abs_err', data=df_pred_err, palette=palette, ax=ax2)\n", 410 | "ax2.set_title('Mean absolute error for rating in test set')\n", 411 | "\n", 412 | "plt.show()" 413 | ] 414 | }, 415 | { 416 | "cell_type": "markdown", 417 | "metadata": {}, 418 | "source": [ 419 | "### Analysis of predicted ratings of a particular user\n", 420 | "\n", 421 | "For this part of the analysis, the user with id 193458 was selected. By analyzing book ratings by this user, it can be noted that he/she likes diverse types of readings: English romantic novels (Pride and Prejudice, Sense and Sensibility), fantasy (Narnia) as well as historical novels (Schindler's List). Among the recommended books there are other works from Narnia's series, two historical novels and one romance which correlates with user's previous preferences." 422 | ] 423 | }, 424 | { 425 | "cell_type": "code", 426 | "execution_count": 17, 427 | "metadata": {}, 428 | "outputs": [], 429 | "source": [ 430 | "df_books = pd.read_csv('data/books.csv')\n", 431 | "\n", 432 | "df_ext = df.merge(df_books[['isbn', 'book_title']], on='isbn', how='left')\n", 433 | "df_ext['book_title_short'] = df_ext['book_title'].apply(f.short_title)\n", 434 | "df_ext = df_ext.merge(df_pred[['isbn', 'user_id', 'pred_rating']], on=['isbn', 'user_id'], how='left')" 435 | ] 436 | }, 437 | { 438 | "cell_type": "code", 439 | "execution_count": 11, 440 | "metadata": {}, 441 | "outputs": [ 442 | { 443 | "data": { 444 | "text/html": [ 445 | "
\n", 446 | "\n", 459 | "\n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | " \n", 533 | " \n", 534 | " \n", 535 | " \n", 536 | " \n", 537 | " \n", 538 | " \n", 539 | " \n", 540 | " \n", 541 | " \n", 542 | " \n", 543 | " \n", 544 | " \n", 545 | " \n", 546 | " \n", 547 | " \n", 548 | " \n", 549 | " \n", 550 | " \n", 551 | " \n", 552 | " \n", 553 | " \n", 554 | " \n", 555 | " \n", 556 | " \n", 557 | " \n", 558 | " \n", 559 | " \n", 560 | " \n", 561 | " \n", 562 | " \n", 563 | "
user_idisbnbook_ratingbook_titlebook_title_shortpred_rating
124989193458185326000210Pride & Prejudice (Wordsworth Classics)Pride & Prejudice (Wordsworth Classics)NaN
12494219345801406201259Wuthering Heights (Penguin Popular Classics)Wuthering Heights (Penguin Popular Classics)NaN
12495219345803453425699Shoeless JoeShoeless JoeNaN
12494019345801402984799Bridget Jones: The Edge of ReasonBridget Jones: The Edge of ReasonNaN
124991193458185326016910Sense and Sensibility (Wordsworth Classics)Sense and Sensibility (Wordsworth Classics)NaN
12497819345806718803149Schindler's ListSchindler's ListNaN
12495119345803303526959Four Letters of LoveFour Letters of LoveNaN
12493219345800644710479The Lion, the Witch, and the Wardrobe (The Chr...The Lion, the Witch, and the Wardrobe (TheNaN
12493819345800644711019The Magician's Nephew (rack) (Narnia)The Magician's Nephew (rack) (Narnia)NaN
124936193458006447108X9The Last BattleThe Last BattleNaN
\n", 564 | "
" 565 | ], 566 | "text/plain": [ 567 | " user_id isbn book_rating \\\n", 568 | "124989 193458 1853260002 10 \n", 569 | "124942 193458 0140620125 9 \n", 570 | "124952 193458 0345342569 9 \n", 571 | "124940 193458 0140298479 9 \n", 572 | "124991 193458 1853260169 10 \n", 573 | "124978 193458 0671880314 9 \n", 574 | "124951 193458 0330352695 9 \n", 575 | "124932 193458 0064471047 9 \n", 576 | "124938 193458 0064471101 9 \n", 577 | "124936 193458 006447108X 9 \n", 578 | "\n", 579 | " book_title \\\n", 580 | "124989 Pride & Prejudice (Wordsworth Classics) \n", 581 | "124942 Wuthering Heights (Penguin Popular Classics) \n", 582 | "124952 Shoeless Joe \n", 583 | "124940 Bridget Jones: The Edge of Reason \n", 584 | "124991 Sense and Sensibility (Wordsworth Classics) \n", 585 | "124978 Schindler's List \n", 586 | "124951 Four Letters of Love \n", 587 | "124932 The Lion, the Witch, and the Wardrobe (The Chr... \n", 588 | "124938 The Magician's Nephew (rack) (Narnia) \n", 589 | "124936 The Last Battle \n", 590 | "\n", 591 | " book_title_short pred_rating \n", 592 | "124989 Pride & Prejudice (Wordsworth Classics) NaN \n", 593 | "124942 Wuthering Heights (Penguin Popular Classics) NaN \n", 594 | "124952 Shoeless Joe NaN \n", 595 | "124940 Bridget Jones: The Edge of Reason NaN \n", 596 | "124991 Sense and Sensibility (Wordsworth Classics) NaN \n", 597 | "124978 Schindler's List NaN \n", 598 | "124951 Four Letters of Love NaN \n", 599 | "124932 The Lion, the Witch, and the Wardrobe (The NaN \n", 600 | "124938 The Magician's Nephew (rack) (Narnia) NaN \n", 601 | "124936 The Last Battle NaN " 602 | ] 603 | }, 604 | "execution_count": 11, 605 | "metadata": {}, 606 | "output_type": "execute_result" 607 | } 608 | ], 609 | "source": [ 610 | "selected_user_id = 193458\n", 611 | "df_user = df_ext[df_ext['user_id']==selected_user_id]\n", 612 | "\n", 613 | "df_user[(df_user['pred_rating'].isna())&(df_user['book_rating']>=9)].sample(10)" 614 | ] 615 | }, 616 | { 617 | "cell_type": "markdown", 618 | "metadata": {}, 619 | "source": [ 620 | "### Train set: Top rated books\n", 621 | "\n", 622 | "![](img/train_actual.jpg)" 623 | ] 624 | }, 625 | { 626 | "cell_type": "code", 627 | "execution_count": 12, 628 | "metadata": {}, 629 | "outputs": [ 630 | { 631 | "data": { 632 | "text/html": [ 633 | "
\n", 634 | "\n", 647 | "\n", 648 | " \n", 649 | " \n", 650 | " \n", 651 | " \n", 652 | " \n", 653 | " \n", 654 | " \n", 655 | " \n", 656 | " \n", 657 | " \n", 658 | " \n", 659 | " \n", 660 | " \n", 661 | " \n", 662 | " \n", 663 | " \n", 664 | " \n", 665 | " \n", 666 | " \n", 667 | " \n", 668 | " \n", 669 | " \n", 670 | " \n", 671 | " \n", 672 | " \n", 673 | " \n", 674 | " \n", 675 | " \n", 676 | " \n", 677 | " \n", 678 | " \n", 679 | " \n", 680 | " \n", 681 | " \n", 682 | " \n", 683 | " \n", 684 | " \n", 685 | " \n", 686 | " \n", 687 | " \n", 688 | " \n", 689 | " \n", 690 | " \n", 691 | " \n", 692 | " \n", 693 | " \n", 694 | " \n", 695 | " \n", 696 | " \n", 697 | " \n", 698 | " \n", 699 | " \n", 700 | " \n", 701 | " \n", 702 | " \n", 703 | " \n", 704 | " \n", 705 | " \n", 706 | "
user_idisbnbook_ratingbook_titlebook_title_shortpred_rating
12494619345801420017409The Secret Life of BeesThe Secret Life of Bees8.281881
12493519345800644710719The Voyage of the Dawn Treader (rack) (Narnia)The Voyage of the Dawn Treader (rack) (Narnia)8.244509
12493719345800644710989The Silver ChairThe Silver Chair8.184727
12497419345805532580019The Cider House RulesThe Cider House Rules8.057183
12495819345803454310579Slaves in the Family (Ballantine Reader's Circle)Slaves in the Family (Ballantine Reader's8.055557
\n", 707 | "
" 708 | ], 709 | "text/plain": [ 710 | " user_id isbn book_rating \\\n", 711 | "124946 193458 0142001740 9 \n", 712 | "124935 193458 0064471071 9 \n", 713 | "124937 193458 0064471098 9 \n", 714 | "124974 193458 0553258001 9 \n", 715 | "124958 193458 0345431057 9 \n", 716 | "\n", 717 | " book_title \\\n", 718 | "124946 The Secret Life of Bees \n", 719 | "124935 The Voyage of the Dawn Treader (rack) (Narnia) \n", 720 | "124937 The Silver Chair \n", 721 | "124974 The Cider House Rules \n", 722 | "124958 Slaves in the Family (Ballantine Reader's Circle) \n", 723 | "\n", 724 | " book_title_short pred_rating \n", 725 | "124946 The Secret Life of Bees 8.281881 \n", 726 | "124935 The Voyage of the Dawn Treader (rack) (Narnia) 8.244509 \n", 727 | "124937 The Silver Chair 8.184727 \n", 728 | "124974 The Cider House Rules 8.057183 \n", 729 | "124958 Slaves in the Family (Ballantine Reader's 8.055557 " 730 | ] 731 | }, 732 | "execution_count": 12, 733 | "metadata": {}, 734 | "output_type": "execute_result" 735 | } 736 | ], 737 | "source": [ 738 | "df_user[df_user['pred_rating'].notna()].sort_values('pred_rating', ascending=False).head(5)" 739 | ] 740 | }, 741 | { 742 | "cell_type": "markdown", 743 | "metadata": {}, 744 | "source": [ 745 | "### Test set: predicted top rated books\n", 746 | "\n", 747 | "![](img/test_pred.jpg)" 748 | ] 749 | }, 750 | { 751 | "cell_type": "code", 752 | "execution_count": 13, 753 | "metadata": {}, 754 | "outputs": [ 755 | { 756 | "data": { 757 | "text/html": [ 758 | "
\n", 759 | "\n", 772 | "\n", 773 | " \n", 774 | " \n", 775 | " \n", 776 | " \n", 777 | " \n", 778 | " \n", 779 | " \n", 780 | " \n", 781 | " \n", 782 | " \n", 783 | " \n", 784 | " \n", 785 | " \n", 786 | " \n", 787 | " \n", 788 | " \n", 789 | " \n", 790 | " \n", 791 | " \n", 792 | " \n", 793 | " \n", 794 | " \n", 795 | " \n", 796 | " \n", 797 | " \n", 798 | " \n", 799 | " \n", 800 | " \n", 801 | " \n", 802 | " \n", 803 | " \n", 804 | " \n", 805 | " \n", 806 | " \n", 807 | " \n", 808 | " \n", 809 | " \n", 810 | " \n", 811 | " \n", 812 | " \n", 813 | " \n", 814 | " \n", 815 | " \n", 816 | " \n", 817 | " \n", 818 | " \n", 819 | " \n", 820 | " \n", 821 | " \n", 822 | " \n", 823 | " \n", 824 | " \n", 825 | " \n", 826 | " \n", 827 | " \n", 828 | " \n", 829 | " \n", 830 | " \n", 831 | "
user_idisbnbook_ratingbook_titlebook_title_shortpred_rating
12493419345800644710639The Horse and His BoyThe Horse and His Boy7.814202
12493519345800644710719The Voyage of the Dawn Treader (rack) (Narnia)The Voyage of the Dawn Treader (rack) (Narnia)8.244509
12493719345800644710989The Silver ChairThe Silver Chair8.184727
12494619345801420017409The Secret Life of BeesThe Secret Life of Bees8.281881
12495819345803454310579Slaves in the Family (Ballantine Reader's Circle)Slaves in the Family (Ballantine Reader's8.055557
\n", 832 | "
" 833 | ], 834 | "text/plain": [ 835 | " user_id isbn book_rating \\\n", 836 | "124934 193458 0064471063 9 \n", 837 | "124935 193458 0064471071 9 \n", 838 | "124937 193458 0064471098 9 \n", 839 | "124946 193458 0142001740 9 \n", 840 | "124958 193458 0345431057 9 \n", 841 | "\n", 842 | " book_title \\\n", 843 | "124934 The Horse and His Boy \n", 844 | "124935 The Voyage of the Dawn Treader (rack) (Narnia) \n", 845 | "124937 The Silver Chair \n", 846 | "124946 The Secret Life of Bees \n", 847 | "124958 Slaves in the Family (Ballantine Reader's Circle) \n", 848 | "\n", 849 | " book_title_short pred_rating \n", 850 | "124934 The Horse and His Boy 7.814202 \n", 851 | "124935 The Voyage of the Dawn Treader (rack) (Narnia) 8.244509 \n", 852 | "124937 The Silver Chair 8.184727 \n", 853 | "124946 The Secret Life of Bees 8.281881 \n", 854 | "124958 Slaves in the Family (Ballantine Reader's 8.055557 " 855 | ] 856 | }, 857 | "execution_count": 13, 858 | "metadata": {}, 859 | "output_type": "execute_result" 860 | } 861 | ], 862 | "source": [ 863 | "df_user[df_user['pred_rating'].notna()].sort_values('book_rating', ascending=False).head(5)" 864 | ] 865 | }, 866 | { 867 | "cell_type": "markdown", 868 | "metadata": {}, 869 | "source": [ 870 | "### Test set: actual top rated books\n", 871 | "\n", 872 | "![](img/test_actual.jpg)" 873 | ] 874 | } 875 | ], 876 | "metadata": { 877 | "kernelspec": { 878 | "display_name": "master", 879 | "language": "python", 880 | "name": "master" 881 | }, 882 | "language_info": { 883 | "codemirror_mode": { 884 | "name": "ipython", 885 | "version": 3 886 | }, 887 | "file_extension": ".py", 888 | "mimetype": "text/x-python", 889 | "name": "python", 890 | "nbconvert_exporter": "python", 891 | "pygments_lexer": "ipython3", 892 | "version": "3.7.6" 893 | } 894 | }, 895 | "nbformat": 4, 896 | "nbformat_minor": 2 897 | } 898 | -------------------------------------------------------------------------------- /functions.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import seaborn as sns 3 | import matplotlib.pyplot as plt 4 | from string import ascii_letters, digits 5 | from surprise.model_selection import cross_validate 6 | 7 | ### DataFrame operations 8 | 9 | def k_from_details(details): 10 | try: 11 | return details['actual_k'] 12 | except KeyError: 13 | return 1000 14 | 15 | def short_title(title, max_len=40): 16 | title = str(title).split(' ') 17 | short_title = '' 18 | 19 | for i in range(len(title)): 20 | if len(short_title) < max_len: 21 | short_title = ' '.join([short_title, title[i]]) 22 | short_title = short_title.strip() 23 | return short_title 24 | 25 | def ascii_check(item): 26 | for letter in str(item): 27 | if letter not in ascii_letters + digits: 28 | return 1 29 | else: 30 | return 0 31 | 32 | def ascii_check_bulk(df): 33 | for col in df.columns: 34 | print('items with non-ascii characters in %s: %d' % (col, df[col].apply(ascii_check).sum())) 35 | print('') 36 | 37 | def colname_fix(colname): 38 | return colname.lower().replace('-','_') 39 | 40 | ### New DataFrames 41 | 42 | def df_dist(df, colname, norm=False): 43 | new_df = df[colname].value_counts(normalize=norm).reset_index() 44 | new_df.columns = [colname, 'count'] 45 | return new_df 46 | 47 | def books_groupby(df, column, new_colname): 48 | df_groupby = df.groupby(column).agg({'isbn': 'count', 'book_rating': 'mean'}).reset_index() 49 | df_groupby.columns = [new_colname, 'count', 'avg_rating'] 50 | return df_groupby 51 | 52 | ### Visualizations 53 | 54 | def draw_distribution(data, title_part, threshold=20): 55 | fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(14, 4)) 56 | 57 | sns.distplot(data['count'], color='#2f6194', ax=ax1) 58 | ax1.set_title('Distribution of number of ratings per %s' % title_part) 59 | 60 | sns.countplot(data[data['count']<=threshold]['count'], color='#2f6194', ax=ax2) 61 | ax2.set_title('Distribution of number of ratings per %s (<= %d ratings)' % (title_part, threshold)) 62 | 63 | plt.show() 64 | 65 | def draw_top_chart(data, x, y_list, title): 66 | fig, ax1 = plt.subplots(figsize=(14, 6)) 67 | plt.xticks(rotation=90) 68 | 69 | palette = sns.color_palette("RdBu", len(data)) 70 | 71 | sns.barplot(x=x, y=y_list[0], data=data, palette=palette, ax=ax1) 72 | ax1.set_title(title) 73 | 74 | ax2 = ax1.twinx() 75 | sns.scatterplot(x=x, y=y_list[1], data=data, color='black', ax=ax2) 76 | 77 | plt.show() 78 | 79 | ### Model-related functions 80 | 81 | def get_model_name(model): 82 | return str(model).split('.')[-1].split(' ')[0].replace("'>", "") 83 | 84 | def cv_multiple_models(data, models_dict, cv=3): 85 | results = pd.DataFrame() 86 | 87 | for model_name, model in models_dict.items(): 88 | print('\n---> CV for %s...' % model_name) 89 | 90 | cv_results = cross_validate(model, data, cv=cv) 91 | tmp = pd.DataFrame(cv_results).mean() 92 | tmp['model'] = model_name 93 | results = results.append(tmp, ignore_index=True) 94 | 95 | return results 96 | 97 | def generate_models_dict(models, sim_names, user_based): 98 | models_dict = {} 99 | 100 | for sim_name in sim_names: 101 | sim_dict = { 102 | 'name': sim_name, 103 | 'user_based': user_based 104 | } 105 | for model in models: 106 | model_name = get_model_name(model) + ' ' + sim_name 107 | models_dict[model_name] = model(sim_options=sim_dict) 108 | 109 | return models_dict 110 | 111 | def draw_model_results(results): 112 | fig, ax1 = plt.subplots(figsize=(10, 6)) 113 | plt.xticks(rotation=90) 114 | 115 | palette = sns.color_palette("RdBu", len(results)) 116 | 117 | sns.barplot(x='model', y='test_rmse', data=results, palette=palette, ax=ax1) 118 | ax1.set_title('Test RMSE and fit time of evaluated models') 119 | 120 | ax2 = ax1.twinx() 121 | sns.scatterplot(x='model', y='fit_time', data=results, color='black', ax=ax2) 122 | ax2.set(ylim=(0, results['fit_time'].max() * 1.1)) 123 | 124 | plt.show() -------------------------------------------------------------------------------- /img/books_header.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klaudia-nazarko/collaborative-filtering-python/8196bce6135bc42a40b36c2f7a1c214dddcf09b1/img/books_header.jpg -------------------------------------------------------------------------------- /img/test_actual.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klaudia-nazarko/collaborative-filtering-python/8196bce6135bc42a40b36c2f7a1c214dddcf09b1/img/test_actual.jpg -------------------------------------------------------------------------------- /img/test_pred.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klaudia-nazarko/collaborative-filtering-python/8196bce6135bc42a40b36c2f7a1c214dddcf09b1/img/test_pred.jpg -------------------------------------------------------------------------------- /img/train_actual.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klaudia-nazarko/collaborative-filtering-python/8196bce6135bc42a40b36c2f7a1c214dddcf09b1/img/train_actual.jpg --------------------------------------------------------------------------------