├── app
    ├── static
    │   ├── favicon.ico
    │   ├── favicon-16x16.png
    │   └── favicon-32x32.png
    ├── templates
    │   ├── 404.html
    │   ├── index.html
    │   └── base.html
    ├── config.py
    ├── views.py
    └── __init__.py
├── application.py
├── .gitignore
├── requirements.txt
├── gunicorn.sh
├── LICENSE
├── README.md
├── review-analysis-A.ipynb
├── textacy-modeling.ipynb
├── analyze-beer-soup.ipynb
└── scrape-all-ba.ipynb


/app/static/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WillNetsky/beer_recommender/HEAD/app/static/favicon.ico


--------------------------------------------------------------------------------
/app/static/favicon-16x16.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WillNetsky/beer_recommender/HEAD/app/static/favicon-16x16.png


--------------------------------------------------------------------------------
/app/static/favicon-32x32.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WillNetsky/beer_recommender/HEAD/app/static/favicon-32x32.png


--------------------------------------------------------------------------------
/application.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | from app import app as application
3 | 
4 | if __name__ == '__main__':
5 |     application.run(port=9000, threaded=True, debug=True)
6 | 


--------------------------------------------------------------------------------
/app/templates/404.html:
--------------------------------------------------------------------------------
1 | {% extends 'base.html' %}
2 | 
3 | {% block body %}
4 |     <div class="content">
5 |         </p>404! Something Went Wrong!</p>
6 |     </div>
7 | {% endblock %}


--------------------------------------------------------------------------------
/app/config.py:
--------------------------------------------------------------------------------
1 | WTF_CSRF_ENABLED = True
2 | SECRET_KEY = 'you-will-never-guess'
3 | 
4 | debug = True
5 | SQL_URL = 'postgresql://Netsky:BeerIsReallyGood@beer-recommender-db.cnkyfxbo8ito.us-west-2.rds.amazonaws.com:5432/beer'


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Python stuff
 2 | *.pyc
 3 | .ipynb_checkpoints
 4 | 
 5 | # OS Stuff
 6 | .DS_Store
 7 | 
 8 | # Model Stuff
 9 | *.pkl
10 | 
11 | # Elastic Beanstalk Files
12 | .elasticbeanstalk/*
13 | !.elasticbeanstalk/*.cfg.yml
14 | !.elasticbeanstalk/*.global.yml
15 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | beautifulsoup4==4.4.1
 2 | boto==2.40.0
 3 | bs4==0.0.1
 4 | bz2file==0.98
 5 | click==6.6
 6 | Flask==0.11.1
 7 | Flask-WTF==0.12
 8 | gensim==0.13.1
 9 | itsdangerous==0.24
10 | Jinja2==2.8
11 | MarkupSafe==0.23
12 | numpy==1.22.0
13 | pandas==0.18.1
14 | python-dateutil==2.5.3
15 | pytz==2016.4
16 | requests==2.10.0
17 | scipy==0.17.1
18 | six==1.10.0
19 | smart-open==1.3.3
20 | Werkzeug==0.11.10
21 | WTForms==2.1
22 | 


--------------------------------------------------------------------------------
/gunicorn.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | NAME="Punctuations-App"
 4 | APPDIR=/home/ubuntu/projects/iris/app
 5 | SOCKFILE=/home/ubuntu/www/sock
 6 | NUM_WORKERS=3
 7 | 
 8 | echo "Starting $NAME"
 9 | 
10 | # activate the virtualenv
11 | source activate env
12 | 
13 | export PYTHONPATH=$APPDIR:$PYTHONPATH
14 | 
15 | # Create the run directory if it doesn't exist
16 | RUNDIR=$(dirname $SOCKFILE)
17 | test -d $RUNDIR || mkdir -p $RUNDIR
18 | 
19 | # Start your unicorn
20 | exec gunicorn run:application -b 0.0.0.0:9000 \
21 |   --name $NAME \
22 |   --workers $NUM_WORKERS \
23 |   --log-level=debug \
24 |   --bind=unix:$SOCKFILE
25 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2016 Will Chernetsky
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # beer_recommender
 2 | beer recommendation engine project for Metis
 3 | 
 4 | # todo  
 5 | - [ ] update this todo, its very behind
 6 | 
 7 | **scraping**
 8 | - [x] take all beer styles  
 9 | - [x] scrape every beer with more than 25 hads (~20.8k)
10 | - [x] take first 100 reviews from top reviewers for those beers  
11 |   
12 | **text processing**
13 | - [x] remove names of breweries  
14 | - [x] remove words common to beer names (IPA etc)  
15 | - [x] remove words with numbers  
16 | - [x] lemmatize all words  
17 | 	- [x] clean up some more stopwords
18 | 	- [x] lemmatize all brewery names and remove them
19 | - [x] remove all words less than 3 chars (sn, rr, etc)  
20 |   
21 | **model**
22 | - [x] similar words between beers  
23 | - [ ] kmeans on the lsi, hopefully this gives something similar to the beer styles  
24 |   
25 | **visualization**
26 | - [ ] force directed graph of beer types  
27 | - [ ] teaswarm  
28 |   
29 | **Flask App**
30 | - [x] beer emoji favicon  
31 | - [x] javascript autocomplete  
32 | 	**result formatting**
33 | 	- [x] links to ba pages  
34 | 	- [x] two columns  
35 | 		- [x]  make this into a table  
36 | 		- [x] one is the beers with ba pages  
37 | 		- [x] one is the keywords between beers  
38 | 			- [ ] the other is the part of the visualization where these beers occur 
39 | - [x] make text input wider  
40 | - [ ] search for both beer and brewery 
41 | 	- [ ] search for style as well 
42 | 
43 | **PRESENTATION**
44 | - [x] diagram of data flow
45 | 
46 | **future additions**
47 | - [ ] hop word descriptions  
48 | - [ ] beer menus integration  
49 | - [ ] taphunter integration  
50 | - [x] sql database
51 | - [ ] pictures
52 | - [x] take out submit button, must click on autocomplete item to submit 
53 | - [ ] search capability
54 | 


--------------------------------------------------------------------------------
/app/views.py:
--------------------------------------------------------------------------------
 1 | 
 2 | from flask import render_template
 3 | from flask import jsonify
 4 | from flask import request
 5 | from flask_wtf import FlaskForm
 6 | from wtforms import fields
 7 | from wtforms.validators import Required, AnyOf
 8 | 
 9 | import pandas as pd
10 | import json
11 | 
12 | from . import app, get_beer_names, get_brewery_names, get_beer_info, get_beer_keywords, get_similar_beer_info, get_search_strings
13 | 
14 | 
15 | beer_names = get_beer_names()
16 | brewery_names = get_brewery_names()
17 | search_strings = get_search_strings()
18 | beer_and_brewery = []
19 | # for beer, brewery in zip(beer_names,brewery_names):
20 | #     beer_and_brewery.append(dict(beer = beer, brewery = brewery))
21 | 
22 | class PredictForm(FlaskForm):
23 |     """Fields for Predict"""
24 |     myChoices = ["one", "two", "three"]
25 |     beer_input = fields.StringField('Search for a Beer and Select from the Menu', validators=[Required(),AnyOf(beer_names)])
26 | 
27 |     submit = fields.SubmitField('Submit')
28 | 
29 | 
30 | @app.route('/', methods=('GET', 'POST'))
31 | def index():
32 | 
33 |     """Index page"""
34 |     form = PredictForm()
35 | 
36 |     similar_beers = [None]*6
37 |     beer_keywords = (None, [None]*6)
38 |     beer_inputted = False
39 |     input_beer_keywords = [None]*5
40 |     input_beer = None
41 |     similar_beer_keywords = None
42 | 
43 |     if form.validate_on_submit():
44 |         # store the submitted values
45 |         submitted_data = form.data
46 |         print(submitted_data)
47 | 
48 |         # Retrieve values from form
49 |         beer_input = submitted_data['beer_input']
50 | 
51 |         # Input beer info
52 |         input_beer = get_beer_info(beer_input)
53 |         input_beer_keywords = get_beer_keywords(beer_input)
54 |         similar_beers = input_beer_keywords[1]
55 |         input_beer_keywords = input_beer_keywords[0]
56 | 
57 |         # Get similar beer recommendations
58 |         beer_inputted = True
59 |         similar_beers = get_similar_beer_info(similar_beers)
60 | 
61 |     return render_template('index.html', form=form, beer_inputted = beer_inputted,
62 |         similar_beers= similar_beers,
63 |         input_beer = input_beer,
64 |         input_beer_keywords = input_beer_keywords)
65 | 
66 | @app.route('/autocomplete', methods=['GET'])
67 | def autocomplete():
68 |     search = request.args.get('term')
69 |     #app.logger.debug(search)
70 | 
71 |     return jsonify(beer=beer_names, brewery = brewery_names, search_string = search_strings)
72 | 


--------------------------------------------------------------------------------
/app/__init__.py:
--------------------------------------------------------------------------------
 1 | from __future__ import print_function
 2 | 
 3 | from flask import Flask
 4 | import pickle
 5 | import pandas as pd
 6 | from gensim import similarities
 7 | 
 8 | import sqlalchemy
 9 | 
10 | import config
11 | 
12 | app = Flask(__name__)
13 | app.config.from_object("app.config")
14 | 
15 | engine = sqlalchemy.create_engine(config.SQL_URL)
16 | beers = pd.read_pickle('app/models/beer_review_df.pkl')
17 | beer_names = list(beers.name)
18 | brewery_names = list(beers.brewery_name)
19 | search_strings = list(beers.search_string)
20 | beers = None
21 | 
22 | def get_beer_names():
23 |     return beer_names
24 | 
25 | def get_brewery_names():
26 |     return brewery_names
27 | 
28 | def get_search_strings():
29 |     return search_strings
30 | 
31 | def get_beer_keywords(text_input):
32 |     connection = engine.connect()
33 |     t = sqlalchemy.sql.expression.text('select * from beers where name = :name')
34 |     sql_result = connection.execute(t,name = text_input)
35 |     result = sql_result.first()
36 |     connection.close()
37 |     return ([result['kw1'],result['kw2'],result['kw3'],result['kw4'],result['kw5']],
38 |             [result['sim_id_1'],result['sim_id_2'],result['sim_id_3'],result['sim_id_4'],result['sim_id_5']])
39 | 
40 | def get_similar_beer_info(similar_beers):
41 |     connection = engine.connect()
42 |     query = 'select * from beers where '
43 |     
44 |     query += 'index = ' + str(similar_beers[0])
45 |     for beer in similar_beers[1:]:
46 |         query += ' or index = ' + str(beer)
47 |         
48 |     sql_result = connection.execute(query)
49 |     
50 |     names = []
51 |     urls = []
52 |     breweries = []
53 |     websites = []
54 |     all_keywords = []
55 |     for row in sql_result:
56 |         names.append(row['name'])
57 |         urls.append(row['url'])
58 |         breweries.append(row['brewery_name'])
59 |         websites.append(row['brewery_website'])
60 |         keywords = [row['kw1'],row['kw2'],row['kw3'],row['kw4'],row['kw5']]
61 |         all_keywords.append(keywords)
62 |     connection.close()
63 |     return zip(zip(names,urls,breweries,websites),all_keywords)
64 | 
65 | def get_beer_info(text_input):
66 |     connection = engine.connect()
67 |     t = sqlalchemy.sql.expression.text('select * from beers where name = :name')
68 |     sql_result = connection.execute(t,name = text_input)
69 |     result = sql_result.first()
70 |     connection.close()
71 |     return (result['name'],result['url'],result['brewery_name'],result['brewery_website'])
72 | 
73 | from .views import *
74 | 
75 | 
76 | # Handle Bad Requests
77 | @app.errorhandler(404)
78 | def page_not_found(e):
79 |     """Page Not Found"""
80 |     return render_template('404.html'), 404
81 | 


--------------------------------------------------------------------------------
/app/templates/index.html:
--------------------------------------------------------------------------------
 1 | {% extends 'base.html' %}
 2 | 
 3 | {% block body %}
 4 |   <section class="header">
 5 |     <h2 class="title"><i class="em em-beers"></i>Beer Recommendation Engine<i class="em em-beers"></i></h2>
 6 |   </section>
 7 |   <hr />
 8 | 
 9 |   <div class="content">
10 | <!--     {% if not beer_inputted %}
11 |       <h3>
12 |         Enter the name of a beer!
13 |       </h3>
14 |     {% endif%} -->
15 |     <form method="POST">
16 |       {{ form.hidden_tag() }}
17 |       <div>
18 |         {{ form.beer_input.label }} {{ form.beer_input() }}
19 |       </div>
20 |     </form>
21 |     <h4>
22 | 
23 |       {% if beer_inputted %}
24 |       <a href=http://www.beeradvocate.com{{input_beer[1]}}>{{input_beer[0]}}</a> by <a href={{input_beer[3]}}>{{input_beer[2]}}</a> can be described as:<br>
25 |         {% for word in input_beer_keywords%}
26 |           {{word}}
27 |         {% endfor %} <br>
28 |     </h4>
29 |     <h5>
30 |       Here are some beers like <a href=http://www.beeradvocate.com{{input_beer[1]}}>{{input_beer[0]}}</a>: 
31 |     </h5>
32 |       <table>
33 |         <thead>
34 |           <th>Similar Beer</th>
35 |           <th>Keywords</th>
36 |         </thead>
37 | 
38 |         <tbody>
39 |           {% for beer in similar_beers %}
40 |           <tr>
41 |             <td>
42 |             <a href=http://www.beeradvocate.com{{beer[0][1]}}>{{beer[0][0]}}</a> by <a href={{beer[0][3]}}>{{beer[0][2]}}</a>
43 |             </td>
44 |             <td>{% for word in beer[1] %}
45 |                   {{word}}
46 |                 {% endfor %}
47 |             </td>
48 |           </tr>
49 |           {% endfor %}
50 |         </tbody>
51 |       </table>
52 |       {% endif %}
53 | 
54 | 
55 |     
56 |   </div>
57 | {% endblock body %}
58 | 
59 | {% block js %}
60 |  <script>
61 |   $(function() {
62 |       $.ajax({
63 |           url: '{{ url_for("autocomplete") }}'
64 |           }).done(function (data) {
65 |               window.data = data;
66 |               $('#beer_input').autocomplete({
67 |                   source: function(request, response) {
68 |                     var results = $.ui.autocomplete.filter(data.search_string, request.term);
69 | 
70 |                     response(results.slice(0, 10));
71 |                   },
72 |                   select: function(event, ui) {
73 |                       //assign value back to the form element
74 |                       if(ui.item){
75 |                           beerIdx = data.search_string.indexOf(ui.item.label)
76 |                           beer = data.beer[beerIdx];
77 |                           $(event.target).val(beer);
78 |                       }
79 |                       //submit the form
80 |                       $(event.target.form).submit();
81 |                   },
82 |                   minLength: 3
83 |               })
84 |               .autocomplete( "instance" )._renderItem = function( ul, item ) {
85 |                 beerIdx = data.search_string.indexOf(item.label);
86 |                 beer    = data.beer[beerIdx];
87 |                 brewery = data.brewery[beerIdx];
88 |                 return $( "<li>" )
89 |                   .append( "<a id=dropdwnBeer>" + beer + "</a><br><a id=dropdwnBrewery> by " + brewery + "</a>" )
90 |                   .appendTo( ul );
91 |               };
92 |           });
93 |   });
94 |   </script>
95 | {% endblock js %}


--------------------------------------------------------------------------------
/app/templates/base.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html lang="en">
  3 |   <head>
  4 |     <title>{% block title %}Beer Recommendation Engine{% endblock title %}</title>
  5 |     <meta charset="utf-8">
  6 |     <meta http-equiv="X-UA-Compatible" content="IE=edge">
  7 |     <meta name="viewport" content="width=device-width, initial-scale=1">
  8 | 
  9 | 
 10 | 
 11 |     <link rel="shortcut icon" href="{{ url_for('static', filename='favicon.ico') }}">
 12 |     <link href="https://afeld.github.io/emoji-css/emoji.css" rel="stylesheet">
 13 | 
 14 |     <script src="https://code.jquery.com/jquery-1.12.4.js"></script>
 15 |     <!-- <script src="//code.jquery.com/ui/1.11.4/jquery-ui.js"></script> -->
 16 |     <script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js"></script>
 17 | 
 18 |     <link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/normalize/4.1.1/normalize.css">
 19 |     <link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/skeleton/2.0.4/skeleton.css">
 20 |     <!-- <link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.17/d3.js"> -->
 21 |     <link rel="stylesheet" href="https://code.jquery.com/ui/1.12.1/themes/smoothness/jquery-ui.css">
 22 |     
 23 |     <script type=text/javascript>
 24 |       $SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
 25 |     </script>
 26 | 
 27 |     <style>
 28 |       #beer_names {
 29 |         float: left ;
 30 |         width: 70% ;
 31 |       }
 32 |       #beer_words {
 33 |         float: left;
 34 |         width: 30%;
 35 |       }
 36 |       #dropdwnBeer{
 37 |         vertical-align: 
 38 |       }
 39 |       #dropdwnBrewery {
 40 |         font-weight: lighter;
 41 |         font-size: 0.5em;
 42 |         color : #999999;
 43 |         vertical-align: text-top;
 44 |       }
 45 |       a {
 46 |         color: #007889; }
 47 |       a:hover {
 48 |         color: #005a66; }
 49 |       .ui-menu {
 50 |         list-style: none;
 51 |         padding: 0;
 52 |         margin: 0;
 53 |         display: block;
 54 |         outline: none;
 55 |       }
 56 |       .ui-menu .ui-menu {
 57 |         position: absolute;
 58 |       }
 59 |       .ui-menu .ui-menu-item {
 60 |         position:  relative;
 61 |         margin: 0px;
 62 |         cursor: pointer;
 63 |         border-style: groove;
 64 |         border-width: 1px;
 65 |         min-height: 0; /* support: IE7 */
 66 |         /* support: IE10, see #8844 */
 67 |         list-style-image: url("data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7");
 68 |       }
 69 |       .ui-menu .ui-menu-divider {
 70 |         margin: 0px 0;
 71 |         height: 0;
 72 |         font-size: 0;
 73 |         line-height: 1;
 74 |         border-width: 3px 0 0 0;
 75 |       }
 76 |       .ui-menu .ui-state-focus,
 77 |       .ui-menu .ui-state-active {
 78 |         margin: -2px;
 79 |         background: #E1E1E1;
 80 |       }
 81 |       .ui-autocomplete-input{
 82 |         width : 80%;
 83 |       }
 84 |       .button.button-primary,
 85 |       button.button-primary,
 86 |       input[type="submit"].button-primary,
 87 |       input[type="reset"].button-primary,
 88 |       input[type="button"].button-primary {
 89 |         color: #FFF;
 90 |         background-color: #f4901e;
 91 |         border-color: #f4901e;
 92 |       }
 93 |       .button.button-primary:hover,
 94 |       button.button-primary:hover,
 95 |       input[type="submit"].button-primary:hover,
 96 |       input[type="reset"].button-primary:hover,
 97 |       input[type="button"].button-primary:hover,
 98 |       .button.button-primary:focus,
 99 |       button.button-primary:focus,
100 |       input[type="submit"].button-primary:focus,
101 |       input[type="reset"].button-primary:focus,
102 |       input[type="button"].button-primary:focus {
103 |         color: #FFF;
104 |         background-color: #db7a0a;
105 |         border-color: #db7a0a;
106 |       }
107 |       th,
108 |       td {
109 |         padding: 6px 6px;
110 |         text-align: left;
111 |         border-bottom: 1px solid #E1E1E1; }
112 |       th:first-child,
113 |       td:first-child {
114 |         border-right: 1px solid #E1E1E1; }
115 |     </style>
116 | 
117 |     {% block header %}{% endblock header %}
118 |   </head>
119 |     <body>
120 |       <div class="container">
121 |         {% block body %}{% endblock body %}
122 |       </div>
123 |       {% block js %}{% endblock js %}
124 |     </body>
125 | 
126 | </html>


--------------------------------------------------------------------------------
/review-analysis-A.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 14,
  6 |    "metadata": {
  7 |     "collapsed": false
  8 |    },
  9 |    "outputs": [],
 10 |    "source": [
 11 |     "from gensim import corpora, models, similarities\n",
 12 |     "import pickle\n",
 13 |     "import string"
 14 |    ]
 15 |   },
 16 |   {
 17 |    "cell_type": "code",
 18 |    "execution_count": 15,
 19 |    "metadata": {
 20 |     "collapsed": false
 21 |    },
 22 |    "outputs": [],
 23 |    "source": [
 24 |     "def remove_punctuation(x):\n",
 25 |     "    x = str(x)\n",
 26 |     "    return x.translate(string.maketrans('',''),string.punctuation)"
 27 |    ]
 28 |   },
 29 |   {
 30 |    "cell_type": "code",
 31 |    "execution_count": 16,
 32 |    "metadata": {
 33 |     "collapsed": false
 34 |    },
 35 |    "outputs": [],
 36 |    "source": [
 37 |     "beer_reviews = pickle.load(open('beer_reviews.pkl','rb'))"
 38 |    ]
 39 |   },
 40 |   {
 41 |    "cell_type": "code",
 42 |    "execution_count": 17,
 43 |    "metadata": {
 44 |     "collapsed": false
 45 |    },
 46 |    "outputs": [],
 47 |    "source": [
 48 |     "documents = [reviews[1] for reviews in beer_reviews.items()]"
 49 |    ]
 50 |   },
 51 |   {
 52 |    "cell_type": "code",
 53 |    "execution_count": 18,
 54 |    "metadata": {
 55 |     "collapsed": true
 56 |    },
 57 |    "outputs": [],
 58 |    "source": [
 59 |     "documents = [' '.join(review) for review in documents]"
 60 |    ]
 61 |   },
 62 |   {
 63 |    "cell_type": "code",
 64 |    "execution_count": 19,
 65 |    "metadata": {
 66 |     "collapsed": false
 67 |    },
 68 |    "outputs": [],
 69 |    "source": [
 70 |     "documents = [remove_punctuation(doc) for doc in documents]"
 71 |    ]
 72 |   },
 73 |   {
 74 |    "cell_type": "code",
 75 |    "execution_count": 20,
 76 |    "metadata": {
 77 |     "collapsed": false
 78 |    },
 79 |    "outputs": [],
 80 |    "source": [
 81 |     "documents = [review.lower() for review in documents]"
 82 |    ]
 83 |   },
 84 |   {
 85 |    "cell_type": "code",
 86 |    "execution_count": 21,
 87 |    "metadata": {
 88 |     "collapsed": false
 89 |    },
 90 |    "outputs": [],
 91 |    "source": [
 92 |     "stoplist = set('for a of the and to in'.split())\n",
 93 |     "texts = [[word for word in document.lower().split() if word not in stoplist]for document in documents]"
 94 |    ]
 95 |   },
 96 |   {
 97 |    "cell_type": "code",
 98 |    "execution_count": 22,
 99 |    "metadata": {
100 |     "collapsed": true
101 |    },
102 |    "outputs": [],
103 |    "source": [
104 |     "from collections import defaultdict"
105 |    ]
106 |   },
107 |   {
108 |    "cell_type": "code",
109 |    "execution_count": 23,
110 |    "metadata": {
111 |     "collapsed": true
112 |    },
113 |    "outputs": [],
114 |    "source": [
115 |     "frequency = defaultdict(int)\n",
116 |     "for text in texts:\n",
117 |     "    for token in text:\n",
118 |     "        frequency[token] += 1\n",
119 |     "        \n",
120 |     "texts = [[token for token in text if frequency[token] > 1]for text in texts]"
121 |    ]
122 |   },
123 |   {
124 |    "cell_type": "code",
125 |    "execution_count": 24,
126 |    "metadata": {
127 |     "collapsed": false
128 |    },
129 |    "outputs": [],
130 |    "source": [
131 |     "dictionary = corpora.Dictionary(texts)"
132 |    ]
133 |   },
134 |   {
135 |    "cell_type": "code",
136 |    "execution_count": 25,
137 |    "metadata": {
138 |     "collapsed": true
139 |    },
140 |    "outputs": [],
141 |    "source": [
142 |     "corpus = [dictionary.doc2bow(text) for text in texts]"
143 |    ]
144 |   },
145 |   {
146 |    "cell_type": "code",
147 |    "execution_count": 26,
148 |    "metadata": {
149 |     "collapsed": true
150 |    },
151 |    "outputs": [],
152 |    "source": [
153 |     "tfidf = models.TfidfModel(corpus)"
154 |    ]
155 |   },
156 |   {
157 |    "cell_type": "code",
158 |    "execution_count": 27,
159 |    "metadata": {
160 |     "collapsed": true
161 |    },
162 |    "outputs": [],
163 |    "source": [
164 |     "corpus_tfidf = tfidf[corpus]"
165 |    ]
166 |   },
167 |   {
168 |    "cell_type": "code",
169 |    "execution_count": 30,
170 |    "metadata": {
171 |     "collapsed": false
172 |    },
173 |    "outputs": [],
174 |    "source": [
175 |     "lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=500)"
176 |    ]
177 |   },
178 |   {
179 |    "cell_type": "code",
180 |    "execution_count": 31,
181 |    "metadata": {
182 |     "collapsed": false
183 |    },
184 |    "outputs": [],
185 |    "source": [
186 |     "index = similarities.MatrixSimilarity(lsi[corpus])"
187 |    ]
188 |   },
189 |   {
190 |    "cell_type": "code",
191 |    "execution_count": 32,
192 |    "metadata": {
193 |     "collapsed": true
194 |    },
195 |    "outputs": [],
196 |    "source": [
197 |     "beer_names = beer_reviews.keys()"
198 |    ]
199 |   },
200 |   {
201 |    "cell_type": "code",
202 |    "execution_count": 33,
203 |    "metadata": {
204 |     "collapsed": false
205 |    },
206 |    "outputs": [],
207 |    "source": [
208 |     "doc = ''\n",
209 |     "vec_bow = dictionary.doc2bow(doc.lower().split())\n",
210 |     "vec_lsi = lsi[vec_bow]"
211 |    ]
212 |   },
213 |   {
214 |    "cell_type": "code",
215 |    "execution_count": 34,
216 |    "metadata": {
217 |     "collapsed": true
218 |    },
219 |    "outputs": [],
220 |    "source": [
221 |     "text_input = 'Heady Topper'"
222 |    ]
223 |   },
224 |   {
225 |    "cell_type": "code",
226 |    "execution_count": 35,
227 |    "metadata": {
228 |     "collapsed": false,
229 |     "scrolled": true
230 |    },
231 |    "outputs": [
232 |     {
233 |      "name": "stdout",
234 |      "output_type": "stream",
235 |      "text": [
236 |       "Pliny The Elder : 92.63\n",
237 |       "Palate Wrecker : 91.26\n",
238 |       "Abrasive Ale : 91.21\n",
239 |       "Stone Enjoy By IPA : 90.74\n",
240 |       "Stone RuinTen Triple IPA : 90.70\n"
241 |      ]
242 |     }
243 |    ],
244 |    "source": [
245 |     "# get the reviews for a beer\n",
246 |     "beer_name_inputted = 1\n",
247 |     "try:\n",
248 |     "    doc= documents[beer_names.index(text_input)]\n",
249 |     "except ValueError:\n",
250 |     "    doc = text_input\n",
251 |     "    beer_name_inputted = 0\n",
252 |     "vec_bow = dictionary.doc2bow(doc.lower().split())\n",
253 |     "vec_lsi = lsi[vec_bow]\n",
254 |     "\n",
255 |     "sims = index[vec_lsi]\n",
256 |     "for beer in sorted(enumerate(sims), key = lambda x: -x[1])[beer_name_inputted:beer_name_inputted+5]:\n",
257 |     "    print(beer_names[beer[0]] + ' : %.2f' % (beer[1]*100))"
258 |    ]
259 |   },
260 |   {
261 |    "cell_type": "code",
262 |    "execution_count": 36,
263 |    "metadata": {
264 |     "collapsed": true
265 |    },
266 |    "outputs": [],
267 |    "source": [
268 |     "# documents\n",
269 |     "# dictionary\n",
270 |     "# lsi    41.2s\n",
271 |     "# index  6.4s --rebuild on app"
272 |    ]
273 |   },
274 |   {
275 |    "cell_type": "code",
276 |    "execution_count": 41,
277 |    "metadata": {
278 |     "collapsed": false
279 |    },
280 |    "outputs": [],
281 |    "source": [
282 |     "pickle.dump(documents,open('flask/app/models/documents.pkl','wb'))"
283 |    ]
284 |   },
285 |   {
286 |    "cell_type": "code",
287 |    "execution_count": 42,
288 |    "metadata": {
289 |     "collapsed": true
290 |    },
291 |    "outputs": [],
292 |    "source": [
293 |     "pickle.dump(dictionary,open('flask/app/models/dictionary.pkl','wb'))"
294 |    ]
295 |   },
296 |   {
297 |    "cell_type": "code",
298 |    "execution_count": 43,
299 |    "metadata": {
300 |     "collapsed": true
301 |    },
302 |    "outputs": [],
303 |    "source": [
304 |     "pickle.dump(lsi,open('flask/app/models/lsi.pkl','wb'))"
305 |    ]
306 |   },
307 |   {
308 |    "cell_type": "code",
309 |    "execution_count": 54,
310 |    "metadata": {
311 |     "collapsed": false
312 |    },
313 |    "outputs": [],
314 |    "source": [
315 |     "pickle.dump(corpus,open('flask/app/models/corpus.pkl','wb'))"
316 |    ]
317 |   },
318 |   {
319 |    "cell_type": "code",
320 |    "execution_count": 55,
321 |    "metadata": {
322 |     "collapsed": true
323 |    },
324 |    "outputs": [],
325 |    "source": [
326 |     "pickle.dump(beer_names,open('flask/app/models/beer_names.pkl','wb'))"
327 |    ]
328 |   },
329 |   {
330 |    "cell_type": "code",
331 |    "execution_count": 56,
332 |    "metadata": {
333 |     "collapsed": true
334 |    },
335 |    "outputs": [],
336 |    "source": [
337 |     "pickle.dump(index,open('flask/app/models/index.pkl','wb'))"
338 |    ]
339 |   },
340 |   {
341 |    "cell_type": "code",
342 |    "execution_count": null,
343 |    "metadata": {
344 |     "collapsed": true
345 |    },
346 |    "outputs": [],
347 |    "source": []
348 |   },
349 |   {
350 |    "cell_type": "code",
351 |    "execution_count": 1,
352 |    "metadata": {
353 |     "collapsed": true
354 |    },
355 |    "outputs": [],
356 |    "source": [
357 |     "from sklearn.cluster import KMeans"
358 |    ]
359 |   },
360 |   {
361 |    "cell_type": "code",
362 |    "execution_count": null,
363 |    "metadata": {
364 |     "collapsed": true
365 |    },
366 |    "outputs": [],
367 |    "source": []
368 |   }
369 |  ],
370 |  "metadata": {
371 |   "kernelspec": {
372 |    "display_name": "Python 2",
373 |    "language": "python",
374 |    "name": "python2"
375 |   },
376 |   "language_info": {
377 |    "codemirror_mode": {
378 |     "name": "ipython",
379 |     "version": 2
380 |    },
381 |    "file_extension": ".py",
382 |    "mimetype": "text/x-python",
383 |    "name": "python",
384 |    "nbconvert_exporter": "python",
385 |    "pygments_lexer": "ipython2",
386 |    "version": "2.7.11"
387 |   }
388 |  },
389 |  "nbformat": 4,
390 |  "nbformat_minor": 0
391 | }
392 | 


--------------------------------------------------------------------------------
/textacy-modeling.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 24,
  6 |    "metadata": {
  7 |     "collapsed": false
  8 |    },
  9 |    "outputs": [],
 10 |    "source": [
 11 |     "import textacy\n",
 12 |     "import pandas as pd\n",
 13 |     "from tqdm import tqdm"
 14 |    ]
 15 |   },
 16 |   {
 17 |    "cell_type": "code",
 18 |    "execution_count": 40,
 19 |    "metadata": {
 20 |     "collapsed": true
 21 |    },
 22 |    "outputs": [],
 23 |    "source": [
 24 |     "beers = pd.read_pickle('all_beer_reviews.pkl')"
 25 |    ]
 26 |   },
 27 |   {
 28 |    "cell_type": "code",
 29 |    "execution_count": 59,
 30 |    "metadata": {
 31 |     "collapsed": false
 32 |    },
 33 |    "outputs": [
 34 |     {
 35 |      "data": {
 36 |       "text/plain": [
 37 |        "[' 2013 version poured into a snifter. Comes out rich, dark brown with bit of mocha foam that fades to a persistent lace. Smell is oak, rye, booze, chocolate, vanilla, slight malt. Very inviting nose on this. Taste is sweet, oak, vanilla, char, and nice rye bite near the tail end. Mouth is very smooth and creamy, though slightly light for the abv. It goes down very easy.This was a really enjoyable beer. Better than the 2012 OF IMO.',\n",
 38 |        " ' Cracked right before the Diaz/McGregor fight, pours as sturdy of thick black liquid as Eclipse ever has.  Plenty of roast, chocolate malt, dark oak barrel notes in the aroma.Taste, easily comes off as the sweetest Eclipse variety.  Less bourbon taste and more bourbon sweetness imparted, the char, vanilla aspect of a barrel is subdued in exchange for a greater, sweeter mouthfeel.  Sweet toffee, mild oak along with that sweetness.',\n",
 39 |        " ' Thanks to far & away my favorite mule/wife for bringing this bottle back from San Diego. Which is funny, because I looked high & low for Eclipse stouts in my seven weeks in San Diego in the last year & came up 100% empty. Luck of the Irish. 22 oz. lime-waxed bottle split with said mule. The pour is eclipse black, which is funny, because you can\\xe2\\x80\\x99t actually see an eclipse. Which isn\\xe2\\x80\\x99t funny at all. The nose is unusually light for both this line & the universe of barrel-aged stouts, really  just a little roast & malt sweetness. Pretty weak. Eclipse - High West Rye is wading in the shallow end of the gene pool. Most of the Eclipse are Emmy-ready, but this is more like the Prince of Bel-Air. Mild to the point of timid. Spare roast, barrel that\\xe2\\x80\\x99s faint, bit of the rye spice as it warms, sweetness, bit of wood. Nothing stands out. Thicker than the average Eclipse, but half a bottle is plenty. Not much to this. The barrel is thin bordering on anorexic, & without the highlight of the barrel, one is left with a base stout that\\xe2\\x80\\x99s $20 overpriced. That brings me down, man. Pretty good.',\n",
 40 |        " \" look: Dark dark brown as an Imp Stout should be, nearly opaque. Some cola highlights. Big ass tan head for a nearly 12%'r and thick creamy lacing.smell: Wow, lots of goodness here. A bit of Rye spice, in a honey roasty milkshake. taste: does not lose anything to the nose. If anything there's a chewy fudgey thing happening on top.  So mellow and easy drinking. Very nice indeed.\",\n",
 41 |        " \" Smooth drinking stout for sure. You get the spiciness from the rye that doesn't translate to overt sweetness as bourbon might which is nice. My only major knock on the eclipse beers is that they are all a bit thin mouthfeel wise...even moreso than say KBS, but that doesn't make them bad. This goes down easy, especially for the abv.\",\n",
 42 |        " ' inky black, thin with a mocha head, good legsnice rye scent, spice notes, vanilla,alcohol is there, dark malts give good chocolate notes,complex charactorthe nose and taste match upgood balance. a bit thin.real good, not great for me. too pricey.',\n",
 43 |        " \" Poured from a 2015 Batch 1 bottle. Poured a dark black color with a minimal head on top. The nose show hints of chocolate, sweetness and coffee with the mild rye whisky in the back. The taste is smooth and follows the nose chocolate coffee and the rye whisky very nicely blended together. The rye whisky tends to me a little more mellow that some of the bourbon does and you really don't feel and alcohol burn. This one of the smoothest drinking barrel aged beers I've had in quite some time.\",\n",
 44 |        " \" 2015 batch, bottle purchases from the brewery at the release partyFrom lime-green wax-sealed bomber to Eclipse snifterInstead of reviewing each variety individually, I like to refer to my first review -http://www.beeradvocate.com/beer/profile/14936/64055/?ba=rand - and note any differences or unique nuances.  Plus, I reviewed that when barrel-aging wasn't as ubiquitous as it is now, and I'd only had a handful of examples, so my palate descriptions were more specific and colorful.This one follows along the same flavor profile of all the Eclipse varieties, with distinct notes of chocolate and licorice and coffee.  You can just subtly make out the rye whisky aging, which isn't quite as sweet as its bourbon counterparts.  For some reason, this is tasty better than it did on tap, probably because my palate was numb from the onslaught of free-pouring whiskey-aged stout, but it doesn't seem as hot.  And this is a clear step up from last years batch -  this one is more drinkable and balanced.\",\n",
 45 |        " \" Bomber labeled Batch 2 (2014) into a snifter at cellar temp, allowed to warm.Look: robust brown, but not at all black; like lightly used motor oil. Minimal light brown head.Smell: at first I get sharp metallic notes, but eventually chocolate, coffee, and dates come through.Taste: bittersweet chocolate, roast, hints of rye spice, mild vanilla. Solid, but not a lot going on.Feel: medium bodied, silky, a bit dry.Overall: a nice tasty stout, and would be in an upper echelon for non-barrel-aged offerings (though still below Monster's Park). But if I didn't know it was barrel-aged, I wouldn't guess it. Maybe the year-plus of age was a bad idea. Nowhere near worth what it cost.\",\n",
 46 |        " ' Fantastic beer! Bourbon is there but nor overpowering at all, lovey sweet honey with dark malts, little coffee taste, dark fruit and rich dark chocolate. Very expensive but very very good',\n",
 47 |        " \" Poured from the bottle into a large snifter glass.  Vintage 2014, lime green wax.Deep classic and simple walnut dark brown body.  Head manages half a finger of off tan brown, slightly into mocha, but dissolves to a fine, and slightly bubbled collar.  A bit of a slick top, but not too much.  Some fine big bubbles manage to stick around on the top as well.Excellent aroma.  Spicy notes and a real sense of sweet and grainy fig quality.  There's even an interesting tropical note going on in here, that sort of hints at a bit of soft papaya or mango.  Light notes of chocolate and woody grain touches.  Soft dark chocolate aromas and almost with the tropical goes to sweet orange as well which is a great combination.  Very complex, very varied, and just fantastic.Palate continues the solid goodness.  Hints almost at fudgy character but takes a step back in thickness but complete oak round flavors and feel.  Mid palate fills with fruited spice, hitting an almost sharp peppered fruit and spicy cinnamon.  Chewy almost lime hop finish, with a crushing and oozy chocolate and char roast character.  First sip feels like a fruited stout, then comes an almost a heavy robust black IPA on the finish.  Light tastes of orange, tangerine, oils and mild bitter zest.  Hardly any sense of booze, and some cocoa powder dryness and honey lip smacking stickiness on the front.What a great play and use of sweetness, unexpected fruity quality, and delicious spice and whiskey character.  Another great feather in the cap for these guys and the Eclipse series.\",\n",
 48 |        " \" Bottle from my Vermont trip. Pours black with nice brown head. Aroma of lots of char, some fruity notes. Flavor follows with big char and sweetness. But this is way thin, even for an Eclipse variant. It's too bad they can't make a base beer that stands up to the barrels, because all the flavors here are really nice. Thicken this beer up, please.\",\n",
 49 |        " ' Aged 5 months, this pours smoothly with a slow rising head about 3/4 inch in a dark caramel. This smells pretty light. Not an overpowering aroma but has hints of bourbon and a light sweetness. The taste is rich with rye, light toffee, dark chocolate, and a creaminess that is reminiscent of a milk stout. The feel exceeds the flavor as it is silky with a medium to heavy body. Great brew and great experience, hands down.',\n",
 50 |        " ' 2014 22 oz bottle poured into snifterA: pure darkness very little head soapyS: bourbon some alcohol light roast T: very smooth light on the tongue nice roast some bourbon very softM: Have to combine the mouth with taste I talked a lot about how it feelsO: Its a good beer but lacks the depth and character that the Elijah Craig version does.   I would pick this version up again.',\n",
 51 |        " ' 2014 Vintage11.9% ABVAppearance: Very dark brown bordering on black with a moderate sized dark beige head that gradually faded.  A nice looking Imperial StoutSmell: Brown sugar, mild Whiskey, and mild roasted malt aromas. Not a bad nose but the barrel aromas are fairly mild.Taste: The rye Whisky bal-aging features prominently with some spicy notes not present in the Bourbon bal-aged Eclipses. The profile also features roast malts, faint vanilla, brown sugar and tannic oak notes. The tannic oak notes feature prominently in the taste profile and the finish is a bit drier than the Bourbon bal-aged variants. Fairy restrained alcohol presence.Mouth-feel: On the light side of medium bodied with a moderate level of carbonation. Overall: This was a good Eclipse variant with the spicy rye whiskey notes featuring prominently.',\n",
 52 |        " ' The beer offers a medium roasted malt, caramel sweetness, onto dark fruit, a spicy licorice, and a heavy on the finish. The beer has a medium mouthfeel. A bit spicy and intense without enough vanilla and chocolate to balance it out.',\n",
 53 |        " \" 2014 vintage. 22oz bottle with lime green wax. Bottle run 2. Pours a pitch black color with over an inch of medium brown head. Lots of lacing. The aroma is lots of wood and booze with dark chocolate, roasted malts, vanilla and earth. The taste is lots of rye flavor up front followed by chocolate syrup, roasted malts, a hint of smoke and some wood. Vanilla and caramel as well. Finishes with lots of the rye flavor. Nice amount of alcohol burn while still being relatively drinkable. Heavy mouthfeel with a good deal of carbonation. Overall, just like the other ones I have had, it's a very good BA Imperial Stout. Pricey, but worth buying once in a while.\",\n",
 54 |        " ' From 03/07/15 notes. 2013 vintage. 22 oz. bottle poured into a tulip.Pours a dark brown, almost black color, with one finger of dark tan head and moderate carbonation evident. Lots of milk chocolate, roasted malts, sweet chocolate, spice, rye harsh whiskey, caramel malts, and burnt malts. Medium body and low to moderate carbonation.Overall not my favorite variant but very good. Worth getting for sure if it is not too much effort.',\n",
 55 |        " \" 22oz, light lime green waxed bottle, BR 2 of 2014.This beer pours a solid black abyss, with the barest of back-lit basal cola edges, and one skinny finger of weakly foamy and mostly bubbly brown head, which leaves a bit of streaky and windblown ocean spray lace around the glass as things slowly abate.It smells of sugary and bready caramel malt, heady Bourbon barrel notes - edgy vanilla, spicy rye grain alcohol, and an acrid woodiness - black licorice, a touch of roasted day-old coffee, bittersweet chocolate, and a soft earthy, musty hop bitterness.  The taste is more roasted graininess and dry cocoa powder, before that big-assed rye whiskey barrel comes 'a, um, barreling through my gentle palate - crisp and yet astringent booze-soaked wood staves, dry vanillan, toasted caramel, and spicy rye bread - over some still perky dry anise and coffee bean esters, and a well-sublimated metallic alcohol measure.The carbonation is still noticeable in its plain and understated frothiness, the body medium-heavy in weight, and mostly smooth, the rye-ness of it all behaving like a perfect southern gentleman.  It finishes on the sweetly caramelized side, and very much like the rye it was aged in - spicy, caramelized, and woody.For a straight rye whiskey barrel aged imperial stout, this comes off as way more drinkable that it has any right to, IMHO.  I guess it helps that said whiskey is one of the higher-end rye products on the shelves of your local liquor purveyor, and keeps (so I've been told) the typical tongue-scraping acerbity to the bare minimum, as such.  Yeah, it's clear that I'm not a fan of the guest spirit in toto, but somehow my attitude changes as far as this product is concerned.  Good stuff and go freaking figure.\",\n",
 56 |        " \" BR/2 2014 batch.A: Pitch-black pour with a nice tall, dense medium brown head. Head lightens quickly but refuses to fade. Great head retention. Marginal lacing, just tiny little spots here and there.S: Spicy rye whiskey is noticeable and dominant right off the bat. Good amount of oak with it. A touch of char and hershey's syrup. T: Oh yeah. Tons of spicy rye whiskey. Huge amount in fact. Initially it tastes like I've sipped on a shot of HW rye (I have some right next to me in fact). Big, sharp and spicy rye character. It fades gradually into the middle where that hershey's syrup shows up again with a bitter coffee backing. Finish and aftertaste are a nice mashup of bittersweet chocolate and spicy rye whiskey. Hint of oak lingers as well.M: Heavy, spicy and bittersweet all at the same time. Not as heavy as some of the other variants, lots of alcohol to cut it down.O: Out of all the variants I've had, this one highlights the spiciness of rye whiskey the best. Huge rye notes and a sharpness that has to be experienced to understand. The combination of rye and alcohol bite really cuts down on the thickness and sweetness of the beer, however, balancing it fairly nicely. Well, almost balanced; it's still a rye whiskey bomb no matter how you look at it. Still, once you get past the first few sips it's really enjoyable. Let this one warm up for sure, any amount of coldness only seems to enhance the harshness. It sweetens up and mellows a bit once it's fully warm.\",\n",
 57 |        " ' A: Thin dark brown in color. Appearance of a thin brown head. Good carbonation.S: Rye on the nose, faint bourbon, chocolate, and grain. Nose is not overwhelming.T: Slightly sweet but cut by the rye whiskey flavor, hints of vanilla and chocolate, wood, earthy grain. Very little booze and a mild warming.M: Medium feel on the mouth. Smooth with good carbonation. O: Very good BA stout. Good representation of rye whiskey barrel aging.',\n",
 58 |        " ' 22oz bottle into snifter. 2013 release. A: Pours a deep black with a thin dark brown head that quickly disappeared into a faint, bubbly film. A ton of tiny, spotty, soapy lace was left down the glass. S: Roasted malts, chocolate, warming booze, and spicy rye. Smells rich and potent, and a bit sweet.T: Toasted and sweet notes of dark chocolate, rye whisky barrel, toasted coconut, vanilla, caramel, brown sugar and woody barrel. Very smooth and decadent, and sweeter than I remember some of the other variants being. The finish is full of spicy rye, toast, and sticky sweet candied sugar and honey, and the finish has a lingering toastedness, mild tobacco, leather, and bitter earthy hops. M: Full bodied, chewy and very sticky-sweet mouthfeel, soft and smooth carbonation. The alcohol is very mild and only slightly warming. O: An all-around great stout and one of the beer, thicker, and sweeter eclipse variants in recent memory. Drinks very well, especially at room temp.',\n",
 59 |        " ' bottle poured into two Sam Adams perfect pints.1 inch of head that dissipates leaving a slight bit of head. dark mahogany color is only revealed using a very bright flashlight.good carbonation.nose is chocolate, booze and bread. taste is boozy but the goodness shines through.I like some other big stouts more but this is one to try.mouthfeel finishes dry and bready. seems a little light for such a strong beverage. some stickiness on the lips.',\n",
 60 |        " ' Bottle opened on 4.21.14Barrel Run #1 2013A - Pours a very dark brown with a fine light brown head with good retention.S - Sweet rye and chocolate, subtle and nice.T - Swwet rye, barrel, chocolate, a touch of toffee.D - Good carbonation with a heavy body.O - Sweet, but nice balance. Rye is there. Good stuff.',\n",
 61 |        " ' Bottle poured into a snifter. 2013 vintage - bottle run #1A - black with dark brown edges. S- whiskey and chocolate very faint vanillaT - whiskey goodness, lots of chocolate and some rye spice but not overpowering. Blends pretty good. M - a little thin compared to the other eclipse variants. O - damn good BA stout. I liked the Evan Williams a bit better, but the rye barrel is definitely very well down. I look forward to the Elijah Craig and four roses.']"
 62 |       ]
 63 |      },
 64 |      "execution_count": 59,
 65 |      "metadata": {},
 66 |      "output_type": "execute_result"
 67 |     }
 68 |    ],
 69 |    "source": [
 70 |     "beers.reviews[0]"
 71 |    ]
 72 |   },
 73 |   {
 74 |    "cell_type": "code",
 75 |    "execution_count": 42,
 76 |    "metadata": {
 77 |     "collapsed": false
 78 |    },
 79 |    "outputs": [],
 80 |    "source": [
 81 |     "documents = [review for review in beers.reviews]"
 82 |    ]
 83 |   },
 84 |   {
 85 |    "cell_type": "code",
 86 |    "execution_count": 45,
 87 |    "metadata": {
 88 |     "collapsed": false
 89 |    },
 90 |    "outputs": [],
 91 |    "source": [
 92 |     "documents = [' '.join(review) for review in documents]"
 93 |    ]
 94 |   },
 95 |   {
 96 |    "cell_type": "code",
 97 |    "execution_count": 47,
 98 |    "metadata": {
 99 |     "collapsed": false
100 |    },
101 |    "outputs": [],
102 |    "source": [
103 |     "documents = [unicode(review,errors='ignore') for review in documents]"
104 |    ]
105 |   },
106 |   {
107 |    "cell_type": "code",
108 |    "execution_count": 57,
109 |    "metadata": {
110 |     "collapsed": false
111 |    },
112 |    "outputs": [
113 |     {
114 |      "name": "stderr",
115 |      "output_type": "stream",
116 |      "text": [
117 |       "100%|██████████| 20381/20381 [01:00<00:00, 338.74it/s]\n"
118 |      ]
119 |     }
120 |    ],
121 |    "source": [
122 |     "documents_processed = []\n",
123 |     "for document in tqdm(documents):\n",
124 |     "    document = textacy.preprocess.remove_punct(document)\n",
125 |     "    document = textacy.preprocess.replace_numbers(document,replace_with='')\n",
126 |     "    documents_processed.append(document)"
127 |    ]
128 |   },
129 |   {
130 |    "cell_type": "code",
131 |    "execution_count": 58,
132 |    "metadata": {
133 |     "collapsed": true
134 |    },
135 |    "outputs": [
136 |     {
137 |      "data": {
138 |       "text/plain": [
139 |        "u' tapPours jet black with a tannish lacing  Smells of mild cocoa and hints of coconut  Bit of cocoa and lactose with a whisper of coconut and a nonastringent coffee element  Medium body with lighter carbonation  Pretty good stout good body for being a lighter stout and the coconut is a nice touch I wish there was a bit more coconut is all Would be willing to try this or other flavored stouts again for sure  All but black with  of head off the pour Soso retention 30Roasty malts dominate with some soily tart aroma like a Baltic Porter would smell Hint of nuttiness 30Wellkilned black malts with a shinethrough freshly grated coconut flavor Earthy with some mild malt bitterness A little greenness from the hops Tickling carb creates the impression of spiciness Hints of vanilla and marshmallow Medium body with overdrive spritz Not a particularly mellow drink  30This works okay Nothing wrong with it Lacking much in the nose and the feel is pretty average Not their best stout   First had on tap at the breweryDark chocolate brown pour with a small tan cap of head that receded to a ringlet minimal lacing My wife took the first sip of this and claimed to detect no coconut whatsoever Well it just goes to show how peoples palates can differ because I got plenty of coconut out of this beer We experienced vice versa palateage for the cherry in the Quadzilla Not overblown but Id hardly call it subtle Nice smooth aroma of roast notes of coffee chocolate coconut Dark roasted malts up front in the taste with a distinct flavor of chocolatecoconut macaroons coming in midpalate sweetness and roastiness find a good equilibrium Maybe its because I detest coconut in most forms that the flavor was so blatant to me but I definitely got a good fix of it here even if it wasnt extremely potent Smooth and creamy mouthfeel not quite fullbodied but quite tasty and drinkable overall One of my favs from the Church  I went for the cask version of the Coconut Stout which in my opinion was a mistake A few of my friends got it from the tap and I believe it was a better beer However it was still good Black pour with a good amount of light brown to white head Once poured the head forms much like a Guinness is poured The smell is almost toasted very sweet which I would expect from combining coconut and a stout The problem with Cask vs Tap for me came with the taste which for me was all over the place I considered the taste of the Tap version to be more controlled and balanced blending the coconut and stout flavors wonderfully whereas my version you didnt know what was going to happen with each drink not in a good way however The beer is full bodied and dry throughout I just think it was served too warm for my preference Like with Quadzilla I like to witness the beer evolve as it warms I sort of missed out with this one My advice If you like cask beers try it If youre not really sure go for the version thats on tap which I thought was nothing short of phenomenal  Got this on tap at local pub for lunch todayA Pitch black and thin foam cap nice and tasty lookingS Light coffee  dark chocolates and virtually no coconut to speak ofT This is a first for me and kind of a drag Lots of roasted malts and some dark chocolates slightly burnt coffee and then a faint coconut tasteM Thin and watery a disappointment hoping this would be rich and thickO Would probably not get another unless it was with the pub special lobster corn dogs the sweetness goes good togetheror something else that would be complimentary  Had this at Ds in Monroeville PA on tap last  weekends A few of the pleasures of life seem to coexist in the description  coffee Mounds bars and Church Brew Works beer I recently had their Belgian Tripel which was topnotch so this was promising But there was concern that it could be cloyingly sweet It wasnt  the coconut is subtle in the afternaste balancing with the rich stout flavors Sold out quickly I nice break from IPAs and summer wheats Ive been trying Worth trying if you find it in the PGH area A Salud  0Had this on tap at the brew pub Review is from notes taken on  and poured from the tap to a pint glassAppearance Pour is a dark coffee brown and starts with a short one finger creamy tan head that fades to a thin and full yet uneven cap a thick creamy wall of lacing is left behind at first but eventually fades to swaths and patches that stick around a while body is dark and nearly entirely opaqueSmell Comes across a little weak some burnt coffee notes and a bit of bittersweet chocolate as it continues to warm the coconut starts to make a presence and comes across warm sweet and toasty like the top on top a pie that is baked for a few extra minutes to brown the coconut and dry it out a bitTaste Dark roasty coffee with a touch of an earthy metallic flavor that is reminiscent of coppery at first I dont really notice much of the coconut but as it sits and warms for a few more minutes it really starts to come out and make a nice creamy and sweet presence and it kind of reminds me of a Mounds bar when it blends in with a bit of bittersweet chocolate from the malts Definite solid and worth trying based upon thisMouthfeel A little thinner than what I like in a stout but for an American Stout its definitely within the style in terms of weight and viscosity body is soft and creamy and a thin dry stickiness is left on the roof of the mouth teeth feel a little gritty when its downOverall Not too bad and the flavor alone makes it worth trying I feel that the nose could be a little bigger and perhaps some time in the secondary on top of some more freshly toasted coconut would really bring it around or perhaps doing this in a more imperial style but then again I prefer that style to begin with  A  Dark brown that you cant see light through with a tan head that leaves a nice lacingS  Lots of chocolate and coconut this beer doesnt pull any punches on aroma there some nuttyness as well but its mostly just the coconutT  Strong chocolate and coconut flavors hit the tongue but quickly fades and leaves little aftertasteMD  Mouthfeel is weak and watery and ruins what could have been a great beer This is reasonably drinkable porter and would probably be a favorite of a coconut lover but left me wanting just a litle bit more  On tap at Smokin Joes Friday July  review is from notes Poured into a pint glassA  Like most normal stouts this one is black Very little head formed at all Just a very thin film of a roasted hazelnut color formed after it was placed in front of meS  Smells like any other run of the mill stout Roasted malts some chocolate aromas and come slight coffee as well Little to no coconut aroma at all I reaaally had to search to try and ascertain any coconut in this beerT  Just an average stout There is almost no coconut flavor in any way in this one quite disappointing Even after letting it warm for almost  minutes the flavor still doesnt make an appearance like one would think it should If anything there is the slightest sweetness on the back end of the taste that could potentially be coconut but it is so incredibly masked I cant say for sure it isM  Nice smooth creamy mouthfeel Very silkyD  The drinkability is fine on this one Unfortunately after being very disappointed in the lack of the most significant flavor advertised in this beer I would not be reaching for another anytime soon  Pint enjoyed  the brew worksA very dark opaque brown Not much head and what is there doesnt last long S roasted malt Sweet chocolate covered fruit Some coffee Noticeable coconut aroma T similar to smell Im not picking up as much coconut flavor as other reviewers have noted but it is there Coffee and bitter chocolate in the finish M very nice soft velvety feel Well suited to to flavors of this beerD high The coconut flavor is in check and not tiresome I could definitely have another Never been a big fan of Church BW but I like this one On top of that its a great place to have a beer The coconut stout isnt worth seeking out but a trip to the brewpub is  Church Brew Works Coconut Stout has a body that is somewhere between dark brown and black it is opaque The offwhite head is small and doesnt have much going on in the lacing or retention departmentsThe aroma has an odd slightly tart fruit smell to it which I find somewhat unpleasant Besides that a ton of roasted malt and a hint of coconut are in thereLuckily the taste omits the strange fruit from the aroma This is actually a pretty simple beer it has a bunch of roasted malt up front and the finish has a good fairly strong coconut flavor It isnt terribly complex but it is one of the better coconut beers that I have sampledThe body on this one is pretty light but I guess that should be expected with the relatively low ABVCoconut stout is simple but very drinkable I would probably order it again  on tap poured into a pub glass at the brewpub in Pittsburgh pours a deep chocolate brownblack color with a half finger of light tan head weak retentionaroma is of roasted malts and a bit of chocolate with a distinct sweetness from the coconut which forms a base nicely proportioned in the nosetaste is of more roasted malts with a sweet coconut base that lingers after the fact somewhat light on the palate with a very satisfying finish well proportioned with a solid treatment of the coconutmouthfeel is medium bodied with a smooth character accented by the sweetness and lasting roasted malt qualities that work well with the coconut character interestingly drinkabledrinkability is great the coconut and roasted malt should not work well together in concept but in the final product do well to compliment one another very drinkable with a sublime aftertaste cheers  Pint on tap at Capones Poured a dark brown color with a very small sizedalmost no off white head Aromas of chocolate roast and very light coconut Tastes of chocolate lighter roast coconut and vanilla Notes from and tasted on   From notes On tap at CaponesAppears a dark brown with a small tan cap that slowly fades into a mild collar Spotty streaks of lacing are left around the glassSmell is of tart fruit caramel vanilla sugar and cocoaTaste is of the mentioned aromas with the standout cocoa caramel and vanilla along wiht a mild coconut flavor coming through in the aftertasteMouthfeel is medium bodied slick sugary with a tart finish  On tap at the Church Brewpub in Pittsburgh Appears like a typical stout dark and creamy Has some of the typical smells dark roasted malt some chocolate and coffee but overwhelming with coconut Taste is dominated by coconut This is something different from any beer I have had before and unique enough that it was worth trying but not something I particularly enjoy There were other much better beers at this brewery  ABlack with a thin tan head Left very spotty laceSSubtle and rather light roasty sweetness Coconut aroma is faintnot really sure if I would guess it was coconut if I didnt knowTSweet roast with a hint of the dryness of the toasted coconut Not very complexMMedium body with a fairly creamy feel and a nice dry finishDDecent but has room for improvement I would like a bit more chocolateroasty flavours to introduce some complexity The coconut adds a nice drying aspect and decent flavourbut that is where it ends I can see hints of a really good brew here but find it lacking  Appearance Arrives at the bar with a nearly black color and a modest head that leaves some lacing while it lastsSmell Unsweetened coconut and roasty elementsTaste Up front the coconut and chocolate clearly come through but there is a developing nut skin bitterness that appears at midpalate the coconut continues into the finish as the bitter flavors fade and the sweet stout elements reassert themselvesMouthfeel Medium body slightly creamy with moderate carbonationDrinkability A nice combination but I cant help but think what this would be like in an Imperial version with a big mouthfeel and a lot more coffeechocolate character'"
140 |       ]
141 |      },
142 |      "execution_count": 58,
143 |      "metadata": {},
144 |      "output_type": "execute_result"
145 |     }
146 |    ],
147 |    "source": [
148 |     "for doc in documents_processed:\n",
149 |     "    "
150 |    ]
151 |   },
152 |   {
153 |    "cell_type": "code",
154 |    "execution_count": 62,
155 |    "metadata": {
156 |     "collapsed": false
157 |    },
158 |    "outputs": [],
159 |    "source": [
160 |     "test_doc = textacy.texts.TextDoc(documents_processed[0])"
161 |    ]
162 |   },
163 |   {
164 |    "cell_type": "code",
165 |    "execution_count": null,
166 |    "metadata": {
167 |     "collapsed": false
168 |    },
169 |    "outputs": [],
170 |    "source": [
171 |     "text_corpus = textacy.texts.TextCorpus.from_texts(lang='en',texts=documents_processed)"
172 |    ]
173 |   },
174 |   {
175 |    "cell_type": "code",
176 |    "execution_count": null,
177 |    "metadata": {
178 |     "collapsed": true
179 |    },
180 |    "outputs": [],
181 |    "source": []
182 |   }
183 |  ],
184 |  "metadata": {
185 |   "kernelspec": {
186 |    "display_name": "Python 2",
187 |    "language": "python",
188 |    "name": "python2"
189 |   },
190 |   "language_info": {
191 |    "codemirror_mode": {
192 |     "name": "ipython",
193 |     "version": 2
194 |    },
195 |    "file_extension": ".py",
196 |    "mimetype": "text/x-python",
197 |    "name": "python",
198 |    "nbconvert_exporter": "python",
199 |    "pygments_lexer": "ipython2",
200 |    "version": "2.7.11"
201 |   }
202 |  },
203 |  "nbformat": 4,
204 |  "nbformat_minor": 0
205 | }
206 | 


--------------------------------------------------------------------------------
/analyze-beer-soup.ipynb:
--------------------------------------------------------------------------------
   1 | {
   2 |  "cells": [
   3 |   {
   4 |    "cell_type": "code",
   5 |    "execution_count": 2,
   6 |    "metadata": {
   7 |     "collapsed": false
   8 |    },
   9 |    "outputs": [
  10 |     {
  11 |      "name": "stderr",
  12 |      "output_type": "stream",
  13 |      "text": [
  14 |       "\r",
  15 |       "0it [00:00, ?it/s]"
  16 |      ]
  17 |     }
  18 |    ],
  19 |    "source": [
  20 |     "import pandas as pd\n",
  21 |     "import numpy as np\n",
  22 |     "from tqdm import tqdm, tqdm_pandas, tqdm_notebook\n",
  23 |     "import re\n",
  24 |     "import requests\n",
  25 |     "from bs4 import BeautifulSoup\n",
  26 |     "import glob\n",
  27 |     "\n",
  28 |     "\n",
  29 |     "from fake_useragent import UserAgent\n",
  30 |     "ua = UserAgent()\n",
  31 |     "\n",
  32 |     "tqdm_pandas(tqdm())"
  33 |    ]
  34 |   },
  35 |   {
  36 |    "cell_type": "code",
  37 |    "execution_count": 3,
  38 |    "metadata": {
  39 |     "collapsed": true
  40 |    },
  41 |    "outputs": [],
  42 |    "source": [
  43 |     "import sys\n",
  44 |     "sys.setrecursionlimit(100000000)"
  45 |    ]
  46 |   },
  47 |   {
  48 |    "cell_type": "code",
  49 |    "execution_count": 6,
  50 |    "metadata": {
  51 |     "collapsed": true
  52 |    },
  53 |    "outputs": [],
  54 |    "source": [
  55 |     "def get_soup(url, timeout=5):\n",
  56 |     "    headers  = {'User-Agent':ua.random}\n",
  57 |     "    try:\n",
  58 |     "        response = requests.get(url,headers=headers)\n",
  59 |     "    except:\n",
  60 |     "        print(\"FAILED \"+ url)\n",
  61 |     "        return 0\n",
  62 |     "    attempts = 0\n",
  63 |     "    while(not response.ok):\n",
  64 |     "            #print((url+' failed with code: '+str(response.status_code)))\n",
  65 |     "            if attempts > timeout:\n",
  66 |     "                print(url+' failed with code: '+str(response.status_code))\n",
  67 |     "                return BeautifulSoup('','lxml')\n",
  68 |     "            response = requests.get(url)\n",
  69 |     "            attempts += 1\n",
  70 |     "    page = response.text\n",
  71 |     "    soup = BeautifulSoup(page,'lxml')\n",
  72 |     "    return soup"
  73 |    ]
  74 |   },
  75 |   {
  76 |    "cell_type": "code",
  77 |    "execution_count": 7,
  78 |    "metadata": {
  79 |     "collapsed": true
  80 |    },
  81 |    "outputs": [],
  82 |    "source": [
  83 |     "def get_beer_stats(row):\n",
  84 |     "    soup = row['soup']\n",
  85 |     "    stats = soup.find(id='item_stats').find('dl')\n",
  86 |     "    row['ba_score']    = soup.find(class_='BAscore_big ba-score').get_text()\n",
  87 |     "    row['num_reviews'] = int(stats.find(class_='ba-reviews').get_text().replace(',',''))\n",
  88 |     "    row['num_ratings'] = int(stats.find(class_='ba-ratings').get_text().replace(',',''))\n",
  89 |     "    row['ravg']        = float(stats.find(class_='ba-ravg').get_text().replace(',',''))\n",
  90 |     "    row['pdev']        = float(stats.find(class_='ba-pdev').get_text().replace(',','').replace('%',''))\n",
  91 |     "    row['wants']       = int(stats.find(class_='ba-wants').get_text().replace(',',''))\n",
  92 |     "    row['gots']        = int(stats.find(class_='ba-gots').get_text().replace(',',''))\n",
  93 |     "    row['for_trade']   = int(stats.find_all('dt')[-1].get_text().replace(',',''))\n",
  94 |     "\n",
  95 |     "    info_links = soup.find('div',style=\"float:right;width:70%;\").find_all('a')\n",
  96 |     "    row['brewery_name'] = info_links[0].get_text()\n",
  97 |     "    row['brewery_loation'] = info_links[1].get_text()\n",
  98 |     "    try:\n",
  99 |     "        row['brewery_website'] = info_links[3]['href']\n",
 100 |     "    except:\n",
 101 |     "        row['brewery_website'] = ''\n",
 102 |     "    row['beer_style'] = info_links[-1].get_text()\n",
 103 |     "    row['style_url'] = info_links[-1]['href']\n",
 104 |     "    if row['brewery_website'] == row['style_url']:\n",
 105 |     "        row['brewery_website'] = ''\n",
 106 |     "    try:\n",
 107 |     "        row['abv'] = float(re.findall(r'(?<=\\(ABV\\): )\\d+\\.\\d+',soup.find('div',style=\"float:right;width:70%;\").get_text())[0])\n",
 108 |     "    except:\n",
 109 |     "        row['abv'] = np.nan\n",
 110 |     "    row['availability'] = re.findall(r'(?<=Availability: )[\\w\\-]*',soup.find('div',style=\"float:right;width:70%;\").get_text())[0]\n",
 111 |     "    return row"
 112 |    ]
 113 |   },
 114 |   {
 115 |    "cell_type": "code",
 116 |    "execution_count": 159,
 117 |    "metadata": {
 118 |     "collapsed": true
 119 |    },
 120 |    "outputs": [],
 121 |    "source": [
 122 |     "beers = pd.read_pickle('beer_soup_229.pkl')"
 123 |    ]
 124 |   },
 125 |   {
 126 |    "cell_type": "code",
 127 |    "execution_count": 160,
 128 |    "metadata": {
 129 |     "collapsed": true
 130 |    },
 131 |    "outputs": [
 132 |     {
 133 |      "name": "stderr",
 134 |      "output_type": "stream",
 135 |      "text": [
 136 |       "\n",
 137 |       "0it [00:00, ?it/s]\u001b[A\n",
 138 |       "100%|██████████| 3/3 [00:00<00:00, 21.46it/s]\u001b[A\n",
 139 |       "5it [00:00, 19.82it/s]                       \u001b[A\n",
 140 |       "7it [00:00, 19.30it/s]\u001b[A\n",
 141 |       "9it [00:00, 18.70it/s]\u001b[A\n",
 142 |       "12it [00:00, 19.55it/s]\u001b[A\n",
 143 |       "14it [00:00, 19.19it/s]\u001b[A\n",
 144 |       "17it [00:00, 20.19it/s]\u001b[A\n",
 145 |       "19it [00:00, 19.93it/s]\u001b[A\n",
 146 |       "21it [00:01, 19.35it/s]\u001b[A\n",
 147 |       "23it [00:01, 19.23it/s]\u001b[A\n",
 148 |       "25it [00:01, 18.03it/s]\u001b[A\n",
 149 |       "27it [00:01, 18.53it/s]\u001b[A\n",
 150 |       "30it [00:01, 19.66it/s]\u001b[A\n",
 151 |       "32it [00:01, 18.74it/s]\u001b[A\n",
 152 |       "34it [00:01, 18.33it/s]\u001b[A\n",
 153 |       "36it [00:01, 18.63it/s]\u001b[A\n",
 154 |       "39it [00:02, 19.38it/s]\u001b[A\n",
 155 |       "41it [00:02, 18.92it/s]\u001b[A\n",
 156 |       "43it [00:02, 18.44it/s]\u001b[A\n",
 157 |       "45it [00:02, 18.30it/s]\u001b[A\n",
 158 |       "47it [00:02, 18.25it/s]\u001b[A\n",
 159 |       "50it [00:02, 19.42it/s]\u001b[A\n",
 160 |       "52it [00:02, 18.97it/s]\u001b[A\n",
 161 |       "54it [00:02, 19.17it/s]\u001b[A\n",
 162 |       "57it [00:02, 20.29it/s]\u001b[A\n",
 163 |       "60it [00:03, 18.44it/s]\u001b[A\n",
 164 |       "62it [00:03, 18.53it/s]\u001b[A\n",
 165 |       "65it [00:03, 19.49it/s]\u001b[A\n",
 166 |       "68it [00:03, 20.20it/s]\u001b[A\n",
 167 |       "71it [00:03, 20.86it/s]\u001b[A\n",
 168 |       "74it [00:03, 21.40it/s]\u001b[A\n",
 169 |       "77it [00:03, 21.17it/s]\u001b[A\n",
 170 |       "80it [00:04, 18.57it/s]\u001b[A\n",
 171 |       "82it [00:04, 17.06it/s]\u001b[A\n",
 172 |       "84it [00:04, 17.60it/s]\u001b[A\n",
 173 |       "87it [00:04, 17.90it/s]\u001b[A\n",
 174 |       "89it [00:04, 17.23it/s]\u001b[A\n",
 175 |       "91it [00:04, 17.93it/s]\u001b[A\n",
 176 |       "93it [00:04, 17.85it/s]\u001b[A\n",
 177 |       "95it [00:04, 18.33it/s]\u001b[A\n",
 178 |       "97it [00:05, 17.85it/s]\u001b[A\n",
 179 |       "99it [00:05, 17.82it/s]\u001b[A\n",
 180 |       "101it [00:05, 15.70it/s]\u001b[A\n",
 181 |       "103it [00:05, 14.99it/s]\u001b[A\n",
 182 |       "105it [00:05, 15.41it/s]\u001b[A\n",
 183 |       "107it [00:05, 15.86it/s]\u001b[A\n",
 184 |       "109it [00:05, 16.13it/s]\u001b[A\n",
 185 |       "111it [00:06, 15.24it/s]\u001b[A\n",
 186 |       "113it [00:06, 15.94it/s]\u001b[A\n",
 187 |       "115it [00:06, 16.84it/s]\u001b[A\n",
 188 |       "117it [00:06, 14.95it/s]\u001b[A\n",
 189 |       "119it [00:06, 13.30it/s]\u001b[A\n",
 190 |       "121it [00:06, 12.87it/s]\u001b[A\n",
 191 |       "123it [00:06, 13.33it/s]\u001b[A\n",
 192 |       "125it [00:07, 13.23it/s]\u001b[A\n",
 193 |       "127it [00:07, 11.97it/s]\u001b[A\n",
 194 |       "129it [00:07, 12.14it/s]\u001b[A\n",
 195 |       "131it [00:07, 12.85it/s]\u001b[A\n",
 196 |       "133it [00:07, 13.26it/s]\u001b[A\n",
 197 |       "135it [00:07, 13.21it/s]\u001b[A\n",
 198 |       "137it [00:08, 13.01it/s]\u001b[A\n",
 199 |       "139it [00:08, 13.03it/s]\u001b[A\n",
 200 |       "141it [00:08, 12.86it/s]\u001b[A\n",
 201 |       "143it [00:08, 12.90it/s]\u001b[A\n",
 202 |       "145it [00:08, 12.71it/s]\u001b[A\n",
 203 |       "147it [00:08, 13.79it/s]\u001b[A\n",
 204 |       "149it [00:08, 13.05it/s]\u001b[A\n",
 205 |       "151it [00:09, 13.26it/s]\u001b[A\n",
 206 |       "153it [00:09, 14.73it/s]\u001b[A\n",
 207 |       "155it [00:09, 14.98it/s]\u001b[A\n",
 208 |       "157it [00:09, 15.99it/s]\u001b[A\n",
 209 |       "159it [00:09, 15.68it/s]\u001b[A\n",
 210 |       "161it [00:09, 14.30it/s]\u001b[A\n",
 211 |       "163it [00:09, 14.26it/s]\u001b[A\n",
 212 |       "165it [00:09, 14.88it/s]\u001b[A\n",
 213 |       "168it [00:10, 15.96it/s]\u001b[A\n",
 214 |       "170it [00:10, 14.63it/s]\u001b[A\n",
 215 |       "172it [00:10, 14.36it/s]\u001b[A\n",
 216 |       "174it [00:10, 12.79it/s]\u001b[A\n",
 217 |       "176it [00:10, 12.96it/s]\u001b[A\n",
 218 |       "179it [00:10, 14.97it/s]\u001b[A\n",
 219 |       "182it [00:11, 16.47it/s]\u001b[A\n",
 220 |       "184it [00:11, 15.74it/s]\u001b[A\n",
 221 |       "186it [00:11, 15.15it/s]\u001b[A\n",
 222 |       "189it [00:11, 16.76it/s]\u001b[A\n",
 223 |       "191it [00:11, 17.57it/s]\u001b[A\n",
 224 |       "193it [00:11, 17.90it/s]\u001b[A\n",
 225 |       "195it [00:11, 18.31it/s]\u001b[A\n",
 226 |       "198it [00:11, 19.43it/s]\u001b[A\n",
 227 |       "201it [00:12, 20.42it/s]\u001b[A\n",
 228 |       "204it [00:12, 18.06it/s]\u001b[A\n",
 229 |       "206it [00:12, 16.31it/s]\u001b[A\n",
 230 |       "208it [00:12, 16.10it/s]\u001b[A\n",
 231 |       "210it [00:12, 16.17it/s]\u001b[A\n",
 232 |       "212it [00:12, 15.91it/s]\u001b[A\n",
 233 |       "214it [00:12, 14.92it/s]\u001b[A\n",
 234 |       "216it [00:13, 15.37it/s]\u001b[A\n",
 235 |       "218it [00:13, 15.42it/s]\u001b[A\n",
 236 |       "220it [00:13, 14.47it/s]\u001b[A\n",
 237 |       "222it [00:13, 14.32it/s]\u001b[A\n",
 238 |       "224it [00:13, 14.62it/s]\u001b[A\n",
 239 |       "226it [00:13, 15.79it/s]\u001b[A\n",
 240 |       "229it [00:13, 17.19it/s]\u001b[A\n",
 241 |       "230it [00:13, 16.43it/s]\u001b[A"
 242 |      ]
 243 |     }
 244 |    ],
 245 |    "source": [
 246 |     "tqdm_pandas(tqdm())\n",
 247 |     "beers = beers.progress_apply(get_beer_stats,axis=1)"
 248 |    ]
 249 |   },
 250 |   {
 251 |    "cell_type": "code",
 252 |    "execution_count": 8,
 253 |    "metadata": {
 254 |     "collapsed": false
 255 |    },
 256 |    "outputs": [],
 257 |    "source": [
 258 |     "def get_beer_df_reviews(row):\n",
 259 |     "    ba_url = 'http://www.beeradvocate.com'\n",
 260 |     "    url_suffix = '?sort=topr&start='\n",
 261 |     "    row['reviews'] = get_beer_reviews(row['soup'])\n",
 262 |     "    if row['num_reviews'] > 25:\n",
 263 |     "        if row['num_reviews'] > 100:\n",
 264 |     "            num_reviews = 100\n",
 265 |     "        else:\n",
 266 |     "            num_reviews = row['num_reviews']\n",
 267 |     "        for i in range(1,num_reviews//25):\n",
 268 |     "            url = ba_url + row['url'] + url_suffix + str(i*25)\n",
 269 |     "            soup = get_soup(url)\n",
 270 |     "            reviews = get_beer_reviews(soup)\n",
 271 |     "            row['reviews'] += reviews\n",
 272 |     "    return row\n",
 273 |     "\n",
 274 |     "def get_beer_reviews(soup):\n",
 275 |     "    reviews = []\n",
 276 |     "    for rating in soup.find_all(id='rating_fullview_content_2'):\n",
 277 |     "        for span in rating.find_all('span'):\n",
 278 |     "            span.extract()\n",
 279 |     "        review = rating.get_text().strip().encode('utf-8')\n",
 280 |     "        review = review.replace('rDev','')\n",
 281 |     "        reviews.append(str(review))\n",
 282 |     "    return reviews"
 283 |    ]
 284 |   },
 285 |   {
 286 |    "cell_type": "code",
 287 |    "execution_count": 162,
 288 |    "metadata": {
 289 |     "collapsed": true
 290 |    },
 291 |    "outputs": [
 292 |     {
 293 |      "name": "stderr",
 294 |      "output_type": "stream",
 295 |      "text": [
 296 |       "\n",
 297 |       "0it [00:00, ?it/s]\u001b[A\n",
 298 |       " 17%|█▋        | 3/18 [00:00<00:00, 21.87it/s]\u001b[A\n",
 299 |       " 33%|███▎      | 6/18 [00:00<00:01,  9.82it/s]\u001b[A\n",
 300 |       " 44%|████▍     | 8/18 [00:02<00:03,  2.94it/s]\u001b[A\n",
 301 |       " 50%|█████     | 9/18 [00:04<00:07,  1.26it/s]\u001b[A\n",
 302 |       " 67%|██████▋   | 12/18 [00:04<00:03,  1.76it/s]\u001b[A\n",
 303 |       " 78%|███████▊  | 14/18 [00:05<00:02,  1.74it/s]\u001b[A\n",
 304 |       " 94%|█████████▍| 17/18 [00:05<00:00,  2.40it/s]\u001b[A\n",
 305 |       "20it [00:07,  2.37it/s]                        \u001b[A\n",
 306 |       "22it [00:08,  1.83it/s]\u001b[A\n",
 307 |       "25it [00:10,  1.75it/s]\u001b[A\n",
 308 |       "27it [00:10,  2.40it/s]\u001b[A\n",
 309 |       "30it [00:11,  3.30it/s]\u001b[A\n",
 310 |       "33it [00:11,  4.47it/s]\u001b[A\n",
 311 |       "35it [00:12,  2.37it/s]\u001b[A\n",
 312 |       "38it [00:13,  3.25it/s]\u001b[A\n",
 313 |       "40it [00:13,  3.31it/s]\u001b[A\n",
 314 |       "42it [00:14,  2.64it/s]\u001b[A\n",
 315 |       "44it [00:15,  2.86it/s]\u001b[A\n",
 316 |       "47it [00:15,  3.91it/s]\u001b[A\n",
 317 |       "49it [00:15,  3.77it/s]\u001b[A\n",
 318 |       "52it [00:16,  5.04it/s]\u001b[A\n",
 319 |       "56it [00:17,  3.72it/s]\u001b[A\n",
 320 |       "59it [00:19,  2.76it/s]\u001b[A\n",
 321 |       "62it [00:20,  3.10it/s]\u001b[A\n",
 322 |       "64it [00:22,  1.63it/s]\u001b[A\n",
 323 |       "65it [00:24,  1.01it/s]\u001b[A\n",
 324 |       "67it [00:26,  1.04it/s]\u001b[A\n",
 325 |       "68it [00:27,  1.18it/s]\u001b[A\n",
 326 |       "71it [00:27,  1.65it/s]\u001b[A\n",
 327 |       "73it [00:27,  2.25it/s]\u001b[A\n",
 328 |       "75it [00:29,  1.69it/s]\u001b[A\n",
 329 |       "76it [00:31,  1.10s/it]\u001b[A\n",
 330 |       "78it [00:33,  1.03s/it]\u001b[A\n",
 331 |       "80it [00:33,  1.36it/s]\u001b[A\n",
 332 |       "83it [00:33,  1.90it/s]\u001b[A\n",
 333 |       "85it [00:36,  1.29it/s]\u001b[A\n",
 334 |       "87it [00:36,  1.79it/s]\u001b[A\n",
 335 |       "90it [00:36,  2.46it/s]\u001b[A\n",
 336 |       "93it [00:37,  2.48it/s]\u001b[A\n",
 337 |       "95it [00:39,  1.90it/s]\u001b[A\n",
 338 |       "96it [00:39,  1.78it/s]\u001b[A\n",
 339 |       "97it [00:41,  1.36it/s]\u001b[A\n",
 340 |       "101it [00:42,  1.52it/s]\u001b[A\n",
 341 |       "102it [00:44,  1.22it/s]\u001b[A\n",
 342 |       "106it [00:44,  1.71it/s]\u001b[A\n",
 343 |       "108it [00:46,  1.43it/s]\u001b[A\n",
 344 |       "111it [00:46,  2.00it/s]\u001b[A\n",
 345 |       "113it [00:47,  1.86it/s]\u001b[A\n",
 346 |       "114it [00:48,  1.36it/s]\u001b[A\n",
 347 |       "116it [00:51,  1.15it/s]\u001b[A\n",
 348 |       "117it [00:52,  1.08s/it]\u001b[A\n",
 349 |       "119it [00:53,  1.18it/s]\u001b[A\n",
 350 |       "122it [00:55,  1.27it/s]\u001b[A\n",
 351 |       "126it [00:55,  1.79it/s]\u001b[A\n",
 352 |       "129it [00:55,  2.47it/s]\u001b[A\n",
 353 |       "131it [00:58,  1.25it/s]\u001b[A\n",
 354 |       "134it [00:59,  1.58it/s]\u001b[A\n",
 355 |       "137it [01:00,  1.77it/s]\u001b[A\n",
 356 |       "138it [01:02,  1.09it/s]\u001b[A\n",
 357 |       "139it [01:03,  1.22it/s]\u001b[A\n",
 358 |       "140it [01:04,  1.06s/it]\u001b[A\n",
 359 |       "143it [01:04,  1.32it/s]\u001b[A\n",
 360 |       "144it [01:06,  1.04s/it]\u001b[A\n",
 361 |       "146it [01:06,  1.33it/s]\u001b[A\n",
 362 |       "149it [01:06,  1.84it/s]\u001b[A\n",
 363 |       "151it [01:08,  1.52it/s]\u001b[A\n",
 364 |       "154it [01:08,  2.12it/s]\u001b[A\n",
 365 |       "156it [01:10,  1.93it/s]\u001b[A\n",
 366 |       "159it [01:10,  2.68it/s]\u001b[A\n",
 367 |       "162it [01:10,  3.66it/s]\u001b[A\n",
 368 |       "164it [01:12,  2.21it/s]\u001b[A\n",
 369 |       "166it [01:13,  1.94it/s]\u001b[A\n",
 370 |       "168it [01:15,  1.53it/s]\u001b[A\n",
 371 |       "170it [01:24,  1.81s/it]\u001b[A\n",
 372 |       "174it [01:24,  1.28s/it]\u001b[A\n",
 373 |       "176it [01:27,  1.32s/it]\u001b[A"
 374 |      ]
 375 |     },
 376 |     {
 377 |      "name": "stdout",
 378 |      "output_type": "stream",
 379 |      "text": [
 380 |       "http://www.beeradvocate.com/beer/profile/13947/30771/?sort=topr&start=75 failed with code: 403\n"
 381 |      ]
 382 |     },
 383 |     {
 384 |      "name": "stderr",
 385 |      "output_type": "stream",
 386 |      "text": [
 387 |       "\n",
 388 |       "178it [01:29,  1.23s/it]\u001b[A\n",
 389 |       "180it [01:31,  1.15s/it]\u001b[A"
 390 |      ]
 391 |     },
 392 |     {
 393 |      "name": "stdout",
 394 |      "output_type": "stream",
 395 |      "text": [
 396 |       "http://www.beeradvocate.com/beer/profile/18134/13906/?sort=topr&start=50 failed with code: 403\n"
 397 |      ]
 398 |     },
 399 |     {
 400 |      "name": "stderr",
 401 |      "output_type": "stream",
 402 |      "text": [
 403 |       "\n",
 404 |       "182it [01:33,  1.06s/it]\u001b[A\n",
 405 |       "184it [01:34,  1.08it/s]\u001b[A\n",
 406 |       "185it [01:36,  1.19s/it]\u001b[A\n",
 407 |       "188it [01:38,  1.03s/it]\u001b[A\n",
 408 |       "191it [01:38,  1.37it/s]\u001b[A\n",
 409 |       "194it [01:38,  1.92it/s]\u001b[A\n",
 410 |       "197it [01:38,  2.67it/s]\u001b[A\n",
 411 |       "199it [01:40,  1.89it/s]\u001b[A\n",
 412 |       "201it [01:40,  2.18it/s]\u001b[A\n",
 413 |       "204it [01:40,  3.01it/s]\u001b[A\n",
 414 |       "207it [01:42,  2.48it/s]\u001b[A\n",
 415 |       "209it [01:44,  1.81it/s]\u001b[A\n",
 416 |       "211it [01:44,  2.10it/s]\u001b[A\n",
 417 |       "214it [01:45,  2.88it/s]\u001b[A\n",
 418 |       "218it [01:45,  3.96it/s]\u001b[A\n",
 419 |       "221it [01:45,  5.32it/s]\u001b[A\n",
 420 |       "225it [01:47,  3.75it/s]\u001b[A\n",
 421 |       "228it [01:49,  2.61it/s]\u001b[A\n",
 422 |       "230it [01:51,  1.57it/s]\u001b[A\n",
 423 |       "\u001b[A"
 424 |      ]
 425 |     },
 426 |     {
 427 |      "data": {
 428 |       "text/html": [
 429 |        "<div>\n",
 430 |        "<table border=\"1\" class=\"dataframe\">\n",
 431 |        "  <thead>\n",
 432 |        "    <tr style=\"text-align: right;\">\n",
 433 |        "      <th></th>\n",
 434 |        "      <th>name</th>\n",
 435 |        "      <th>url</th>\n",
 436 |        "      <th>soup</th>\n",
 437 |        "      <th>ba_score</th>\n",
 438 |        "      <th>num_reviews</th>\n",
 439 |        "      <th>num_ratings</th>\n",
 440 |        "      <th>ravg</th>\n",
 441 |        "      <th>pdev</th>\n",
 442 |        "      <th>wants</th>\n",
 443 |        "      <th>gots</th>\n",
 444 |        "      <th>for_trade</th>\n",
 445 |        "      <th>brewery_name</th>\n",
 446 |        "      <th>brewery_loation</th>\n",
 447 |        "      <th>brewery_website</th>\n",
 448 |        "      <th>beer_style</th>\n",
 449 |        "      <th>style_url</th>\n",
 450 |        "      <th>abv</th>\n",
 451 |        "      <th>availability</th>\n",
 452 |        "      <th>reviews</th>\n",
 453 |        "    </tr>\n",
 454 |        "  </thead>\n",
 455 |        "  <tbody>\n",
 456 |        "    <tr>\n",
 457 |        "      <th>0</th>\n",
 458 |        "      <td>Imperial Eclipse Stout - High West Rye</td>\n",
 459 |        "      <td>/beer/profile/14936/107388/</td>\n",
 460 |        "      <td>&lt;!DOCTYPE html&gt;\n",
 461 |        "&lt;html class=\"Public NoJs Logge...</td>\n",
 462 |        "      <td>92</td>\n",
 463 |        "      <td>30</td>\n",
 464 |        "      <td>162</td>\n",
 465 |        "      <td>4.24</td>\n",
 466 |        "      <td>17.92</td>\n",
 467 |        "      <td>27</td>\n",
 468 |        "      <td>64</td>\n",
 469 |        "      <td>17</td>\n",
 470 |        "      <td>FiftyFifty Brewing Co.</td>\n",
 471 |        "      <td>California</td>\n",
 472 |        "      <td>http://www.fiftyfiftybrewing.com</td>\n",
 473 |        "      <td>American Double / Imperial Stout</td>\n",
 474 |        "      <td>/beer/style/157/</td>\n",
 475 |        "      <td>11.9</td>\n",
 476 |        "      <td>Fall</td>\n",
 477 |        "      <td>[ 2013 version poured into a snifter. Comes ou...</td>\n",
 478 |        "    </tr>\n",
 479 |        "    <tr>\n",
 480 |        "      <th>1</th>\n",
 481 |        "      <td>River Horse India Pale Ale</td>\n",
 482 |        "      <td>/beer/profile/877/138007/</td>\n",
 483 |        "      <td>&lt;!DOCTYPE html&gt;\n",
 484 |        "&lt;html class=\"Public NoJs Logge...</td>\n",
 485 |        "      <td>83</td>\n",
 486 |        "      <td>14</td>\n",
 487 |        "      <td>90</td>\n",
 488 |        "      <td>3.60</td>\n",
 489 |        "      <td>31.94</td>\n",
 490 |        "      <td>0</td>\n",
 491 |        "      <td>32</td>\n",
 492 |        "      <td>0</td>\n",
 493 |        "      <td>River Horse Brewing Co.</td>\n",
 494 |        "      <td>New Jersey</td>\n",
 495 |        "      <td>http://www.riverhorse.com</td>\n",
 496 |        "      <td>American IPA</td>\n",
 497 |        "      <td>/beer/style/116/</td>\n",
 498 |        "      <td>5.7</td>\n",
 499 |        "      <td>Year-round</td>\n",
 500 |        "      <td>[ No date on the bottle, but purchased from th...</td>\n",
 501 |        "    </tr>\n",
 502 |        "    <tr>\n",
 503 |        "      <th>2</th>\n",
 504 |        "      <td>Hibiscus Wit</td>\n",
 505 |        "      <td>/beer/profile/24428/66018/</td>\n",
 506 |        "      <td>&lt;!DOCTYPE html&gt;\n",
 507 |        "&lt;html class=\"Public NoJs Logge...</td>\n",
 508 |        "      <td>84</td>\n",
 509 |        "      <td>10</td>\n",
 510 |        "      <td>57</td>\n",
 511 |        "      <td>3.75</td>\n",
 512 |        "      <td>16.80</td>\n",
 513 |        "      <td>2</td>\n",
 514 |        "      <td>7</td>\n",
 515 |        "      <td>0</td>\n",
 516 |        "      <td>2nd Shift Brewing</td>\n",
 517 |        "      <td>Missouri</td>\n",
 518 |        "      <td>http://www.2ndshiftbrewing.com</td>\n",
 519 |        "      <td>Witbier</td>\n",
 520 |        "      <td>/beer/style/48/</td>\n",
 521 |        "      <td>5.2</td>\n",
 522 |        "      <td>Rotating</td>\n",
 523 |        "      <td>[ Gold with pure white headLight fresh stone f...</td>\n",
 524 |        "    </tr>\n",
 525 |        "    <tr>\n",
 526 |        "      <th>3</th>\n",
 527 |        "      <td>Kozel</td>\n",
 528 |        "      <td>/beer/profile/448/5430/</td>\n",
 529 |        "      <td>&lt;!DOCTYPE html&gt;\n",
 530 |        "&lt;html class=\"Public NoJs Logge...</td>\n",
 531 |        "      <td>78</td>\n",
 532 |        "      <td>59</td>\n",
 533 |        "      <td>134</td>\n",
 534 |        "      <td>3.36</td>\n",
 535 |        "      <td>24.70</td>\n",
 536 |        "      <td>2</td>\n",
 537 |        "      <td>21</td>\n",
 538 |        "      <td>0</td>\n",
 539 |        "      <td>Pivovar Velké Popovice a.s.</td>\n",
 540 |        "      <td>Czech Republic</td>\n",
 541 |        "      <td></td>\n",
 542 |        "      <td>Czech Pilsener</td>\n",
 543 |        "      <td>/beer/style/40/</td>\n",
 544 |        "      <td>5.0</td>\n",
 545 |        "      <td>Year-round</td>\n",
 546 |        "      <td>[ On tap into a dimple beer mugA - Beer is cle...</td>\n",
 547 |        "    </tr>\n",
 548 |        "    <tr>\n",
 549 |        "      <th>4</th>\n",
 550 |        "      <td>Summer Wheat Ale</td>\n",
 551 |        "      <td>/beer/profile/12375/61223/</td>\n",
 552 |        "      <td>&lt;!DOCTYPE html&gt;\n",
 553 |        "&lt;html class=\"Public NoJs Logge...</td>\n",
 554 |        "      <td>84</td>\n",
 555 |        "      <td>19</td>\n",
 556 |        "      <td>45</td>\n",
 557 |        "      <td>3.74</td>\n",
 558 |        "      <td>15.24</td>\n",
 559 |        "      <td>1</td>\n",
 560 |        "      <td>9</td>\n",
 561 |        "      <td>0</td>\n",
 562 |        "      <td>Mt. Carmel Brewing Company</td>\n",
 563 |        "      <td>Ohio</td>\n",
 564 |        "      <td>http://www.mtcarmelbrewingcompany.com</td>\n",
 565 |        "      <td>American Pale Wheat Ale</td>\n",
 566 |        "      <td>/beer/style/93/</td>\n",
 567 |        "      <td>4.8</td>\n",
 568 |        "      <td>Summer</td>\n",
 569 |        "      <td>[ A: The beer is slightly hazy light yellow in...</td>\n",
 570 |        "    </tr>\n",
 571 |        "  </tbody>\n",
 572 |        "</table>\n",
 573 |        "</div>"
 574 |       ],
 575 |       "text/plain": [
 576 |        "                                     name                          url  \\\n",
 577 |        "0  Imperial Eclipse Stout - High West Rye  /beer/profile/14936/107388/   \n",
 578 |        "1              River Horse India Pale Ale    /beer/profile/877/138007/   \n",
 579 |        "2                            Hibiscus Wit   /beer/profile/24428/66018/   \n",
 580 |        "3                                   Kozel      /beer/profile/448/5430/   \n",
 581 |        "4                        Summer Wheat Ale   /beer/profile/12375/61223/   \n",
 582 |        "\n",
 583 |        "                                                soup ba_score  num_reviews  \\\n",
 584 |        "0  <!DOCTYPE html>\n",
 585 |        "<html class=\"Public NoJs Logge...       92           30   \n",
 586 |        "1  <!DOCTYPE html>\n",
 587 |        "<html class=\"Public NoJs Logge...       83           14   \n",
 588 |        "2  <!DOCTYPE html>\n",
 589 |        "<html class=\"Public NoJs Logge...       84           10   \n",
 590 |        "3  <!DOCTYPE html>\n",
 591 |        "<html class=\"Public NoJs Logge...       78           59   \n",
 592 |        "4  <!DOCTYPE html>\n",
 593 |        "<html class=\"Public NoJs Logge...       84           19   \n",
 594 |        "\n",
 595 |        "   num_ratings  ravg   pdev  wants  gots  for_trade  \\\n",
 596 |        "0          162  4.24  17.92     27    64         17   \n",
 597 |        "1           90  3.60  31.94      0    32          0   \n",
 598 |        "2           57  3.75  16.80      2     7          0   \n",
 599 |        "3          134  3.36  24.70      2    21          0   \n",
 600 |        "4           45  3.74  15.24      1     9          0   \n",
 601 |        "\n",
 602 |        "                  brewery_name brewery_loation  \\\n",
 603 |        "0       FiftyFifty Brewing Co.      California   \n",
 604 |        "1      River Horse Brewing Co.      New Jersey   \n",
 605 |        "2            2nd Shift Brewing        Missouri   \n",
 606 |        "3  Pivovar Velké Popovice a.s.  Czech Republic   \n",
 607 |        "4   Mt. Carmel Brewing Company            Ohio   \n",
 608 |        "\n",
 609 |        "                         brewery_website                        beer_style  \\\n",
 610 |        "0       http://www.fiftyfiftybrewing.com  American Double / Imperial Stout   \n",
 611 |        "1              http://www.riverhorse.com                      American IPA   \n",
 612 |        "2         http://www.2ndshiftbrewing.com                           Witbier   \n",
 613 |        "3                                                           Czech Pilsener   \n",
 614 |        "4  http://www.mtcarmelbrewingcompany.com           American Pale Wheat Ale   \n",
 615 |        "\n",
 616 |        "          style_url   abv availability  \\\n",
 617 |        "0  /beer/style/157/  11.9         Fall   \n",
 618 |        "1  /beer/style/116/   5.7   Year-round   \n",
 619 |        "2   /beer/style/48/   5.2     Rotating   \n",
 620 |        "3   /beer/style/40/   5.0   Year-round   \n",
 621 |        "4   /beer/style/93/   4.8       Summer   \n",
 622 |        "\n",
 623 |        "                                             reviews  \n",
 624 |        "0  [ 2013 version poured into a snifter. Comes ou...  \n",
 625 |        "1  [ No date on the bottle, but purchased from th...  \n",
 626 |        "2  [ Gold with pure white headLight fresh stone f...  \n",
 627 |        "3  [ On tap into a dimple beer mugA - Beer is cle...  \n",
 628 |        "4  [ A: The beer is slightly hazy light yellow in...  "
 629 |       ]
 630 |      },
 631 |      "execution_count": 162,
 632 |      "metadata": {},
 633 |      "output_type": "execute_result"
 634 |     }
 635 |    ],
 636 |    "source": [
 637 |     "tqdm_pandas(tqdm())\n",
 638 |     "beers = beers.progress_apply(get_beer_df_reviews,axis = 1)\n",
 639 |     "beers.head()"
 640 |    ]
 641 |   },
 642 |   {
 643 |    "cell_type": "code",
 644 |    "execution_count": 165,
 645 |    "metadata": {
 646 |     "collapsed": false
 647 |    },
 648 |    "outputs": [],
 649 |    "source": [
 650 |     "beers.to_pickle('test.pkl')"
 651 |    ]
 652 |   },
 653 |   {
 654 |    "cell_type": "code",
 655 |    "execution_count": 180,
 656 |    "metadata": {
 657 |     "collapsed": true
 658 |    },
 659 |    "outputs": [
 660 |     {
 661 |      "name": "stdout",
 662 |      "output_type": "stream",
 663 |      "text": [
 664 |       "beer_soup_10119.pkl\n",
 665 |       "beer_soup_10349.pkl\n",
 666 |       "beer_soup_10579.pkl\n",
 667 |       "beer_soup_10809.pkl\n",
 668 |       "beer_soup_11039.pkl\n",
 669 |       "beer_soup_11269.pkl\n",
 670 |       "beer_soup_1149.pkl\n",
 671 |       "beer_soup_11499.pkl\n",
 672 |       "beer_soup_11729.pkl\n",
 673 |       "beer_soup_11959.pkl\n",
 674 |       "beer_soup_12189.pkl\n",
 675 |       "beer_soup_12419.pkl\n",
 676 |       "beer_soup_12649.pkl\n",
 677 |       "beer_soup_12879.pkl\n",
 678 |       "beer_soup_13109.pkl\n",
 679 |       "beer_soup_13339.pkl\n",
 680 |       "beer_soup_13569.pkl\n",
 681 |       "beer_soup_1379.pkl\n",
 682 |       "beer_soup_13799.pkl\n",
 683 |       "beer_soup_14029.pkl\n",
 684 |       "beer_soup_14259.pkl\n",
 685 |       "beer_soup_14489.pkl\n",
 686 |       "beer_soup_14719.pkl\n",
 687 |       "beer_soup_14949.pkl\n",
 688 |       "beer_soup_15179.pkl\n",
 689 |       "beer_soup_15409.pkl\n",
 690 |       "beer_soup_15639.pkl\n",
 691 |       "beer_soup_15869.pkl\n",
 692 |       "beer_soup_1609.pkl\n",
 693 |       "beer_soup_16099.pkl\n",
 694 |       "beer_soup_16329.pkl\n",
 695 |       "beer_soup_16559.pkl\n",
 696 |       "beer_soup_16789.pkl\n",
 697 |       "beer_soup_17019.pkl\n",
 698 |       "beer_soup_17249.pkl\n",
 699 |       "beer_soup_17479.pkl\n",
 700 |       "beer_soup_17709.pkl\n",
 701 |       "beer_soup_17939.pkl\n",
 702 |       "beer_soup_18169.pkl\n",
 703 |       "beer_soup_1839.pkl\n",
 704 |       "beer_soup_18399.pkl\n",
 705 |       "beer_soup_18629.pkl\n",
 706 |       "beer_soup_18859.pkl\n",
 707 |       "beer_soup_19089.pkl\n",
 708 |       "beer_soup_19319.pkl\n",
 709 |       "beer_soup_19549.pkl\n",
 710 |       "beer_soup_19779.pkl\n",
 711 |       "beer_soup_20009.pkl\n",
 712 |       "beer_soup_20239.pkl\n",
 713 |       "beer_soup_20469.pkl\n",
 714 |       "beer_soup_2069.pkl\n",
 715 |       "beer_soup_229.pkl\n",
 716 |       "beer_soup_2299.pkl\n",
 717 |       "beer_soup_2529.pkl\n",
 718 |       "beer_soup_2759.pkl\n",
 719 |       "beer_soup_2989.pkl\n",
 720 |       "beer_soup_3219.pkl\n",
 721 |       "beer_soup_3449.pkl\n",
 722 |       "beer_soup_3679.pkl\n",
 723 |       "beer_soup_3909.pkl\n",
 724 |       "beer_soup_4139.pkl\n",
 725 |       "beer_soup_4369.pkl\n",
 726 |       "beer_soup_459.pkl\n",
 727 |       "beer_soup_4599.pkl\n",
 728 |       "beer_soup_4829.pkl\n",
 729 |       "beer_soup_5059.pkl\n",
 730 |       "beer_soup_5289.pkl\n",
 731 |       "beer_soup_5519.pkl\n",
 732 |       "beer_soup_5749.pkl\n",
 733 |       "beer_soup_5979.pkl\n",
 734 |       "beer_soup_6209.pkl\n",
 735 |       "beer_soup_6439.pkl\n",
 736 |       "beer_soup_6669.pkl\n",
 737 |       "beer_soup_689.pkl\n",
 738 |       "beer_soup_6899.pkl\n",
 739 |       "beer_soup_7129.pkl\n",
 740 |       "beer_soup_7359.pkl\n",
 741 |       "beer_soup_7589.pkl\n",
 742 |       "beer_soup_7819.pkl\n",
 743 |       "beer_soup_8049.pkl\n",
 744 |       "beer_soup_8279.pkl\n",
 745 |       "beer_soup_8509.pkl\n",
 746 |       "beer_soup_8739.pkl\n",
 747 |       "beer_soup_8969.pkl\n",
 748 |       "beer_soup_919.pkl\n",
 749 |       "beer_soup_9199.pkl\n",
 750 |       "beer_soup_9429.pkl\n",
 751 |       "beer_soup_9659.pkl\n",
 752 |       "beer_soup_9889.pkl\n"
 753 |      ]
 754 |     }
 755 |    ],
 756 |    "source": [
 757 |     "for pkl in glob.glob('data/*.pkl'):\n",
 758 |     "    filename = pkl.split('/')[1]\n"
 759 |    ]
 760 |   },
 761 |   {
 762 |    "cell_type": "code",
 763 |    "execution_count": 181,
 764 |    "metadata": {
 765 |     "collapsed": true
 766 |    },
 767 |    "outputs": [
 768 |     {
 769 |      "name": "stderr",
 770 |      "output_type": "stream",
 771 |      "text": [
 772 |       "\n",
 773 |       "\n",
 774 |       "\n",
 775 |       "\n",
 776 |       "\n",
 777 |       "0it [00:00, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 778 |       "\n",
 779 |       "\n",
 780 |       "\n",
 781 |       "\u001b[A\u001b[A\u001b[A\u001b[A\n",
 782 |       "\n",
 783 |       "\n",
 784 |       "\n",
 785 |       "\n",
 786 |       "\n",
 787 |       "  0%|          | 0/89 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 788 |       "\n",
 789 |       "\n",
 790 |       "\n",
 791 |       "\n",
 792 |       "\n",
 793 |       "  1%|          | 1/89 [04:50<7:05:45, 290.29s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 794 |       "\n",
 795 |       "\n",
 796 |       "\n",
 797 |       "\n",
 798 |       "\n",
 799 |       "  2%|▏         | 2/89 [08:44<6:36:36, 273.53s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 800 |       "\n",
 801 |       "\n",
 802 |       "\n",
 803 |       "\n",
 804 |       "\n",
 805 |       "  3%|▎         | 3/89 [11:50<5:54:11, 247.11s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 806 |       "\n",
 807 |       "\n",
 808 |       "\n",
 809 |       "\n",
 810 |       "\n",
 811 |       "  4%|▍         | 4/89 [14:18<5:08:03, 217.45s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 812 |       "\n",
 813 |       "\n",
 814 |       "\n",
 815 |       "\n",
 816 |       "\n",
 817 |       "  6%|▌         | 5/89 [19:27<5:43:01, 245.01s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 818 |       "\n",
 819 |       "\n",
 820 |       "\n",
 821 |       "\n",
 822 |       "\n",
 823 |       "  7%|▋         | 6/89 [22:48<5:20:25, 231.63s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 824 |       "\n",
 825 |       "\n",
 826 |       "\n",
 827 |       "\n",
 828 |       "\n",
 829 |       "  8%|▊         | 7/89 [25:45<4:54:25, 215.44s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 830 |       "\n",
 831 |       "\n",
 832 |       "\n",
 833 |       "\n",
 834 |       "\n",
 835 |       "  9%|▉         | 8/89 [29:49<5:02:19, 223.95s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 836 |       "\n",
 837 |       "\n",
 838 |       "\n",
 839 |       "\n",
 840 |       "\n",
 841 |       " 10%|█         | 9/89 [32:37<4:36:13, 207.17s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 842 |       "\n",
 843 |       "\n",
 844 |       "\n",
 845 |       "\n",
 846 |       "\n",
 847 |       " 11%|█         | 10/89 [35:23<4:16:20, 194.68s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 848 |       "\n",
 849 |       "\n",
 850 |       "\n",
 851 |       "\n",
 852 |       "\n",
 853 |       " 12%|█▏        | 11/89 [38:28<4:09:28, 191.90s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 854 |       "\n",
 855 |       "\n",
 856 |       "\n",
 857 |       "\n",
 858 |       "\n",
 859 |       " 13%|█▎        | 12/89 [41:23<3:59:51, 186.90s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 860 |       "\n",
 861 |       "\n",
 862 |       "\n",
 863 |       "\n",
 864 |       "\n",
 865 |       " 15%|█▍        | 13/89 [43:37<3:36:35, 170.99s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 866 |       "\n",
 867 |       "\n",
 868 |       "\n",
 869 |       "\n",
 870 |       "\n",
 871 |       " 16%|█▌        | 14/89 [46:44<3:39:31, 175.62s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 872 |       "\n",
 873 |       "\n",
 874 |       "\n",
 875 |       "\n",
 876 |       "\n",
 877 |       " 17%|█▋        | 15/89 [50:08<3:47:25, 184.40s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 878 |       "\n",
 879 |       "\n",
 880 |       "\n",
 881 |       "\n",
 882 |       "\n",
 883 |       " 18%|█▊        | 16/89 [53:31<3:51:03, 189.91s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 884 |       "\n",
 885 |       "\n",
 886 |       "\n",
 887 |       "\n",
 888 |       "\n",
 889 |       " 19%|█▉        | 17/89 [57:20<4:01:55, 201.60s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 890 |       "\n",
 891 |       "\n",
 892 |       "\n",
 893 |       "\n",
 894 |       "\n",
 895 |       " 20%|██        | 18/89 [1:00:48<4:00:39, 203.37s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 896 |       "\n",
 897 |       "\n",
 898 |       "\n",
 899 |       "\n",
 900 |       "\n",
 901 |       " 21%|██▏       | 19/89 [1:05:40<4:28:29, 230.14s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 902 |       "\n",
 903 |       "\n",
 904 |       "\n",
 905 |       "\n",
 906 |       "\n",
 907 |       " 22%|██▏       | 20/89 [1:09:19<4:20:40, 226.68s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 908 |       "\n",
 909 |       "\n",
 910 |       "\n",
 911 |       "\n",
 912 |       "\n",
 913 |       " 24%|██▎       | 21/89 [1:12:34<4:06:01, 217.08s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 914 |       "\n",
 915 |       "\n",
 916 |       "\n",
 917 |       "\n",
 918 |       "\n",
 919 |       " 25%|██▍       | 22/89 [1:16:44<4:13:28, 226.99s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 920 |       "\n",
 921 |       "\n",
 922 |       "\n",
 923 |       "\n",
 924 |       "\n",
 925 |       " 26%|██▌       | 23/89 [1:20:29<4:09:07, 226.48s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 926 |       "\n",
 927 |       "\n",
 928 |       "\n",
 929 |       "\n",
 930 |       "\n",
 931 |       " 27%|██▋       | 24/89 [1:24:15<4:05:11, 226.34s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 932 |       "\n",
 933 |       "\n",
 934 |       "\n",
 935 |       "\n",
 936 |       "\n",
 937 |       " 28%|██▊       | 25/89 [1:28:27<4:09:32, 233.95s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 938 |       "\n",
 939 |       "\n",
 940 |       "\n",
 941 |       "\n",
 942 |       "\n",
 943 |       " 29%|██▉       | 26/89 [1:31:54<3:57:20, 226.04s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 944 |       "\n",
 945 |       "\n",
 946 |       "\n",
 947 |       "\n",
 948 |       "\n",
 949 |       " 30%|███       | 27/89 [1:35:30<3:50:27, 223.02s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 950 |       "\n",
 951 |       "\n",
 952 |       "\n",
 953 |       "\n",
 954 |       "\n",
 955 |       " 31%|███▏      | 28/89 [1:39:28<3:51:07, 227.34s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 956 |       "\n",
 957 |       "\n",
 958 |       "\n",
 959 |       "\n",
 960 |       "\n",
 961 |       " 33%|███▎      | 29/89 [1:43:30<3:51:53, 231.89s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 962 |       "\n",
 963 |       "\n",
 964 |       "\n",
 965 |       "\n",
 966 |       "\n",
 967 |       " 34%|███▎      | 30/89 [1:47:02<3:42:10, 225.95s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 968 |       "\n",
 969 |       "\n",
 970 |       "\n",
 971 |       "\n",
 972 |       "\n",
 973 |       " 35%|███▍      | 31/89 [1:50:49<3:38:45, 226.30s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 974 |       "\n",
 975 |       "\n",
 976 |       "\n",
 977 |       "\n",
 978 |       "\n",
 979 |       " 36%|███▌      | 32/89 [1:54:28<3:32:52, 224.07s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 980 |       "\n",
 981 |       "\n",
 982 |       "\n",
 983 |       "\n",
 984 |       "\n",
 985 |       " 37%|███▋      | 33/89 [1:57:20<3:14:30, 208.41s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 986 |       "\n",
 987 |       "\n",
 988 |       "\n",
 989 |       "\n",
 990 |       "\n",
 991 |       " 38%|███▊      | 34/89 [2:00:10<3:00:19, 196.72s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 992 |       "\n",
 993 |       "\n",
 994 |       "\n",
 995 |       "\n",
 996 |       "\n",
 997 |       " 39%|███▉      | 35/89 [2:03:15<2:54:06, 193.45s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
 998 |       "\n",
 999 |       "\n",
1000 |       "\n",
1001 |       "\n",
1002 |       "\n",
1003 |       " 40%|████      | 36/89 [2:06:23<2:49:28, 191.86s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1004 |       "\n",
1005 |       "\n",
1006 |       "\n",
1007 |       "\n",
1008 |       "\n",
1009 |       " 42%|████▏     | 37/89 [2:09:32<2:45:29, 190.96s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1010 |       "\n",
1011 |       "\n",
1012 |       "\n",
1013 |       "\n",
1014 |       "\n",
1015 |       " 43%|████▎     | 38/89 [2:12:29<2:38:47, 186.81s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1016 |       "\n",
1017 |       "\n",
1018 |       "\n",
1019 |       "\n",
1020 |       "\n",
1021 |       " 44%|████▍     | 39/89 [2:16:21<2:46:54, 200.29s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1022 |       "\n",
1023 |       "\n",
1024 |       "\n",
1025 |       "\n",
1026 |       "\n",
1027 |       " 45%|████▍     | 40/89 [2:18:59<2:33:06, 187.48s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1028 |       "\n",
1029 |       "\n",
1030 |       "\n",
1031 |       "\n",
1032 |       "\n",
1033 |       " 46%|████▌     | 41/89 [2:21:58<2:27:55, 184.90s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1034 |       "\n",
1035 |       "\n",
1036 |       "\n",
1037 |       "\n",
1038 |       "\n",
1039 |       " 47%|████▋     | 42/89 [2:25:42<2:34:02, 196.65s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1040 |       "\n",
1041 |       "\n",
1042 |       "\n",
1043 |       "\n",
1044 |       "\n",
1045 |       " 48%|████▊     | 43/89 [2:28:28<2:23:45, 187.50s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1046 |       "\n",
1047 |       "\n",
1048 |       "\n",
1049 |       "\n",
1050 |       "\n",
1051 |       " 49%|████▉     | 44/89 [2:31:47<2:23:13, 190.97s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1052 |       "\n",
1053 |       "\n",
1054 |       "\n",
1055 |       "\n",
1056 |       "\n",
1057 |       " 51%|█████     | 45/89 [2:35:16<2:23:55, 196.26s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1058 |       "\n",
1059 |       "\n",
1060 |       "\n",
1061 |       "\n",
1062 |       "\n",
1063 |       " 52%|█████▏    | 46/89 [2:38:46<2:23:36, 200.39s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1064 |       "\n",
1065 |       "\n",
1066 |       "\n",
1067 |       "\n",
1068 |       "\n",
1069 |       " 53%|█████▎    | 47/89 [2:42:56<2:30:49, 215.46s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1070 |       "\n",
1071 |       "\n",
1072 |       "\n",
1073 |       "\n",
1074 |       "\n",
1075 |       " 54%|█████▍    | 48/89 [2:46:22<2:25:11, 212.47s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1076 |       "\n",
1077 |       "\n",
1078 |       "\n",
1079 |       "\n",
1080 |       "\n",
1081 |       " 55%|█████▌    | 49/89 [2:49:48<2:20:29, 210.73s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1082 |       "\n",
1083 |       "\n",
1084 |       "\n",
1085 |       "\n",
1086 |       "\n",
1087 |       " 56%|█████▌    | 50/89 [2:53:40<2:21:01, 216.96s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1088 |       "\n",
1089 |       "\n",
1090 |       "\n",
1091 |       "\n",
1092 |       "\n",
1093 |       " 57%|█████▋    | 51/89 [2:58:00<2:25:31, 229.77s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1094 |       "\n",
1095 |       "\n",
1096 |       "\n",
1097 |       "\n",
1098 |       "\n",
1099 |       " 58%|█████▊    | 52/89 [3:01:46<2:21:03, 228.75s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1100 |       "\n",
1101 |       "\n",
1102 |       "\n",
1103 |       "\n",
1104 |       "\n",
1105 |       " 60%|█████▉    | 53/89 [3:04:49<2:09:02, 215.07s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1106 |       "\n",
1107 |       "\n",
1108 |       "\n",
1109 |       "\n",
1110 |       "\n",
1111 |       " 61%|██████    | 54/89 [3:07:27<1:55:29, 197.97s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1112 |       "\n",
1113 |       "\n",
1114 |       "\n",
1115 |       "\n",
1116 |       "\n",
1117 |       " 62%|██████▏   | 55/89 [3:09:53<1:43:14, 182.19s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1118 |       "\n",
1119 |       "\n",
1120 |       "\n",
1121 |       "\n",
1122 |       "\n",
1123 |       " 63%|██████▎   | 56/89 [3:12:16<1:33:52, 170.69s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1124 |       "\n",
1125 |       "\n",
1126 |       "\n",
1127 |       "\n",
1128 |       "\n",
1129 |       " 64%|██████▍   | 57/89 [3:14:52<1:28:36, 166.15s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1130 |       "\n",
1131 |       "\n",
1132 |       "\n",
1133 |       "\n",
1134 |       "\n",
1135 |       " 65%|██████▌   | 58/89 [3:17:40<1:26:07, 166.69s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1136 |       "\n",
1137 |       "\n",
1138 |       "\n",
1139 |       "\n",
1140 |       "\n",
1141 |       " 66%|██████▋   | 59/89 [3:20:08<1:20:37, 161.25s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1142 |       "\n",
1143 |       "\n",
1144 |       "\n",
1145 |       "\n",
1146 |       "\n",
1147 |       " 67%|██████▋   | 60/89 [3:22:39<1:16:24, 158.09s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1148 |       "\n",
1149 |       "\n",
1150 |       "\n",
1151 |       "\n",
1152 |       "\n",
1153 |       " 69%|██████▊   | 61/89 [3:25:17<1:13:46, 158.10s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1154 |       "\n",
1155 |       "\n",
1156 |       "\n",
1157 |       "\n",
1158 |       "\n",
1159 |       " 70%|██████▉   | 62/89 [3:27:39<1:08:53, 153.08s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1160 |       "\n",
1161 |       "\n",
1162 |       "\n",
1163 |       "\n",
1164 |       "\n",
1165 |       " 71%|███████   | 63/89 [3:29:54<1:03:59, 147.69s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1166 |       "\n",
1167 |       "\n",
1168 |       "\n",
1169 |       "\n",
1170 |       "\n",
1171 |       " 72%|███████▏  | 64/89 [3:32:41<1:03:58, 153.53s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1172 |       "\n",
1173 |       "\n",
1174 |       "\n",
1175 |       "\n",
1176 |       "\n",
1177 |       " 73%|███████▎  | 65/89 [3:35:25<1:02:40, 156.69s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1178 |       "\n",
1179 |       "\n",
1180 |       "\n",
1181 |       "\n",
1182 |       "\n",
1183 |       " 74%|███████▍  | 66/89 [3:37:44<58:02, 151.42s/it]  \u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1184 |       "\n",
1185 |       "\n",
1186 |       "\n",
1187 |       "\n",
1188 |       "\n",
1189 |       " 75%|███████▌  | 67/89 [3:39:57<53:31, 145.96s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1190 |       "\n",
1191 |       "\n",
1192 |       "\n",
1193 |       "\n",
1194 |       "\n",
1195 |       " 76%|███████▋  | 68/89 [3:42:37<52:34, 150.23s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1196 |       "\n",
1197 |       "\n",
1198 |       "\n",
1199 |       "\n",
1200 |       "\n",
1201 |       " 78%|███████▊  | 69/89 [3:44:46<47:53, 143.66s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1202 |       "\n",
1203 |       "\n",
1204 |       "\n",
1205 |       "\n",
1206 |       "\n",
1207 |       " 79%|███████▊  | 70/89 [3:47:10<45:31, 143.76s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1208 |       "\n",
1209 |       "\n",
1210 |       "\n",
1211 |       "\n",
1212 |       "\n",
1213 |       " 80%|███████▉  | 71/89 [3:49:49<44:31, 148.43s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1214 |       "\n",
1215 |       "\n",
1216 |       "\n",
1217 |       "\n",
1218 |       "\n",
1219 |       " 81%|████████  | 72/89 [3:52:16<41:52, 147.81s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1220 |       "\n",
1221 |       "\n",
1222 |       "\n",
1223 |       "\n",
1224 |       "\n",
1225 |       " 82%|████████▏ | 73/89 [3:54:26<38:02, 142.68s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1226 |       "\n",
1227 |       "\n",
1228 |       "\n",
1229 |       "\n",
1230 |       "\n",
1231 |       " 83%|████████▎ | 74/89 [3:56:35<34:36, 138.40s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1232 |       "\n",
1233 |       "\n",
1234 |       "\n",
1235 |       "\n",
1236 |       "\n",
1237 |       " 84%|████████▍ | 75/89 [3:59:01<32:53, 140.93s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1238 |       "\n",
1239 |       "\n",
1240 |       "\n",
1241 |       "\n",
1242 |       "\n",
1243 |       " 85%|████████▌ | 76/89 [4:01:30<31:01, 143.21s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1244 |       "\n",
1245 |       "\n",
1246 |       "\n",
1247 |       "\n",
1248 |       "\n",
1249 |       " 87%|████████▋ | 77/89 [4:04:00<29:02, 145.21s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1250 |       "\n",
1251 |       "\n",
1252 |       "\n",
1253 |       "\n",
1254 |       "\n",
1255 |       " 88%|████████▊ | 78/89 [4:06:41<27:30, 150.05s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1256 |       "\n",
1257 |       "\n",
1258 |       "\n",
1259 |       "\n",
1260 |       "\n",
1261 |       " 89%|████████▉ | 79/89 [4:08:32<23:03, 138.31s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1262 |       "\n",
1263 |       "\n",
1264 |       "\n",
1265 |       "\n",
1266 |       "\n",
1267 |       " 90%|████████▉ | 80/89 [4:11:03<21:17, 141.97s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1268 |       "\n",
1269 |       "\n",
1270 |       "\n",
1271 |       "\n",
1272 |       "\n",
1273 |       " 91%|█████████ | 81/89 [4:12:59<17:53, 134.22s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1274 |       "\n",
1275 |       "\n",
1276 |       "\n",
1277 |       "\n",
1278 |       "\n",
1279 |       " 92%|█████████▏| 82/89 [4:15:48<16:52, 144.65s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1280 |       "\n",
1281 |       "\n",
1282 |       "\n",
1283 |       "\n",
1284 |       "\n",
1285 |       " 93%|█████████▎| 83/89 [4:18:39<15:15, 152.59s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1286 |       "\n",
1287 |       "\n",
1288 |       "\n",
1289 |       "\n",
1290 |       "\n",
1291 |       " 94%|█████████▍| 84/89 [4:20:56<12:20, 148.02s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1292 |       "\n",
1293 |       "\n",
1294 |       "\n",
1295 |       "\n",
1296 |       "\n",
1297 |       " 96%|█████████▌| 85/89 [4:23:23<09:50, 147.60s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1298 |       "\n",
1299 |       "\n",
1300 |       "\n",
1301 |       "\n",
1302 |       "\n",
1303 |       " 97%|█████████▋| 86/89 [4:25:35<07:09, 143.09s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1304 |       "\n",
1305 |       "\n",
1306 |       "\n",
1307 |       "\n",
1308 |       "\n",
1309 |       " 98%|█████████▊| 87/89 [4:28:05<04:49, 144.89s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1310 |       "\n",
1311 |       "\n",
1312 |       "\n",
1313 |       "\n",
1314 |       "\n",
1315 |       " 99%|█████████▉| 88/89 [4:29:53<02:13, 133.88s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1316 |       "\n",
1317 |       "\n",
1318 |       "\n",
1319 |       "\n",
1320 |       "\n",
1321 |       "100%|██████████| 89/89 [4:32:37<00:00, 143.00s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\n",
1322 |       "\n",
1323 |       "\n",
1324 |       "\n",
1325 |       "\n",
1326 |       "\n",
1327 |       "\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A"
1328 |      ]
1329 |     }
1330 |    ],
1331 |    "source": [
1332 |     "tqdm_pandas(tqdm())\n",
1333 |     "for pkl in tqdm(glob.glob('data/*.pkl')):\n",
1334 |     "    temp = pd.read_pickle(pkl)\n",
1335 |     "    temp = temp.apply(get_beer_stats, axis=1)\n",
1336 |     "    temp = temp.apply(get_beer_df_reviews, axis=1)\n",
1337 |     "    filename = pkl.split('/')[1]\n",
1338 |     "    temp.to_pickle('temp/'+filename)"
1339 |    ]
1340 |   },
1341 |   {
1342 |    "cell_type": "code",
1343 |    "execution_count": 6,
1344 |    "metadata": {
1345 |     "collapsed": false
1346 |    },
1347 |    "outputs": [],
1348 |    "source": [
1349 |     "beers = pd.read_pickle(glob.glob('temp/*.pkl')[0])"
1350 |    ]
1351 |   },
1352 |   {
1353 |    "cell_type": "code",
1354 |    "execution_count": 11,
1355 |    "metadata": {
1356 |     "collapsed": false
1357 |    },
1358 |    "outputs": [],
1359 |    "source": [
1360 |     "pkl = 'data/beer_soup_missing.pkl'\n",
1361 |     "temp = pd.read_pickle(pkl)\n",
1362 |     "temp = temp.apply(get_beer_stats, axis=1)\n",
1363 |     "temp = temp.apply(get_beer_df_reviews, axis=1)"
1364 |    ]
1365 |   },
1366 |   {
1367 |    "cell_type": "code",
1368 |    "execution_count": 12,
1369 |    "metadata": {
1370 |     "collapsed": true
1371 |    },
1372 |    "outputs": [
1373 |     {
1374 |      "name": "stderr",
1375 |      "output_type": "stream",
1376 |      "text": [
1377 |       "\n",
1378 |       "  0%|          | 0/89 [00:00<?, ?it/s]\u001b[A\n",
1379 |       "  1%|          | 1/89 [00:10<15:53, 10.84s/it]\u001b[A\n",
1380 |       "  2%|▏         | 2/89 [00:22<15:57, 11.00s/it]\u001b[A\n",
1381 |       "  3%|▎         | 3/89 [00:34<16:13, 11.32s/it]\u001b[A\n",
1382 |       "  4%|▍         | 4/89 [00:45<15:51, 11.19s/it]\u001b[A\n",
1383 |       "  6%|▌         | 5/89 [00:57<16:05, 11.49s/it]\u001b[A\n",
1384 |       "  7%|▋         | 6/89 [01:08<15:39, 11.32s/it]\u001b[A\n",
1385 |       "  8%|▊         | 7/89 [01:21<16:04, 11.76s/it]\u001b[A\n",
1386 |       "  9%|▉         | 8/89 [01:34<16:27, 12.19s/it]\u001b[A\n",
1387 |       " 10%|█         | 9/89 [01:47<16:35, 12.44s/it]\u001b[A\n",
1388 |       " 11%|█         | 10/89 [01:59<16:12, 12.31s/it]\u001b[A\n",
1389 |       " 12%|█▏        | 11/89 [02:10<15:45, 12.12s/it]\u001b[A\n",
1390 |       " 13%|█▎        | 12/89 [02:24<15:57, 12.43s/it]\u001b[A\n",
1391 |       " 15%|█▍        | 13/89 [02:36<15:31, 12.26s/it]\u001b[A\n",
1392 |       " 16%|█▌        | 14/89 [02:48<15:28, 12.38s/it]\u001b[A\n",
1393 |       " 17%|█▋        | 15/89 [03:00<14:59, 12.15s/it]\u001b[A\n",
1394 |       " 18%|█▊        | 16/89 [03:13<14:59, 12.32s/it]\u001b[A\n",
1395 |       " 19%|█▉        | 17/89 [03:25<14:47, 12.32s/it]\u001b[A\n",
1396 |       " 20%|██        | 18/89 [03:36<14:14, 12.04s/it]\u001b[A\n",
1397 |       " 21%|██▏       | 19/89 [03:49<14:27, 12.39s/it]\u001b[A\n",
1398 |       " 22%|██▏       | 20/89 [04:01<14:03, 12.23s/it]\u001b[A\n",
1399 |       " 24%|██▎       | 21/89 [04:13<13:39, 12.05s/it]\u001b[A\n",
1400 |       " 25%|██▍       | 22/89 [04:25<13:38, 12.21s/it]\u001b[A\n",
1401 |       " 26%|██▌       | 23/89 [04:38<13:22, 12.16s/it]\u001b[A\n",
1402 |       " 27%|██▋       | 24/89 [04:51<13:29, 12.45s/it]\u001b[A\n",
1403 |       " 28%|██▊       | 25/89 [05:02<13:01, 12.21s/it]\u001b[A\n",
1404 |       " 29%|██▉       | 26/89 [05:16<13:09, 12.52s/it]\u001b[A\n",
1405 |       " 30%|███       | 27/89 [05:29<13:10, 12.75s/it]\u001b[A\n",
1406 |       " 31%|███▏      | 28/89 [05:40<12:37, 12.41s/it]\u001b[A\n",
1407 |       " 33%|███▎      | 29/89 [05:53<12:20, 12.35s/it]\u001b[A\n",
1408 |       " 34%|███▎      | 30/89 [06:05<12:05, 12.29s/it]\u001b[A\n",
1409 |       " 35%|███▍      | 31/89 [06:17<11:51, 12.27s/it]\u001b[A\n",
1410 |       " 36%|███▌      | 32/89 [06:29<11:42, 12.32s/it]\u001b[A\n",
1411 |       " 37%|███▋      | 33/89 [06:42<11:35, 12.43s/it]\u001b[A\n",
1412 |       " 38%|███▊      | 34/89 [06:54<11:11, 12.21s/it]\u001b[A\n",
1413 |       " 39%|███▉      | 35/89 [07:07<11:06, 12.35s/it]\u001b[A\n",
1414 |       " 40%|████      | 36/89 [07:18<10:40, 12.09s/it]\u001b[A\n",
1415 |       " 42%|████▏     | 37/89 [07:31<10:39, 12.29s/it]\u001b[A\n",
1416 |       " 43%|████▎     | 38/89 [07:43<10:20, 12.17s/it]\u001b[A\n",
1417 |       " 44%|████▍     | 39/89 [07:54<09:57, 11.95s/it]\u001b[A\n",
1418 |       " 45%|████▍     | 40/89 [08:07<09:55, 12.15s/it]\u001b[A\n",
1419 |       " 46%|████▌     | 41/89 [08:18<09:37, 12.03s/it]\u001b[A\n",
1420 |       " 47%|████▋     | 42/89 [08:30<09:18, 11.89s/it]\u001b[A\n",
1421 |       " 48%|████▊     | 43/89 [08:42<09:10, 11.97s/it]\u001b[A\n",
1422 |       " 49%|████▉     | 44/89 [08:54<08:57, 11.94s/it]\u001b[A\n",
1423 |       " 51%|█████     | 45/89 [09:07<09:00, 12.28s/it]\u001b[A\n",
1424 |       " 52%|█████▏    | 46/89 [09:18<08:34, 11.96s/it]\u001b[A\n",
1425 |       " 53%|█████▎    | 47/89 [09:30<08:17, 11.84s/it]\u001b[A\n",
1426 |       " 54%|█████▍    | 48/89 [09:42<08:06, 11.88s/it]\u001b[A\n",
1427 |       " 55%|█████▌    | 49/89 [09:54<07:58, 11.97s/it]\u001b[A\n",
1428 |       " 56%|█████▌    | 50/89 [10:07<07:54, 12.16s/it]\u001b[A\n",
1429 |       " 57%|█████▋    | 51/89 [10:18<07:32, 11.91s/it]\u001b[A\n",
1430 |       " 58%|█████▊    | 52/89 [10:29<07:09, 11.62s/it]\u001b[A\n",
1431 |       " 60%|█████▉    | 53/89 [10:40<06:56, 11.58s/it]\u001b[A\n",
1432 |       " 61%|██████    | 54/89 [10:51<06:32, 11.22s/it]\u001b[A\n",
1433 |       " 62%|██████▏   | 55/89 [11:02<06:19, 11.15s/it]\u001b[A\n",
1434 |       " 63%|██████▎   | 56/89 [11:13<06:05, 11.06s/it]\u001b[A\n",
1435 |       " 64%|██████▍   | 57/89 [11:24<05:59, 11.24s/it]\u001b[A\n",
1436 |       " 65%|██████▌   | 58/89 [11:35<05:47, 11.22s/it]\u001b[A\n",
1437 |       " 66%|██████▋   | 59/89 [11:47<05:38, 11.28s/it]\u001b[A\n",
1438 |       " 67%|██████▋   | 60/89 [11:58<05:22, 11.13s/it]\u001b[A\n",
1439 |       " 69%|██████▊   | 61/89 [12:08<05:06, 10.94s/it]\u001b[A\n",
1440 |       " 70%|██████▉   | 62/89 [12:19<04:57, 11.03s/it]\u001b[A\n",
1441 |       " 71%|███████   | 63/89 [12:31<04:49, 11.12s/it]\u001b[A\n",
1442 |       " 72%|███████▏  | 64/89 [12:42<04:38, 11.13s/it]\u001b[A\n",
1443 |       " 73%|███████▎  | 65/89 [12:53<04:29, 11.24s/it]\u001b[A\n",
1444 |       " 74%|███████▍  | 66/89 [13:04<04:12, 10.99s/it]\u001b[A\n",
1445 |       " 75%|███████▌  | 67/89 [13:14<03:59, 10.90s/it]\u001b[A\n",
1446 |       " 76%|███████▋  | 68/89 [13:27<03:56, 11.25s/it]\u001b[A\n",
1447 |       " 78%|███████▊  | 69/89 [13:37<03:41, 11.06s/it]\u001b[A\n",
1448 |       " 79%|███████▊  | 70/89 [13:48<03:29, 11.00s/it]\u001b[A\n",
1449 |       " 80%|███████▉  | 71/89 [14:00<03:22, 11.26s/it]\u001b[A\n",
1450 |       " 81%|████████  | 72/89 [14:11<03:08, 11.07s/it]\u001b[A\n",
1451 |       " 82%|████████▏ | 73/89 [14:22<03:00, 11.28s/it]\u001b[A\n",
1452 |       " 83%|████████▎ | 74/89 [14:34<02:50, 11.35s/it]\u001b[A\n",
1453 |       " 84%|████████▍ | 75/89 [14:45<02:36, 11.18s/it]\u001b[A\n",
1454 |       " 85%|████████▌ | 76/89 [14:57<02:30, 11.59s/it]\u001b[A\n",
1455 |       " 87%|████████▋ | 77/89 [15:08<02:17, 11.44s/it]\u001b[A\n",
1456 |       " 88%|████████▊ | 78/89 [15:19<02:02, 11.17s/it]\u001b[A\n",
1457 |       " 89%|████████▉ | 79/89 [15:31<01:55, 11.52s/it]\u001b[A\n",
1458 |       " 90%|████████▉ | 80/89 [15:42<01:41, 11.31s/it]\u001b[A\n",
1459 |       " 91%|█████████ | 81/89 [15:53<01:30, 11.30s/it]\u001b[A\n",
1460 |       " 92%|█████████▏| 82/89 [16:05<01:19, 11.31s/it]\u001b[A\n",
1461 |       " 93%|█████████▎| 83/89 [16:16<01:07, 11.29s/it]\u001b[A\n",
1462 |       " 94%|█████████▍| 84/89 [16:27<00:56, 11.32s/it]\u001b[A\n",
1463 |       " 96%|█████████▌| 85/89 [16:39<00:45, 11.36s/it]\u001b[A\n",
1464 |       " 97%|█████████▋| 86/89 [16:50<00:34, 11.41s/it]\u001b[A\n",
1465 |       " 98%|█████████▊| 87/89 [17:01<00:22, 11.27s/it]\u001b[A\n",
1466 |       " 99%|█████████▉| 88/89 [17:12<00:11, 11.31s/it]\u001b[A\n",
1467 |       "100%|██████████| 89/89 [17:23<00:00, 11.20s/it]\u001b[A\n",
1468 |       "\u001b[A"
1469 |      ]
1470 |     },
1471 |     {
1472 |      "ename": "NameError",
1473 |      "evalue": "name 'beer_reviews' is not defined",
1474 |      "output_type": "error",
1475 |      "traceback": [
1476 |       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
1477 |       "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
1478 |       "\u001b[0;32m<ipython-input-12-2528f3e88433>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mbeers_reviews\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdfs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mdfs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mbeer_reviews\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_pickle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'all_beers_reviews.pkl'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
1479 |       "\u001b[0;31mNameError\u001b[0m: name 'beer_reviews' is not defined"
1480 |      ]
1481 |     }
1482 |    ],
1483 |    "source": [
1484 |     "dfs = [temp]\n",
1485 |     "for pkl in tqdm(glob.glob('temp/*.pkl')):\n",
1486 |     "    temp = pd.read_pickle(pkl)\n",
1487 |     "    temp.drop('soup',axis='columns',inplace=True)\n",
1488 |     "    dfs.append(temp)\n",
1489 |     "    \n",
1490 |     "beer_reviews = pd.concat(dfs)\n",
1491 |     "dfs = []\n",
1492 |     "beer_reviews.to_pickle('all_beers_reviews.pkl')"
1493 |    ]
1494 |   },
1495 |   {
1496 |    "cell_type": "code",
1497 |    "execution_count": 20,
1498 |    "metadata": {
1499 |     "collapsed": false
1500 |    },
1501 |    "outputs": [
1502 |     {
1503 |      "ename": "NameError",
1504 |      "evalue": "name 'beer_reviews' is not defined",
1505 |      "output_type": "error",
1506 |      "traceback": [
1507 |       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
1508 |       "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
1509 |       "\u001b[0;32m<ipython-input-20-3c01785abe90>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mbeer_reviews\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
1510 |       "\u001b[0;31mNameError\u001b[0m: name 'beer_reviews' is not defined"
1511 |      ]
1512 |     }
1513 |    ],
1514 |    "source": []
1515 |   },
1516 |   {
1517 |    "cell_type": "code",
1518 |    "execution_count": null,
1519 |    "metadata": {
1520 |     "collapsed": true
1521 |    },
1522 |    "outputs": [],
1523 |    "source": []
1524 |   }
1525 |  ],
1526 |  "metadata": {
1527 |   "kernelspec": {
1528 |    "display_name": "Python 2",
1529 |    "language": "python",
1530 |    "name": "python2"
1531 |   },
1532 |   "language_info": {
1533 |    "codemirror_mode": {
1534 |     "name": "ipython",
1535 |     "version": 2
1536 |    },
1537 |    "file_extension": ".py",
1538 |    "mimetype": "text/x-python",
1539 |    "name": "python",
1540 |    "nbconvert_exporter": "python",
1541 |    "pygments_lexer": "ipython2",
1542 |    "version": "2.7.11"
1543 |   }
1544 |  },
1545 |  "nbformat": 4,
1546 |  "nbformat_minor": 0
1547 | }
1548 | 


--------------------------------------------------------------------------------
/scrape-all-ba.ipynb:
--------------------------------------------------------------------------------
   1 | {
   2 |  "cells": [
   3 |   {
   4 |    "cell_type": "code",
   5 |    "execution_count": 3,
   6 |    "metadata": {
   7 |     "collapsed": false
   8 |    },
   9 |    "outputs": [],
  10 |    "source": [
  11 |     "from bs4 import BeautifulSoup\n",
  12 |     "import string\n",
  13 |     "import requests\n",
  14 |     "import pandas as pd\n",
  15 |     "import re\n",
  16 |     "import pickle\n",
  17 |     "\n",
  18 |     "from fake_useragent import UserAgent\n",
  19 |     "ua = UserAgent()"
  20 |    ]
  21 |   },
  22 |   {
  23 |    "cell_type": "code",
  24 |    "execution_count": 4,
  25 |    "metadata": {
  26 |     "collapsed": true
  27 |    },
  28 |    "outputs": [],
  29 |    "source": [
  30 |     "def remove_punctuation(x):\n",
  31 |     "    x = str(x)\n",
  32 |     "    return x.translate(str.maketrans({a:None for a in string.punctuation}))"
  33 |    ]
  34 |   },
  35 |   {
  36 |    "cell_type": "code",
  37 |    "execution_count": 5,
  38 |    "metadata": {
  39 |     "collapsed": true
  40 |    },
  41 |    "outputs": [],
  42 |    "source": [
  43 |     "def get_soup(url, timeout=5):\n",
  44 |     "    headers  = {'User-Agent':ua.random}\n",
  45 |     "    try:\n",
  46 |     "        response = requests.get(url,headers=headers)\n",
  47 |     "    except:\n",
  48 |     "        print(\"FAILED \"+ url)\n",
  49 |     "        return 0\n",
  50 |     "    attempts = 0\n",
  51 |     "    while(not response.ok):\n",
  52 |     "            print((url+' failed with code: '+str(response.status_code)))\n",
  53 |     "            if attempts > timeout:\n",
  54 |     "                print(url+' failed with code: '+str(response.status_code))\n",
  55 |     "                return BeautifulSoup('')\n",
  56 |     "            response = requests.get(url)\n",
  57 |     "            attempts += 1\n",
  58 |     "    page = response.text\n",
  59 |     "    soup = BeautifulSoup(page)\n",
  60 |     "    return soup"
  61 |    ]
  62 |   },
  63 |   {
  64 |    "cell_type": "code",
  65 |    "execution_count": null,
  66 |    "metadata": {
  67 |     "collapsed": false
  68 |    },
  69 |    "outputs": [],
  70 |    "source": [
  71 |     "# Get all urls for each beer style\n",
  72 |     "\n",
  73 |     "url = 'http://www.beeradvocate.com/beer/style/'\n",
  74 |     "soup = get_soup(url)\n",
  75 |     "\n",
  76 |     "beer_styles = {}\n",
  77 |     "for style in soup.find('table').find_all('a'):\n",
  78 |     "    beer_styles[style.get_text()] = style['href']"
  79 |    ]
  80 |   },
  81 |   {
  82 |    "cell_type": "code",
  83 |    "execution_count": null,
  84 |    "metadata": {
  85 |     "collapsed": false
  86 |    },
  87 |    "outputs": [],
  88 |    "source": [
  89 |     "ba_url = 'http://www.beeradvocate.com'\n",
  90 |     "style_suffix = '?sort=revsD&start=0'\n",
  91 |     "soup = get_soup(ba_url+style_url+style_suffix)"
  92 |    ]
  93 |   },
  94 |   {
  95 |    "cell_type": "code",
  96 |    "execution_count": null,
  97 |    "metadata": {
  98 |     "collapsed": false
  99 |    },
 100 |    "outputs": [],
 101 |    "source": [
 102 |     "int(re.findall(r'(?<=\\(out of )\\d*',soup.find('tr').get_text())[0])"
 103 |    ]
 104 |   },
 105 |   {
 106 |    "cell_type": "code",
 107 |    "execution_count": null,
 108 |    "metadata": {
 109 |     "collapsed": false
 110 |    },
 111 |    "outputs": [],
 112 |    "source": []
 113 |   },
 114 |   {
 115 |    "cell_type": "code",
 116 |    "execution_count": null,
 117 |    "metadata": {
 118 |     "collapsed": false
 119 |    },
 120 |    "outputs": [],
 121 |    "source": [
 122 |     "beer_styles.items()[0]"
 123 |    ]
 124 |   },
 125 |   {
 126 |    "cell_type": "code",
 127 |    "execution_count": null,
 128 |    "metadata": {
 129 |     "collapsed": true
 130 |    },
 131 |    "outputs": [],
 132 |    "source": [
 133 |     "# Pulls the name and url to all beers with more than 25 'hads'\n",
 134 |     "\n",
 135 |     "ba_url = 'http://www.beeradvocate.com'\n",
 136 |     "style_suffix = '?sort=revsD&start='\n",
 137 |     "columns = ['name','url']\n",
 138 |     "temp = {}\n",
 139 |     "for style in beer_styles.items():\n",
 140 |     "    print(url)\n",
 141 |     "    url = ba_url+style[1]+style_suffix\n",
 142 |     "    soup = get_soup(url)\n",
 143 |     "    num_beers = int(re.findall(r'(?<=\\(out of )\\d*',soup.find('tr').get_text())[0])\n",
 144 |     "    print num_beers\n",
 145 |     "    min_beer = False\n",
 146 |     "    for i in range(num_beers//50):\n",
 147 |     "        if min_beer:\n",
 148 |     "                break\n",
 149 |     "        url = ba_url+style[1]+style_suffix+str(i*50)\n",
 150 |     "        soup = get_soup(url)\n",
 151 |     "        for row in soup.find_all('tr')[3:-1]:\n",
 152 |     "            cells = row.find_all('td')\n",
 153 |     "            if int(cells[4].get_text().replace(',','')) < 25:\n",
 154 |     "                min_beer = True\n",
 155 |     "                break\n",
 156 |     "            temp[cells[0].find('a').get_text()] = cells[0].find('a')['href']"
 157 |    ]
 158 |   },
 159 |   {
 160 |    "cell_type": "code",
 161 |    "execution_count": 7,
 162 |    "metadata": {
 163 |     "collapsed": true
 164 |    },
 165 |    "outputs": [
 166 |     {
 167 |      "name": "stdout",
 168 |      "output_type": "stream",
 169 |      "text": [
 170 |       "538\n"
 171 |      ]
 172 |     },
 173 |     {
 174 |      "data": {
 175 |       "text/plain": [
 176 |        "{u'1554 Black Lager': '/beer/profile/192/111828/',\n",
 177 |        " u'3 Best Friends': '/beer/profile/219/116077/',\n",
 178 |        " u'Alhambra Negra': '/beer/profile/9262/22584/',\n",
 179 |        " u'Asahi Dry Black': '/beer/profile/716/89545/',\n",
 180 |        " u'B.B. Dark Bohemia Beer - 1795 Original Czech Dark Lager': '/beer/profile/303/37361/',\n",
 181 |        " u'Baltika #4 Original (Dark)': '/beer/profile/401/2235/',\n",
 182 |        " u'Beerlao Dark': '/beer/profile/2970/27607/',\n",
 183 |        " u'Bernard Cerne': '/beer/profile/2055/21521/',\n",
 184 |        " u'Big Rock Honey Brown Lager': '/beer/profile/391/11175/',\n",
 185 |        " u'Black Lager (\\u010cern\\xe9 Pivo)': '/beer/profile/22723/76132/',\n",
 186 |        " u'Black Licorice Lager': '/beer/profile/9629/37147/',\n",
 187 |        " u'Bohemia': '/beer/profile/301/7078/',\n",
 188 |        " u'Bohemia Regent Lager Dark': '/beer/profile/7366/14349/',\n",
 189 |        " u'Bony Fingers': '/beer/profile/763/20446/',\n",
 190 |        " u'Brick Waterloo Dark Lager': '/beer/profile/416/5196/',\n",
 191 |        " u'Budweiser Budvar Czech Dark Lager': '/beer/profile/304/35967/',\n",
 192 |        " u'California Black Beer': '/beer/profile/8818/40370/',\n",
 193 |        " u'Celestial Meridian Cascadian Dark Lager': '/beer/profile/29619/118880/',\n",
 194 |        " u'Dark 266': '/beer/profile/3912/27684/',\n",
 195 |        " u'Death & Taxes Black Beer': '/beer/profile/763/2306/',\n",
 196 |        " u'Efes Dark': '/beer/profile/569/12362/',\n",
 197 |        " u'El Steinber Dark Lager': '/beer/profile/193/83646/',\n",
 198 |        " u'Elevator Dark Horse Lager': '/beer/profile/1464/40147/',\n",
 199 |        " u'Faxe Amber': '/beer/profile/783/11961/',\n",
 200 |        " u'Fischer Tradition Amber': '/beer/profile/197/710/',\n",
 201 |        " u'Fix Dark': '/beer/profile/3963/85171/',\n",
 202 |        " u'Gigi': '/beer/profile/29250/115954/',\n",
 203 |        " u'Gran\\xe1t - BrouCzech Dark': '/beer/profile/21333/58010/',\n",
 204 |        " u'Guinness Black Lager': '/beer/profile/209/57285/',\n",
 205 |        " u'Heineken Dark Lager': '/beer/profile/81/1167/',\n",
 206 |        " u'Heineken Oud Bruin': '/beer/profile/81/4087/',\n",
 207 |        " u'Hot Rocks Lager': '/beer/profile/1337/48508/',\n",
 208 |        " u'John Michael Dark Lyric Lagrrr!': '/beer/profile/30452/98186/',\n",
 209 |        " u'Kelso Nut Brown Lager': '/beer/profile/8768/33357/',\n",
 210 |        " u'Kilikia Dark': '/beer/profile/671/2061/',\n",
 211 |        " u'LTD Series - 06': '/beer/profile/5316/86044/',\n",
 212 |        " u'Layla Dirty Blonde Lager': '/beer/profile/1939/12687/',\n",
 213 |        " u\"Leinenkugel's Creamy Dark\": '/beer/profile/710/2940/',\n",
 214 |        " u'Lev Black Lion': '/beer/profile/168/9395/',\n",
 215 |        " u\"McSorley's Dark Lager\": '/beer/profile/447/42663/',\n",
 216 |        " u'Moa Noir': '/beer/profile/15922/36001/',\n",
 217 |        " u'Mythos Red': '/beer/profile/1084/45087/',\n",
 218 |        " u'Nightfall Lager': '/beer/profile/32426/97943/',\n",
 219 |        " u'Obolon Oksamytove (Deep Velvet)': '/beer/profile/601/35035/',\n",
 220 |        " u'Oldgott': '/beer/profile/12215/25835/',\n",
 221 |        " u'Palone': '/beer/profile/568/25468/',\n",
 222 |        " u'Pietra': '/beer/profile/2977/7195/',\n",
 223 |        " u'Pils Noir': '/beer/profile/29196/86001/',\n",
 224 |        " u'Podkovan Dark': '/beer/profile/11206/28049/',\n",
 225 |        " u'Praga Dark Lager': '/beer/profile/303/82289/',\n",
 226 |        " u'Primator Dark Lager': '/beer/profile/707/14874/',\n",
 227 |        " u'Sagres Cerveja Preta (Dark)': '/beer/profile/301/6187/',\n",
 228 |        " u'San Miguel Dark Lager': '/beer/profile/355/7102/',\n",
 229 |        " u'Saranac Chocolate Amber Lager': '/beer/profile/99/6796/',\n",
 230 |        " u'Schwarzer Kristall': '/beer/profile/5687/45326/',\n",
 231 |        " u'Silva Strong Dark Beer': '/beer/profile/1705/4808/',\n",
 232 |        " u'St. Pauli Girl Special Dark': '/beer/profile/224/698/',\n",
 233 |        " u'Staropramen Granat Beer': '/beer/profile/437/14167/',\n",
 234 |        " u'Staropramen \\u010cern\\xfd': '/beer/profile/437/8689/',\n",
 235 |        " u'Super Bock Stout': '/beer/profile/439/37864/',\n",
 236 |        " u'Telenn Du': '/beer/profile/2520/6245/',\n",
 237 |        " u'Tomislav Pivo': '/beer/profile/1720/42822/',\n",
 238 |        " u'Tooheys Red': '/beer/profile/839/4001/',\n",
 239 |        " u\"Trafalgar Paddy's Irish Red Lager\": '/beer/profile/765/8590/',\n",
 240 |        " u'U Fleku Dark Lager': '/beer/profile/2805/6614/',\n",
 241 |        " u'U Rousse': '/beer/profile/22/2054/',\n",
 242 |        " u'Wolverine Dark': '/beer/profile/24808/72163/',\n",
 243 |        " u'Yebisu Black Beer': '/beer/profile/284/11490/',\n",
 244 |        " u'\\x8e\\u017datec Dark Lager': '/beer/profile/4106/57920/'}"
 245 |       ]
 246 |      },
 247 |      "execution_count": 7,
 248 |      "metadata": {},
 249 |      "output_type": "execute_result"
 250 |     }
 251 |    ],
 252 |    "source": [
 253 |     "# Pulls the name and url to all beers with more than 25 'hads'\n",
 254 |     "\n",
 255 |     "ba_url = 'http://www.beeradvocate.com'\n",
 256 |     "style_suffix = '?sort=revsD&start='\n",
 257 |     "columns = ['name','url']\n",
 258 |     "temp = {}\n",
 259 |     "\n",
 260 |     "url = ba_url+'/beer/style/149/'+style_suffix\n",
 261 |     "soup = get_soup(url)\n",
 262 |     "num_beers = int(re.findall(r'(?<=\\(out of )\\d*',soup.find('tr').get_text())[0])\n",
 263 |     "print num_beers\n",
 264 |     "min_beer = False\n",
 265 |     "for i in range(num_beers//50):\n",
 266 |     "    if min_beer:\n",
 267 |     "            break\n",
 268 |     "    url = ba_url+'/beer/style/149/'+style_suffix+str(i*50)\n",
 269 |     "    soup = get_soup(url)\n",
 270 |     "    for row in soup.find_all('tr')[3:-1]:\n",
 271 |     "        cells = row.find_all('td')\n",
 272 |     "        if int(cells[4].get_text().replace(',','')) < 25:\n",
 273 |     "            min_beer = True\n",
 274 |     "            break\n",
 275 |     "        temp[cells[0].find('a').get_text()] = cells[0].find('a')['href']\n",
 276 |     "        \n",
 277 |     "temp"
 278 |    ]
 279 |   },
 280 |   {
 281 |    "cell_type": "code",
 282 |    "execution_count": null,
 283 |    "metadata": {
 284 |     "collapsed": false
 285 |    },
 286 |    "outputs": [],
 287 |    "source": [
 288 |     "pickle.dump(temp,open('beer_list.pkl','wb'))"
 289 |    ]
 290 |   },
 291 |   {
 292 |    "cell_type": "code",
 293 |    "execution_count": null,
 294 |    "metadata": {
 295 |     "collapsed": false
 296 |    },
 297 |    "outputs": [],
 298 |    "source": [
 299 |     "beer_urls = pickle.load(open('beer_list.pkl','rb'))"
 300 |    ]
 301 |   },
 302 |   {
 303 |    "cell_type": "code",
 304 |    "execution_count": null,
 305 |    "metadata": {
 306 |     "collapsed": false
 307 |    },
 308 |    "outputs": [],
 309 |    "source": [
 310 |     "columns = ['name','url']\n",
 311 |     "beers = pd.DataFrame(columns=columns)\n",
 312 |     "\n",
 313 |     "beers['name'] = beer_urls.keys()\n",
 314 |     "beers['url']  = beer_urls.values()\n",
 315 |     "pd.to_pickle(beers,'beers.pkl')"
 316 |    ]
 317 |   },
 318 |   {
 319 |    "cell_type": "code",
 320 |    "execution_count": 8,
 321 |    "metadata": {
 322 |     "collapsed": true
 323 |    },
 324 |    "outputs": [],
 325 |    "source": [
 326 |     "beers = pd.read_pickle('beers.pkl')"
 327 |    ]
 328 |   },
 329 |   {
 330 |    "cell_type": "code",
 331 |    "execution_count": 9,
 332 |    "metadata": {
 333 |     "collapsed": false
 334 |    },
 335 |    "outputs": [
 336 |     {
 337 |      "data": {
 338 |       "text/html": [
 339 |        "<div>\n",
 340 |        "<table border=\"1\" class=\"dataframe\">\n",
 341 |        "  <thead>\n",
 342 |        "    <tr style=\"text-align: right;\">\n",
 343 |        "      <th></th>\n",
 344 |        "      <th>name</th>\n",
 345 |        "      <th>url</th>\n",
 346 |        "    </tr>\n",
 347 |        "  </thead>\n",
 348 |        "  <tbody>\n",
 349 |        "    <tr>\n",
 350 |        "      <th>2299</th>\n",
 351 |        "      <td>Death &amp; Taxes Black Beer</td>\n",
 352 |        "      <td>/beer/profile/763/2306/</td>\n",
 353 |        "    </tr>\n",
 354 |        "  </tbody>\n",
 355 |        "</table>\n",
 356 |        "</div>"
 357 |       ],
 358 |       "text/plain": [
 359 |        "                          name                      url\n",
 360 |        "2299  Death & Taxes Black Beer  /beer/profile/763/2306/"
 361 |       ]
 362 |      },
 363 |      "execution_count": 9,
 364 |      "metadata": {},
 365 |      "output_type": "execute_result"
 366 |     }
 367 |    ],
 368 |    "source": [
 369 |     "beers[beers.name == 'Death & Taxes Black Beer']"
 370 |    ]
 371 |   },
 372 |   {
 373 |    "cell_type": "code",
 374 |    "execution_count": 18,
 375 |    "metadata": {
 376 |     "collapsed": false
 377 |    },
 378 |    "outputs": [],
 379 |    "source": [
 380 |     "def get_beer_soup(url):\n",
 381 |     "    ba_url = 'http://www.beeradvocate.com'\n",
 382 |     "    url = ba_url+url\n",
 383 |     "    soup = get_soup(url)\n",
 384 |     "    return soup"
 385 |    ]
 386 |   },
 387 |   {
 388 |    "cell_type": "code",
 389 |    "execution_count": 20,
 390 |    "metadata": {
 391 |     "collapsed": false
 392 |    },
 393 |    "outputs": [],
 394 |    "source": [
 395 |     "import sys\n",
 396 |     "sys.setrecursionlimit(100000000)"
 397 |    ]
 398 |   },
 399 |   {
 400 |    "cell_type": "code",
 401 |    "execution_count": 31,
 402 |    "metadata": {
 403 |     "collapsed": true
 404 |    },
 405 |    "outputs": [
 406 |     {
 407 |      "name": "stderr",
 408 |      "output_type": "stream",
 409 |      "text": [
 410 |       "\r",
 411 |       "  0%|          | 0/89 [00:00<?, ?it/s]"
 412 |      ]
 413 |     },
 414 |     {
 415 |      "name": "stdout",
 416 |      "output_type": "stream",
 417 |      "text": [
 418 |       "http://www.beeradvocate.com/beer/profile/26676/104914/ failed with code: 403\n",
 419 |       "http://www.beeradvocate.com/beer/profile/2055/13583/ failed with code: 403\n",
 420 |       "http://www.beeradvocate.com/beer/profile/27021/124612/ failed with code: 403\n"
 421 |      ]
 422 |     },
 423 |     {
 424 |      "name": "stderr",
 425 |      "output_type": "stream",
 426 |      "text": [
 427 |       "/usr/local/lib/python2.7/site-packages/ipykernel/__main__.py:3: SettingWithCopyWarning: \n",
 428 |       "A value is trying to be set on a copy of a slice from a DataFrame.\n",
 429 |       "Try using .loc[row_indexer,col_indexer] = value instead\n",
 430 |       "\n",
 431 |       "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
 432 |       "  app.launch_new_instance()\n",
 433 |       "  1%|          | 1/89 [02:04<3:02:00, 124.10s/it]"
 434 |      ]
 435 |     },
 436 |     {
 437 |      "name": "stdout",
 438 |      "output_type": "stream",
 439 |      "text": [
 440 |       "http://www.beeradvocate.com/beer/profile/16773/104438/ failed with code: 403\n",
 441 |       "http://www.beeradvocate.com/beer/profile/395/35378/ failed with code: 403\n"
 442 |      ]
 443 |     },
 444 |     {
 445 |      "name": "stderr",
 446 |      "output_type": "stream",
 447 |      "text": [
 448 |       "\r",
 449 |       "  2%|▏         | 2/89 [04:09<3:00:19, 124.36s/it]"
 450 |      ]
 451 |     },
 452 |     {
 453 |      "name": "stdout",
 454 |      "output_type": "stream",
 455 |      "text": [
 456 |       "http://www.beeradvocate.com/beer/profile/113/17284/ failed with code: 403\n",
 457 |       "http://www.beeradvocate.com/beer/profile/24764/99037/ failed with code: 403\n"
 458 |      ]
 459 |     },
 460 |     {
 461 |      "name": "stderr",
 462 |      "output_type": "stream",
 463 |      "text": [
 464 |       "\r",
 465 |       "  3%|▎         | 3/89 [06:18<3:00:29, 125.93s/it]"
 466 |      ]
 467 |     },
 468 |     {
 469 |      "name": "stdout",
 470 |      "output_type": "stream",
 471 |      "text": [
 472 |       "http://www.beeradvocate.com/beer/profile/34318/156937/ failed with code: 403\n",
 473 |       "http://www.beeradvocate.com/beer/profile/23/43060/ failed with code: 403\n",
 474 |       "http://www.beeradvocate.com/beer/profile/22655/101507/ failed with code: 403\n",
 475 |       "http://www.beeradvocate.com/beer/profile/173/45936/ failed with code: 403\n"
 476 |      ]
 477 |     },
 478 |     {
 479 |      "name": "stderr",
 480 |      "output_type": "stream",
 481 |      "text": [
 482 |       "\r",
 483 |       "  4%|▍         | 4/89 [08:20<2:56:29, 124.58s/it]"
 484 |      ]
 485 |     },
 486 |     {
 487 |      "name": "stdout",
 488 |      "output_type": "stream",
 489 |      "text": [
 490 |       "http://www.beeradvocate.com/beer/profile/1636/4603/ failed with code: 403\n"
 491 |      ]
 492 |     },
 493 |     {
 494 |      "name": "stderr",
 495 |      "output_type": "stream",
 496 |      "text": [
 497 |       "\r",
 498 |       "  6%|▌         | 5/89 [10:23<2:54:07, 124.38s/it]"
 499 |      ]
 500 |     },
 501 |     {
 502 |      "name": "stdout",
 503 |      "output_type": "stream",
 504 |      "text": [
 505 |       "http://www.beeradvocate.com/beer/profile/26/86165/ failed with code: 403\n",
 506 |       "http://www.beeradvocate.com/beer/profile/19897/51045/ failed with code: 403\n"
 507 |      ]
 508 |     },
 509 |     {
 510 |      "name": "stderr",
 511 |      "output_type": "stream",
 512 |      "text": [
 513 |       "\r",
 514 |       "  7%|▋         | 6/89 [12:27<2:51:31, 124.00s/it]"
 515 |      ]
 516 |     },
 517 |     {
 518 |      "name": "stdout",
 519 |      "output_type": "stream",
 520 |      "text": [
 521 |       "http://www.beeradvocate.com/beer/profile/28383/81870/ failed with code: 403\n",
 522 |       "http://www.beeradvocate.com/beer/profile/1623/4584/ failed with code: 403\n",
 523 |       "http://www.beeradvocate.com/beer/profile/21154/57481/ failed with code: 403\n",
 524 |       "http://www.beeradvocate.com/beer/profile/433/1313/ failed with code: 500\n"
 525 |      ]
 526 |     },
 527 |     {
 528 |      "name": "stderr",
 529 |      "output_type": "stream",
 530 |      "text": [
 531 |       "\r",
 532 |       "  8%|▊         | 7/89 [14:26<2:47:23, 122.48s/it]"
 533 |      ]
 534 |     },
 535 |     {
 536 |      "name": "stdout",
 537 |      "output_type": "stream",
 538 |      "text": [
 539 |       "http://www.beeradvocate.com/beer/profile/703/6135/ failed with code: 403\n"
 540 |      ]
 541 |     },
 542 |     {
 543 |      "name": "stderr",
 544 |      "output_type": "stream",
 545 |      "text": [
 546 |       "\r",
 547 |       "  9%|▉         | 8/89 [16:24<2:43:53, 121.40s/it]"
 548 |      ]
 549 |     },
 550 |     {
 551 |      "name": "stdout",
 552 |      "output_type": "stream",
 553 |      "text": [
 554 |       "http://www.beeradvocate.com/beer/profile/396/189097/ failed with code: 403\n",
 555 |       "http://www.beeradvocate.com/beer/profile/24247/81721/ failed with code: 403\n",
 556 |       "http://www.beeradvocate.com/beer/profile/32908/114626/ failed with code: 403\n",
 557 |       "http://www.beeradvocate.com/beer/profile/789/35291/ failed with code: 403\n",
 558 |       "http://www.beeradvocate.com/beer/profile/28178/203498/ failed with code: 403\n"
 559 |      ]
 560 |     },
 561 |     {
 562 |      "name": "stderr",
 563 |      "output_type": "stream",
 564 |      "text": [
 565 |       "\r",
 566 |       " 10%|█         | 9/89 [18:23<2:40:46, 120.58s/it]"
 567 |      ]
 568 |     },
 569 |     {
 570 |      "name": "stdout",
 571 |      "output_type": "stream",
 572 |      "text": [
 573 |       "http://www.beeradvocate.com/beer/profile/16866/200131/ failed with code: 403\n",
 574 |       "http://www.beeradvocate.com/beer/profile/31644/180672/ failed with code: 403\n"
 575 |      ]
 576 |     },
 577 |     {
 578 |      "name": "stderr",
 579 |      "output_type": "stream",
 580 |      "text": [
 581 |       "\r",
 582 |       " 11%|█         | 10/89 [20:25<2:39:10, 120.89s/it]"
 583 |      ]
 584 |     },
 585 |     {
 586 |      "name": "stdout",
 587 |      "output_type": "stream",
 588 |      "text": [
 589 |       "http://www.beeradvocate.com/beer/profile/889/23531/ failed with code: 403\n"
 590 |      ]
 591 |     },
 592 |     {
 593 |      "name": "stderr",
 594 |      "output_type": "stream",
 595 |      "text": [
 596 |       " 13%|█▎        | 12/89 [24:37<2:38:54, 123.82s/it]"
 597 |      ]
 598 |     },
 599 |     {
 600 |      "name": "stdout",
 601 |      "output_type": "stream",
 602 |      "text": [
 603 |       "http://www.beeradvocate.com/beer/profile/1457/5136/ failed with code: 403\n",
 604 |       "http://www.beeradvocate.com/beer/profile/684/26207/ failed with code: 403\n",
 605 |       "http://www.beeradvocate.com/beer/profile/22524/61859/ failed with code: 403\n"
 606 |      ]
 607 |     },
 608 |     {
 609 |      "name": "stderr",
 610 |      "output_type": "stream",
 611 |      "text": [
 612 |       "\r",
 613 |       " 15%|█▍        | 13/89 [26:38<2:35:35, 122.84s/it]"
 614 |      ]
 615 |     },
 616 |     {
 617 |      "name": "stdout",
 618 |      "output_type": "stream",
 619 |      "text": [
 620 |       "http://www.beeradvocate.com/beer/profile/14879/118321/ failed with code: 403\n",
 621 |       "http://www.beeradvocate.com/beer/profile/28255/111924/ failed with code: 403\n"
 622 |      ]
 623 |     },
 624 |     {
 625 |      "name": "stderr",
 626 |      "output_type": "stream",
 627 |      "text": [
 628 |       "\r",
 629 |       " 16%|█▌        | 14/89 [28:46<2:35:33, 124.44s/it]"
 630 |      ]
 631 |     },
 632 |     {
 633 |      "name": "stdout",
 634 |      "output_type": "stream",
 635 |      "text": [
 636 |       "http://www.beeradvocate.com/beer/profile/158/88572/ failed with code: 403\n",
 637 |       "http://www.beeradvocate.com/beer/profile/907/3592/ failed with code: 403\n",
 638 |       "http://www.beeradvocate.com/beer/profile/19241/52046/ failed with code: 403\n",
 639 |       "http://www.beeradvocate.com/beer/profile/28019/84491/ failed with code: 403\n"
 640 |      ]
 641 |     },
 642 |     {
 643 |      "name": "stderr",
 644 |      "output_type": "stream",
 645 |      "text": [
 646 |       "\r",
 647 |       " 17%|█▋        | 15/89 [30:54<2:34:41, 125.42s/it]"
 648 |      ]
 649 |     },
 650 |     {
 651 |      "name": "stdout",
 652 |      "output_type": "stream",
 653 |      "text": [
 654 |       "http://www.beeradvocate.com/beer/profile/139/103831/ failed with code: 403\n",
 655 |       "http://www.beeradvocate.com/beer/profile/28726/90099/ failed with code: 403\n"
 656 |      ]
 657 |     },
 658 |     {
 659 |      "name": "stderr",
 660 |      "output_type": "stream",
 661 |      "text": [
 662 |       "\r",
 663 |       " 18%|█▊        | 16/89 [32:51<2:29:42, 123.05s/it]"
 664 |      ]
 665 |     },
 666 |     {
 667 |      "name": "stdout",
 668 |      "output_type": "stream",
 669 |      "text": [
 670 |       "http://www.beeradvocate.com/beer/profile/9139/43579/ failed with code: 403\n",
 671 |       "http://www.beeradvocate.com/beer/profile/16333/211556/ failed with code: 403\n",
 672 |       "http://www.beeradvocate.com/beer/profile/30542/118621/ failed with code: 403\n"
 673 |      ]
 674 |     },
 675 |     {
 676 |      "name": "stderr",
 677 |      "output_type": "stream",
 678 |      "text": [
 679 |       " 20%|██        | 18/89 [36:58<2:25:56, 123.33s/it]"
 680 |      ]
 681 |     },
 682 |     {
 683 |      "name": "stdout",
 684 |      "output_type": "stream",
 685 |      "text": [
 686 |       "http://www.beeradvocate.com/beer/profile/1037/6565/ failed with code: 500\n",
 687 |       "http://www.beeradvocate.com/beer/profile/26/184050/ failed with code: 403\n",
 688 |       "http://www.beeradvocate.com/beer/profile/596/26613/ failed with code: 403\n"
 689 |      ]
 690 |     },
 691 |     {
 692 |      "name": "stderr",
 693 |      "output_type": "stream",
 694 |      "text": [
 695 |       "\r",
 696 |       " 21%|██▏       | 19/89 [39:21<2:30:46, 129.24s/it]"
 697 |      ]
 698 |     },
 699 |     {
 700 |      "name": "stdout",
 701 |      "output_type": "stream",
 702 |      "text": [
 703 |       "http://www.beeradvocate.com/beer/profile/23315/71064/ failed with code: 403\n",
 704 |       "http://www.beeradvocate.com/beer/profile/10607/77299/ failed with code: 403\n"
 705 |      ]
 706 |     },
 707 |     {
 708 |      "name": "stderr",
 709 |      "output_type": "stream",
 710 |      "text": [
 711 |       "\r",
 712 |       " 22%|██▏       | 20/89 [41:31<2:28:59, 129.55s/it]"
 713 |      ]
 714 |     },
 715 |     {
 716 |      "name": "stdout",
 717 |      "output_type": "stream",
 718 |      "text": [
 719 |       "http://www.beeradvocate.com/beer/profile/24940/132380/ failed with code: 403\n",
 720 |       "http://www.beeradvocate.com/beer/profile/1935/7920/ failed with code: 403\n",
 721 |       "http://www.beeradvocate.com/beer/profile/700/83742/ failed with code: 403\n"
 722 |      ]
 723 |     },
 724 |     {
 725 |      "name": "stderr",
 726 |      "output_type": "stream",
 727 |      "text": [
 728 |       " 26%|██▌       | 23/89 [47:41<2:17:58, 125.43s/it]"
 729 |      ]
 730 |     },
 731 |     {
 732 |      "name": "stdout",
 733 |      "output_type": "stream",
 734 |      "text": [
 735 |       "http://www.beeradvocate.com/beer/profile/263/29145/ failed with code: 403\n",
 736 |       "http://www.beeradvocate.com/beer/profile/2346/7052/ failed with code: 403\n",
 737 |       "http://www.beeradvocate.com/beer/profile/25710/177881/ failed with code: 403\n"
 738 |      ]
 739 |     },
 740 |     {
 741 |      "name": "stderr",
 742 |      "output_type": "stream",
 743 |      "text": [
 744 |       "\r",
 745 |       " 27%|██▋       | 24/89 [49:58<2:19:38, 128.90s/it]"
 746 |      ]
 747 |     },
 748 |     {
 749 |      "name": "stdout",
 750 |      "output_type": "stream",
 751 |      "text": [
 752 |       "http://www.beeradvocate.com/beer/profile/26565/82744/ failed with code: 403\n",
 753 |       "http://www.beeradvocate.com/beer/profile/5316/7200/ failed with code: 403\n"
 754 |      ]
 755 |     },
 756 |     {
 757 |      "name": "stderr",
 758 |      "output_type": "stream",
 759 |      "text": [
 760 |       " 30%|███       | 27/89 [56:35<2:15:35, 131.22s/it]"
 761 |      ]
 762 |     },
 763 |     {
 764 |      "name": "stdout",
 765 |      "output_type": "stream",
 766 |      "text": [
 767 |       "http://www.beeradvocate.com/beer/profile/16393/146756/ failed with code: 403\n",
 768 |       "http://www.beeradvocate.com/beer/profile/20430/81744/ failed with code: 403\n"
 769 |      ]
 770 |     },
 771 |     {
 772 |      "name": "stderr",
 773 |      "output_type": "stream",
 774 |      "text": [
 775 |       "\r",
 776 |       " 31%|███▏      | 28/89 [58:52<2:15:08, 132.93s/it]"
 777 |      ]
 778 |     },
 779 |     {
 780 |      "name": "stdout",
 781 |      "output_type": "stream",
 782 |      "text": [
 783 |       "http://www.beeradvocate.com/beer/profile/436/19809/ failed with code: 403\n",
 784 |       "http://www.beeradvocate.com/beer/profile/4051/40865/ failed with code: 403\n",
 785 |       "http://www.beeradvocate.com/beer/profile/13884/63180/ failed with code: 403\n",
 786 |       "http://www.beeradvocate.com/beer/profile/2372/113505/ failed with code: 403\n"
 787 |      ]
 788 |     },
 789 |     {
 790 |      "name": "stderr",
 791 |      "output_type": "stream",
 792 |      "text": [
 793 |       "\r",
 794 |       " 33%|███▎      | 29/89 [1:00:59<2:11:01, 131.03s/it]"
 795 |      ]
 796 |     },
 797 |     {
 798 |      "name": "stdout",
 799 |      "output_type": "stream",
 800 |      "text": [
 801 |       "http://www.beeradvocate.com/beer/profile/30/1830/ failed with code: 403\n"
 802 |      ]
 803 |     },
 804 |     {
 805 |      "name": "stderr",
 806 |      "output_type": "stream",
 807 |      "text": [
 808 |       "\r",
 809 |       " 34%|███▎      | 30/89 [1:03:10<2:09:02, 131.23s/it]"
 810 |      ]
 811 |     },
 812 |     {
 813 |      "name": "stdout",
 814 |      "output_type": "stream",
 815 |      "text": [
 816 |       "http://www.beeradvocate.com/beer/profile/40579/218624/ failed with code: 403\n",
 817 |       "http://www.beeradvocate.com/beer/profile/71/84562/ failed with code: 403\n",
 818 |       "http://www.beeradvocate.com/beer/profile/1675/8052/ failed with code: 403\n",
 819 |       "http://www.beeradvocate.com/beer/profile/147/109789/ failed with code: 403\n"
 820 |      ]
 821 |     },
 822 |     {
 823 |      "name": "stderr",
 824 |      "output_type": "stream",
 825 |      "text": [
 826 |       "\r",
 827 |       " 35%|███▍      | 31/89 [1:05:14<2:04:41, 128.99s/it]"
 828 |      ]
 829 |     },
 830 |     {
 831 |      "name": "stdout",
 832 |      "output_type": "stream",
 833 |      "text": [
 834 |       "http://www.beeradvocate.com/beer/profile/15237/62070/ failed with code: 403\n"
 835 |      ]
 836 |     },
 837 |     {
 838 |      "name": "stderr",
 839 |      "output_type": "stream",
 840 |      "text": [
 841 |       "\r",
 842 |       " 36%|███▌      | 32/89 [1:07:22<2:02:19, 128.77s/it]"
 843 |      ]
 844 |     },
 845 |     {
 846 |      "name": "stdout",
 847 |      "output_type": "stream",
 848 |      "text": [
 849 |       "http://www.beeradvocate.com/beer/profile/17981/180992/ failed with code: 403\n"
 850 |      ]
 851 |     },
 852 |     {
 853 |      "name": "stderr",
 854 |      "output_type": "stream",
 855 |      "text": [
 856 |       "\r",
 857 |       " 37%|███▋      | 33/89 [1:09:28<1:59:12, 127.72s/it]"
 858 |      ]
 859 |     },
 860 |     {
 861 |      "name": "stdout",
 862 |      "output_type": "stream",
 863 |      "text": [
 864 |       "http://www.beeradvocate.com/beer/profile/31840/96859/ failed with code: 403\n",
 865 |       "http://www.beeradvocate.com/beer/profile/515/18895/ failed with code: 403\n",
 866 |       "http://www.beeradvocate.com/beer/profile/26932/93416/ failed with code: 403\n"
 867 |      ]
 868 |     },
 869 |     {
 870 |      "name": "stderr",
 871 |      "output_type": "stream",
 872 |      "text": [
 873 |       "\r",
 874 |       " 38%|███▊      | 34/89 [1:11:44<1:59:32, 130.41s/it]"
 875 |      ]
 876 |     },
 877 |     {
 878 |      "name": "stdout",
 879 |      "output_type": "stream",
 880 |      "text": [
 881 |       "http://www.beeradvocate.com/beer/profile/33064/148605/ failed with code: 403\n"
 882 |      ]
 883 |     },
 884 |     {
 885 |      "name": "stderr",
 886 |      "output_type": "stream",
 887 |      "text": [
 888 |       " 40%|████      | 36/89 [1:15:48<1:51:15, 125.95s/it]"
 889 |      ]
 890 |     },
 891 |     {
 892 |      "name": "stdout",
 893 |      "output_type": "stream",
 894 |      "text": [
 895 |       "http://www.beeradvocate.com/beer/profile/29656/144719/ failed with code: 403\n",
 896 |       "http://www.beeradvocate.com/beer/profile/43/467/ failed with code: 403\n",
 897 |       "http://www.beeradvocate.com/beer/profile/21872/55347/ failed with code: 403\n"
 898 |      ]
 899 |     },
 900 |     {
 901 |      "name": "stderr",
 902 |      "output_type": "stream",
 903 |      "text": [
 904 |       "\r",
 905 |       " 42%|████▏     | 37/89 [1:17:56<1:49:40, 126.54s/it]"
 906 |      ]
 907 |     },
 908 |     {
 909 |      "name": "stdout",
 910 |      "output_type": "stream",
 911 |      "text": [
 912 |       "http://www.beeradvocate.com/beer/profile/24168/72411/ failed with code: 403\n",
 913 |       "http://www.beeradvocate.com/beer/profile/267/66436/ failed with code: 403\n",
 914 |       "http://www.beeradvocate.com/beer/profile/583/39397/ failed with code: 403\n",
 915 |       "http://www.beeradvocate.com/beer/profile/18823/166022/ failed with code: 403\n"
 916 |      ]
 917 |     },
 918 |     {
 919 |      "name": "stderr",
 920 |      "output_type": "stream",
 921 |      "text": [
 922 |       "\r",
 923 |       " 43%|████▎     | 38/89 [1:20:00<1:46:54, 125.78s/it]"
 924 |      ]
 925 |     },
 926 |     {
 927 |      "name": "stdout",
 928 |      "output_type": "stream",
 929 |      "text": [
 930 |       "http://www.beeradvocate.com/beer/profile/2772/8725/ failed with code: 403\n",
 931 |       "http://www.beeradvocate.com/beer/profile/26520/79898/ failed with code: 403\n"
 932 |      ]
 933 |     },
 934 |     {
 935 |      "name": "stderr",
 936 |      "output_type": "stream",
 937 |      "text": [
 938 |       "\r",
 939 |       " 44%|████▍     | 39/89 [1:22:00<1:43:24, 124.09s/it]"
 940 |      ]
 941 |     },
 942 |     {
 943 |      "name": "stdout",
 944 |      "output_type": "stream",
 945 |      "text": [
 946 |       "http://www.beeradvocate.com/beer/profile/825/3587/ failed with code: 500\n"
 947 |      ]
 948 |     },
 949 |     {
 950 |      "name": "stderr",
 951 |      "output_type": "stream",
 952 |      "text": [
 953 |       "\r",
 954 |       " 45%|████▍     | 40/89 [1:24:04<1:41:16, 124.01s/it]"
 955 |      ]
 956 |     },
 957 |     {
 958 |      "name": "stdout",
 959 |      "output_type": "stream",
 960 |      "text": [
 961 |       "http://www.beeradvocate.com/beer/profile/22564/79840/ failed with code: 403\n",
 962 |       "http://www.beeradvocate.com/beer/profile/36798/210557/ failed with code: 403\n"
 963 |      ]
 964 |     },
 965 |     {
 966 |      "name": "stderr",
 967 |      "output_type": "stream",
 968 |      "text": [
 969 |       "\r",
 970 |       " 46%|████▌     | 41/89 [1:26:10<1:39:40, 124.58s/it]"
 971 |      ]
 972 |     },
 973 |     {
 974 |      "name": "stdout",
 975 |      "output_type": "stream",
 976 |      "text": [
 977 |       "http://www.beeradvocate.com/beer/profile/31272/96578/ failed with code: 403\n"
 978 |      ]
 979 |     },
 980 |     {
 981 |      "name": "stderr",
 982 |      "output_type": "stream",
 983 |      "text": [
 984 |       "\r",
 985 |       " 47%|████▋     | 42/89 [1:28:18<1:38:29, 125.74s/it]"
 986 |      ]
 987 |     },
 988 |     {
 989 |      "name": "stdout",
 990 |      "output_type": "stream",
 991 |      "text": [
 992 |       "http://www.beeradvocate.com/beer/profile/1628/34956/ failed with code: 403\n",
 993 |       "http://www.beeradvocate.com/beer/profile/566/13070/ failed with code: 403\n"
 994 |      ]
 995 |     },
 996 |     {
 997 |      "name": "stderr",
 998 |      "output_type": "stream",
 999 |      "text": [
1000 |       "\r",
1001 |       " 48%|████▊     | 43/89 [1:30:18<1:35:04, 124.02s/it]"
1002 |      ]
1003 |     },
1004 |     {
1005 |      "name": "stdout",
1006 |      "output_type": "stream",
1007 |      "text": [
1008 |       "http://www.beeradvocate.com/beer/profile/33160/112140/ failed with code: 403\n",
1009 |       "http://www.beeradvocate.com/beer/profile/105/68248/ failed with code: 403\n",
1010 |       "http://www.beeradvocate.com/beer/profile/2432/27792/ failed with code: 403\n",
1011 |       "http://www.beeradvocate.com/beer/profile/347/30518/ failed with code: 403\n"
1012 |      ]
1013 |     },
1014 |     {
1015 |      "name": "stderr",
1016 |      "output_type": "stream",
1017 |      "text": [
1018 |       "\r",
1019 |       " 49%|████▉     | 44/89 [1:32:22<1:33:02, 124.05s/it]"
1020 |      ]
1021 |     },
1022 |     {
1023 |      "name": "stdout",
1024 |      "output_type": "stream",
1025 |      "text": [
1026 |       "http://www.beeradvocate.com/beer/profile/1530/10277/ failed with code: 403\n",
1027 |       "http://www.beeradvocate.com/beer/profile/31678/189327/ failed with code: 403\n"
1028 |      ]
1029 |     },
1030 |     {
1031 |      "name": "stderr",
1032 |      "output_type": "stream",
1033 |      "text": [
1034 |       " 52%|█████▏    | 46/89 [1:36:42<1:30:52, 126.81s/it]"
1035 |      ]
1036 |     },
1037 |     {
1038 |      "name": "stdout",
1039 |      "output_type": "stream",
1040 |      "text": [
1041 |       "http://www.beeradvocate.com/beer/profile/13371/118229/ failed with code: 403\n",
1042 |       "http://www.beeradvocate.com/beer/profile/604/21197/ failed with code: 403\n",
1043 |       "http://www.beeradvocate.com/beer/profile/28383/95633/ failed with code: 403\n"
1044 |      ]
1045 |     },
1046 |     {
1047 |      "name": "stderr",
1048 |      "output_type": "stream",
1049 |      "text": [
1050 |       "\r",
1051 |       " 53%|█████▎    | 47/89 [1:39:06<1:32:24, 132.01s/it]"
1052 |      ]
1053 |     },
1054 |     {
1055 |      "name": "stdout",
1056 |      "output_type": "stream",
1057 |      "text": [
1058 |       "http://www.beeradvocate.com/beer/profile/37961/161406/ failed with code: 403\n",
1059 |       "http://www.beeradvocate.com/beer/profile/215/1811/ failed with code: 403\n"
1060 |      ]
1061 |     },
1062 |     {
1063 |      "name": "stderr",
1064 |      "output_type": "stream",
1065 |      "text": [
1066 |       "\r",
1067 |       " 54%|█████▍    | 48/89 [1:41:14<1:29:25, 130.86s/it]"
1068 |      ]
1069 |     },
1070 |     {
1071 |      "name": "stdout",
1072 |      "output_type": "stream",
1073 |      "text": [
1074 |       "http://www.beeradvocate.com/beer/profile/12516/42983/ failed with code: 403\n",
1075 |       "http://www.beeradvocate.com/beer/profile/598/65060/ failed with code: 403\n"
1076 |      ]
1077 |     },
1078 |     {
1079 |      "name": "stderr",
1080 |      "output_type": "stream",
1081 |      "text": [
1082 |       "\r",
1083 |       " 55%|█████▌    | 49/89 [1:43:51<1:32:30, 138.77s/it]"
1084 |      ]
1085 |     },
1086 |     {
1087 |      "name": "stdout",
1088 |      "output_type": "stream",
1089 |      "text": [
1090 |       "http://www.beeradvocate.com/beer/profile/697/96243/ failed with code: 403\n",
1091 |       "http://www.beeradvocate.com/beer/profile/9629/37483/ failed with code: 403\n",
1092 |       "http://www.beeradvocate.com/beer/profile/31478/96766/ failed with code: 403\n"
1093 |      ]
1094 |     },
1095 |     {
1096 |      "name": "stderr",
1097 |      "output_type": "stream",
1098 |      "text": [
1099 |       "\r",
1100 |       " 56%|█████▌    | 50/89 [1:46:03<1:28:46, 136.57s/it]"
1101 |      ]
1102 |     },
1103 |     {
1104 |      "name": "stdout",
1105 |      "output_type": "stream",
1106 |      "text": [
1107 |       "http://www.beeradvocate.com/beer/profile/696/125502/ failed with code: 403\n"
1108 |      ]
1109 |     },
1110 |     {
1111 |      "name": "stderr",
1112 |      "output_type": "stream",
1113 |      "text": [
1114 |       "\r",
1115 |       " 57%|█████▋    | 51/89 [1:48:25<1:27:36, 138.33s/it]"
1116 |      ]
1117 |     },
1118 |     {
1119 |      "name": "stdout",
1120 |      "output_type": "stream",
1121 |      "text": [
1122 |       "http://www.beeradvocate.com/beer/profile/32406/106852/ failed with code: 403\n",
1123 |       "http://www.beeradvocate.com/beer/profile/30807/93480/ failed with code: 403\n",
1124 |       "http://www.beeradvocate.com/beer/profile/27947/86596/ failed with code: 403\n"
1125 |      ]
1126 |     },
1127 |     {
1128 |      "name": "stderr",
1129 |      "output_type": "stream",
1130 |      "text": [
1131 |       "\r",
1132 |       " 58%|█████▊    | 52/89 [1:50:42<1:25:02, 137.90s/it]"
1133 |      ]
1134 |     },
1135 |     {
1136 |      "name": "stdout",
1137 |      "output_type": "stream",
1138 |      "text": [
1139 |       "http://www.beeradvocate.com/beer/profile/48/19363/ failed with code: 403\n",
1140 |       "http://www.beeradvocate.com/beer/profile/158/76886/ failed with code: 403\n",
1141 |       "http://www.beeradvocate.com/beer/profile/31805/102492/ failed with code: 403\n"
1142 |      ]
1143 |     },
1144 |     {
1145 |      "name": "stderr",
1146 |      "output_type": "stream",
1147 |      "text": [
1148 |       "\r",
1149 |       " 60%|█████▉    | 53/89 [1:52:51<1:21:06, 135.18s/it]"
1150 |      ]
1151 |     },
1152 |     {
1153 |      "name": "stdout",
1154 |      "output_type": "stream",
1155 |      "text": [
1156 |       "http://www.beeradvocate.com/beer/profile/11031/42445/ failed with code: 403\n",
1157 |       "http://www.beeradvocate.com/beer/profile/16206/48375/ failed with code: 403\n",
1158 |       "http://www.beeradvocate.com/beer/profile/33607/154162/ failed with code: 403\n"
1159 |      ]
1160 |     },
1161 |     {
1162 |      "name": "stderr",
1163 |      "output_type": "stream",
1164 |      "text": [
1165 |       " 63%|██████▎   | 56/89 [1:59:16<1:11:55, 130.78s/it]"
1166 |      ]
1167 |     },
1168 |     {
1169 |      "name": "stdout",
1170 |      "output_type": "stream",
1171 |      "text": [
1172 |       "http://www.beeradvocate.com/beer/profile/1141/42958/ failed with code: 403\n",
1173 |       "http://www.beeradvocate.com/beer/profile/664/2731/ failed with code: 403\n"
1174 |      ]
1175 |     },
1176 |     {
1177 |      "name": "stderr",
1178 |      "output_type": "stream",
1179 |      "text": [
1180 |       "\r",
1181 |       " 64%|██████▍   | 57/89 [2:01:26<1:09:35, 130.48s/it]"
1182 |      ]
1183 |     },
1184 |     {
1185 |      "name": "stdout",
1186 |      "output_type": "stream",
1187 |      "text": [
1188 |       "http://www.beeradvocate.com/beer/profile/5266/17026/ failed with code: 403\n",
1189 |       "http://www.beeradvocate.com/beer/profile/34149/115118/ failed with code: 403\n",
1190 |       "http://www.beeradvocate.com/beer/profile/30815/99003/ failed with code: 403\n"
1191 |      ]
1192 |     },
1193 |     {
1194 |      "name": "stderr",
1195 |      "output_type": "stream",
1196 |      "text": [
1197 |       "\r",
1198 |       " 65%|██████▌   | 58/89 [2:03:51<1:09:39, 134.83s/it]"
1199 |      ]
1200 |     },
1201 |     {
1202 |      "name": "stdout",
1203 |      "output_type": "stream",
1204 |      "text": [
1205 |       "http://www.beeradvocate.com/beer/profile/17033/42695/ failed with code: 403\n",
1206 |       "http://www.beeradvocate.com/beer/profile/12881/61689/ failed with code: 500\n"
1207 |      ]
1208 |     },
1209 |     {
1210 |      "name": "stderr",
1211 |      "output_type": "stream",
1212 |      "text": [
1213 |       "\r",
1214 |       " 66%|██████▋   | 59/89 [2:06:02<1:06:48, 133.63s/it]"
1215 |      ]
1216 |     },
1217 |     {
1218 |      "name": "stdout",
1219 |      "output_type": "stream",
1220 |      "text": [
1221 |       "http://www.beeradvocate.com/beer/profile/24018/184060/ failed with code: 403\n"
1222 |      ]
1223 |     },
1224 |     {
1225 |      "name": "stderr",
1226 |      "output_type": "stream",
1227 |      "text": [
1228 |       "\r",
1229 |       " 67%|██████▋   | 60/89 [2:08:08<1:03:29, 131.35s/it]"
1230 |      ]
1231 |     },
1232 |     {
1233 |      "name": "stdout",
1234 |      "output_type": "stream",
1235 |      "text": [
1236 |       "http://www.beeradvocate.com/beer/profile/28556/89820/ failed with code: 403\n",
1237 |       "http://www.beeradvocate.com/beer/profile/29434/105721/ failed with code: 403\n",
1238 |       "http://www.beeradvocate.com/beer/profile/10485/178506/ failed with code: 403\n",
1239 |       "http://www.beeradvocate.com/beer/profile/27917/150504/ failed with code: 403\n"
1240 |      ]
1241 |     },
1242 |     {
1243 |      "name": "stderr",
1244 |      "output_type": "stream",
1245 |      "text": [
1246 |       "\r",
1247 |       " 69%|██████▊   | 61/89 [2:10:14<1:00:29, 129.63s/it]"
1248 |      ]
1249 |     },
1250 |     {
1251 |      "name": "stdout",
1252 |      "output_type": "stream",
1253 |      "text": [
1254 |       "http://www.beeradvocate.com/beer/profile/23459/64493/ failed with code: 403\n"
1255 |      ]
1256 |     },
1257 |     {
1258 |      "name": "stderr",
1259 |      "output_type": "stream",
1260 |      "text": [
1261 |       "\r",
1262 |       " 70%|██████▉   | 62/89 [2:12:27<58:50, 130.77s/it]  "
1263 |      ]
1264 |     },
1265 |     {
1266 |      "name": "stdout",
1267 |      "output_type": "stream",
1268 |      "text": [
1269 |       "http://www.beeradvocate.com/beer/profile/3/60284/ failed with code: 403\n",
1270 |       "http://www.beeradvocate.com/beer/profile/294/815/ failed with code: 403\n",
1271 |       "http://www.beeradvocate.com/beer/profile/22072/179142/ failed with code: 403\n",
1272 |       "http://www.beeradvocate.com/beer/profile/24428/73210/ failed with code: 403\n"
1273 |      ]
1274 |     },
1275 |     {
1276 |      "name": "stderr",
1277 |      "output_type": "stream",
1278 |      "text": [
1279 |       " 72%|███████▏  | 64/89 [2:16:49<54:27, 130.69s/it]"
1280 |      ]
1281 |     },
1282 |     {
1283 |      "name": "stdout",
1284 |      "output_type": "stream",
1285 |      "text": [
1286 |       "http://www.beeradvocate.com/beer/profile/23222/76304/ failed with code: 403\n",
1287 |       "http://www.beeradvocate.com/beer/profile/75/88536/ failed with code: 403\n",
1288 |       "http://www.beeradvocate.com/beer/profile/864/112280/ failed with code: 403\n"
1289 |      ]
1290 |     },
1291 |     {
1292 |      "name": "stderr",
1293 |      "output_type": "stream",
1294 |      "text": [
1295 |       "\r",
1296 |       " 73%|███████▎  | 65/89 [2:19:00<52:25, 131.05s/it]"
1297 |      ]
1298 |     },
1299 |     {
1300 |      "name": "stdout",
1301 |      "output_type": "stream",
1302 |      "text": [
1303 |       "http://www.beeradvocate.com/beer/profile/21154/54966/ failed with code: 500\n",
1304 |       "http://www.beeradvocate.com/beer/profile/2391/55504/ failed with code: 403\n",
1305 |       "http://www.beeradvocate.com/beer/profile/28178/112878/ failed with code: 403\n",
1306 |       "http://www.beeradvocate.com/beer/profile/25818/145776/ failed with code: 403\n"
1307 |      ]
1308 |     },
1309 |     {
1310 |      "name": "stderr",
1311 |      "output_type": "stream",
1312 |      "text": [
1313 |       "\r",
1314 |       " 74%|███████▍  | 66/89 [2:21:06<49:38, 129.49s/it]"
1315 |      ]
1316 |     },
1317 |     {
1318 |      "name": "stdout",
1319 |      "output_type": "stream",
1320 |      "text": [
1321 |       "http://www.beeradvocate.com/beer/profile/1805/13701/ failed with code: 403\n",
1322 |       "http://www.beeradvocate.com/beer/profile/4/88815/ failed with code: 500\n"
1323 |      ]
1324 |     },
1325 |     {
1326 |      "name": "stderr",
1327 |      "output_type": "stream",
1328 |      "text": [
1329 |       "\r",
1330 |       " 75%|███████▌  | 67/89 [2:23:17<47:37, 129.88s/it]"
1331 |      ]
1332 |     },
1333 |     {
1334 |      "name": "stdout",
1335 |      "output_type": "stream",
1336 |      "text": [
1337 |       "http://www.beeradvocate.com/beer/profile/22/1693/ failed with code: 403\n"
1338 |      ]
1339 |     },
1340 |     {
1341 |      "name": "stderr",
1342 |      "output_type": "stream",
1343 |      "text": [
1344 |       " 78%|███████▊  | 69/89 [2:27:45<43:59, 131.98s/it]"
1345 |      ]
1346 |     },
1347 |     {
1348 |      "name": "stdout",
1349 |      "output_type": "stream",
1350 |      "text": [
1351 |       "http://www.beeradvocate.com/beer/profile/33437/109428/ failed with code: 403\n"
1352 |      ]
1353 |     },
1354 |     {
1355 |      "name": "stderr",
1356 |      "output_type": "stream",
1357 |      "text": [
1358 |       "\r",
1359 |       " 79%|███████▊  | 70/89 [2:29:56<41:44, 131.82s/it]"
1360 |      ]
1361 |     },
1362 |     {
1363 |      "name": "stdout",
1364 |      "output_type": "stream",
1365 |      "text": [
1366 |       "http://www.beeradvocate.com/beer/profile/195/617/ failed with code: 403\n",
1367 |       "http://www.beeradvocate.com/beer/profile/30949/101709/ failed with code: 403\n"
1368 |      ]
1369 |     },
1370 |     {
1371 |      "name": "stderr",
1372 |      "output_type": "stream",
1373 |      "text": [
1374 |       "\r",
1375 |       " 80%|███████▉  | 71/89 [2:32:11<39:48, 132.71s/it]"
1376 |      ]
1377 |     },
1378 |     {
1379 |      "name": "stdout",
1380 |      "output_type": "stream",
1381 |      "text": [
1382 |       "http://www.beeradvocate.com/beer/profile/48/3490/ failed with code: 403\n",
1383 |       "http://www.beeradvocate.com/beer/profile/16354/39238/ failed with code: 403\n"
1384 |      ]
1385 |     },
1386 |     {
1387 |      "name": "stderr",
1388 |      "output_type": "stream",
1389 |      "text": [
1390 |       "\r",
1391 |       " 81%|████████  | 72/89 [2:34:22<37:28, 132.26s/it]"
1392 |      ]
1393 |     },
1394 |     {
1395 |      "name": "stdout",
1396 |      "output_type": "stream",
1397 |      "text": [
1398 |       "http://www.beeradvocate.com/beer/profile/775/14027/ failed with code: 403\n",
1399 |       "http://www.beeradvocate.com/beer/profile/16352/149857/ failed with code: 403\n",
1400 |       "http://www.beeradvocate.com/beer/profile/13014/212431/ failed with code: 403\n"
1401 |      ]
1402 |     },
1403 |     {
1404 |      "name": "stderr",
1405 |      "output_type": "stream",
1406 |      "text": [
1407 |       "\r",
1408 |       " 82%|████████▏ | 73/89 [2:36:42<35:53, 134.57s/it]"
1409 |      ]
1410 |     },
1411 |     {
1412 |      "name": "stdout",
1413 |      "output_type": "stream",
1414 |      "text": [
1415 |       "http://www.beeradvocate.com/beer/profile/27025/101083/ failed with code: 500\n",
1416 |       "http://www.beeradvocate.com/beer/profile/32893/212780/ failed with code: 403\n"
1417 |      ]
1418 |     },
1419 |     {
1420 |      "name": "stderr",
1421 |      "output_type": "stream",
1422 |      "text": [
1423 |       " 84%|████████▍ | 75/89 [2:41:03<30:52, 132.32s/it]"
1424 |      ]
1425 |     },
1426 |     {
1427 |      "name": "stdout",
1428 |      "output_type": "stream",
1429 |      "text": [
1430 |       "http://www.beeradvocate.com/beer/profile/32171/105040/ failed with code: 403\n",
1431 |       "http://www.beeradvocate.com/beer/profile/24018/128242/ failed with code: 500\n"
1432 |      ]
1433 |     },
1434 |     {
1435 |      "name": "stderr",
1436 |      "output_type": "stream",
1437 |      "text": [
1438 |       "\r",
1439 |       " 85%|████████▌ | 76/89 [2:43:22<29:05, 134.29s/it]"
1440 |      ]
1441 |     },
1442 |     {
1443 |      "name": "stdout",
1444 |      "output_type": "stream",
1445 |      "text": [
1446 |       "http://www.beeradvocate.com/beer/profile/33018/148547/ failed with code: 403\n",
1447 |       "http://www.beeradvocate.com/beer/profile/25775/82169/ failed with code: 403\n",
1448 |       "http://www.beeradvocate.com/beer/profile/33901/118920/ failed with code: 403\n"
1449 |      ]
1450 |     },
1451 |     {
1452 |      "name": "stderr",
1453 |      "output_type": "stream",
1454 |      "text": [
1455 |       "\r",
1456 |       " 87%|████████▋ | 77/89 [2:45:34<26:44, 133.70s/it]"
1457 |      ]
1458 |     },
1459 |     {
1460 |      "name": "stdout",
1461 |      "output_type": "stream",
1462 |      "text": [
1463 |       "http://www.beeradvocate.com/beer/profile/402/45914/ failed with code: 403\n"
1464 |      ]
1465 |     },
1466 |     {
1467 |      "name": "stderr",
1468 |      "output_type": "stream",
1469 |      "text": [
1470 |       " 89%|████████▉ | 79/89 [2:49:55<22:03, 132.34s/it]"
1471 |      ]
1472 |     },
1473 |     {
1474 |      "name": "stdout",
1475 |      "output_type": "stream",
1476 |      "text": [
1477 |       "http://www.beeradvocate.com/beer/profile/17223/55942/ failed with code: 403\n",
1478 |       "http://www.beeradvocate.com/beer/profile/23326/69298/ failed with code: 403\n",
1479 |       "http://www.beeradvocate.com/beer/profile/138/8081/ failed with code: 403\n",
1480 |       "http://www.beeradvocate.com/beer/profile/14967/118297/ failed with code: 403\n"
1481 |      ]
1482 |     },
1483 |     {
1484 |      "name": "stderr",
1485 |      "output_type": "stream",
1486 |      "text": [
1487 |       "\r",
1488 |       " 90%|████████▉ | 80/89 [2:52:13<20:05, 133.98s/it]"
1489 |      ]
1490 |     },
1491 |     {
1492 |      "name": "stdout",
1493 |      "output_type": "stream",
1494 |      "text": [
1495 |       "http://www.beeradvocate.com/beer/profile/2743/86841/ failed with code: 403\n",
1496 |       "http://www.beeradvocate.com/beer/profile/640/1738/ failed with code: 403\n",
1497 |       "http://www.beeradvocate.com/beer/profile/506/16406/ failed with code: 403\n"
1498 |      ]
1499 |     },
1500 |     {
1501 |      "name": "stderr",
1502 |      "output_type": "stream",
1503 |      "text": [
1504 |       "\r",
1505 |       " 91%|█████████ | 81/89 [2:54:21<17:38, 132.31s/it]"
1506 |      ]
1507 |     },
1508 |     {
1509 |      "name": "stdout",
1510 |      "output_type": "stream",
1511 |      "text": [
1512 |       "http://www.beeradvocate.com/beer/profile/60/1598/ failed with code: 403\n"
1513 |      ]
1514 |     },
1515 |     {
1516 |      "name": "stderr",
1517 |      "output_type": "stream",
1518 |      "text": [
1519 |       "\r",
1520 |       " 92%|█████████▏| 82/89 [2:56:34<15:26, 132.34s/it]"
1521 |      ]
1522 |     },
1523 |     {
1524 |      "name": "stdout",
1525 |      "output_type": "stream",
1526 |      "text": [
1527 |       "http://www.beeradvocate.com/beer/profile/19378/98544/ failed with code: 403\n"
1528 |      ]
1529 |     },
1530 |     {
1531 |      "name": "stderr",
1532 |      "output_type": "stream",
1533 |      "text": [
1534 |       " 94%|█████████▍| 84/89 [3:00:49<10:49, 129.81s/it]"
1535 |      ]
1536 |     },
1537 |     {
1538 |      "name": "stdout",
1539 |      "output_type": "stream",
1540 |      "text": [
1541 |       "http://www.beeradvocate.com/beer/profile/583/1994/ failed with code: 403\n"
1542 |      ]
1543 |     },
1544 |     {
1545 |      "name": "stderr",
1546 |      "output_type": "stream",
1547 |      "text": [
1548 |       "\r",
1549 |       " 96%|█████████▌| 85/89 [3:02:52<08:31, 127.77s/it]"
1550 |      ]
1551 |     },
1552 |     {
1553 |      "name": "stdout",
1554 |      "output_type": "stream",
1555 |      "text": [
1556 |       "http://www.beeradvocate.com/beer/profile/1195/3349/ failed with code: 500\n",
1557 |       "http://www.beeradvocate.com/beer/profile/28/197461/ failed with code: 403\n"
1558 |      ]
1559 |     },
1560 |     {
1561 |      "name": "stderr",
1562 |      "output_type": "stream",
1563 |      "text": [
1564 |       "\r",
1565 |       " 97%|█████████▋| 86/89 [3:04:55<06:19, 126.40s/it]"
1566 |      ]
1567 |     },
1568 |     {
1569 |      "name": "stdout",
1570 |      "output_type": "stream",
1571 |      "text": [
1572 |       "http://www.beeradvocate.com/beer/profile/30746/90736/ failed with code: 500\n",
1573 |       "http://www.beeradvocate.com/beer/profile/24299/92753/ failed with code: 403\n"
1574 |      ]
1575 |     },
1576 |     {
1577 |      "name": "stderr",
1578 |      "output_type": "stream",
1579 |      "text": [
1580 |       " 99%|█████████▉| 88/89 [3:09:07<02:05, 125.93s/it]"
1581 |      ]
1582 |     },
1583 |     {
1584 |      "name": "stdout",
1585 |      "output_type": "stream",
1586 |      "text": [
1587 |       "http://www.beeradvocate.com/beer/profile/17980/64545/ failed with code: 403\n",
1588 |       "http://www.beeradvocate.com/beer/profile/25959/70761/ failed with code: 403\n"
1589 |      ]
1590 |     },
1591 |     {
1592 |      "name": "stderr",
1593 |      "output_type": "stream",
1594 |      "text": [
1595 |       "100%|██████████| 89/89 [3:11:12<00:00, 125.82s/it]\n"
1596 |      ]
1597 |     }
1598 |    ],
1599 |    "source": [
1600 |     "for i in tqdm(range(0,beers.shape[0],230)):\n",
1601 |     "    temp = beers.iloc[i:i+230-1,:]\n",
1602 |     "    temp['soup'] = temp.url.map(get_beer_soup)\n",
1603 |     "    temp.to_pickle('beer_soup_'+str(i+230-1)+'.pkl')"
1604 |    ]
1605 |   },
1606 |   {
1607 |    "cell_type": "code",
1608 |    "execution_count": null,
1609 |    "metadata": {
1610 |     "collapsed": true
1611 |    },
1612 |    "outputs": [],
1613 |    "source": [
1614 |     "temp = beers.iloc[229:beers.shape[0]:230]\n",
1615 |     "temp['soup'] = temp.url.map(get_beer_soup)"
1616 |    ]
1617 |   },
1618 |   {
1619 |    "cell_type": "code",
1620 |    "execution_count": 21,
1621 |    "metadata": {
1622 |     "collapsed": false
1623 |    },
1624 |    "outputs": [
1625 |     {
1626 |      "data": {
1627 |       "text/html": [
1628 |        "<div>\n",
1629 |        "<table border=\"1\" class=\"dataframe\">\n",
1630 |        "  <thead>\n",
1631 |        "    <tr style=\"text-align: right;\">\n",
1632 |        "      <th></th>\n",
1633 |        "      <th>name</th>\n",
1634 |        "      <th>url</th>\n",
1635 |        "      <th>soup</th>\n",
1636 |        "    </tr>\n",
1637 |        "  </thead>\n",
1638 |        "  <tbody>\n",
1639 |        "    <tr>\n",
1640 |        "      <th>229</th>\n",
1641 |        "      <td>Franconia Amber</td>\n",
1642 |        "      <td>/beer/profile/17033/43115/</td>\n",
1643 |        "      <td>&lt;!DOCTYPE html&gt;\n",
1644 |        "&lt;html class=\"Public NoJs Logge...</td>\n",
1645 |        "    </tr>\n",
1646 |        "    <tr>\n",
1647 |        "      <th>459</th>\n",
1648 |        "      <td>Big American Stout</td>\n",
1649 |        "      <td>/beer/profile/26824/113395/</td>\n",
1650 |        "      <td>&lt;!DOCTYPE html&gt;\n",
1651 |        "&lt;html class=\"Public NoJs Logge...</td>\n",
1652 |        "    </tr>\n",
1653 |        "    <tr>\n",
1654 |        "      <th>689</th>\n",
1655 |        "      <td>Love's Armor</td>\n",
1656 |        "      <td>/beer/profile/28019/90275/</td>\n",
1657 |        "      <td>&lt;!DOCTYPE html&gt;\n",
1658 |        "&lt;html class=\"Public NoJs Logge...</td>\n",
1659 |        "    </tr>\n",
1660 |        "    <tr>\n",
1661 |        "      <th>919</th>\n",
1662 |        "      <td>Deadeye Jack Porter</td>\n",
1663 |        "      <td>/beer/profile/19126/47303/</td>\n",
1664 |        "      <td>&lt;!DOCTYPE html&gt;\n",
1665 |        "&lt;html class=\"Public NoJs Logge...</td>\n",
1666 |        "    </tr>\n",
1667 |        "    <tr>\n",
1668 |        "      <th>1149</th>\n",
1669 |        "      <td>Peach Lambic</td>\n",
1670 |        "      <td>/beer/profile/1170/45334/</td>\n",
1671 |        "      <td>&lt;!DOCTYPE html&gt;\n",
1672 |        "&lt;html class=\"Public NoJs Logge...</td>\n",
1673 |        "    </tr>\n",
1674 |        "  </tbody>\n",
1675 |        "</table>\n",
1676 |        "</div>"
1677 |       ],
1678 |       "text/plain": [
1679 |        "                     name                          url  \\\n",
1680 |        "229       Franconia Amber   /beer/profile/17033/43115/   \n",
1681 |        "459    Big American Stout  /beer/profile/26824/113395/   \n",
1682 |        "689          Love's Armor   /beer/profile/28019/90275/   \n",
1683 |        "919   Deadeye Jack Porter   /beer/profile/19126/47303/   \n",
1684 |        "1149         Peach Lambic    /beer/profile/1170/45334/   \n",
1685 |        "\n",
1686 |        "                                                   soup  \n",
1687 |        "229   <!DOCTYPE html>\n",
1688 |        "<html class=\"Public NoJs Logge...  \n",
1689 |        "459   <!DOCTYPE html>\n",
1690 |        "<html class=\"Public NoJs Logge...  \n",
1691 |        "689   <!DOCTYPE html>\n",
1692 |        "<html class=\"Public NoJs Logge...  \n",
1693 |        "919   <!DOCTYPE html>\n",
1694 |        "<html class=\"Public NoJs Logge...  \n",
1695 |        "1149  <!DOCTYPE html>\n",
1696 |        "<html class=\"Public NoJs Logge...  "
1697 |       ]
1698 |      },
1699 |      "execution_count": 21,
1700 |      "metadata": {},
1701 |      "output_type": "execute_result"
1702 |     }
1703 |    ],
1704 |    "source": [
1705 |     "temp.head()"
1706 |    ]
1707 |   },
1708 |   {
1709 |    "cell_type": "code",
1710 |    "execution_count": 22,
1711 |    "metadata": {
1712 |     "collapsed": false
1713 |    },
1714 |    "outputs": [],
1715 |    "source": [
1716 |     "temp.to_pickle('beer_soup_missing.pkl')"
1717 |    ]
1718 |   },
1719 |   {
1720 |    "cell_type": "code",
1721 |    "execution_count": 14,
1722 |    "metadata": {
1723 |     "collapsed": false
1724 |    },
1725 |    "outputs": [
1726 |     {
1727 |      "name": "stderr",
1728 |      "output_type": "stream",
1729 |      "text": [
1730 |       "100%|██████████| 89/89 [00:00<00:00, 46279.82it/s]"
1731 |      ]
1732 |     },
1733 |     {
1734 |      "name": "stdout",
1735 |      "output_type": "stream",
1736 |      "text": [
1737 |       "229\n",
1738 |       "459\n",
1739 |       "689\n",
1740 |       "919\n",
1741 |       "1149\n",
1742 |       "1379\n",
1743 |       "1609\n",
1744 |       "1839\n",
1745 |       "2069\n",
1746 |       "2299\n",
1747 |       "2529\n",
1748 |       "2759\n",
1749 |       "2989\n",
1750 |       "3219\n",
1751 |       "3449\n",
1752 |       "3679\n",
1753 |       "3909\n",
1754 |       "4139\n",
1755 |       "4369\n",
1756 |       "4599\n",
1757 |       "4829\n",
1758 |       "5059\n",
1759 |       "5289\n",
1760 |       "5519\n",
1761 |       "5749\n",
1762 |       "5979\n",
1763 |       "6209\n",
1764 |       "6439\n",
1765 |       "6669\n",
1766 |       "6899\n",
1767 |       "7129\n",
1768 |       "7359\n",
1769 |       "7589\n",
1770 |       "7819\n",
1771 |       "8049\n",
1772 |       "8279\n",
1773 |       "8509\n",
1774 |       "8739\n",
1775 |       "8969\n",
1776 |       "9199\n",
1777 |       "9429\n",
1778 |       "9659\n",
1779 |       "9889\n",
1780 |       "10119\n",
1781 |       "10349\n",
1782 |       "10579\n",
1783 |       "10809\n",
1784 |       "11039\n",
1785 |       "11269\n",
1786 |       "11499\n",
1787 |       "11729\n",
1788 |       "11959\n",
1789 |       "12189\n",
1790 |       "12419\n",
1791 |       "12649\n",
1792 |       "12879\n",
1793 |       "13109\n",
1794 |       "13339\n",
1795 |       "13569\n",
1796 |       "13799\n",
1797 |       "14029\n",
1798 |       "14259\n",
1799 |       "14489\n",
1800 |       "14719\n",
1801 |       "14949\n",
1802 |       "15179\n",
1803 |       "15409\n",
1804 |       "15639\n",
1805 |       "15869\n",
1806 |       "16099\n",
1807 |       "16329\n",
1808 |       "16559\n",
1809 |       "16789\n",
1810 |       "17019\n",
1811 |       "17249\n",
1812 |       "17479\n",
1813 |       "17709\n",
1814 |       "17939\n",
1815 |       "18169\n",
1816 |       "18399\n",
1817 |       "18629\n",
1818 |       "18859\n",
1819 |       "19089\n",
1820 |       "19319\n",
1821 |       "19549\n",
1822 |       "19779\n",
1823 |       "20009\n",
1824 |       "20239\n",
1825 |       "20469\n"
1826 |      ]
1827 |     },
1828 |     {
1829 |      "name": "stderr",
1830 |      "output_type": "stream",
1831 |      "text": [
1832 |       "\n"
1833 |      ]
1834 |     }
1835 |    ],
1836 |    "source": [
1837 |     "from tqdm import tqdm\n",
1838 |     "for i in tqdm(range(229,beers.shape[0],230)):\n",
1839 |     "    print(i)"
1840 |    ]
1841 |   },
1842 |   {
1843 |    "cell_type": "code",
1844 |    "execution_count": null,
1845 |    "metadata": {
1846 |     "collapsed": true
1847 |    },
1848 |    "outputs": [],
1849 |    "source": []
1850 |   },
1851 |   {
1852 |    "cell_type": "code",
1853 |    "execution_count": null,
1854 |    "metadata": {
1855 |     "collapsed": false
1856 |    },
1857 |    "outputs": [],
1858 |    "source": []
1859 |   },
1860 |   {
1861 |    "cell_type": "code",
1862 |    "execution_count": null,
1863 |    "metadata": {
1864 |     "collapsed": false
1865 |    },
1866 |    "outputs": [],
1867 |    "source": [
1868 |     "beer_reviews = {}\n",
1869 |     "ba_url = 'http://www.beeradvocate.com'\n",
1870 |     "\n",
1871 |     "for beer in beers.items():\n",
1872 |     "    soup = get_soup(ba_url+beer[1])\n",
1873 |     "    print(beer[0])\n",
1874 |     "    beer_reviews[beer[0]] = get_beer_reviews(soup)"
1875 |    ]
1876 |   }
1877 |  ],
1878 |  "metadata": {
1879 |   "anaconda-cloud": {},
1880 |   "kernelspec": {
1881 |    "display_name": "Python 2",
1882 |    "language": "python",
1883 |    "name": "python2"
1884 |   },
1885 |   "language_info": {
1886 |    "codemirror_mode": {
1887 |     "name": "ipython",
1888 |     "version": 2
1889 |    },
1890 |    "file_extension": ".py",
1891 |    "mimetype": "text/x-python",
1892 |    "name": "python",
1893 |    "nbconvert_exporter": "python",
1894 |    "pygments_lexer": "ipython2",
1895 |    "version": "2.7.11"
1896 |   }
1897 |  },
1898 |  "nbformat": 4,
1899 |  "nbformat_minor": 0
1900 | }
1901 | 


--------------------------------------------------------------------------------