├── Google2Csv.ipynb
└── README.md


/Google2Csv.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |   "nbformat": 4,
  3 |   "nbformat_minor": 0,
  4 |   "metadata": {
  5 |     "colab": {
  6 |       "name": "Google2Csv.ipynb",
  7 |       "provenance": [],
  8 |       "toc_visible": true,
  9 |       "authorship_tag": "ABX9TyN/EIlGRKp/NPVnoqRZtLV/",
 10 |       "include_colab_link": true
 11 |     },
 12 |     "kernelspec": {
 13 |       "name": "python3",
 14 |       "display_name": "Python 3"
 15 |     }
 16 |   },
 17 |   "cells": [
 18 |     {
 19 |       "cell_type": "markdown",
 20 |       "metadata": {
 21 |         "id": "view-in-github",
 22 |         "colab_type": "text"
 23 |       },
 24 |       "source": [
 25 |         "<a href=\"https://colab.research.google.com/github/psalias2006/Google2Csv/blob/master/Google2Csv.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
 26 |       ]
 27 |     },
 28 |     {
 29 |       "cell_type": "markdown",
 30 |       "metadata": {
 31 |         "id": "3w-fyktSoKxu",
 32 |         "colab_type": "text"
 33 |       },
 34 |       "source": [
 35 |         "##GoogleScraper"
 36 |       ]
 37 |     },
 38 |     {
 39 |       "cell_type": "code",
 40 |       "metadata": {
 41 |         "id": "zCQtYO-Po_kf",
 42 |         "colab_type": "code",
 43 |         "colab": {
 44 |           "base_uri": "https://localhost:8080/",
 45 |           "height": 34
 46 |         },
 47 |         "outputId": "2d2221b1-b425-4723-adbc-7aa415ecf254"
 48 |       },
 49 |       "source": [
 50 |         "!pip install beautifulsoup4\n",
 51 |         "\n",
 52 |         "from bs4 import BeautifulSoup\n",
 53 |         "import requests\n",
 54 |         "import pandas as pd"
 55 |       ],
 56 |       "execution_count": 1,
 57 |       "outputs": [
 58 |         {
 59 |           "output_type": "stream",
 60 |           "text": [
 61 |             "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.6/dist-packages (4.6.3)\n"
 62 |           ],
 63 |           "name": "stdout"
 64 |         }
 65 |       ]
 66 |     },
 67 |     {
 68 |       "cell_type": "code",
 69 |       "metadata": {
 70 |         "id": "6JZjJa-joXEh",
 71 |         "colab_type": "code",
 72 |         "colab": {}
 73 |       },
 74 |       "source": [
 75 |         "def simpleGoogleSearch(query, start):\n",
 76 |         "  results = []\n",
 77 |         "\n",
 78 |         "  query = query.replace(' ', '+')\n",
 79 |         "  URL = f\"https://google.com/search?q={query}&start={start}\"\n",
 80 |         "\n",
 81 |         "  # desktop user-agent\n",
 82 |         "  USER_AGENT = \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0\"\n",
 83 |         "  \n",
 84 |         "  headers = {\"user-agent\" : USER_AGENT}\n",
 85 |         "  resp = requests.get(URL, headers=headers)\n",
 86 |         "\n",
 87 |         "  if resp.status_code == 200:\n",
 88 |         "    soup = BeautifulSoup(resp.content, \"html.parser\")\n",
 89 |         "\n",
 90 |         "    for g in soup.find_all('div', class_='r'):\n",
 91 |         "      anchors = g.find_all('a')\n",
 92 |         "\n",
 93 |         "      if anchors:\n",
 94 |         "        link  = anchors[0]['href']\n",
 95 |         "        title = g.find('h3').text\n",
 96 |         "        item  = {\"title\": title, \"link\": link}\n",
 97 |         "        results.append(item)\n",
 98 |         "\n",
 99 |         "  return results"
100 |       ],
101 |       "execution_count": 0,
102 |       "outputs": []
103 |     },
104 |     {
105 |       "cell_type": "code",
106 |       "metadata": {
107 |         "id": "kBCwCEcGob2M",
108 |         "colab_type": "code",
109 |         "colab": {}
110 |       },
111 |       "source": [
112 |         "def googleToPandas(googleQuery):\n",
113 |         "  resultsCounter  = 0\n",
114 |         "  resultsList     = []\n",
115 |         "\n",
116 |         "  while True:\n",
117 |         "    pageResults = simpleGoogleSearch(googleQuery, resultsCounter)\n",
118 |         "    \n",
119 |         "    if not pageResults: break\n",
120 |         "    else: \n",
121 |         "      resultsList.extend(pageResults)\n",
122 |         "      resultsCounter = resultsCounter + 10\n",
123 |         "\n",
124 |         "  return pd.DataFrame(resultsList)"
125 |       ],
126 |       "execution_count": 0,
127 |       "outputs": []
128 |     },
129 |     {
130 |       "cell_type": "code",
131 |       "metadata": {
132 |         "id": "SoUutC4DojV7",
133 |         "colab_type": "code",
134 |         "colab": {}
135 |       },
136 |       "source": [
137 |         "googleSearchQuery   = \"Covid 19\""
138 |       ],
139 |       "execution_count": 0,
140 |       "outputs": []
141 |     },
142 |     {
143 |       "cell_type": "code",
144 |       "metadata": {
145 |         "id": "r44AZDaBpjGO",
146 |         "colab_type": "code",
147 |         "colab": {
148 |           "base_uri": "https://localhost:8080/",
149 |           "height": 419
150 |         },
151 |         "outputId": "d7a82dce-8463-45fa-c088-8cac5dba3fa5"
152 |       },
153 |       "source": [
154 |         "results = googleToPandas(googleSearchQuery)\n",
155 |         "results"
156 |       ],
157 |       "execution_count": 5,
158 |       "outputs": [
159 |         {
160 |           "output_type": "execute_result",
161 |           "data": {
162 |             "text/html": [
163 |               "<div>\n",
164 |               "<style scoped>\n",
165 |               "    .dataframe tbody tr th:only-of-type {\n",
166 |               "        vertical-align: middle;\n",
167 |               "    }\n",
168 |               "\n",
169 |               "    .dataframe tbody tr th {\n",
170 |               "        vertical-align: top;\n",
171 |               "    }\n",
172 |               "\n",
173 |               "    .dataframe thead th {\n",
174 |               "        text-align: right;\n",
175 |               "    }\n",
176 |               "</style>\n",
177 |               "<table border=\"1\" class=\"dataframe\">\n",
178 |               "  <thead>\n",
179 |               "    <tr style=\"text-align: right;\">\n",
180 |               "      <th></th>\n",
181 |               "      <th>title</th>\n",
182 |               "      <th>link</th>\n",
183 |               "    </tr>\n",
184 |               "  </thead>\n",
185 |               "  <tbody>\n",
186 |               "    <tr>\n",
187 |               "      <th>0</th>\n",
188 |               "      <td>Coronavirus Disease 2019 (COVID-19) | CDC</td>\n",
189 |               "      <td>https://www.cdc.gov/coronavirus/2019-ncov/inde...</td>\n",
190 |               "    </tr>\n",
191 |               "    <tr>\n",
192 |               "      <th>1</th>\n",
193 |               "      <td>Cases in the U.S. | CDC</td>\n",
194 |               "      <td>https://www.cdc.gov/coronavirus/2019-ncov/case...</td>\n",
195 |               "    </tr>\n",
196 |               "    <tr>\n",
197 |               "      <th>2</th>\n",
198 |               "      <td>Coronavirus (COVID-19) frequently asked questi...</td>\n",
199 |               "      <td>https://www.cdc.gov/coronavirus/2019-ncov/faq....</td>\n",
200 |               "    </tr>\n",
201 |               "    <tr>\n",
202 |               "      <th>3</th>\n",
203 |               "      <td>Coronavirus disease 2019 - World Health Organi...</td>\n",
204 |               "      <td>https://www.who.int/emergencies/diseases/novel...</td>\n",
205 |               "    </tr>\n",
206 |               "    <tr>\n",
207 |               "      <th>4</th>\n",
208 |               "      <td>Coronavirus (COVID-19) events as they happen</td>\n",
209 |               "      <td>https://www.who.int/emergencies/diseases/novel...</td>\n",
210 |               "    </tr>\n",
211 |               "    <tr>\n",
212 |               "      <th>...</th>\n",
213 |               "      <td>...</td>\n",
214 |               "      <td>...</td>\n",
215 |               "    </tr>\n",
216 |               "    <tr>\n",
217 |               "      <th>231</th>\n",
218 |               "      <td>This web page is parked FREE, courtesy of GoDa...</td>\n",
219 |               "      <td>http://covid-2019.me/</td>\n",
220 |               "    </tr>\n",
221 |               "    <tr>\n",
222 |               "      <th>232</th>\n",
223 |               "      <td>COVID-19 - Doctors eBook</td>\n",
224 |               "      <td>http://doctorsebook.com/covid-19/</td>\n",
225 |               "    </tr>\n",
226 |               "    <tr>\n",
227 |               "      <th>233</th>\n",
228 |               "      <td>COVID-19 Update</td>\n",
229 |               "      <td>https://www.oepreview.com/covid19sample/covid-...</td>\n",
230 |               "    </tr>\n",
231 |               "    <tr>\n",
232 |               "      <th>234</th>\n",
233 |               "      <td>Page 2 – Corona Virus (Covid-19) - Covid-19 Ou...</td>\n",
234 |               "      <td>https://covid-19outbreak.co/page/2/</td>\n",
235 |               "    </tr>\n",
236 |               "    <tr>\n",
237 |               "      <th>235</th>\n",
238 |               "      <td>This web page is parked FREE, courtesy of GoDa...</td>\n",
239 |               "      <td>http://covid-2019.me/</td>\n",
240 |               "    </tr>\n",
241 |               "  </tbody>\n",
242 |               "</table>\n",
243 |               "<p>236 rows × 2 columns</p>\n",
244 |               "</div>"
245 |             ],
246 |             "text/plain": [
247 |               "                                                 title                                               link\n",
248 |               "0            Coronavirus Disease 2019 (COVID-19) | CDC  https://www.cdc.gov/coronavirus/2019-ncov/inde...\n",
249 |               "1                              Cases in the U.S. | CDC  https://www.cdc.gov/coronavirus/2019-ncov/case...\n",
250 |               "2    Coronavirus (COVID-19) frequently asked questi...  https://www.cdc.gov/coronavirus/2019-ncov/faq....\n",
251 |               "3    Coronavirus disease 2019 - World Health Organi...  https://www.who.int/emergencies/diseases/novel...\n",
252 |               "4         Coronavirus (COVID-19) events as they happen  https://www.who.int/emergencies/diseases/novel...\n",
253 |               "..                                                 ...                                                ...\n",
254 |               "231  This web page is parked FREE, courtesy of GoDa...                              http://covid-2019.me/\n",
255 |               "232                           COVID-19 - Doctors eBook                  http://doctorsebook.com/covid-19/\n",
256 |               "233                                    COVID-19 Update  https://www.oepreview.com/covid19sample/covid-...\n",
257 |               "234  Page 2 – Corona Virus (Covid-19) - Covid-19 Ou...                https://covid-19outbreak.co/page/2/\n",
258 |               "235  This web page is parked FREE, courtesy of GoDa...                              http://covid-2019.me/\n",
259 |               "\n",
260 |               "[236 rows x 2 columns]"
261 |             ]
262 |           },
263 |           "metadata": {
264 |             "tags": []
265 |           },
266 |           "execution_count": 5
267 |         }
268 |       ]
269 |     },
270 |     {
271 |       "cell_type": "code",
272 |       "metadata": {
273 |         "id": "y-wbjJuYqnOT",
274 |         "colab_type": "code",
275 |         "colab": {}
276 |       },
277 |       "source": [
278 |         "results.to_csv('GoogleResults.csv', index=False)\n",
279 |         "results.to_excel('GoogleResults.xlsx', index=False)\n",
280 |         "results.to_json('GoogleResults.jsonl', orient='records', lines=True)"
281 |       ],
282 |       "execution_count": 0,
283 |       "outputs": []
284 |     }
285 |   ]
286 | }


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Google2Csv
 2 | [![Colab badge](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/psalias2006/Google2Csv/blob/master/Google2Csv.ipynb)
 3 | 
 4 | This is a dead simple tutorial on how to scrape google using BeautifulSoup and save the results on a csv/xlsx/jsonl file
 5 | 
 6 | 
 7 | ![alt text](https://i.imgur.com/G8acZQf.png)
 8 | 
 9 | 
10 | ## License
11 | [MIT](https://choosealicense.com/licenses/mit/)
12 | 
13 | #### Legal & Disclaimer
14 | *This script is just a proof of concept. The author isn't responsible for the actions of the end users.*
15 | 


--------------------------------------------------------------------------------