├── #1 Load a CSV in Python Pandas.ipynb
├── #2 Extract a sub-string in Python Pandas.ipynb
├── #3 Pivot tables in Python Pandas.ipynb
├── #4 Header cleanup (reset_index, droplevel, rename) in Python Pandas.ipynb
├── #5 Read Excel in Python Pandas.ipynb
├── #6 Merge two tables in Python Pandas.ipynb
├── #7 Filter and export in Python Pandas.ipynb
├── Analytics Big Beach Spring Break.ipynb
├── Cleanup-2-Analytics-API-and-Custom-Search-Crawl.ipynb
├── Cleanup-Analytics.ipynb
├── Cleanup.ipynb
├── README.md
└── Read News XML Sitemap with Python + Pandas.ipynb


/#1 Load a CSV in Python Pandas.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": null,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "# Check: \n",
 10 |     "# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html\n",
 11 |     "#\n",
 12 |     "# Video about this notebook: \n",
 13 |     "# https://youtu.be/iq1qNmsBlXg "
 14 |    ]
 15 |   },
 16 |   {
 17 |    "cell_type": "code",
 18 |    "execution_count": 38,
 19 |    "metadata": {},
 20 |    "outputs": [],
 21 |    "source": [
 22 |     "import pandas as pd"
 23 |    ]
 24 |   },
 25 |   {
 26 |    "cell_type": "code",
 27 |    "execution_count": 39,
 28 |    "metadata": {},
 29 |    "outputs": [],
 30 |    "source": [
 31 |     "dfTestCSV = pd.DataFrame(pd.read_csv('./dfanalyticstestdata1.csv', header=2, sep=\"\\t\", index_col=0))"
 32 |    ]
 33 |   },
 34 |   {
 35 |    "cell_type": "code",
 36 |    "execution_count": 40,
 37 |    "metadata": {},
 38 |    "outputs": [
 39 |     {
 40 |      "data": {
 41 |       "text/html": [
 42 |        "<div>\n",
 43 |        "<style scoped>\n",
 44 |        "    .dataframe tbody tr th:only-of-type {\n",
 45 |        "        vertical-align: middle;\n",
 46 |        "    }\n",
 47 |        "\n",
 48 |        "    .dataframe tbody tr th {\n",
 49 |        "        vertical-align: top;\n",
 50 |        "    }\n",
 51 |        "\n",
 52 |        "    .dataframe thead th {\n",
 53 |        "        text-align: right;\n",
 54 |        "    }\n",
 55 |        "</style>\n",
 56 |        "<table border=\"1\" class=\"dataframe\">\n",
 57 |        "  <thead>\n",
 58 |        "    <tr style=\"text-align: right;\">\n",
 59 |        "      <th></th>\n",
 60 |        "      <th>ga:pagePath</th>\n",
 61 |        "      <th>ga:pageviews</th>\n",
 62 |        "    </tr>\n",
 63 |        "  </thead>\n",
 64 |        "  <tbody>\n",
 65 |        "    <tr>\n",
 66 |        "      <th>0</th>\n",
 67 |        "      <td>/auto/-Oh-slug-slug-why-are-you-slug-id31218.html</td>\n",
 68 |        "      <td>3</td>\n",
 69 |        "    </tr>\n",
 70 |        "    <tr>\n",
 71 |        "      <th>1</th>\n",
 72 |        "      <td>/auto/-onboarding-story-id26545.html</td>\n",
 73 |        "      <td>1</td>\n",
 74 |        "    </tr>\n",
 75 |        "    <tr>\n",
 76 |        "      <th>2</th>\n",
 77 |        "      <td>/auto/abgefahren/Test-Zattoo-Sandro-Überzeile-...</td>\n",
 78 |        "      <td>10</td>\n",
 79 |        "    </tr>\n",
 80 |        "    <tr>\n",
 81 |        "      <th>3</th>\n",
 82 |        "      <td>/auto/autosalon/autosalon-story-ferrari-id3146...</td>\n",
 83 |        "      <td>8</td>\n",
 84 |        "    </tr>\n",
 85 |        "    <tr>\n",
 86 |        "      <th>4</th>\n",
 87 |        "      <td>/auto/Über-999-Stufen-und-durch-99-Haarnadel-K...</td>\n",
 88 |        "      <td>2</td>\n",
 89 |        "    </tr>\n",
 90 |        "    <tr>\n",
 91 |        "      <th>5</th>\n",
 92 |        "      <td>/auto/ueber-999-stufen-und-durch-99-haarnadel-...</td>\n",
 93 |        "      <td>4</td>\n",
 94 |        "    </tr>\n",
 95 |        "    <tr>\n",
 96 |        "      <th>6</th>\n",
 97 |        "      <td>/digital/Superlange-Überzeile-eines-Artikels-a...</td>\n",
 98 |        "      <td>2</td>\n",
 99 |        "    </tr>\n",
100 |        "    <tr>\n",
101 |        "      <th>7</th>\n",
102 |        "      <td>/digital/thomas-probiert-was-labeltest-id31705...</td>\n",
103 |        "      <td>3</td>\n",
104 |        "    </tr>\n",
105 |        "    <tr>\n",
106 |        "      <th>8</th>\n",
107 |        "      <td>/incoming/-CAS-id31002.html</td>\n",
108 |        "      <td>1</td>\n",
109 |        "    </tr>\n",
110 |        "    <tr>\n",
111 |        "      <th>9</th>\n",
112 |        "      <td>/incoming/-kathy-Slug-id31000.html</td>\n",
113 |        "      <td>3</td>\n",
114 |        "    </tr>\n",
115 |        "    <tr>\n",
116 |        "      <th>10</th>\n",
117 |        "      <td>/incoming/-Kathy-slug-id31004.html</td>\n",
118 |        "      <td>5</td>\n",
119 |        "    </tr>\n",
120 |        "    <tr>\n",
121 |        "      <th>11</th>\n",
122 |        "      <td>/incoming/-Zattoo-test-id30914.html</td>\n",
123 |        "      <td>1</td>\n",
124 |        "    </tr>\n",
125 |        "    <tr>\n",
126 |        "      <th>12</th>\n",
127 |        "      <td>/incoming/59-characters-59-characters-id30984....</td>\n",
128 |        "      <td>7</td>\n",
129 |        "    </tr>\n",
130 |        "    <tr>\n",
131 |        "      <th>13</th>\n",
132 |        "      <td>/incoming/75-characters-title-75-characters-ti...</td>\n",
133 |        "      <td>2</td>\n",
134 |        "    </tr>\n",
135 |        "    <tr>\n",
136 |        "      <th>14</th>\n",
137 |        "      <td>/incoming/8-Vermisste-in-Bondo-8-Vermisste-in-...</td>\n",
138 |        "      <td>1</td>\n",
139 |        "    </tr>\n",
140 |        "    <tr>\n",
141 |        "      <th>15</th>\n",
142 |        "      <td>/incoming/A-test-4-A-test-4-id31112.html</td>\n",
143 |        "      <td>1</td>\n",
144 |        "    </tr>\n",
145 |        "    <tr>\n",
146 |        "      <th>16</th>\n",
147 |        "      <td>/incoming/Abstimmungskampf-um-die-Rentenreform...</td>\n",
148 |        "      <td>2</td>\n",
149 |        "    </tr>\n",
150 |        "    <tr>\n",
151 |        "      <th>17</th>\n",
152 |        "      <td>/incoming/ade-testing-id31460.html</td>\n",
153 |        "      <td>1</td>\n",
154 |        "    </tr>\n",
155 |        "    <tr>\n",
156 |        "      <th>18</th>\n",
157 |        "      <td>/incoming/da-id31653.html</td>\n",
158 |        "      <td>11</td>\n",
159 |        "    </tr>\n",
160 |        "    <tr>\n",
161 |        "      <th>19</th>\n",
162 |        "      <td>/incoming/das-ist-ein-test-artikel-iframe-id31...</td>\n",
163 |        "      <td>1</td>\n",
164 |        "    </tr>\n",
165 |        "    <tr>\n",
166 |        "      <th>20</th>\n",
167 |        "      <td>/incoming/Detailhandel-verliert-1-Prozent-Umsa...</td>\n",
168 |        "      <td>1</td>\n",
169 |        "    </tr>\n",
170 |        "    <tr>\n",
171 |        "      <th>21</th>\n",
172 |        "      <td>/incoming/e2e-test-catchword-e2e-id31001.html</td>\n",
173 |        "      <td>11</td>\n",
174 |        "    </tr>\n",
175 |        "    <tr>\n",
176 |        "      <th>22</th>\n",
177 |        "      <td>/incoming/e2e-test-catchword-e2e-id31001.html'</td>\n",
178 |        "      <td>1</td>\n",
179 |        "    </tr>\n",
180 |        "    <tr>\n",
181 |        "      <th>23</th>\n",
182 |        "      <td>/incoming/e2e-test-catchword-e2e-testsite-plea...</td>\n",
183 |        "      <td>96</td>\n",
184 |        "    </tr>\n",
185 |        "    <tr>\n",
186 |        "      <th>24</th>\n",
187 |        "      <td>/incoming/form-story-form-story-id31887.html</td>\n",
188 |        "      <td>3</td>\n",
189 |        "    </tr>\n",
190 |        "    <tr>\n",
191 |        "      <th>25</th>\n",
192 |        "      <td>/incoming/fux-ueber-sex-f-id6260.html</td>\n",
193 |        "      <td>1</td>\n",
194 |        "    </tr>\n",
195 |        "    <tr>\n",
196 |        "      <th>26</th>\n",
197 |        "      <td>/incoming/iframe-article-super-iframe-article-...</td>\n",
198 |        "      <td>2</td>\n",
199 |        "    </tr>\n",
200 |        "    <tr>\n",
201 |        "      <th>27</th>\n",
202 |        "      <td>/incoming/instagram-test-embeded-main-element-...</td>\n",
203 |        "      <td>1</td>\n",
204 |        "    </tr>\n",
205 |        "    <tr>\n",
206 |        "      <th>28</th>\n",
207 |        "      <td>/incoming/ise-embedded-content-test-id31878.html</td>\n",
208 |        "      <td>1</td>\n",
209 |        "    </tr>\n",
210 |        "    <tr>\n",
211 |        "      <th>29</th>\n",
212 |        "      <td>/incoming/ise-publish-date-test-ise-test-artic...</td>\n",
213 |        "      <td>6</td>\n",
214 |        "    </tr>\n",
215 |        "    <tr>\n",
216 |        "      <th>...</th>\n",
217 |        "      <td>...</td>\n",
218 |        "      <td>...</td>\n",
219 |        "    </tr>\n",
220 |        "    <tr>\n",
221 |        "      <th>164</th>\n",
222 |        "      <td>/sport/fussball/wm/fussball-experte-ilja-kaenz...</td>\n",
223 |        "      <td>2</td>\n",
224 |        "    </tr>\n",
225 |        "    <tr>\n",
226 |        "      <th>165</th>\n",
227 |        "      <td>/sport/fussball/wm/gehts-gegen-die-schweiz-alb...</td>\n",
228 |        "      <td>12</td>\n",
229 |        "    </tr>\n",
230 |        "    <tr>\n",
231 |        "      <th>166</th>\n",
232 |        "      <td>/sport/fussball/wm/nach-geplatztem-wm-final-tr...</td>\n",
233 |        "      <td>1</td>\n",
234 |        "    </tr>\n",
235 |        "    <tr>\n",
236 |        "      <th>167</th>\n",
237 |        "      <td>/sport/fussball/wm/wm-2018-schweden-schweiz-id...</td>\n",
238 |        "      <td>1</td>\n",
239 |        "    </tr>\n",
240 |        "    <tr>\n",
241 |        "      <th>168</th>\n",
242 |        "      <td>/sport/mundgeruch-bei-der-royal-wedding-meghan...</td>\n",
243 |        "      <td>10</td>\n",
244 |        "    </tr>\n",
245 |        "    <tr>\n",
246 |        "      <th>169</th>\n",
247 |        "      <td>/sport/presse-muss-demuetiger-sein-wm-2018-hat...</td>\n",
248 |        "      <td>12</td>\n",
249 |        "    </tr>\n",
250 |        "    <tr>\n",
251 |        "      <th>170</th>\n",
252 |        "      <td>/sport/rad/dadsa-id31271.html</td>\n",
253 |        "      <td>6</td>\n",
254 |        "    </tr>\n",
255 |        "    <tr>\n",
256 |        "      <th>171</th>\n",
257 |        "      <td>/sport/rad/Giro-Zoff-vor-Zeitfahren-Dumoulin-«...</td>\n",
258 |        "      <td>7</td>\n",
259 |        "    </tr>\n",
260 |        "    <tr>\n",
261 |        "      <th>172</th>\n",
262 |        "      <td>/sport/reiten/Was-für-ein-Exploit-Monkey-Video...</td>\n",
263 |        "      <td>2</td>\n",
264 |        "    </tr>\n",
265 |        "    <tr>\n",
266 |        "      <th>173</th>\n",
267 |        "      <td>/sport/schwingen/Knatsch-im-Reich-der-Bösen-Ka...</td>\n",
268 |        "      <td>5</td>\n",
269 |        "    </tr>\n",
270 |        "    <tr>\n",
271 |        "      <th>174</th>\n",
272 |        "      <td>/sport/so-lernten-sich-lara-und-valon-kennen-l...</td>\n",
273 |        "      <td>1</td>\n",
274 |        "    </tr>\n",
275 |        "    <tr>\n",
276 |        "      <th>175</th>\n",
277 |        "      <td>/sport/sport-tagesticker-id5539965.html1</td>\n",
278 |        "      <td>4</td>\n",
279 |        "    </tr>\n",
280 |        "    <tr>\n",
281 |        "      <th>176</th>\n",
282 |        "      <td>/sport/tennis/asdasd-id31325.html</td>\n",
283 |        "      <td>9</td>\n",
284 |        "    </tr>\n",
285 |        "    <tr>\n",
286 |        "      <th>177</th>\n",
287 |        "      <td>/sport/tennis/glanzloser-sieg-gegen-qualifikan...</td>\n",
288 |        "      <td>1</td>\n",
289 |        "    </tr>\n",
290 |        "    <tr>\n",
291 |        "      <th>178</th>\n",
292 |        "      <td>/sport/tennis/Regen-verschiebt-Duell-mit-Donal...</td>\n",
293 |        "      <td>1</td>\n",
294 |        "    </tr>\n",
295 |        "    <tr>\n",
296 |        "      <th>179</th>\n",
297 |        "      <td>/sport/tennis/Timea-Bacsinszky-im-Interview-Ic...</td>\n",
298 |        "      <td>4</td>\n",
299 |        "    </tr>\n",
300 |        "    <tr>\n",
301 |        "      <th>180</th>\n",
302 |        "      <td>/sport/tennis/timea-bacsinszky-im-interview-me...</td>\n",
303 |        "      <td>43</td>\n",
304 |        "    </tr>\n",
305 |        "    <tr>\n",
306 |        "      <th>181</th>\n",
307 |        "      <td>/sport/tennis/timea-bacsinszky-im-interview-me...</td>\n",
308 |        "      <td>1</td>\n",
309 |        "    </tr>\n",
310 |        "    <tr>\n",
311 |        "      <th>182</th>\n",
312 |        "      <td>/sport/tennis/timea-bacsinszky-im-interview-me...</td>\n",
313 |        "      <td>18</td>\n",
314 |        "    </tr>\n",
315 |        "    <tr>\n",
316 |        "      <th>183</th>\n",
317 |        "      <td>/sport/tennis/timea-bacsinszky-im-interview-me...</td>\n",
318 |        "      <td>1</td>\n",
319 |        "    </tr>\n",
320 |        "    <tr>\n",
321 |        "      <th>184</th>\n",
322 |        "      <td>/sport/testing-those-teasers-super-title-testi...</td>\n",
323 |        "      <td>38</td>\n",
324 |        "    </tr>\n",
325 |        "    <tr>\n",
326 |        "      <th>185</th>\n",
327 |        "      <td>/sport/this-is-a-super-title-this-is-a-meta-ti...</td>\n",
328 |        "      <td>1</td>\n",
329 |        "    </tr>\n",
330 |        "    <tr>\n",
331 |        "      <th>186</th>\n",
332 |        "      <td>/sport/thomas-markle-laesst-es-sich-nach-herz-...</td>\n",
333 |        "      <td>4</td>\n",
334 |        "    </tr>\n",
335 |        "    <tr>\n",
336 |        "      <th>187</th>\n",
337 |        "      <td>/sport/Touristiker-jubeln-Trumps-Besuch-ist-20...</td>\n",
338 |        "      <td>1</td>\n",
339 |        "    </tr>\n",
340 |        "    <tr>\n",
341 |        "      <th>188</th>\n",
342 |        "      <td>/sport/Tunisia-Gamiz-vs-Kennel-long-title-lore...</td>\n",
343 |        "      <td>2</td>\n",
344 |        "    </tr>\n",
345 |        "    <tr>\n",
346 |        "      <th>189</th>\n",
347 |        "      <td>/sport/videoformate/dino/-id2483959.html</td>\n",
348 |        "      <td>1</td>\n",
349 |        "    </tr>\n",
350 |        "    <tr>\n",
351 |        "      <th>190</th>\n",
352 |        "      <td>/video/viral/ein-mathe-model-und-eine-wunder-w...</td>\n",
353 |        "      <td>1</td>\n",
354 |        "    </tr>\n",
355 |        "    <tr>\n",
356 |        "      <th>191</th>\n",
357 |        "      <td>/video/viral/ein-mathe-model-und-eine-wunder-w...</td>\n",
358 |        "      <td>3</td>\n",
359 |        "    </tr>\n",
360 |        "    <tr>\n",
361 |        "      <th>192</th>\n",
362 |        "      <td>/vr/ein-tauziehen-um-die-trepa-mine-in-mitrovi...</td>\n",
363 |        "      <td>1</td>\n",
364 |        "    </tr>\n",
365 |        "    <tr>\n",
366 |        "      <th>193</th>\n",
367 |        "      <td>/wirtschaft/wirtschaft-test-story-id31609.html</td>\n",
368 |        "      <td>5</td>\n",
369 |        "    </tr>\n",
370 |        "  </tbody>\n",
371 |        "</table>\n",
372 |        "<p>194 rows × 2 columns</p>\n",
373 |        "</div>"
374 |       ],
375 |       "text/plain": [
376 |        "                                           ga:pagePath  ga:pageviews\n",
377 |        "0    /auto/-Oh-slug-slug-why-are-you-slug-id31218.html             3\n",
378 |        "1                 /auto/-onboarding-story-id26545.html             1\n",
379 |        "2    /auto/abgefahren/Test-Zattoo-Sandro-Überzeile-...            10\n",
380 |        "3    /auto/autosalon/autosalon-story-ferrari-id3146...             8\n",
381 |        "4    /auto/Über-999-Stufen-und-durch-99-Haarnadel-K...             2\n",
382 |        "5    /auto/ueber-999-stufen-und-durch-99-haarnadel-...             4\n",
383 |        "6    /digital/Superlange-Überzeile-eines-Artikels-a...             2\n",
384 |        "7    /digital/thomas-probiert-was-labeltest-id31705...             3\n",
385 |        "8                          /incoming/-CAS-id31002.html             1\n",
386 |        "9                   /incoming/-kathy-Slug-id31000.html             3\n",
387 |        "10                  /incoming/-Kathy-slug-id31004.html             5\n",
388 |        "11                 /incoming/-Zattoo-test-id30914.html             1\n",
389 |        "12   /incoming/59-characters-59-characters-id30984....             7\n",
390 |        "13   /incoming/75-characters-title-75-characters-ti...             2\n",
391 |        "14   /incoming/8-Vermisste-in-Bondo-8-Vermisste-in-...             1\n",
392 |        "15            /incoming/A-test-4-A-test-4-id31112.html             1\n",
393 |        "16   /incoming/Abstimmungskampf-um-die-Rentenreform...             2\n",
394 |        "17                  /incoming/ade-testing-id31460.html             1\n",
395 |        "18                           /incoming/da-id31653.html            11\n",
396 |        "19   /incoming/das-ist-ein-test-artikel-iframe-id31...             1\n",
397 |        "20   /incoming/Detailhandel-verliert-1-Prozent-Umsa...             1\n",
398 |        "21       /incoming/e2e-test-catchword-e2e-id31001.html            11\n",
399 |        "22      /incoming/e2e-test-catchword-e2e-id31001.html'             1\n",
400 |        "23   /incoming/e2e-test-catchword-e2e-testsite-plea...            96\n",
401 |        "24        /incoming/form-story-form-story-id31887.html             3\n",
402 |        "25               /incoming/fux-ueber-sex-f-id6260.html             1\n",
403 |        "26   /incoming/iframe-article-super-iframe-article-...             2\n",
404 |        "27   /incoming/instagram-test-embeded-main-element-...             1\n",
405 |        "28    /incoming/ise-embedded-content-test-id31878.html             1\n",
406 |        "29   /incoming/ise-publish-date-test-ise-test-artic...             6\n",
407 |        "..                                                 ...           ...\n",
408 |        "164  /sport/fussball/wm/fussball-experte-ilja-kaenz...             2\n",
409 |        "165  /sport/fussball/wm/gehts-gegen-die-schweiz-alb...            12\n",
410 |        "166  /sport/fussball/wm/nach-geplatztem-wm-final-tr...             1\n",
411 |        "167  /sport/fussball/wm/wm-2018-schweden-schweiz-id...             1\n",
412 |        "168  /sport/mundgeruch-bei-der-royal-wedding-meghan...            10\n",
413 |        "169  /sport/presse-muss-demuetiger-sein-wm-2018-hat...            12\n",
414 |        "170                      /sport/rad/dadsa-id31271.html             6\n",
415 |        "171  /sport/rad/Giro-Zoff-vor-Zeitfahren-Dumoulin-«...             7\n",
416 |        "172  /sport/reiten/Was-für-ein-Exploit-Monkey-Video...             2\n",
417 |        "173  /sport/schwingen/Knatsch-im-Reich-der-Bösen-Ka...             5\n",
418 |        "174  /sport/so-lernten-sich-lara-und-valon-kennen-l...             1\n",
419 |        "175           /sport/sport-tagesticker-id5539965.html1             4\n",
420 |        "176                  /sport/tennis/asdasd-id31325.html             9\n",
421 |        "177  /sport/tennis/glanzloser-sieg-gegen-qualifikan...             1\n",
422 |        "178  /sport/tennis/Regen-verschiebt-Duell-mit-Donal...             1\n",
423 |        "179  /sport/tennis/Timea-Bacsinszky-im-Interview-Ic...             4\n",
424 |        "180  /sport/tennis/timea-bacsinszky-im-interview-me...            43\n",
425 |        "181  /sport/tennis/timea-bacsinszky-im-interview-me...             1\n",
426 |        "182  /sport/tennis/timea-bacsinszky-im-interview-me...            18\n",
427 |        "183  /sport/tennis/timea-bacsinszky-im-interview-me...             1\n",
428 |        "184  /sport/testing-those-teasers-super-title-testi...            38\n",
429 |        "185  /sport/this-is-a-super-title-this-is-a-meta-ti...             1\n",
430 |        "186  /sport/thomas-markle-laesst-es-sich-nach-herz-...             4\n",
431 |        "187  /sport/Touristiker-jubeln-Trumps-Besuch-ist-20...             1\n",
432 |        "188  /sport/Tunisia-Gamiz-vs-Kennel-long-title-lore...             2\n",
433 |        "189           /sport/videoformate/dino/-id2483959.html             1\n",
434 |        "190  /video/viral/ein-mathe-model-und-eine-wunder-w...             1\n",
435 |        "191  /video/viral/ein-mathe-model-und-eine-wunder-w...             3\n",
436 |        "192  /vr/ein-tauziehen-um-die-trepa-mine-in-mitrovi...             1\n",
437 |        "193     /wirtschaft/wirtschaft-test-story-id31609.html             5\n",
438 |        "\n",
439 |        "[194 rows x 2 columns]"
440 |       ]
441 |      },
442 |      "execution_count": 40,
443 |      "metadata": {},
444 |      "output_type": "execute_result"
445 |     }
446 |    ],
447 |    "source": [
448 |     "dfTestCSV"
449 |    ]
450 |   },
451 |   {
452 |    "cell_type": "code",
453 |    "execution_count": null,
454 |    "metadata": {},
455 |    "outputs": [],
456 |    "source": []
457 |   }
458 |  ],
459 |  "metadata": {
460 |   "kernelspec": {
461 |    "display_name": "Python 3",
462 |    "language": "python",
463 |    "name": "python3"
464 |   },
465 |   "language_info": {
466 |    "codemirror_mode": {
467 |     "name": "ipython",
468 |     "version": 3
469 |    },
470 |    "file_extension": ".py",
471 |    "mimetype": "text/x-python",
472 |    "name": "python",
473 |    "nbconvert_exporter": "python",
474 |    "pygments_lexer": "ipython3",
475 |    "version": "3.7.2"
476 |   }
477 |  },
478 |  "nbformat": 4,
479 |  "nbformat_minor": 2
480 | }
481 | 


--------------------------------------------------------------------------------
/#2 Extract a sub-string in Python Pandas.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": null,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "# Check documentation: \n",
 10 |     "# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.extract.html\n",
 11 |     "#\n",
 12 |     "# Video about this notebook: \n",
 13 |     "# https://www.youtube.com/watch?v=xhNSDQ1dbBA"
 14 |    ]
 15 |   },
 16 |   {
 17 |    "cell_type": "code",
 18 |    "execution_count": 58,
 19 |    "metadata": {},
 20 |    "outputs": [],
 21 |    "source": [
 22 |     "import pandas as pd"
 23 |    ]
 24 |   },
 25 |   {
 26 |    "cell_type": "code",
 27 |    "execution_count": 59,
 28 |    "metadata": {},
 29 |    "outputs": [],
 30 |    "source": [
 31 |     "dfTestCSV = pd.DataFrame(pd.read_csv('./dfanalyticstestdata1.csv', header=2, sep=\"\\t\", index_col=0))"
 32 |    ]
 33 |   },
 34 |   {
 35 |    "cell_type": "code",
 36 |    "execution_count": 60,
 37 |    "metadata": {},
 38 |    "outputs": [
 39 |     {
 40 |      "data": {
 41 |       "text/html": [
 42 |        "<div>\n",
 43 |        "<style scoped>\n",
 44 |        "    .dataframe tbody tr th:only-of-type {\n",
 45 |        "        vertical-align: middle;\n",
 46 |        "    }\n",
 47 |        "\n",
 48 |        "    .dataframe tbody tr th {\n",
 49 |        "        vertical-align: top;\n",
 50 |        "    }\n",
 51 |        "\n",
 52 |        "    .dataframe thead th {\n",
 53 |        "        text-align: right;\n",
 54 |        "    }\n",
 55 |        "</style>\n",
 56 |        "<table border=\"1\" class=\"dataframe\">\n",
 57 |        "  <thead>\n",
 58 |        "    <tr style=\"text-align: right;\">\n",
 59 |        "      <th></th>\n",
 60 |        "      <th>ga:pagePath</th>\n",
 61 |        "      <th>ga:pageviews</th>\n",
 62 |        "      <th>ID_GA</th>\n",
 63 |        "    </tr>\n",
 64 |        "  </thead>\n",
 65 |        "  <tbody>\n",
 66 |        "    <tr>\n",
 67 |        "      <th>0</th>\n",
 68 |        "      <td>/auto/-Oh-slug-slug-why-are-you-slug-id31218.html</td>\n",
 69 |        "      <td>3</td>\n",
 70 |        "      <td>31218</td>\n",
 71 |        "    </tr>\n",
 72 |        "    <tr>\n",
 73 |        "      <th>1</th>\n",
 74 |        "      <td>/auto/-onboarding-story-id26545.html</td>\n",
 75 |        "      <td>1</td>\n",
 76 |        "      <td>26545</td>\n",
 77 |        "    </tr>\n",
 78 |        "    <tr>\n",
 79 |        "      <th>2</th>\n",
 80 |        "      <td>/auto/abgefahren/Test-Zattoo-Sandro-Überzeile-...</td>\n",
 81 |        "      <td>10</td>\n",
 82 |        "      <td>30770</td>\n",
 83 |        "    </tr>\n",
 84 |        "    <tr>\n",
 85 |        "      <th>3</th>\n",
 86 |        "      <td>/auto/autosalon/autosalon-story-ferrari-id3146...</td>\n",
 87 |        "      <td>8</td>\n",
 88 |        "      <td>31463</td>\n",
 89 |        "    </tr>\n",
 90 |        "    <tr>\n",
 91 |        "      <th>4</th>\n",
 92 |        "      <td>/auto/Über-999-Stufen-und-durch-99-Haarnadel-K...</td>\n",
 93 |        "      <td>2</td>\n",
 94 |        "      <td>31288</td>\n",
 95 |        "    </tr>\n",
 96 |        "  </tbody>\n",
 97 |        "</table>\n",
 98 |        "</div>"
 99 |       ],
100 |       "text/plain": [
101 |        "                                         ga:pagePath  ga:pageviews  ID_GA\n",
102 |        "0  /auto/-Oh-slug-slug-why-are-you-slug-id31218.html             3  31218\n",
103 |        "1               /auto/-onboarding-story-id26545.html             1  26545\n",
104 |        "2  /auto/abgefahren/Test-Zattoo-Sandro-Überzeile-...            10  30770\n",
105 |        "3  /auto/autosalon/autosalon-story-ferrari-id3146...             8  31463\n",
106 |        "4  /auto/Über-999-Stufen-und-durch-99-Haarnadel-K...             2  31288"
107 |       ]
108 |      },
109 |      "execution_count": 60,
110 |      "metadata": {},
111 |      "output_type": "execute_result"
112 |     }
113 |    ],
114 |    "source": [
115 |     "dfTestCSV['ID_GA'] = dfTestCSV['ga:pagePath'].str.extract('(?:.*id)([0-9]+)(?:.html)')\n",
116 |     "dfTestCSV.head(5)"
117 |    ]
118 |   },
119 |   {
120 |    "cell_type": "code",
121 |    "execution_count": null,
122 |    "metadata": {},
123 |    "outputs": [],
124 |    "source": []
125 |   },
126 |   {
127 |    "cell_type": "code",
128 |    "execution_count": null,
129 |    "metadata": {},
130 |    "outputs": [],
131 |    "source": []
132 |   },
133 |   {
134 |    "cell_type": "code",
135 |    "execution_count": null,
136 |    "metadata": {},
137 |    "outputs": [],
138 |    "source": []
139 |   }
140 |  ],
141 |  "metadata": {
142 |   "kernelspec": {
143 |    "display_name": "Python 3",
144 |    "language": "python",
145 |    "name": "python3"
146 |   },
147 |   "language_info": {
148 |    "codemirror_mode": {
149 |     "name": "ipython",
150 |     "version": 3
151 |    },
152 |    "file_extension": ".py",
153 |    "mimetype": "text/x-python",
154 |    "name": "python",
155 |    "nbconvert_exporter": "python",
156 |    "pygments_lexer": "ipython3",
157 |    "version": "3.7.2"
158 |   }
159 |  },
160 |  "nbformat": 4,
161 |  "nbformat_minor": 2
162 | }
163 | 


--------------------------------------------------------------------------------
/#3 Pivot tables in Python Pandas.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": null,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "# Check documentation: \n",
 10 |     "# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.pivot.html\n",
 11 |     "#\n",
 12 |     "# Video about this notebook: \n",
 13 |     "# https://youtu.be/E5v-kUi_NUI"
 14 |    ]
 15 |   },
 16 |   {
 17 |    "cell_type": "code",
 18 |    "execution_count": 96,
 19 |    "metadata": {},
 20 |    "outputs": [],
 21 |    "source": [
 22 |     "import pandas as pd"
 23 |    ]
 24 |   },
 25 |   {
 26 |    "cell_type": "code",
 27 |    "execution_count": 97,
 28 |    "metadata": {},
 29 |    "outputs": [],
 30 |    "source": [
 31 |     "dfTestCSV = pd.DataFrame(pd.read_csv('./dfanalyticstestdata1.csv', header=2, sep=\"\\t\", index_col=0))"
 32 |    ]
 33 |   },
 34 |   {
 35 |    "cell_type": "code",
 36 |    "execution_count": 98,
 37 |    "metadata": {},
 38 |    "outputs": [
 39 |     {
 40 |      "data": {
 41 |       "text/html": [
 42 |        "<div>\n",
 43 |        "<style scoped>\n",
 44 |        "    .dataframe tbody tr th:only-of-type {\n",
 45 |        "        vertical-align: middle;\n",
 46 |        "    }\n",
 47 |        "\n",
 48 |        "    .dataframe tbody tr th {\n",
 49 |        "        vertical-align: top;\n",
 50 |        "    }\n",
 51 |        "\n",
 52 |        "    .dataframe thead th {\n",
 53 |        "        text-align: right;\n",
 54 |        "    }\n",
 55 |        "</style>\n",
 56 |        "<table border=\"1\" class=\"dataframe\">\n",
 57 |        "  <thead>\n",
 58 |        "    <tr style=\"text-align: right;\">\n",
 59 |        "      <th></th>\n",
 60 |        "      <th>ga:pagePath</th>\n",
 61 |        "      <th>ga:pageviews</th>\n",
 62 |        "      <th>ID_GA</th>\n",
 63 |        "    </tr>\n",
 64 |        "  </thead>\n",
 65 |        "  <tbody>\n",
 66 |        "    <tr>\n",
 67 |        "      <th>66</th>\n",
 68 |        "      <td>/incoming/Tochter-Alexis-Olympia-Das-Geheimnis...</td>\n",
 69 |        "      <td>7</td>\n",
 70 |        "      <td>10061</td>\n",
 71 |        "    </tr>\n",
 72 |        "    <tr>\n",
 73 |        "      <th>135</th>\n",
 74 |        "      <td>/people-tv/international/tochter-alexis-olympi...</td>\n",
 75 |        "      <td>1</td>\n",
 76 |        "      <td>10061</td>\n",
 77 |        "    </tr>\n",
 78 |        "    <tr>\n",
 79 |        "      <th>91</th>\n",
 80 |        "      <td>/news/-id1510256.html</td>\n",
 81 |        "      <td>1</td>\n",
 82 |        "      <td>1510256</td>\n",
 83 |        "    </tr>\n",
 84 |        "    <tr>\n",
 85 |        "      <th>151</th>\n",
 86 |        "      <td>/services/datenschutzbestimmungen-id151553.html</td>\n",
 87 |        "      <td>5</td>\n",
 88 |        "      <td>151553</td>\n",
 89 |        "    </tr>\n",
 90 |        "    <tr>\n",
 91 |        "      <th>83</th>\n",
 92 |        "      <td>/life/besten-seite-sind-meine-ferien-meta-titl...</td>\n",
 93 |        "      <td>1</td>\n",
 94 |        "      <td>1742</td>\n",
 95 |        "    </tr>\n",
 96 |        "    <tr>\n",
 97 |        "      <th>51</th>\n",
 98 |        "      <td>/incoming/schreiben-sie-uns-id19186.html1</td>\n",
 99 |        "      <td>28</td>\n",
100 |        "      <td>19186</td>\n",
101 |        "    </tr>\n",
102 |        "    <tr>\n",
103 |        "      <th>109</th>\n",
104 |        "      <td>/news/politik/Gegen-Gleichstellung-Homosexuell...</td>\n",
105 |        "      <td>2</td>\n",
106 |        "      <td>19969</td>\n",
107 |        "    </tr>\n",
108 |        "    <tr>\n",
109 |        "      <th>154</th>\n",
110 |        "      <td>/services/marktplatz/ticino-weekend-id2241980....</td>\n",
111 |        "      <td>1</td>\n",
112 |        "      <td>2241980</td>\n",
113 |        "    </tr>\n",
114 |        "    <tr>\n",
115 |        "      <th>20</th>\n",
116 |        "      <td>/incoming/Detailhandel-verliert-1-Prozent-Umsa...</td>\n",
117 |        "      <td>1</td>\n",
118 |        "      <td>22672</td>\n",
119 |        "    </tr>\n",
120 |        "    <tr>\n",
121 |        "      <th>189</th>\n",
122 |        "      <td>/sport/videoformate/dino/-id2483959.html</td>\n",
123 |        "      <td>1</td>\n",
124 |        "      <td>2483959</td>\n",
125 |        "    </tr>\n",
126 |        "  </tbody>\n",
127 |        "</table>\n",
128 |        "</div>"
129 |       ],
130 |       "text/plain": [
131 |        "                                           ga:pagePath  ga:pageviews    ID_GA\n",
132 |        "66   /incoming/Tochter-Alexis-Olympia-Das-Geheimnis...             7    10061\n",
133 |        "135  /people-tv/international/tochter-alexis-olympi...             1    10061\n",
134 |        "91                               /news/-id1510256.html             1  1510256\n",
135 |        "151    /services/datenschutzbestimmungen-id151553.html             5   151553\n",
136 |        "83   /life/besten-seite-sind-meine-ferien-meta-titl...             1     1742\n",
137 |        "51           /incoming/schreiben-sie-uns-id19186.html1            28    19186\n",
138 |        "109  /news/politik/Gegen-Gleichstellung-Homosexuell...             2    19969\n",
139 |        "154  /services/marktplatz/ticino-weekend-id2241980....             1  2241980\n",
140 |        "20   /incoming/Detailhandel-verliert-1-Prozent-Umsa...             1    22672\n",
141 |        "189           /sport/videoformate/dino/-id2483959.html             1  2483959"
142 |       ]
143 |      },
144 |      "execution_count": 98,
145 |      "metadata": {},
146 |      "output_type": "execute_result"
147 |     }
148 |    ],
149 |    "source": [
150 |     "dfTestCSV['ID_GA'] = dfTestCSV['ga:pagePath'].str.extract('(?:.*id)([0-9]+)(?:.html)')\n",
151 |     "dfTestCSV = dfTestCSV.sort_values(by='ID_GA')\n",
152 |     "dfTestCSV.head(10)"
153 |    ]
154 |   },
155 |   {
156 |    "cell_type": "code",
157 |    "execution_count": 99,
158 |    "metadata": {},
159 |    "outputs": [
160 |     {
161 |      "data": {
162 |       "text/plain": [
163 |        "194"
164 |       ]
165 |      },
166 |      "execution_count": 99,
167 |      "metadata": {},
168 |      "output_type": "execute_result"
169 |     }
170 |    ],
171 |    "source": [
172 |     "len(dfTestCSV)\n"
173 |    ]
174 |   },
175 |   {
176 |    "cell_type": "code",
177 |    "execution_count": 100,
178 |    "metadata": {},
179 |    "outputs": [
180 |     {
181 |      "data": {
182 |       "text/html": [
183 |        "<div>\n",
184 |        "<style scoped>\n",
185 |        "    .dataframe tbody tr th:only-of-type {\n",
186 |        "        vertical-align: middle;\n",
187 |        "    }\n",
188 |        "\n",
189 |        "    .dataframe tbody tr th {\n",
190 |        "        vertical-align: top;\n",
191 |        "    }\n",
192 |        "\n",
193 |        "    .dataframe thead tr th {\n",
194 |        "        text-align: left;\n",
195 |        "    }\n",
196 |        "\n",
197 |        "    .dataframe thead tr:last-of-type th {\n",
198 |        "        text-align: right;\n",
199 |        "    }\n",
200 |        "</style>\n",
201 |        "<table border=\"1\" class=\"dataframe\">\n",
202 |        "  <thead>\n",
203 |        "    <tr>\n",
204 |        "      <th></th>\n",
205 |        "      <th>count</th>\n",
206 |        "      <th>sum</th>\n",
207 |        "    </tr>\n",
208 |        "    <tr>\n",
209 |        "      <th></th>\n",
210 |        "      <th>ga:pageviews</th>\n",
211 |        "      <th>ga:pageviews</th>\n",
212 |        "    </tr>\n",
213 |        "    <tr>\n",
214 |        "      <th>ID_GA</th>\n",
215 |        "      <th></th>\n",
216 |        "      <th></th>\n",
217 |        "    </tr>\n",
218 |        "  </thead>\n",
219 |        "  <tbody>\n",
220 |        "    <tr>\n",
221 |        "      <th>10061</th>\n",
222 |        "      <td>2</td>\n",
223 |        "      <td>8</td>\n",
224 |        "    </tr>\n",
225 |        "    <tr>\n",
226 |        "      <th>1510256</th>\n",
227 |        "      <td>1</td>\n",
228 |        "      <td>1</td>\n",
229 |        "    </tr>\n",
230 |        "    <tr>\n",
231 |        "      <th>151553</th>\n",
232 |        "      <td>1</td>\n",
233 |        "      <td>5</td>\n",
234 |        "    </tr>\n",
235 |        "    <tr>\n",
236 |        "      <th>1742</th>\n",
237 |        "      <td>1</td>\n",
238 |        "      <td>1</td>\n",
239 |        "    </tr>\n",
240 |        "    <tr>\n",
241 |        "      <th>19186</th>\n",
242 |        "      <td>1</td>\n",
243 |        "      <td>28</td>\n",
244 |        "    </tr>\n",
245 |        "    <tr>\n",
246 |        "      <th>19969</th>\n",
247 |        "      <td>1</td>\n",
248 |        "      <td>2</td>\n",
249 |        "    </tr>\n",
250 |        "    <tr>\n",
251 |        "      <th>2241980</th>\n",
252 |        "      <td>1</td>\n",
253 |        "      <td>1</td>\n",
254 |        "    </tr>\n",
255 |        "    <tr>\n",
256 |        "      <th>22672</th>\n",
257 |        "      <td>1</td>\n",
258 |        "      <td>1</td>\n",
259 |        "    </tr>\n",
260 |        "    <tr>\n",
261 |        "      <th>2483959</th>\n",
262 |        "      <td>1</td>\n",
263 |        "      <td>1</td>\n",
264 |        "    </tr>\n",
265 |        "    <tr>\n",
266 |        "      <th>26545</th>\n",
267 |        "      <td>1</td>\n",
268 |        "      <td>1</td>\n",
269 |        "    </tr>\n",
270 |        "  </tbody>\n",
271 |        "</table>\n",
272 |        "</div>"
273 |       ],
274 |       "text/plain": [
275 |        "               count          sum\n",
276 |        "        ga:pageviews ga:pageviews\n",
277 |        "ID_GA                            \n",
278 |        "10061              2            8\n",
279 |        "1510256            1            1\n",
280 |        "151553             1            5\n",
281 |        "1742               1            1\n",
282 |        "19186              1           28\n",
283 |        "19969              1            2\n",
284 |        "2241980            1            1\n",
285 |        "22672              1            1\n",
286 |        "2483959            1            1\n",
287 |        "26545              1            1"
288 |       ]
289 |      },
290 |      "execution_count": 100,
291 |      "metadata": {},
292 |      "output_type": "execute_result"
293 |     }
294 |    ],
295 |    "source": [
296 |     "dfTestCSV = pd.pivot_table(dfTestCSV,index='ID_GA', values='ga:pageviews', aggfunc=['count', 'sum'])\n",
297 |     "dfTestCSV = dfTestCSV.sort_values(by='ID_GA')\n",
298 |     "dfTestCSV.head(10)"
299 |    ]
300 |   },
301 |   {
302 |    "cell_type": "code",
303 |    "execution_count": 94,
304 |    "metadata": {},
305 |    "outputs": [
306 |     {
307 |      "data": {
308 |       "text/plain": [
309 |        "147"
310 |       ]
311 |      },
312 |      "execution_count": 94,
313 |      "metadata": {},
314 |      "output_type": "execute_result"
315 |     }
316 |    ],
317 |    "source": [
318 |     "len(dfTestCSV)\n"
319 |    ]
320 |   },
321 |   {
322 |    "cell_type": "code",
323 |    "execution_count": null,
324 |    "metadata": {},
325 |    "outputs": [],
326 |    "source": []
327 |   },
328 |   {
329 |    "cell_type": "code",
330 |    "execution_count": null,
331 |    "metadata": {},
332 |    "outputs": [],
333 |    "source": []
334 |   },
335 |   {
336 |    "cell_type": "code",
337 |    "execution_count": null,
338 |    "metadata": {},
339 |    "outputs": [],
340 |    "source": []
341 |   }
342 |  ],
343 |  "metadata": {
344 |   "kernelspec": {
345 |    "display_name": "Python 3",
346 |    "language": "python",
347 |    "name": "python3"
348 |   },
349 |   "language_info": {
350 |    "codemirror_mode": {
351 |     "name": "ipython",
352 |     "version": 3
353 |    },
354 |    "file_extension": ".py",
355 |    "mimetype": "text/x-python",
356 |    "name": "python",
357 |    "nbconvert_exporter": "python",
358 |    "pygments_lexer": "ipython3",
359 |    "version": "3.7.2"
360 |   }
361 |  },
362 |  "nbformat": 4,
363 |  "nbformat_minor": 2
364 | }
365 | 


--------------------------------------------------------------------------------
/#4 Header cleanup (reset_index, droplevel, rename) in Python Pandas.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": null,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "# Check documentation: \n",
 10 |     "# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.reset_index.html\n",
 11 |     "# https://pandas-docs.github.io/pandas-docs-travis/reference/api/pandas.DataFrame.droplevel.html\n",
 12 |     "# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.rename.html\n",
 13 |     "#\n",
 14 |     "# Video about this notebook: \n",
 15 |     "# https://youtu.be/AvJ1YrdSKzQ"
 16 |    ]
 17 |   },
 18 |   {
 19 |    "cell_type": "code",
 20 |    "execution_count": 51,
 21 |    "metadata": {},
 22 |    "outputs": [],
 23 |    "source": [
 24 |     "import pandas as pd"
 25 |    ]
 26 |   },
 27 |   {
 28 |    "cell_type": "code",
 29 |    "execution_count": 52,
 30 |    "metadata": {},
 31 |    "outputs": [],
 32 |    "source": [
 33 |     "dfTestCSV = pd.DataFrame(pd.read_csv('./dfanalyticstestdata1.csv', header=2, sep=\"\\t\", index_col=0))"
 34 |    ]
 35 |   },
 36 |   {
 37 |    "cell_type": "code",
 38 |    "execution_count": 53,
 39 |    "metadata": {},
 40 |    "outputs": [],
 41 |    "source": [
 42 |     "dfTestCSV['ID_GA'] = dfTestCSV['ga:pagePath'].str.extract('(?:.*id)([0-9]+)(?:.html)')"
 43 |    ]
 44 |   },
 45 |   {
 46 |    "cell_type": "code",
 47 |    "execution_count": 54,
 48 |    "metadata": {},
 49 |    "outputs": [
 50 |     {
 51 |      "data": {
 52 |       "text/html": [
 53 |        "<div>\n",
 54 |        "<style scoped>\n",
 55 |        "    .dataframe tbody tr th:only-of-type {\n",
 56 |        "        vertical-align: middle;\n",
 57 |        "    }\n",
 58 |        "\n",
 59 |        "    .dataframe tbody tr th {\n",
 60 |        "        vertical-align: top;\n",
 61 |        "    }\n",
 62 |        "\n",
 63 |        "    .dataframe thead tr th {\n",
 64 |        "        text-align: left;\n",
 65 |        "    }\n",
 66 |        "\n",
 67 |        "    .dataframe thead tr:last-of-type th {\n",
 68 |        "        text-align: right;\n",
 69 |        "    }\n",
 70 |        "</style>\n",
 71 |        "<table border=\"1\" class=\"dataframe\">\n",
 72 |        "  <thead>\n",
 73 |        "    <tr>\n",
 74 |        "      <th></th>\n",
 75 |        "      <th>count</th>\n",
 76 |        "      <th>sum</th>\n",
 77 |        "    </tr>\n",
 78 |        "    <tr>\n",
 79 |        "      <th></th>\n",
 80 |        "      <th>ga:pageviews</th>\n",
 81 |        "      <th>ga:pageviews</th>\n",
 82 |        "    </tr>\n",
 83 |        "    <tr>\n",
 84 |        "      <th>ID_GA</th>\n",
 85 |        "      <th></th>\n",
 86 |        "      <th></th>\n",
 87 |        "    </tr>\n",
 88 |        "  </thead>\n",
 89 |        "  <tbody>\n",
 90 |        "    <tr>\n",
 91 |        "      <th>10061</th>\n",
 92 |        "      <td>2</td>\n",
 93 |        "      <td>8</td>\n",
 94 |        "    </tr>\n",
 95 |        "    <tr>\n",
 96 |        "      <th>1510256</th>\n",
 97 |        "      <td>1</td>\n",
 98 |        "      <td>1</td>\n",
 99 |        "    </tr>\n",
100 |        "    <tr>\n",
101 |        "      <th>151553</th>\n",
102 |        "      <td>1</td>\n",
103 |        "      <td>5</td>\n",
104 |        "    </tr>\n",
105 |        "    <tr>\n",
106 |        "      <th>1742</th>\n",
107 |        "      <td>1</td>\n",
108 |        "      <td>1</td>\n",
109 |        "    </tr>\n",
110 |        "    <tr>\n",
111 |        "      <th>19186</th>\n",
112 |        "      <td>1</td>\n",
113 |        "      <td>28</td>\n",
114 |        "    </tr>\n",
115 |        "    <tr>\n",
116 |        "      <th>19969</th>\n",
117 |        "      <td>1</td>\n",
118 |        "      <td>2</td>\n",
119 |        "    </tr>\n",
120 |        "    <tr>\n",
121 |        "      <th>2241980</th>\n",
122 |        "      <td>1</td>\n",
123 |        "      <td>1</td>\n",
124 |        "    </tr>\n",
125 |        "    <tr>\n",
126 |        "      <th>22672</th>\n",
127 |        "      <td>1</td>\n",
128 |        "      <td>1</td>\n",
129 |        "    </tr>\n",
130 |        "    <tr>\n",
131 |        "      <th>2483959</th>\n",
132 |        "      <td>1</td>\n",
133 |        "      <td>1</td>\n",
134 |        "    </tr>\n",
135 |        "    <tr>\n",
136 |        "      <th>26545</th>\n",
137 |        "      <td>1</td>\n",
138 |        "      <td>1</td>\n",
139 |        "    </tr>\n",
140 |        "  </tbody>\n",
141 |        "</table>\n",
142 |        "</div>"
143 |       ],
144 |       "text/plain": [
145 |        "               count          sum\n",
146 |        "        ga:pageviews ga:pageviews\n",
147 |        "ID_GA                            \n",
148 |        "10061              2            8\n",
149 |        "1510256            1            1\n",
150 |        "151553             1            5\n",
151 |        "1742               1            1\n",
152 |        "19186              1           28\n",
153 |        "19969              1            2\n",
154 |        "2241980            1            1\n",
155 |        "22672              1            1\n",
156 |        "2483959            1            1\n",
157 |        "26545              1            1"
158 |       ]
159 |      },
160 |      "execution_count": 54,
161 |      "metadata": {},
162 |      "output_type": "execute_result"
163 |     }
164 |    ],
165 |    "source": [
166 |     "dfTestCSV = pd.pivot_table(dfTestCSV,index='ID_GA', values='ga:pageviews', aggfunc=['count', 'sum'])\n",
167 |     "dfTestCSV.head(10)"
168 |    ]
169 |   },
170 |   {
171 |    "cell_type": "code",
172 |    "execution_count": 55,
173 |    "metadata": {},
174 |    "outputs": [
175 |     {
176 |      "data": {
177 |       "text/html": [
178 |        "<div>\n",
179 |        "<style scoped>\n",
180 |        "    .dataframe tbody tr th:only-of-type {\n",
181 |        "        vertical-align: middle;\n",
182 |        "    }\n",
183 |        "\n",
184 |        "    .dataframe tbody tr th {\n",
185 |        "        vertical-align: top;\n",
186 |        "    }\n",
187 |        "\n",
188 |        "    .dataframe thead tr th {\n",
189 |        "        text-align: left;\n",
190 |        "    }\n",
191 |        "</style>\n",
192 |        "<table border=\"1\" class=\"dataframe\">\n",
193 |        "  <thead>\n",
194 |        "    <tr>\n",
195 |        "      <th></th>\n",
196 |        "      <th>ID_GA</th>\n",
197 |        "      <th>count</th>\n",
198 |        "      <th>sum</th>\n",
199 |        "    </tr>\n",
200 |        "    <tr>\n",
201 |        "      <th></th>\n",
202 |        "      <th></th>\n",
203 |        "      <th>ga:pageviews</th>\n",
204 |        "      <th>ga:pageviews</th>\n",
205 |        "    </tr>\n",
206 |        "  </thead>\n",
207 |        "  <tbody>\n",
208 |        "    <tr>\n",
209 |        "      <th>0</th>\n",
210 |        "      <td>10061</td>\n",
211 |        "      <td>2</td>\n",
212 |        "      <td>8</td>\n",
213 |        "    </tr>\n",
214 |        "    <tr>\n",
215 |        "      <th>1</th>\n",
216 |        "      <td>1510256</td>\n",
217 |        "      <td>1</td>\n",
218 |        "      <td>1</td>\n",
219 |        "    </tr>\n",
220 |        "    <tr>\n",
221 |        "      <th>2</th>\n",
222 |        "      <td>151553</td>\n",
223 |        "      <td>1</td>\n",
224 |        "      <td>5</td>\n",
225 |        "    </tr>\n",
226 |        "    <tr>\n",
227 |        "      <th>3</th>\n",
228 |        "      <td>1742</td>\n",
229 |        "      <td>1</td>\n",
230 |        "      <td>1</td>\n",
231 |        "    </tr>\n",
232 |        "    <tr>\n",
233 |        "      <th>4</th>\n",
234 |        "      <td>19186</td>\n",
235 |        "      <td>1</td>\n",
236 |        "      <td>28</td>\n",
237 |        "    </tr>\n",
238 |        "    <tr>\n",
239 |        "      <th>5</th>\n",
240 |        "      <td>19969</td>\n",
241 |        "      <td>1</td>\n",
242 |        "      <td>2</td>\n",
243 |        "    </tr>\n",
244 |        "    <tr>\n",
245 |        "      <th>6</th>\n",
246 |        "      <td>2241980</td>\n",
247 |        "      <td>1</td>\n",
248 |        "      <td>1</td>\n",
249 |        "    </tr>\n",
250 |        "    <tr>\n",
251 |        "      <th>7</th>\n",
252 |        "      <td>22672</td>\n",
253 |        "      <td>1</td>\n",
254 |        "      <td>1</td>\n",
255 |        "    </tr>\n",
256 |        "    <tr>\n",
257 |        "      <th>8</th>\n",
258 |        "      <td>2483959</td>\n",
259 |        "      <td>1</td>\n",
260 |        "      <td>1</td>\n",
261 |        "    </tr>\n",
262 |        "    <tr>\n",
263 |        "      <th>9</th>\n",
264 |        "      <td>26545</td>\n",
265 |        "      <td>1</td>\n",
266 |        "      <td>1</td>\n",
267 |        "    </tr>\n",
268 |        "  </tbody>\n",
269 |        "</table>\n",
270 |        "</div>"
271 |       ],
272 |       "text/plain": [
273 |        "     ID_GA        count          sum\n",
274 |        "           ga:pageviews ga:pageviews\n",
275 |        "0    10061            2            8\n",
276 |        "1  1510256            1            1\n",
277 |        "2   151553            1            5\n",
278 |        "3     1742            1            1\n",
279 |        "4    19186            1           28\n",
280 |        "5    19969            1            2\n",
281 |        "6  2241980            1            1\n",
282 |        "7    22672            1            1\n",
283 |        "8  2483959            1            1\n",
284 |        "9    26545            1            1"
285 |       ]
286 |      },
287 |      "execution_count": 55,
288 |      "metadata": {},
289 |      "output_type": "execute_result"
290 |     }
291 |    ],
292 |    "source": [
293 |     "dfTestCSV.reset_index(inplace=True)\n",
294 |     "dfTestCSV.head(10)"
295 |    ]
296 |   },
297 |   {
298 |    "cell_type": "code",
299 |    "execution_count": 56,
300 |    "metadata": {},
301 |    "outputs": [
302 |     {
303 |      "name": "stdout",
304 |      "output_type": "stream",
305 |      "text": [
306 |       "('ID_GA', '')\n",
307 |       "('count', 'ga:pageviews')\n",
308 |       "('sum', 'ga:pageviews')\n"
309 |      ]
310 |     }
311 |    ],
312 |    "source": [
313 |     "for col in dfTestCSV.columns: \n",
314 |     "    print(col)"
315 |    ]
316 |   },
317 |   {
318 |    "cell_type": "code",
319 |    "execution_count": 57,
320 |    "metadata": {},
321 |    "outputs": [
322 |     {
323 |      "data": {
324 |       "text/html": [
325 |        "<div>\n",
326 |        "<style scoped>\n",
327 |        "    .dataframe tbody tr th:only-of-type {\n",
328 |        "        vertical-align: middle;\n",
329 |        "    }\n",
330 |        "\n",
331 |        "    .dataframe tbody tr th {\n",
332 |        "        vertical-align: top;\n",
333 |        "    }\n",
334 |        "\n",
335 |        "    .dataframe thead th {\n",
336 |        "        text-align: right;\n",
337 |        "    }\n",
338 |        "</style>\n",
339 |        "<table border=\"1\" class=\"dataframe\">\n",
340 |        "  <thead>\n",
341 |        "    <tr style=\"text-align: right;\">\n",
342 |        "      <th></th>\n",
343 |        "      <th>ID_GA</th>\n",
344 |        "      <th>count</th>\n",
345 |        "      <th>sum</th>\n",
346 |        "    </tr>\n",
347 |        "  </thead>\n",
348 |        "  <tbody>\n",
349 |        "    <tr>\n",
350 |        "      <th>0</th>\n",
351 |        "      <td>10061</td>\n",
352 |        "      <td>2</td>\n",
353 |        "      <td>8</td>\n",
354 |        "    </tr>\n",
355 |        "    <tr>\n",
356 |        "      <th>1</th>\n",
357 |        "      <td>1510256</td>\n",
358 |        "      <td>1</td>\n",
359 |        "      <td>1</td>\n",
360 |        "    </tr>\n",
361 |        "    <tr>\n",
362 |        "      <th>2</th>\n",
363 |        "      <td>151553</td>\n",
364 |        "      <td>1</td>\n",
365 |        "      <td>5</td>\n",
366 |        "    </tr>\n",
367 |        "    <tr>\n",
368 |        "      <th>3</th>\n",
369 |        "      <td>1742</td>\n",
370 |        "      <td>1</td>\n",
371 |        "      <td>1</td>\n",
372 |        "    </tr>\n",
373 |        "    <tr>\n",
374 |        "      <th>4</th>\n",
375 |        "      <td>19186</td>\n",
376 |        "      <td>1</td>\n",
377 |        "      <td>28</td>\n",
378 |        "    </tr>\n",
379 |        "    <tr>\n",
380 |        "      <th>5</th>\n",
381 |        "      <td>19969</td>\n",
382 |        "      <td>1</td>\n",
383 |        "      <td>2</td>\n",
384 |        "    </tr>\n",
385 |        "    <tr>\n",
386 |        "      <th>6</th>\n",
387 |        "      <td>2241980</td>\n",
388 |        "      <td>1</td>\n",
389 |        "      <td>1</td>\n",
390 |        "    </tr>\n",
391 |        "    <tr>\n",
392 |        "      <th>7</th>\n",
393 |        "      <td>22672</td>\n",
394 |        "      <td>1</td>\n",
395 |        "      <td>1</td>\n",
396 |        "    </tr>\n",
397 |        "    <tr>\n",
398 |        "      <th>8</th>\n",
399 |        "      <td>2483959</td>\n",
400 |        "      <td>1</td>\n",
401 |        "      <td>1</td>\n",
402 |        "    </tr>\n",
403 |        "    <tr>\n",
404 |        "      <th>9</th>\n",
405 |        "      <td>26545</td>\n",
406 |        "      <td>1</td>\n",
407 |        "      <td>1</td>\n",
408 |        "    </tr>\n",
409 |        "  </tbody>\n",
410 |        "</table>\n",
411 |        "</div>"
412 |       ],
413 |       "text/plain": [
414 |        "     ID_GA  count  sum\n",
415 |        "0    10061      2    8\n",
416 |        "1  1510256      1    1\n",
417 |        "2   151553      1    5\n",
418 |        "3     1742      1    1\n",
419 |        "4    19186      1   28\n",
420 |        "5    19969      1    2\n",
421 |        "6  2241980      1    1\n",
422 |        "7    22672      1    1\n",
423 |        "8  2483959      1    1\n",
424 |        "9    26545      1    1"
425 |       ]
426 |      },
427 |      "execution_count": 57,
428 |      "metadata": {},
429 |      "output_type": "execute_result"
430 |     }
431 |    ],
432 |    "source": [
433 |     "dfTestCSV.columns = dfTestCSV.columns.droplevel(1)\n",
434 |     "dfTestCSV.head(10)"
435 |    ]
436 |   },
437 |   {
438 |    "cell_type": "code",
439 |    "execution_count": 59,
440 |    "metadata": {},
441 |    "outputs": [
442 |     {
443 |      "data": {
444 |       "text/html": [
445 |        "<div>\n",
446 |        "<style scoped>\n",
447 |        "    .dataframe tbody tr th:only-of-type {\n",
448 |        "        vertical-align: middle;\n",
449 |        "    }\n",
450 |        "\n",
451 |        "    .dataframe tbody tr th {\n",
452 |        "        vertical-align: top;\n",
453 |        "    }\n",
454 |        "\n",
455 |        "    .dataframe thead th {\n",
456 |        "        text-align: right;\n",
457 |        "    }\n",
458 |        "</style>\n",
459 |        "<table border=\"1\" class=\"dataframe\">\n",
460 |        "  <thead>\n",
461 |        "    <tr style=\"text-align: right;\">\n",
462 |        "      <th></th>\n",
463 |        "      <th>ID_GA</th>\n",
464 |        "      <th>id count</th>\n",
465 |        "      <th>pageviews per id</th>\n",
466 |        "    </tr>\n",
467 |        "  </thead>\n",
468 |        "  <tbody>\n",
469 |        "    <tr>\n",
470 |        "      <th>0</th>\n",
471 |        "      <td>10061</td>\n",
472 |        "      <td>2</td>\n",
473 |        "      <td>8</td>\n",
474 |        "    </tr>\n",
475 |        "    <tr>\n",
476 |        "      <th>1</th>\n",
477 |        "      <td>1510256</td>\n",
478 |        "      <td>1</td>\n",
479 |        "      <td>1</td>\n",
480 |        "    </tr>\n",
481 |        "    <tr>\n",
482 |        "      <th>2</th>\n",
483 |        "      <td>151553</td>\n",
484 |        "      <td>1</td>\n",
485 |        "      <td>5</td>\n",
486 |        "    </tr>\n",
487 |        "    <tr>\n",
488 |        "      <th>3</th>\n",
489 |        "      <td>1742</td>\n",
490 |        "      <td>1</td>\n",
491 |        "      <td>1</td>\n",
492 |        "    </tr>\n",
493 |        "    <tr>\n",
494 |        "      <th>4</th>\n",
495 |        "      <td>19186</td>\n",
496 |        "      <td>1</td>\n",
497 |        "      <td>28</td>\n",
498 |        "    </tr>\n",
499 |        "    <tr>\n",
500 |        "      <th>5</th>\n",
501 |        "      <td>19969</td>\n",
502 |        "      <td>1</td>\n",
503 |        "      <td>2</td>\n",
504 |        "    </tr>\n",
505 |        "    <tr>\n",
506 |        "      <th>6</th>\n",
507 |        "      <td>2241980</td>\n",
508 |        "      <td>1</td>\n",
509 |        "      <td>1</td>\n",
510 |        "    </tr>\n",
511 |        "    <tr>\n",
512 |        "      <th>7</th>\n",
513 |        "      <td>22672</td>\n",
514 |        "      <td>1</td>\n",
515 |        "      <td>1</td>\n",
516 |        "    </tr>\n",
517 |        "    <tr>\n",
518 |        "      <th>8</th>\n",
519 |        "      <td>2483959</td>\n",
520 |        "      <td>1</td>\n",
521 |        "      <td>1</td>\n",
522 |        "    </tr>\n",
523 |        "    <tr>\n",
524 |        "      <th>9</th>\n",
525 |        "      <td>26545</td>\n",
526 |        "      <td>1</td>\n",
527 |        "      <td>1</td>\n",
528 |        "    </tr>\n",
529 |        "    <tr>\n",
530 |        "      <th>10</th>\n",
531 |        "      <td>26828</td>\n",
532 |        "      <td>1</td>\n",
533 |        "      <td>1</td>\n",
534 |        "    </tr>\n",
535 |        "    <tr>\n",
536 |        "      <th>11</th>\n",
537 |        "      <td>30291</td>\n",
538 |        "      <td>2</td>\n",
539 |        "      <td>2</td>\n",
540 |        "    </tr>\n",
541 |        "    <tr>\n",
542 |        "      <th>12</th>\n",
543 |        "      <td>30519</td>\n",
544 |        "      <td>1</td>\n",
545 |        "      <td>1</td>\n",
546 |        "    </tr>\n",
547 |        "    <tr>\n",
548 |        "      <th>13</th>\n",
549 |        "      <td>30527</td>\n",
550 |        "      <td>1</td>\n",
551 |        "      <td>2</td>\n",
552 |        "    </tr>\n",
553 |        "    <tr>\n",
554 |        "      <th>14</th>\n",
555 |        "      <td>30568</td>\n",
556 |        "      <td>1</td>\n",
557 |        "      <td>1</td>\n",
558 |        "    </tr>\n",
559 |        "    <tr>\n",
560 |        "      <th>15</th>\n",
561 |        "      <td>30615</td>\n",
562 |        "      <td>1</td>\n",
563 |        "      <td>1</td>\n",
564 |        "    </tr>\n",
565 |        "    <tr>\n",
566 |        "      <th>16</th>\n",
567 |        "      <td>30625</td>\n",
568 |        "      <td>1</td>\n",
569 |        "      <td>2</td>\n",
570 |        "    </tr>\n",
571 |        "    <tr>\n",
572 |        "      <th>17</th>\n",
573 |        "      <td>30702</td>\n",
574 |        "      <td>1</td>\n",
575 |        "      <td>2</td>\n",
576 |        "    </tr>\n",
577 |        "    <tr>\n",
578 |        "      <th>18</th>\n",
579 |        "      <td>30715</td>\n",
580 |        "      <td>1</td>\n",
581 |        "      <td>3</td>\n",
582 |        "    </tr>\n",
583 |        "    <tr>\n",
584 |        "      <th>19</th>\n",
585 |        "      <td>30758</td>\n",
586 |        "      <td>1</td>\n",
587 |        "      <td>1</td>\n",
588 |        "    </tr>\n",
589 |        "    <tr>\n",
590 |        "      <th>20</th>\n",
591 |        "      <td>30770</td>\n",
592 |        "      <td>1</td>\n",
593 |        "      <td>10</td>\n",
594 |        "    </tr>\n",
595 |        "    <tr>\n",
596 |        "      <th>21</th>\n",
597 |        "      <td>30787</td>\n",
598 |        "      <td>1</td>\n",
599 |        "      <td>60</td>\n",
600 |        "    </tr>\n",
601 |        "    <tr>\n",
602 |        "      <th>22</th>\n",
603 |        "      <td>30790</td>\n",
604 |        "      <td>2</td>\n",
605 |        "      <td>8</td>\n",
606 |        "    </tr>\n",
607 |        "    <tr>\n",
608 |        "      <th>23</th>\n",
609 |        "      <td>30842</td>\n",
610 |        "      <td>1</td>\n",
611 |        "      <td>2</td>\n",
612 |        "    </tr>\n",
613 |        "    <tr>\n",
614 |        "      <th>24</th>\n",
615 |        "      <td>30859</td>\n",
616 |        "      <td>1</td>\n",
617 |        "      <td>2</td>\n",
618 |        "    </tr>\n",
619 |        "    <tr>\n",
620 |        "      <th>25</th>\n",
621 |        "      <td>30914</td>\n",
622 |        "      <td>1</td>\n",
623 |        "      <td>1</td>\n",
624 |        "    </tr>\n",
625 |        "    <tr>\n",
626 |        "      <th>26</th>\n",
627 |        "      <td>30919</td>\n",
628 |        "      <td>1</td>\n",
629 |        "      <td>1</td>\n",
630 |        "    </tr>\n",
631 |        "    <tr>\n",
632 |        "      <th>27</th>\n",
633 |        "      <td>30947</td>\n",
634 |        "      <td>1</td>\n",
635 |        "      <td>5</td>\n",
636 |        "    </tr>\n",
637 |        "    <tr>\n",
638 |        "      <th>28</th>\n",
639 |        "      <td>30950</td>\n",
640 |        "      <td>1</td>\n",
641 |        "      <td>1</td>\n",
642 |        "    </tr>\n",
643 |        "    <tr>\n",
644 |        "      <th>29</th>\n",
645 |        "      <td>30952</td>\n",
646 |        "      <td>1</td>\n",
647 |        "      <td>1</td>\n",
648 |        "    </tr>\n",
649 |        "    <tr>\n",
650 |        "      <th>...</th>\n",
651 |        "      <td>...</td>\n",
652 |        "      <td>...</td>\n",
653 |        "      <td>...</td>\n",
654 |        "    </tr>\n",
655 |        "    <tr>\n",
656 |        "      <th>117</th>\n",
657 |        "      <td>32168</td>\n",
658 |        "      <td>1</td>\n",
659 |        "      <td>3</td>\n",
660 |        "    </tr>\n",
661 |        "    <tr>\n",
662 |        "      <th>118</th>\n",
663 |        "      <td>32177</td>\n",
664 |        "      <td>1</td>\n",
665 |        "      <td>2</td>\n",
666 |        "    </tr>\n",
667 |        "    <tr>\n",
668 |        "      <th>119</th>\n",
669 |        "      <td>32179</td>\n",
670 |        "      <td>1</td>\n",
671 |        "      <td>1</td>\n",
672 |        "    </tr>\n",
673 |        "    <tr>\n",
674 |        "      <th>120</th>\n",
675 |        "      <td>32188</td>\n",
676 |        "      <td>1</td>\n",
677 |        "      <td>12</td>\n",
678 |        "    </tr>\n",
679 |        "    <tr>\n",
680 |        "      <th>121</th>\n",
681 |        "      <td>4725725</td>\n",
682 |        "      <td>1</td>\n",
683 |        "      <td>1</td>\n",
684 |        "    </tr>\n",
685 |        "    <tr>\n",
686 |        "      <th>122</th>\n",
687 |        "      <td>5049878</td>\n",
688 |        "      <td>1</td>\n",
689 |        "      <td>1</td>\n",
690 |        "    </tr>\n",
691 |        "    <tr>\n",
692 |        "      <th>123</th>\n",
693 |        "      <td>5421492</td>\n",
694 |        "      <td>1</td>\n",
695 |        "      <td>3</td>\n",
696 |        "    </tr>\n",
697 |        "    <tr>\n",
698 |        "      <th>124</th>\n",
699 |        "      <td>5539965</td>\n",
700 |        "      <td>1</td>\n",
701 |        "      <td>4</td>\n",
702 |        "    </tr>\n",
703 |        "    <tr>\n",
704 |        "      <th>125</th>\n",
705 |        "      <td>5636781</td>\n",
706 |        "      <td>2</td>\n",
707 |        "      <td>4</td>\n",
708 |        "    </tr>\n",
709 |        "    <tr>\n",
710 |        "      <th>126</th>\n",
711 |        "      <td>5697230</td>\n",
712 |        "      <td>1</td>\n",
713 |        "      <td>1</td>\n",
714 |        "    </tr>\n",
715 |        "    <tr>\n",
716 |        "      <th>127</th>\n",
717 |        "      <td>5703102</td>\n",
718 |        "      <td>1</td>\n",
719 |        "      <td>1</td>\n",
720 |        "    </tr>\n",
721 |        "    <tr>\n",
722 |        "      <th>128</th>\n",
723 |        "      <td>6181</td>\n",
724 |        "      <td>1</td>\n",
725 |        "      <td>3</td>\n",
726 |        "    </tr>\n",
727 |        "    <tr>\n",
728 |        "      <th>129</th>\n",
729 |        "      <td>6243</td>\n",
730 |        "      <td>1</td>\n",
731 |        "      <td>2</td>\n",
732 |        "    </tr>\n",
733 |        "    <tr>\n",
734 |        "      <th>130</th>\n",
735 |        "      <td>6255</td>\n",
736 |        "      <td>1</td>\n",
737 |        "      <td>1</td>\n",
738 |        "    </tr>\n",
739 |        "    <tr>\n",
740 |        "      <th>131</th>\n",
741 |        "      <td>6256</td>\n",
742 |        "      <td>1</td>\n",
743 |        "      <td>1</td>\n",
744 |        "    </tr>\n",
745 |        "    <tr>\n",
746 |        "      <th>132</th>\n",
747 |        "      <td>6260</td>\n",
748 |        "      <td>1</td>\n",
749 |        "      <td>1</td>\n",
750 |        "    </tr>\n",
751 |        "    <tr>\n",
752 |        "      <th>133</th>\n",
753 |        "      <td>6271</td>\n",
754 |        "      <td>1</td>\n",
755 |        "      <td>2</td>\n",
756 |        "    </tr>\n",
757 |        "    <tr>\n",
758 |        "      <th>134</th>\n",
759 |        "      <td>6323</td>\n",
760 |        "      <td>1</td>\n",
761 |        "      <td>1</td>\n",
762 |        "    </tr>\n",
763 |        "    <tr>\n",
764 |        "      <th>135</th>\n",
765 |        "      <td>6328</td>\n",
766 |        "      <td>1</td>\n",
767 |        "      <td>1</td>\n",
768 |        "    </tr>\n",
769 |        "    <tr>\n",
770 |        "      <th>136</th>\n",
771 |        "      <td>6448519</td>\n",
772 |        "      <td>1</td>\n",
773 |        "      <td>5</td>\n",
774 |        "    </tr>\n",
775 |        "    <tr>\n",
776 |        "      <th>137</th>\n",
777 |        "      <td>6759728</td>\n",
778 |        "      <td>1</td>\n",
779 |        "      <td>1</td>\n",
780 |        "    </tr>\n",
781 |        "    <tr>\n",
782 |        "      <th>138</th>\n",
783 |        "      <td>6923005</td>\n",
784 |        "      <td>1</td>\n",
785 |        "      <td>1</td>\n",
786 |        "    </tr>\n",
787 |        "    <tr>\n",
788 |        "      <th>139</th>\n",
789 |        "      <td>6974</td>\n",
790 |        "      <td>1</td>\n",
791 |        "      <td>1</td>\n",
792 |        "    </tr>\n",
793 |        "    <tr>\n",
794 |        "      <th>140</th>\n",
795 |        "      <td>7662601</td>\n",
796 |        "      <td>1</td>\n",
797 |        "      <td>2</td>\n",
798 |        "    </tr>\n",
799 |        "    <tr>\n",
800 |        "      <th>141</th>\n",
801 |        "      <td>7748855</td>\n",
802 |        "      <td>1</td>\n",
803 |        "      <td>1</td>\n",
804 |        "    </tr>\n",
805 |        "    <tr>\n",
806 |        "      <th>142</th>\n",
807 |        "      <td>7758574</td>\n",
808 |        "      <td>1</td>\n",
809 |        "      <td>4</td>\n",
810 |        "    </tr>\n",
811 |        "    <tr>\n",
812 |        "      <th>143</th>\n",
813 |        "      <td>7831155</td>\n",
814 |        "      <td>1</td>\n",
815 |        "      <td>2</td>\n",
816 |        "    </tr>\n",
817 |        "    <tr>\n",
818 |        "      <th>144</th>\n",
819 |        "      <td>7860</td>\n",
820 |        "      <td>1</td>\n",
821 |        "      <td>1</td>\n",
822 |        "    </tr>\n",
823 |        "    <tr>\n",
824 |        "      <th>145</th>\n",
825 |        "      <td>7863</td>\n",
826 |        "      <td>1</td>\n",
827 |        "      <td>2</td>\n",
828 |        "    </tr>\n",
829 |        "    <tr>\n",
830 |        "      <th>146</th>\n",
831 |        "      <td>7987426</td>\n",
832 |        "      <td>1</td>\n",
833 |        "      <td>1</td>\n",
834 |        "    </tr>\n",
835 |        "  </tbody>\n",
836 |        "</table>\n",
837 |        "<p>147 rows × 3 columns</p>\n",
838 |        "</div>"
839 |       ],
840 |       "text/plain": [
841 |        "       ID_GA  id count  pageviews per id\n",
842 |        "0      10061         2                 8\n",
843 |        "1    1510256         1                 1\n",
844 |        "2     151553         1                 5\n",
845 |        "3       1742         1                 1\n",
846 |        "4      19186         1                28\n",
847 |        "5      19969         1                 2\n",
848 |        "6    2241980         1                 1\n",
849 |        "7      22672         1                 1\n",
850 |        "8    2483959         1                 1\n",
851 |        "9      26545         1                 1\n",
852 |        "10     26828         1                 1\n",
853 |        "11     30291         2                 2\n",
854 |        "12     30519         1                 1\n",
855 |        "13     30527         1                 2\n",
856 |        "14     30568         1                 1\n",
857 |        "15     30615         1                 1\n",
858 |        "16     30625         1                 2\n",
859 |        "17     30702         1                 2\n",
860 |        "18     30715         1                 3\n",
861 |        "19     30758         1                 1\n",
862 |        "20     30770         1                10\n",
863 |        "21     30787         1                60\n",
864 |        "22     30790         2                 8\n",
865 |        "23     30842         1                 2\n",
866 |        "24     30859         1                 2\n",
867 |        "25     30914         1                 1\n",
868 |        "26     30919         1                 1\n",
869 |        "27     30947         1                 5\n",
870 |        "28     30950         1                 1\n",
871 |        "29     30952         1                 1\n",
872 |        "..       ...       ...               ...\n",
873 |        "117    32168         1                 3\n",
874 |        "118    32177         1                 2\n",
875 |        "119    32179         1                 1\n",
876 |        "120    32188         1                12\n",
877 |        "121  4725725         1                 1\n",
878 |        "122  5049878         1                 1\n",
879 |        "123  5421492         1                 3\n",
880 |        "124  5539965         1                 4\n",
881 |        "125  5636781         2                 4\n",
882 |        "126  5697230         1                 1\n",
883 |        "127  5703102         1                 1\n",
884 |        "128     6181         1                 3\n",
885 |        "129     6243         1                 2\n",
886 |        "130     6255         1                 1\n",
887 |        "131     6256         1                 1\n",
888 |        "132     6260         1                 1\n",
889 |        "133     6271         1                 2\n",
890 |        "134     6323         1                 1\n",
891 |        "135     6328         1                 1\n",
892 |        "136  6448519         1                 5\n",
893 |        "137  6759728         1                 1\n",
894 |        "138  6923005         1                 1\n",
895 |        "139     6974         1                 1\n",
896 |        "140  7662601         1                 2\n",
897 |        "141  7748855         1                 1\n",
898 |        "142  7758574         1                 4\n",
899 |        "143  7831155         1                 2\n",
900 |        "144     7860         1                 1\n",
901 |        "145     7863         1                 2\n",
902 |        "146  7987426         1                 1\n",
903 |        "\n",
904 |        "[147 rows x 3 columns]"
905 |       ]
906 |      },
907 |      "execution_count": 59,
908 |      "metadata": {},
909 |      "output_type": "execute_result"
910 |     }
911 |    ],
912 |    "source": [
913 |     "dfTestCSV = dfTestCSV.rename(columns={\"count\": \"id count\", \"sum\": \"pageviews per id\"})\n",
914 |     "dfTestCSV"
915 |    ]
916 |   },
917 |   {
918 |    "cell_type": "code",
919 |    "execution_count": null,
920 |    "metadata": {},
921 |    "outputs": [],
922 |    "source": []
923 |   },
924 |   {
925 |    "cell_type": "code",
926 |    "execution_count": null,
927 |    "metadata": {},
928 |    "outputs": [],
929 |    "source": []
930 |   }
931 |  ],
932 |  "metadata": {
933 |   "kernelspec": {
934 |    "display_name": "Python 3",
935 |    "language": "python",
936 |    "name": "python3"
937 |   },
938 |   "language_info": {
939 |    "codemirror_mode": {
940 |     "name": "ipython",
941 |     "version": 3
942 |    },
943 |    "file_extension": ".py",
944 |    "mimetype": "text/x-python",
945 |    "name": "python",
946 |    "nbconvert_exporter": "python",
947 |    "pygments_lexer": "ipython3",
948 |    "version": "3.7.2"
949 |   }
950 |  },
951 |  "nbformat": 4,
952 |  "nbformat_minor": 2
953 | }
954 | 


--------------------------------------------------------------------------------
/#5 Read Excel in Python Pandas.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": null,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "# Check documentation: \n",
 10 |     "# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_excel.html\n",
 11 |     "#\n",
 12 |     "# Video about this notebook: \n",
 13 |     "# https://youtu.be/--zFZjSlfK8"
 14 |    ]
 15 |   },
 16 |   {
 17 |    "cell_type": "code",
 18 |    "execution_count": 10,
 19 |    "metadata": {},
 20 |    "outputs": [],
 21 |    "source": [
 22 |     "import pandas as pd"
 23 |    ]
 24 |   },
 25 |   {
 26 |    "cell_type": "code",
 27 |    "execution_count": 11,
 28 |    "metadata": {},
 29 |    "outputs": [
 30 |     {
 31 |      "data": {
 32 |       "text/html": [
 33 |        "<div>\n",
 34 |        "<style scoped>\n",
 35 |        "    .dataframe tbody tr th:only-of-type {\n",
 36 |        "        vertical-align: middle;\n",
 37 |        "    }\n",
 38 |        "\n",
 39 |        "    .dataframe tbody tr th {\n",
 40 |        "        vertical-align: top;\n",
 41 |        "    }\n",
 42 |        "\n",
 43 |        "    .dataframe thead th {\n",
 44 |        "        text-align: right;\n",
 45 |        "    }\n",
 46 |        "</style>\n",
 47 |        "<table border=\"1\" class=\"dataframe\">\n",
 48 |        "  <thead>\n",
 49 |        "    <tr style=\"text-align: right;\">\n",
 50 |        "      <th></th>\n",
 51 |        "      <th>Address</th>\n",
 52 |        "      <th>Content</th>\n",
 53 |        "      <th>Status Code</th>\n",
 54 |        "      <th>Status</th>\n",
 55 |        "      <th>Contains: '\"article-text-box\"'</th>\n",
 56 |        "      <th>Contains: '\"video-player\"'</th>\n",
 57 |        "      <th>Contains: '\"article-body'</th>\n",
 58 |        "      <th>Contains: '\\(sda\\)'</th>\n",
 59 |        "      <th>Contains: '\"scribble\"'</th>\n",
 60 |        "      <th>Contains: 'basic-image--free-desktop'</th>\n",
 61 |        "      <th>Contains: '&lt;iframe'</th>\n",
 62 |        "      <th>Contains: 'layout-item--flex-desktop-4'</th>\n",
 63 |        "      <th>Contains: '\\(sda\\/'</th>\n",
 64 |        "    </tr>\n",
 65 |        "  </thead>\n",
 66 |        "  <tbody>\n",
 67 |        "    <tr>\n",
 68 |        "      <th>0</th>\n",
 69 |        "      <td>https://www.blick.ch/people-tv/international/r...</td>\n",
 70 |        "      <td>text/html; charset=utf-8</td>\n",
 71 |        "      <td>200</td>\n",
 72 |        "      <td>Connection Refused</td>\n",
 73 |        "      <td>0</td>\n",
 74 |        "      <td>0</td>\n",
 75 |        "      <td>0</td>\n",
 76 |        "      <td>0</td>\n",
 77 |        "      <td>0</td>\n",
 78 |        "      <td>0</td>\n",
 79 |        "      <td>0</td>\n",
 80 |        "      <td>0</td>\n",
 81 |        "      <td>0</td>\n",
 82 |        "    </tr>\n",
 83 |        "    <tr>\n",
 84 |        "      <th>1</th>\n",
 85 |        "      <td>https://www.blick.ch/sport/eishockey/overtime-...</td>\n",
 86 |        "      <td>text/html; charset=utf-8</td>\n",
 87 |        "      <td>200</td>\n",
 88 |        "      <td>Connection Timeout</td>\n",
 89 |        "      <td>0</td>\n",
 90 |        "      <td>0</td>\n",
 91 |        "      <td>0</td>\n",
 92 |        "      <td>0</td>\n",
 93 |        "      <td>0</td>\n",
 94 |        "      <td>0</td>\n",
 95 |        "      <td>0</td>\n",
 96 |        "      <td>0</td>\n",
 97 |        "      <td>0</td>\n",
 98 |        "    </tr>\n",
 99 |        "    <tr>\n",
100 |        "      <th>2</th>\n",
101 |        "      <td>https://www.blick.ch/news/ausland/neben-berueh...</td>\n",
102 |        "      <td>text/html; charset=utf-8</td>\n",
103 |        "      <td>200</td>\n",
104 |        "      <td>Connection Timeout</td>\n",
105 |        "      <td>0</td>\n",
106 |        "      <td>0</td>\n",
107 |        "      <td>0</td>\n",
108 |        "      <td>0</td>\n",
109 |        "      <td>0</td>\n",
110 |        "      <td>0</td>\n",
111 |        "      <td>0</td>\n",
112 |        "      <td>0</td>\n",
113 |        "      <td>0</td>\n",
114 |        "    </tr>\n",
115 |        "    <tr>\n",
116 |        "      <th>3</th>\n",
117 |        "      <td>https://www.blick.ch/news/grausiger-fund-in-an...</td>\n",
118 |        "      <td>text/html; charset=utf-8</td>\n",
119 |        "      <td>200</td>\n",
120 |        "      <td>Connection Timeout</td>\n",
121 |        "      <td>0</td>\n",
122 |        "      <td>0</td>\n",
123 |        "      <td>0</td>\n",
124 |        "      <td>0</td>\n",
125 |        "      <td>0</td>\n",
126 |        "      <td>0</td>\n",
127 |        "      <td>0</td>\n",
128 |        "      <td>0</td>\n",
129 |        "      <td>0</td>\n",
130 |        "    </tr>\n",
131 |        "    <tr>\n",
132 |        "      <th>4</th>\n",
133 |        "      <td>https://www.blick.ch/news/mahamaya-bern-die-we...</td>\n",
134 |        "      <td>text/html; charset=utf-8</td>\n",
135 |        "      <td>200</td>\n",
136 |        "      <td>Connection Timeout</td>\n",
137 |        "      <td>0</td>\n",
138 |        "      <td>0</td>\n",
139 |        "      <td>0</td>\n",
140 |        "      <td>0</td>\n",
141 |        "      <td>0</td>\n",
142 |        "      <td>0</td>\n",
143 |        "      <td>0</td>\n",
144 |        "      <td>0</td>\n",
145 |        "      <td>0</td>\n",
146 |        "    </tr>\n",
147 |        "    <tr>\n",
148 |        "      <th>5</th>\n",
149 |        "      <td>https://www.blick.ch/people-tv/musik/liechtens...</td>\n",
150 |        "      <td>text/html; charset=utf-8</td>\n",
151 |        "      <td>200</td>\n",
152 |        "      <td>Connection Timeout</td>\n",
153 |        "      <td>0</td>\n",
154 |        "      <td>0</td>\n",
155 |        "      <td>0</td>\n",
156 |        "      <td>0</td>\n",
157 |        "      <td>0</td>\n",
158 |        "      <td>0</td>\n",
159 |        "      <td>0</td>\n",
160 |        "      <td>0</td>\n",
161 |        "      <td>0</td>\n",
162 |        "    </tr>\n",
163 |        "    <tr>\n",
164 |        "      <th>6</th>\n",
165 |        "      <td>https://www.blick.ch/people-tv/tv/bachelorette...</td>\n",
166 |        "      <td>text/html; charset=utf-8</td>\n",
167 |        "      <td>200</td>\n",
168 |        "      <td>Connection Timeout</td>\n",
169 |        "      <td>0</td>\n",
170 |        "      <td>0</td>\n",
171 |        "      <td>0</td>\n",
172 |        "      <td>0</td>\n",
173 |        "      <td>0</td>\n",
174 |        "      <td>0</td>\n",
175 |        "      <td>0</td>\n",
176 |        "      <td>0</td>\n",
177 |        "      <td>0</td>\n",
178 |        "    </tr>\n",
179 |        "    <tr>\n",
180 |        "      <th>7</th>\n",
181 |        "      <td>https://www.blick.ch/news/das-ist-die-swiss-bo...</td>\n",
182 |        "      <td>text/html; charset=utf-8</td>\n",
183 |        "      <td>200</td>\n",
184 |        "      <td>Connection Timeout</td>\n",
185 |        "      <td>0</td>\n",
186 |        "      <td>0</td>\n",
187 |        "      <td>0</td>\n",
188 |        "      <td>0</td>\n",
189 |        "      <td>0</td>\n",
190 |        "      <td>0</td>\n",
191 |        "      <td>0</td>\n",
192 |        "      <td>0</td>\n",
193 |        "      <td>0</td>\n",
194 |        "    </tr>\n",
195 |        "    <tr>\n",
196 |        "      <th>8</th>\n",
197 |        "      <td>https://www.blick.ch/news/schweiz/vergangenhei...</td>\n",
198 |        "      <td>text/html; charset=utf-8</td>\n",
199 |        "      <td>200</td>\n",
200 |        "      <td>Connection Timeout</td>\n",
201 |        "      <td>0</td>\n",
202 |        "      <td>0</td>\n",
203 |        "      <td>0</td>\n",
204 |        "      <td>0</td>\n",
205 |        "      <td>0</td>\n",
206 |        "      <td>0</td>\n",
207 |        "      <td>0</td>\n",
208 |        "      <td>0</td>\n",
209 |        "      <td>0</td>\n",
210 |        "    </tr>\n",
211 |        "    <tr>\n",
212 |        "      <th>9</th>\n",
213 |        "      <td>https://www.blick.ch/news/wirtschaft/das-sind-...</td>\n",
214 |        "      <td>text/html; charset=utf-8</td>\n",
215 |        "      <td>200</td>\n",
216 |        "      <td>Connection Timeout</td>\n",
217 |        "      <td>0</td>\n",
218 |        "      <td>0</td>\n",
219 |        "      <td>0</td>\n",
220 |        "      <td>0</td>\n",
221 |        "      <td>0</td>\n",
222 |        "      <td>0</td>\n",
223 |        "      <td>0</td>\n",
224 |        "      <td>0</td>\n",
225 |        "      <td>0</td>\n",
226 |        "    </tr>\n",
227 |        "  </tbody>\n",
228 |        "</table>\n",
229 |        "</div>"
230 |       ],
231 |       "text/plain": [
232 |        "                                             Address  \\\n",
233 |        "0  https://www.blick.ch/people-tv/international/r...   \n",
234 |        "1  https://www.blick.ch/sport/eishockey/overtime-...   \n",
235 |        "2  https://www.blick.ch/news/ausland/neben-berueh...   \n",
236 |        "3  https://www.blick.ch/news/grausiger-fund-in-an...   \n",
237 |        "4  https://www.blick.ch/news/mahamaya-bern-die-we...   \n",
238 |        "5  https://www.blick.ch/people-tv/musik/liechtens...   \n",
239 |        "6  https://www.blick.ch/people-tv/tv/bachelorette...   \n",
240 |        "7  https://www.blick.ch/news/das-ist-die-swiss-bo...   \n",
241 |        "8  https://www.blick.ch/news/schweiz/vergangenhei...   \n",
242 |        "9  https://www.blick.ch/news/wirtschaft/das-sind-...   \n",
243 |        "\n",
244 |        "                    Content  Status Code              Status  \\\n",
245 |        "0  text/html; charset=utf-8          200  Connection Refused   \n",
246 |        "1  text/html; charset=utf-8          200  Connection Timeout   \n",
247 |        "2  text/html; charset=utf-8          200  Connection Timeout   \n",
248 |        "3  text/html; charset=utf-8          200  Connection Timeout   \n",
249 |        "4  text/html; charset=utf-8          200  Connection Timeout   \n",
250 |        "5  text/html; charset=utf-8          200  Connection Timeout   \n",
251 |        "6  text/html; charset=utf-8          200  Connection Timeout   \n",
252 |        "7  text/html; charset=utf-8          200  Connection Timeout   \n",
253 |        "8  text/html; charset=utf-8          200  Connection Timeout   \n",
254 |        "9  text/html; charset=utf-8          200  Connection Timeout   \n",
255 |        "\n",
256 |        "   Contains: '\"article-text-box\"'  Contains: '\"video-player\"'  \\\n",
257 |        "0                               0                           0   \n",
258 |        "1                               0                           0   \n",
259 |        "2                               0                           0   \n",
260 |        "3                               0                           0   \n",
261 |        "4                               0                           0   \n",
262 |        "5                               0                           0   \n",
263 |        "6                               0                           0   \n",
264 |        "7                               0                           0   \n",
265 |        "8                               0                           0   \n",
266 |        "9                               0                           0   \n",
267 |        "\n",
268 |        "   Contains: '\"article-body'  Contains: '\\(sda\\)'  Contains: '\"scribble\"'  \\\n",
269 |        "0                          0                    0                       0   \n",
270 |        "1                          0                    0                       0   \n",
271 |        "2                          0                    0                       0   \n",
272 |        "3                          0                    0                       0   \n",
273 |        "4                          0                    0                       0   \n",
274 |        "5                          0                    0                       0   \n",
275 |        "6                          0                    0                       0   \n",
276 |        "7                          0                    0                       0   \n",
277 |        "8                          0                    0                       0   \n",
278 |        "9                          0                    0                       0   \n",
279 |        "\n",
280 |        "   Contains: 'basic-image--free-desktop'  Contains: '<iframe'  \\\n",
281 |        "0                                      0                    0   \n",
282 |        "1                                      0                    0   \n",
283 |        "2                                      0                    0   \n",
284 |        "3                                      0                    0   \n",
285 |        "4                                      0                    0   \n",
286 |        "5                                      0                    0   \n",
287 |        "6                                      0                    0   \n",
288 |        "7                                      0                    0   \n",
289 |        "8                                      0                    0   \n",
290 |        "9                                      0                    0   \n",
291 |        "\n",
292 |        "   Contains: 'layout-item--flex-desktop-4'  Contains: '\\(sda\\/'  \n",
293 |        "0                                        0                    0  \n",
294 |        "1                                        0                    0  \n",
295 |        "2                                        0                    0  \n",
296 |        "3                                        0                    0  \n",
297 |        "4                                        0                    0  \n",
298 |        "5                                        0                    0  \n",
299 |        "6                                        0                    0  \n",
300 |        "7                                        0                    0  \n",
301 |        "8                                        0                    0  \n",
302 |        "9                                        0                    0  "
303 |       ]
304 |      },
305 |      "execution_count": 11,
306 |      "metadata": {},
307 |      "output_type": "execute_result"
308 |     }
309 |    ],
310 |    "source": [
311 |     "dfCrawled = pd.DataFrame(pd.read_excel('./custom_search_all-test.xlsx', header=0, sheet_name='1 - All'))\n",
312 |     "dfCrawled.head(10)"
313 |    ]
314 |   },
315 |   {
316 |    "cell_type": "code",
317 |    "execution_count": 12,
318 |    "metadata": {},
319 |    "outputs": [
320 |     {
321 |      "data": {
322 |       "text/html": [
323 |        "<div>\n",
324 |        "<style scoped>\n",
325 |        "    .dataframe tbody tr th:only-of-type {\n",
326 |        "        vertical-align: middle;\n",
327 |        "    }\n",
328 |        "\n",
329 |        "    .dataframe tbody tr th {\n",
330 |        "        vertical-align: top;\n",
331 |        "    }\n",
332 |        "\n",
333 |        "    .dataframe thead th {\n",
334 |        "        text-align: right;\n",
335 |        "    }\n",
336 |        "</style>\n",
337 |        "<table border=\"1\" class=\"dataframe\">\n",
338 |        "  <thead>\n",
339 |        "    <tr style=\"text-align: right;\">\n",
340 |        "      <th></th>\n",
341 |        "      <th>Address</th>\n",
342 |        "      <th>Content</th>\n",
343 |        "      <th>Status Code</th>\n",
344 |        "      <th>Status</th>\n",
345 |        "      <th>article-text-box</th>\n",
346 |        "      <th>video-player</th>\n",
347 |        "      <th>article-body</th>\n",
348 |        "      <th>sda</th>\n",
349 |        "      <th>scribble</th>\n",
350 |        "      <th>has-image</th>\n",
351 |        "      <th>iframe</th>\n",
352 |        "      <th>normal-article-layout</th>\n",
353 |        "      <th>sda2</th>\n",
354 |        "    </tr>\n",
355 |        "  </thead>\n",
356 |        "  <tbody>\n",
357 |        "    <tr>\n",
358 |        "      <th>0</th>\n",
359 |        "      <td>https://www.blick.ch/people-tv/international/r...</td>\n",
360 |        "      <td>text/html; charset=utf-8</td>\n",
361 |        "      <td>200</td>\n",
362 |        "      <td>Connection Refused</td>\n",
363 |        "      <td>0</td>\n",
364 |        "      <td>0</td>\n",
365 |        "      <td>0</td>\n",
366 |        "      <td>0</td>\n",
367 |        "      <td>0</td>\n",
368 |        "      <td>0</td>\n",
369 |        "      <td>0</td>\n",
370 |        "      <td>0</td>\n",
371 |        "      <td>0</td>\n",
372 |        "    </tr>\n",
373 |        "    <tr>\n",
374 |        "      <th>1</th>\n",
375 |        "      <td>https://www.blick.ch/sport/eishockey/overtime-...</td>\n",
376 |        "      <td>text/html; charset=utf-8</td>\n",
377 |        "      <td>200</td>\n",
378 |        "      <td>Connection Timeout</td>\n",
379 |        "      <td>0</td>\n",
380 |        "      <td>0</td>\n",
381 |        "      <td>0</td>\n",
382 |        "      <td>0</td>\n",
383 |        "      <td>0</td>\n",
384 |        "      <td>0</td>\n",
385 |        "      <td>0</td>\n",
386 |        "      <td>0</td>\n",
387 |        "      <td>0</td>\n",
388 |        "    </tr>\n",
389 |        "    <tr>\n",
390 |        "      <th>2</th>\n",
391 |        "      <td>https://www.blick.ch/news/ausland/neben-berueh...</td>\n",
392 |        "      <td>text/html; charset=utf-8</td>\n",
393 |        "      <td>200</td>\n",
394 |        "      <td>Connection Timeout</td>\n",
395 |        "      <td>0</td>\n",
396 |        "      <td>0</td>\n",
397 |        "      <td>0</td>\n",
398 |        "      <td>0</td>\n",
399 |        "      <td>0</td>\n",
400 |        "      <td>0</td>\n",
401 |        "      <td>0</td>\n",
402 |        "      <td>0</td>\n",
403 |        "      <td>0</td>\n",
404 |        "    </tr>\n",
405 |        "    <tr>\n",
406 |        "      <th>3</th>\n",
407 |        "      <td>https://www.blick.ch/news/grausiger-fund-in-an...</td>\n",
408 |        "      <td>text/html; charset=utf-8</td>\n",
409 |        "      <td>200</td>\n",
410 |        "      <td>Connection Timeout</td>\n",
411 |        "      <td>0</td>\n",
412 |        "      <td>0</td>\n",
413 |        "      <td>0</td>\n",
414 |        "      <td>0</td>\n",
415 |        "      <td>0</td>\n",
416 |        "      <td>0</td>\n",
417 |        "      <td>0</td>\n",
418 |        "      <td>0</td>\n",
419 |        "      <td>0</td>\n",
420 |        "    </tr>\n",
421 |        "    <tr>\n",
422 |        "      <th>4</th>\n",
423 |        "      <td>https://www.blick.ch/news/mahamaya-bern-die-we...</td>\n",
424 |        "      <td>text/html; charset=utf-8</td>\n",
425 |        "      <td>200</td>\n",
426 |        "      <td>Connection Timeout</td>\n",
427 |        "      <td>0</td>\n",
428 |        "      <td>0</td>\n",
429 |        "      <td>0</td>\n",
430 |        "      <td>0</td>\n",
431 |        "      <td>0</td>\n",
432 |        "      <td>0</td>\n",
433 |        "      <td>0</td>\n",
434 |        "      <td>0</td>\n",
435 |        "      <td>0</td>\n",
436 |        "    </tr>\n",
437 |        "    <tr>\n",
438 |        "      <th>5</th>\n",
439 |        "      <td>https://www.blick.ch/people-tv/musik/liechtens...</td>\n",
440 |        "      <td>text/html; charset=utf-8</td>\n",
441 |        "      <td>200</td>\n",
442 |        "      <td>Connection Timeout</td>\n",
443 |        "      <td>0</td>\n",
444 |        "      <td>0</td>\n",
445 |        "      <td>0</td>\n",
446 |        "      <td>0</td>\n",
447 |        "      <td>0</td>\n",
448 |        "      <td>0</td>\n",
449 |        "      <td>0</td>\n",
450 |        "      <td>0</td>\n",
451 |        "      <td>0</td>\n",
452 |        "    </tr>\n",
453 |        "    <tr>\n",
454 |        "      <th>6</th>\n",
455 |        "      <td>https://www.blick.ch/people-tv/tv/bachelorette...</td>\n",
456 |        "      <td>text/html; charset=utf-8</td>\n",
457 |        "      <td>200</td>\n",
458 |        "      <td>Connection Timeout</td>\n",
459 |        "      <td>0</td>\n",
460 |        "      <td>0</td>\n",
461 |        "      <td>0</td>\n",
462 |        "      <td>0</td>\n",
463 |        "      <td>0</td>\n",
464 |        "      <td>0</td>\n",
465 |        "      <td>0</td>\n",
466 |        "      <td>0</td>\n",
467 |        "      <td>0</td>\n",
468 |        "    </tr>\n",
469 |        "    <tr>\n",
470 |        "      <th>7</th>\n",
471 |        "      <td>https://www.blick.ch/news/das-ist-die-swiss-bo...</td>\n",
472 |        "      <td>text/html; charset=utf-8</td>\n",
473 |        "      <td>200</td>\n",
474 |        "      <td>Connection Timeout</td>\n",
475 |        "      <td>0</td>\n",
476 |        "      <td>0</td>\n",
477 |        "      <td>0</td>\n",
478 |        "      <td>0</td>\n",
479 |        "      <td>0</td>\n",
480 |        "      <td>0</td>\n",
481 |        "      <td>0</td>\n",
482 |        "      <td>0</td>\n",
483 |        "      <td>0</td>\n",
484 |        "    </tr>\n",
485 |        "    <tr>\n",
486 |        "      <th>8</th>\n",
487 |        "      <td>https://www.blick.ch/news/schweiz/vergangenhei...</td>\n",
488 |        "      <td>text/html; charset=utf-8</td>\n",
489 |        "      <td>200</td>\n",
490 |        "      <td>Connection Timeout</td>\n",
491 |        "      <td>0</td>\n",
492 |        "      <td>0</td>\n",
493 |        "      <td>0</td>\n",
494 |        "      <td>0</td>\n",
495 |        "      <td>0</td>\n",
496 |        "      <td>0</td>\n",
497 |        "      <td>0</td>\n",
498 |        "      <td>0</td>\n",
499 |        "      <td>0</td>\n",
500 |        "    </tr>\n",
501 |        "    <tr>\n",
502 |        "      <th>9</th>\n",
503 |        "      <td>https://www.blick.ch/news/wirtschaft/das-sind-...</td>\n",
504 |        "      <td>text/html; charset=utf-8</td>\n",
505 |        "      <td>200</td>\n",
506 |        "      <td>Connection Timeout</td>\n",
507 |        "      <td>0</td>\n",
508 |        "      <td>0</td>\n",
509 |        "      <td>0</td>\n",
510 |        "      <td>0</td>\n",
511 |        "      <td>0</td>\n",
512 |        "      <td>0</td>\n",
513 |        "      <td>0</td>\n",
514 |        "      <td>0</td>\n",
515 |        "      <td>0</td>\n",
516 |        "    </tr>\n",
517 |        "  </tbody>\n",
518 |        "</table>\n",
519 |        "</div>"
520 |       ],
521 |       "text/plain": [
522 |        "                                             Address  \\\n",
523 |        "0  https://www.blick.ch/people-tv/international/r...   \n",
524 |        "1  https://www.blick.ch/sport/eishockey/overtime-...   \n",
525 |        "2  https://www.blick.ch/news/ausland/neben-berueh...   \n",
526 |        "3  https://www.blick.ch/news/grausiger-fund-in-an...   \n",
527 |        "4  https://www.blick.ch/news/mahamaya-bern-die-we...   \n",
528 |        "5  https://www.blick.ch/people-tv/musik/liechtens...   \n",
529 |        "6  https://www.blick.ch/people-tv/tv/bachelorette...   \n",
530 |        "7  https://www.blick.ch/news/das-ist-die-swiss-bo...   \n",
531 |        "8  https://www.blick.ch/news/schweiz/vergangenhei...   \n",
532 |        "9  https://www.blick.ch/news/wirtschaft/das-sind-...   \n",
533 |        "\n",
534 |        "                    Content  Status Code              Status  \\\n",
535 |        "0  text/html; charset=utf-8          200  Connection Refused   \n",
536 |        "1  text/html; charset=utf-8          200  Connection Timeout   \n",
537 |        "2  text/html; charset=utf-8          200  Connection Timeout   \n",
538 |        "3  text/html; charset=utf-8          200  Connection Timeout   \n",
539 |        "4  text/html; charset=utf-8          200  Connection Timeout   \n",
540 |        "5  text/html; charset=utf-8          200  Connection Timeout   \n",
541 |        "6  text/html; charset=utf-8          200  Connection Timeout   \n",
542 |        "7  text/html; charset=utf-8          200  Connection Timeout   \n",
543 |        "8  text/html; charset=utf-8          200  Connection Timeout   \n",
544 |        "9  text/html; charset=utf-8          200  Connection Timeout   \n",
545 |        "\n",
546 |        "   article-text-box  video-player  article-body  sda  scribble  has-image  \\\n",
547 |        "0                 0             0             0    0         0          0   \n",
548 |        "1                 0             0             0    0         0          0   \n",
549 |        "2                 0             0             0    0         0          0   \n",
550 |        "3                 0             0             0    0         0          0   \n",
551 |        "4                 0             0             0    0         0          0   \n",
552 |        "5                 0             0             0    0         0          0   \n",
553 |        "6                 0             0             0    0         0          0   \n",
554 |        "7                 0             0             0    0         0          0   \n",
555 |        "8                 0             0             0    0         0          0   \n",
556 |        "9                 0             0             0    0         0          0   \n",
557 |        "\n",
558 |        "   iframe  normal-article-layout  sda2  \n",
559 |        "0       0                      0     0  \n",
560 |        "1       0                      0     0  \n",
561 |        "2       0                      0     0  \n",
562 |        "3       0                      0     0  \n",
563 |        "4       0                      0     0  \n",
564 |        "5       0                      0     0  \n",
565 |        "6       0                      0     0  \n",
566 |        "7       0                      0     0  \n",
567 |        "8       0                      0     0  \n",
568 |        "9       0                      0     0  "
569 |       ]
570 |      },
571 |      "execution_count": 12,
572 |      "metadata": {},
573 |      "output_type": "execute_result"
574 |     }
575 |    ],
576 |    "source": [
577 |     "#Rename columns\n",
578 |     "dfCrawled = dfCrawled.rename(columns={\"Contains: \\'\\\"article-text-box\\\"\\'\": \"article-text-box\", \n",
579 |     "                                         \"Contains: \\'\\\"video-player\\\"\\'\":\"video-player\", \n",
580 |     "                                         \"Contains: \\'\\\"article-body\\'\":\"article-body\",\n",
581 |     "                                         \"Contains: \\'\\(sda\\)\\'\":\"sda\",\n",
582 |     "                                         \"Contains: \\'\\\"scribble\\\"\\'\":\"scribble\",\n",
583 |     "                                         \"Contains: \\'basic-image--free-desktop\\'\":\"has-image\", \n",
584 |     "                                         \"Contains: \\'<iframe\\'\":\"iframe\", \n",
585 |     "                                         \"Contains: \\'layout-item--flex-desktop-4'\":\"normal-article-layout\", \n",
586 |     "                                         \"Contains: \\'\\(sda\\/\\'\":\"sda2\", \n",
587 |     "                                     })\n",
588 |     "dfCrawled.head(10)"
589 |    ]
590 |   },
591 |   {
592 |    "cell_type": "code",
593 |    "execution_count": 13,
594 |    "metadata": {},
595 |    "outputs": [
596 |     {
597 |      "data": {
598 |       "text/html": [
599 |        "<div>\n",
600 |        "<style scoped>\n",
601 |        "    .dataframe tbody tr th:only-of-type {\n",
602 |        "        vertical-align: middle;\n",
603 |        "    }\n",
604 |        "\n",
605 |        "    .dataframe tbody tr th {\n",
606 |        "        vertical-align: top;\n",
607 |        "    }\n",
608 |        "\n",
609 |        "    .dataframe thead th {\n",
610 |        "        text-align: right;\n",
611 |        "    }\n",
612 |        "</style>\n",
613 |        "<table border=\"1\" class=\"dataframe\">\n",
614 |        "  <thead>\n",
615 |        "    <tr style=\"text-align: right;\">\n",
616 |        "      <th></th>\n",
617 |        "      <th>Address</th>\n",
618 |        "      <th>article-text-box</th>\n",
619 |        "      <th>video-player</th>\n",
620 |        "      <th>article-body</th>\n",
621 |        "      <th>sda</th>\n",
622 |        "      <th>scribble</th>\n",
623 |        "      <th>has-image</th>\n",
624 |        "      <th>iframe</th>\n",
625 |        "      <th>normal-article-layout</th>\n",
626 |        "      <th>sda2</th>\n",
627 |        "      <th>ID_Crawl</th>\n",
628 |        "    </tr>\n",
629 |        "  </thead>\n",
630 |        "  <tbody>\n",
631 |        "    <tr>\n",
632 |        "      <th>0</th>\n",
633 |        "      <td>https://www.blick.ch/people-tv/international/r...</td>\n",
634 |        "      <td>0</td>\n",
635 |        "      <td>0</td>\n",
636 |        "      <td>0</td>\n",
637 |        "      <td>0</td>\n",
638 |        "      <td>0</td>\n",
639 |        "      <td>0</td>\n",
640 |        "      <td>0</td>\n",
641 |        "      <td>0</td>\n",
642 |        "      <td>0</td>\n",
643 |        "      <td>4270807</td>\n",
644 |        "    </tr>\n",
645 |        "    <tr>\n",
646 |        "      <th>1</th>\n",
647 |        "      <td>https://www.blick.ch/sport/eishockey/overtime-...</td>\n",
648 |        "      <td>0</td>\n",
649 |        "      <td>0</td>\n",
650 |        "      <td>0</td>\n",
651 |        "      <td>0</td>\n",
652 |        "      <td>0</td>\n",
653 |        "      <td>0</td>\n",
654 |        "      <td>0</td>\n",
655 |        "      <td>0</td>\n",
656 |        "      <td>0</td>\n",
657 |        "      <td>106244</td>\n",
658 |        "    </tr>\n",
659 |        "    <tr>\n",
660 |        "      <th>2</th>\n",
661 |        "      <td>https://www.blick.ch/news/ausland/neben-berueh...</td>\n",
662 |        "      <td>0</td>\n",
663 |        "      <td>0</td>\n",
664 |        "      <td>0</td>\n",
665 |        "      <td>0</td>\n",
666 |        "      <td>0</td>\n",
667 |        "      <td>0</td>\n",
668 |        "      <td>0</td>\n",
669 |        "      <td>0</td>\n",
670 |        "      <td>0</td>\n",
671 |        "      <td>3896239</td>\n",
672 |        "    </tr>\n",
673 |        "    <tr>\n",
674 |        "      <th>3</th>\n",
675 |        "      <td>https://www.blick.ch/news/grausiger-fund-in-an...</td>\n",
676 |        "      <td>0</td>\n",
677 |        "      <td>0</td>\n",
678 |        "      <td>0</td>\n",
679 |        "      <td>0</td>\n",
680 |        "      <td>0</td>\n",
681 |        "      <td>0</td>\n",
682 |        "      <td>0</td>\n",
683 |        "      <td>0</td>\n",
684 |        "      <td>0</td>\n",
685 |        "      <td>2024473</td>\n",
686 |        "    </tr>\n",
687 |        "    <tr>\n",
688 |        "      <th>4</th>\n",
689 |        "      <td>https://www.blick.ch/news/mahamaya-bern-die-we...</td>\n",
690 |        "      <td>0</td>\n",
691 |        "      <td>0</td>\n",
692 |        "      <td>0</td>\n",
693 |        "      <td>0</td>\n",
694 |        "      <td>0</td>\n",
695 |        "      <td>0</td>\n",
696 |        "      <td>0</td>\n",
697 |        "      <td>0</td>\n",
698 |        "      <td>0</td>\n",
699 |        "      <td>110094</td>\n",
700 |        "    </tr>\n",
701 |        "    <tr>\n",
702 |        "      <th>5</th>\n",
703 |        "      <td>https://www.blick.ch/people-tv/musik/liechtens...</td>\n",
704 |        "      <td>0</td>\n",
705 |        "      <td>0</td>\n",
706 |        "      <td>0</td>\n",
707 |        "      <td>0</td>\n",
708 |        "      <td>0</td>\n",
709 |        "      <td>0</td>\n",
710 |        "      <td>0</td>\n",
711 |        "      <td>0</td>\n",
712 |        "      <td>0</td>\n",
713 |        "      <td>2131248</td>\n",
714 |        "    </tr>\n",
715 |        "    <tr>\n",
716 |        "      <th>6</th>\n",
717 |        "      <td>https://www.blick.ch/people-tv/tv/bachelorette...</td>\n",
718 |        "      <td>0</td>\n",
719 |        "      <td>0</td>\n",
720 |        "      <td>0</td>\n",
721 |        "      <td>0</td>\n",
722 |        "      <td>0</td>\n",
723 |        "      <td>0</td>\n",
724 |        "      <td>0</td>\n",
725 |        "      <td>0</td>\n",
726 |        "      <td>0</td>\n",
727 |        "      <td>3790667</td>\n",
728 |        "    </tr>\n",
729 |        "    <tr>\n",
730 |        "      <th>7</th>\n",
731 |        "      <td>https://www.blick.ch/news/das-ist-die-swiss-bo...</td>\n",
732 |        "      <td>0</td>\n",
733 |        "      <td>0</td>\n",
734 |        "      <td>0</td>\n",
735 |        "      <td>0</td>\n",
736 |        "      <td>0</td>\n",
737 |        "      <td>0</td>\n",
738 |        "      <td>0</td>\n",
739 |        "      <td>0</td>\n",
740 |        "      <td>0</td>\n",
741 |        "      <td>2238005</td>\n",
742 |        "    </tr>\n",
743 |        "    <tr>\n",
744 |        "      <th>8</th>\n",
745 |        "      <td>https://www.blick.ch/news/schweiz/vergangenhei...</td>\n",
746 |        "      <td>0</td>\n",
747 |        "      <td>0</td>\n",
748 |        "      <td>0</td>\n",
749 |        "      <td>0</td>\n",
750 |        "      <td>0</td>\n",
751 |        "      <td>0</td>\n",
752 |        "      <td>0</td>\n",
753 |        "      <td>0</td>\n",
754 |        "      <td>0</td>\n",
755 |        "      <td>5550392</td>\n",
756 |        "    </tr>\n",
757 |        "    <tr>\n",
758 |        "      <th>9</th>\n",
759 |        "      <td>https://www.blick.ch/news/wirtschaft/das-sind-...</td>\n",
760 |        "      <td>0</td>\n",
761 |        "      <td>0</td>\n",
762 |        "      <td>0</td>\n",
763 |        "      <td>0</td>\n",
764 |        "      <td>0</td>\n",
765 |        "      <td>0</td>\n",
766 |        "      <td>0</td>\n",
767 |        "      <td>0</td>\n",
768 |        "      <td>0</td>\n",
769 |        "      <td>4955860</td>\n",
770 |        "    </tr>\n",
771 |        "  </tbody>\n",
772 |        "</table>\n",
773 |        "</div>"
774 |       ],
775 |       "text/plain": [
776 |        "                                             Address  article-text-box  \\\n",
777 |        "0  https://www.blick.ch/people-tv/international/r...                 0   \n",
778 |        "1  https://www.blick.ch/sport/eishockey/overtime-...                 0   \n",
779 |        "2  https://www.blick.ch/news/ausland/neben-berueh...                 0   \n",
780 |        "3  https://www.blick.ch/news/grausiger-fund-in-an...                 0   \n",
781 |        "4  https://www.blick.ch/news/mahamaya-bern-die-we...                 0   \n",
782 |        "5  https://www.blick.ch/people-tv/musik/liechtens...                 0   \n",
783 |        "6  https://www.blick.ch/people-tv/tv/bachelorette...                 0   \n",
784 |        "7  https://www.blick.ch/news/das-ist-die-swiss-bo...                 0   \n",
785 |        "8  https://www.blick.ch/news/schweiz/vergangenhei...                 0   \n",
786 |        "9  https://www.blick.ch/news/wirtschaft/das-sind-...                 0   \n",
787 |        "\n",
788 |        "   video-player  article-body  sda  scribble  has-image  iframe  \\\n",
789 |        "0             0             0    0         0          0       0   \n",
790 |        "1             0             0    0         0          0       0   \n",
791 |        "2             0             0    0         0          0       0   \n",
792 |        "3             0             0    0         0          0       0   \n",
793 |        "4             0             0    0         0          0       0   \n",
794 |        "5             0             0    0         0          0       0   \n",
795 |        "6             0             0    0         0          0       0   \n",
796 |        "7             0             0    0         0          0       0   \n",
797 |        "8             0             0    0         0          0       0   \n",
798 |        "9             0             0    0         0          0       0   \n",
799 |        "\n",
800 |        "   normal-article-layout  sda2 ID_Crawl  \n",
801 |        "0                      0     0  4270807  \n",
802 |        "1                      0     0   106244  \n",
803 |        "2                      0     0  3896239  \n",
804 |        "3                      0     0  2024473  \n",
805 |        "4                      0     0   110094  \n",
806 |        "5                      0     0  2131248  \n",
807 |        "6                      0     0  3790667  \n",
808 |        "7                      0     0  2238005  \n",
809 |        "8                      0     0  5550392  \n",
810 |        "9                      0     0  4955860  "
811 |       ]
812 |      },
813 |      "execution_count": 13,
814 |      "metadata": {},
815 |      "output_type": "execute_result"
816 |     }
817 |    ],
818 |    "source": [
819 |     "dfCrawled['ID_Crawl'] = dfCrawled['Address'].str.extract('(?:.*id)([0-9]+)(?:.html)', expand=False)\n",
820 |     "dfCrawled = dfCrawled.drop(\"Status Code\",1)\n",
821 |     "dfCrawled = dfCrawled.drop(\"Status\",1)\n",
822 |     "dfCrawled = dfCrawled.drop(\"Content\",1)\n",
823 |     "dfCrawled.head(10)"
824 |    ]
825 |   },
826 |   {
827 |    "cell_type": "code",
828 |    "execution_count": null,
829 |    "metadata": {},
830 |    "outputs": [],
831 |    "source": []
832 |   },
833 |   {
834 |    "cell_type": "code",
835 |    "execution_count": null,
836 |    "metadata": {},
837 |    "outputs": [],
838 |    "source": []
839 |   },
840 |   {
841 |    "cell_type": "code",
842 |    "execution_count": null,
843 |    "metadata": {},
844 |    "outputs": [],
845 |    "source": []
846 |   },
847 |   {
848 |    "cell_type": "code",
849 |    "execution_count": null,
850 |    "metadata": {},
851 |    "outputs": [],
852 |    "source": []
853 |   },
854 |   {
855 |    "cell_type": "code",
856 |    "execution_count": null,
857 |    "metadata": {},
858 |    "outputs": [],
859 |    "source": []
860 |   },
861 |   {
862 |    "cell_type": "code",
863 |    "execution_count": null,
864 |    "metadata": {},
865 |    "outputs": [],
866 |    "source": []
867 |   },
868 |   {
869 |    "cell_type": "code",
870 |    "execution_count": null,
871 |    "metadata": {},
872 |    "outputs": [],
873 |    "source": []
874 |   }
875 |  ],
876 |  "metadata": {
877 |   "kernelspec": {
878 |    "display_name": "Python 3",
879 |    "language": "python",
880 |    "name": "python3"
881 |   },
882 |   "language_info": {
883 |    "codemirror_mode": {
884 |     "name": "ipython",
885 |     "version": 3
886 |    },
887 |    "file_extension": ".py",
888 |    "mimetype": "text/x-python",
889 |    "name": "python",
890 |    "nbconvert_exporter": "python",
891 |    "pygments_lexer": "ipython3",
892 |    "version": "3.7.2"
893 |   }
894 |  },
895 |  "nbformat": 4,
896 |  "nbformat_minor": 2
897 | }
898 | 


--------------------------------------------------------------------------------
/#6 Merge two tables in Python Pandas.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": null,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "# Check documentation: \n",
 10 |     "# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.merge.html\n",
 11 |     "#\n",
 12 |     "# Video about this notebook: \n",
 13 |     "# https://youtu.be/l9S6sWCykDo"
 14 |    ]
 15 |   },
 16 |   {
 17 |    "cell_type": "code",
 18 |    "execution_count": 24,
 19 |    "metadata": {},
 20 |    "outputs": [],
 21 |    "source": [
 22 |     "import pandas as pd"
 23 |    ]
 24 |   },
 25 |   {
 26 |    "cell_type": "code",
 27 |    "execution_count": 25,
 28 |    "metadata": {},
 29 |    "outputs": [],
 30 |    "source": [
 31 |     "dfTestCSV = pd.DataFrame(pd.read_csv('./dfanalyticstestdata1.csv', header=2, sep=\"\\t\", index_col=0))"
 32 |    ]
 33 |   },
 34 |   {
 35 |    "cell_type": "code",
 36 |    "execution_count": 26,
 37 |    "metadata": {},
 38 |    "outputs": [],
 39 |    "source": [
 40 |     "dfTestCSV['ID_GA'] = dfTestCSV['ga:pagePath'].str.extract('(?:.*id)([0-9]+)(?:.html)')"
 41 |    ]
 42 |   },
 43 |   {
 44 |    "cell_type": "code",
 45 |    "execution_count": 27,
 46 |    "metadata": {},
 47 |    "outputs": [
 48 |     {
 49 |      "data": {
 50 |       "text/html": [
 51 |        "<div>\n",
 52 |        "<style scoped>\n",
 53 |        "    .dataframe tbody tr th:only-of-type {\n",
 54 |        "        vertical-align: middle;\n",
 55 |        "    }\n",
 56 |        "\n",
 57 |        "    .dataframe tbody tr th {\n",
 58 |        "        vertical-align: top;\n",
 59 |        "    }\n",
 60 |        "\n",
 61 |        "    .dataframe thead th {\n",
 62 |        "        text-align: right;\n",
 63 |        "    }\n",
 64 |        "</style>\n",
 65 |        "<table border=\"1\" class=\"dataframe\">\n",
 66 |        "  <thead>\n",
 67 |        "    <tr style=\"text-align: right;\">\n",
 68 |        "      <th></th>\n",
 69 |        "      <th>ID_GA</th>\n",
 70 |        "      <th>id count</th>\n",
 71 |        "      <th>pageviews per id</th>\n",
 72 |        "    </tr>\n",
 73 |        "  </thead>\n",
 74 |        "  <tbody>\n",
 75 |        "    <tr>\n",
 76 |        "      <th>0</th>\n",
 77 |        "      <td>10061</td>\n",
 78 |        "      <td>2</td>\n",
 79 |        "      <td>8</td>\n",
 80 |        "    </tr>\n",
 81 |        "    <tr>\n",
 82 |        "      <th>1</th>\n",
 83 |        "      <td>1510256</td>\n",
 84 |        "      <td>1</td>\n",
 85 |        "      <td>1</td>\n",
 86 |        "    </tr>\n",
 87 |        "    <tr>\n",
 88 |        "      <th>2</th>\n",
 89 |        "      <td>151553</td>\n",
 90 |        "      <td>1</td>\n",
 91 |        "      <td>5</td>\n",
 92 |        "    </tr>\n",
 93 |        "    <tr>\n",
 94 |        "      <th>3</th>\n",
 95 |        "      <td>1742</td>\n",
 96 |        "      <td>1</td>\n",
 97 |        "      <td>1</td>\n",
 98 |        "    </tr>\n",
 99 |        "    <tr>\n",
100 |        "      <th>4</th>\n",
101 |        "      <td>19186</td>\n",
102 |        "      <td>1</td>\n",
103 |        "      <td>28</td>\n",
104 |        "    </tr>\n",
105 |        "    <tr>\n",
106 |        "      <th>5</th>\n",
107 |        "      <td>19969</td>\n",
108 |        "      <td>1</td>\n",
109 |        "      <td>2</td>\n",
110 |        "    </tr>\n",
111 |        "    <tr>\n",
112 |        "      <th>6</th>\n",
113 |        "      <td>2241980</td>\n",
114 |        "      <td>1</td>\n",
115 |        "      <td>1</td>\n",
116 |        "    </tr>\n",
117 |        "    <tr>\n",
118 |        "      <th>7</th>\n",
119 |        "      <td>22672</td>\n",
120 |        "      <td>1</td>\n",
121 |        "      <td>1</td>\n",
122 |        "    </tr>\n",
123 |        "    <tr>\n",
124 |        "      <th>8</th>\n",
125 |        "      <td>2483959</td>\n",
126 |        "      <td>1</td>\n",
127 |        "      <td>1</td>\n",
128 |        "    </tr>\n",
129 |        "    <tr>\n",
130 |        "      <th>9</th>\n",
131 |        "      <td>26545</td>\n",
132 |        "      <td>1</td>\n",
133 |        "      <td>1</td>\n",
134 |        "    </tr>\n",
135 |        "  </tbody>\n",
136 |        "</table>\n",
137 |        "</div>"
138 |       ],
139 |       "text/plain": [
140 |        "     ID_GA  id count  pageviews per id\n",
141 |        "0    10061         2                 8\n",
142 |        "1  1510256         1                 1\n",
143 |        "2   151553         1                 5\n",
144 |        "3     1742         1                 1\n",
145 |        "4    19186         1                28\n",
146 |        "5    19969         1                 2\n",
147 |        "6  2241980         1                 1\n",
148 |        "7    22672         1                 1\n",
149 |        "8  2483959         1                 1\n",
150 |        "9    26545         1                 1"
151 |       ]
152 |      },
153 |      "execution_count": 27,
154 |      "metadata": {},
155 |      "output_type": "execute_result"
156 |     }
157 |    ],
158 |    "source": [
159 |     "dfTestCSV = pd.pivot_table(dfTestCSV,index='ID_GA', values='ga:pageviews', aggfunc=['count', 'sum'])\n",
160 |     "dfTestCSV.reset_index(inplace=True)\n",
161 |     "dfTestCSV.columns = dfTestCSV.columns.droplevel(1)\n",
162 |     "dfTestCSV = dfTestCSV.rename(columns={\"count\": \"id count\", \"sum\": \"pageviews per id\"})\n",
163 |     "dfTestCSV.head(10)"
164 |    ]
165 |   },
166 |   {
167 |    "cell_type": "code",
168 |    "execution_count": 30,
169 |    "metadata": {},
170 |    "outputs": [
171 |     {
172 |      "data": {
173 |       "text/html": [
174 |        "<div>\n",
175 |        "<style scoped>\n",
176 |        "    .dataframe tbody tr th:only-of-type {\n",
177 |        "        vertical-align: middle;\n",
178 |        "    }\n",
179 |        "\n",
180 |        "    .dataframe tbody tr th {\n",
181 |        "        vertical-align: top;\n",
182 |        "    }\n",
183 |        "\n",
184 |        "    .dataframe thead th {\n",
185 |        "        text-align: right;\n",
186 |        "    }\n",
187 |        "</style>\n",
188 |        "<table border=\"1\" class=\"dataframe\">\n",
189 |        "  <thead>\n",
190 |        "    <tr style=\"text-align: right;\">\n",
191 |        "      <th></th>\n",
192 |        "      <th>Address</th>\n",
193 |        "      <th>article-text-box</th>\n",
194 |        "      <th>video-player</th>\n",
195 |        "      <th>article-body</th>\n",
196 |        "      <th>sda</th>\n",
197 |        "      <th>scribble</th>\n",
198 |        "      <th>has-image</th>\n",
199 |        "      <th>iframe</th>\n",
200 |        "      <th>normal-article-layout</th>\n",
201 |        "      <th>sda2</th>\n",
202 |        "      <th>ID_Crawl</th>\n",
203 |        "    </tr>\n",
204 |        "  </thead>\n",
205 |        "  <tbody>\n",
206 |        "    <tr>\n",
207 |        "      <th>0</th>\n",
208 |        "      <td>https://www.blick.ch/people-tv/international/r...</td>\n",
209 |        "      <td>0</td>\n",
210 |        "      <td>0</td>\n",
211 |        "      <td>0</td>\n",
212 |        "      <td>0</td>\n",
213 |        "      <td>0</td>\n",
214 |        "      <td>0</td>\n",
215 |        "      <td>0</td>\n",
216 |        "      <td>0</td>\n",
217 |        "      <td>0</td>\n",
218 |        "      <td>4270807</td>\n",
219 |        "    </tr>\n",
220 |        "    <tr>\n",
221 |        "      <th>1</th>\n",
222 |        "      <td>https://www.blick.ch/sport/eishockey/overtime-...</td>\n",
223 |        "      <td>0</td>\n",
224 |        "      <td>0</td>\n",
225 |        "      <td>0</td>\n",
226 |        "      <td>0</td>\n",
227 |        "      <td>0</td>\n",
228 |        "      <td>0</td>\n",
229 |        "      <td>0</td>\n",
230 |        "      <td>0</td>\n",
231 |        "      <td>0</td>\n",
232 |        "      <td>106244</td>\n",
233 |        "    </tr>\n",
234 |        "    <tr>\n",
235 |        "      <th>2</th>\n",
236 |        "      <td>https://www.blick.ch/news/ausland/neben-berueh...</td>\n",
237 |        "      <td>0</td>\n",
238 |        "      <td>0</td>\n",
239 |        "      <td>0</td>\n",
240 |        "      <td>0</td>\n",
241 |        "      <td>0</td>\n",
242 |        "      <td>0</td>\n",
243 |        "      <td>0</td>\n",
244 |        "      <td>0</td>\n",
245 |        "      <td>0</td>\n",
246 |        "      <td>3896239</td>\n",
247 |        "    </tr>\n",
248 |        "    <tr>\n",
249 |        "      <th>3</th>\n",
250 |        "      <td>https://www.blick.ch/news/grausiger-fund-in-an...</td>\n",
251 |        "      <td>0</td>\n",
252 |        "      <td>0</td>\n",
253 |        "      <td>0</td>\n",
254 |        "      <td>0</td>\n",
255 |        "      <td>0</td>\n",
256 |        "      <td>0</td>\n",
257 |        "      <td>0</td>\n",
258 |        "      <td>0</td>\n",
259 |        "      <td>0</td>\n",
260 |        "      <td>2024473</td>\n",
261 |        "    </tr>\n",
262 |        "    <tr>\n",
263 |        "      <th>4</th>\n",
264 |        "      <td>https://www.blick.ch/news/mahamaya-bern-die-we...</td>\n",
265 |        "      <td>0</td>\n",
266 |        "      <td>0</td>\n",
267 |        "      <td>0</td>\n",
268 |        "      <td>0</td>\n",
269 |        "      <td>0</td>\n",
270 |        "      <td>0</td>\n",
271 |        "      <td>0</td>\n",
272 |        "      <td>0</td>\n",
273 |        "      <td>0</td>\n",
274 |        "      <td>110094</td>\n",
275 |        "    </tr>\n",
276 |        "    <tr>\n",
277 |        "      <th>5</th>\n",
278 |        "      <td>https://www.blick.ch/people-tv/musik/liechtens...</td>\n",
279 |        "      <td>0</td>\n",
280 |        "      <td>0</td>\n",
281 |        "      <td>0</td>\n",
282 |        "      <td>0</td>\n",
283 |        "      <td>0</td>\n",
284 |        "      <td>0</td>\n",
285 |        "      <td>0</td>\n",
286 |        "      <td>0</td>\n",
287 |        "      <td>0</td>\n",
288 |        "      <td>2131248</td>\n",
289 |        "    </tr>\n",
290 |        "    <tr>\n",
291 |        "      <th>6</th>\n",
292 |        "      <td>https://www.blick.ch/people-tv/tv/bachelorette...</td>\n",
293 |        "      <td>0</td>\n",
294 |        "      <td>0</td>\n",
295 |        "      <td>0</td>\n",
296 |        "      <td>0</td>\n",
297 |        "      <td>0</td>\n",
298 |        "      <td>0</td>\n",
299 |        "      <td>0</td>\n",
300 |        "      <td>0</td>\n",
301 |        "      <td>0</td>\n",
302 |        "      <td>3790667</td>\n",
303 |        "    </tr>\n",
304 |        "    <tr>\n",
305 |        "      <th>7</th>\n",
306 |        "      <td>https://www.blick.ch/news/das-ist-die-swiss-bo...</td>\n",
307 |        "      <td>0</td>\n",
308 |        "      <td>0</td>\n",
309 |        "      <td>0</td>\n",
310 |        "      <td>0</td>\n",
311 |        "      <td>0</td>\n",
312 |        "      <td>0</td>\n",
313 |        "      <td>0</td>\n",
314 |        "      <td>0</td>\n",
315 |        "      <td>0</td>\n",
316 |        "      <td>2238005</td>\n",
317 |        "    </tr>\n",
318 |        "    <tr>\n",
319 |        "      <th>8</th>\n",
320 |        "      <td>https://www.blick.ch/news/schweiz/vergangenhei...</td>\n",
321 |        "      <td>0</td>\n",
322 |        "      <td>0</td>\n",
323 |        "      <td>0</td>\n",
324 |        "      <td>0</td>\n",
325 |        "      <td>0</td>\n",
326 |        "      <td>0</td>\n",
327 |        "      <td>0</td>\n",
328 |        "      <td>0</td>\n",
329 |        "      <td>0</td>\n",
330 |        "      <td>5550392</td>\n",
331 |        "    </tr>\n",
332 |        "    <tr>\n",
333 |        "      <th>9</th>\n",
334 |        "      <td>https://www.blick.ch/news/wirtschaft/das-sind-...</td>\n",
335 |        "      <td>0</td>\n",
336 |        "      <td>0</td>\n",
337 |        "      <td>0</td>\n",
338 |        "      <td>0</td>\n",
339 |        "      <td>0</td>\n",
340 |        "      <td>0</td>\n",
341 |        "      <td>0</td>\n",
342 |        "      <td>0</td>\n",
343 |        "      <td>0</td>\n",
344 |        "      <td>4955860</td>\n",
345 |        "    </tr>\n",
346 |        "  </tbody>\n",
347 |        "</table>\n",
348 |        "</div>"
349 |       ],
350 |       "text/plain": [
351 |        "                                             Address  article-text-box  \\\n",
352 |        "0  https://www.blick.ch/people-tv/international/r...                 0   \n",
353 |        "1  https://www.blick.ch/sport/eishockey/overtime-...                 0   \n",
354 |        "2  https://www.blick.ch/news/ausland/neben-berueh...                 0   \n",
355 |        "3  https://www.blick.ch/news/grausiger-fund-in-an...                 0   \n",
356 |        "4  https://www.blick.ch/news/mahamaya-bern-die-we...                 0   \n",
357 |        "5  https://www.blick.ch/people-tv/musik/liechtens...                 0   \n",
358 |        "6  https://www.blick.ch/people-tv/tv/bachelorette...                 0   \n",
359 |        "7  https://www.blick.ch/news/das-ist-die-swiss-bo...                 0   \n",
360 |        "8  https://www.blick.ch/news/schweiz/vergangenhei...                 0   \n",
361 |        "9  https://www.blick.ch/news/wirtschaft/das-sind-...                 0   \n",
362 |        "\n",
363 |        "   video-player  article-body  sda  scribble  has-image  iframe  \\\n",
364 |        "0             0             0    0         0          0       0   \n",
365 |        "1             0             0    0         0          0       0   \n",
366 |        "2             0             0    0         0          0       0   \n",
367 |        "3             0             0    0         0          0       0   \n",
368 |        "4             0             0    0         0          0       0   \n",
369 |        "5             0             0    0         0          0       0   \n",
370 |        "6             0             0    0         0          0       0   \n",
371 |        "7             0             0    0         0          0       0   \n",
372 |        "8             0             0    0         0          0       0   \n",
373 |        "9             0             0    0         0          0       0   \n",
374 |        "\n",
375 |        "   normal-article-layout  sda2 ID_Crawl  \n",
376 |        "0                      0     0  4270807  \n",
377 |        "1                      0     0   106244  \n",
378 |        "2                      0     0  3896239  \n",
379 |        "3                      0     0  2024473  \n",
380 |        "4                      0     0   110094  \n",
381 |        "5                      0     0  2131248  \n",
382 |        "6                      0     0  3790667  \n",
383 |        "7                      0     0  2238005  \n",
384 |        "8                      0     0  5550392  \n",
385 |        "9                      0     0  4955860  "
386 |       ]
387 |      },
388 |      "execution_count": 30,
389 |      "metadata": {},
390 |      "output_type": "execute_result"
391 |     }
392 |    ],
393 |    "source": [
394 |     "dfCrawled = pd.DataFrame(pd.read_excel('./custom_search_all-test.xlsx', header=0, sheet_name='1 - All'))\n",
395 |     "dfCrawled = dfCrawled.rename(columns={\"Contains: \\'\\\"article-text-box\\\"\\'\": \"article-text-box\", \n",
396 |     "                                         \"Contains: \\'\\\"video-player\\\"\\'\":\"video-player\", \n",
397 |     "                                         \"Contains: \\'\\\"article-body\\'\":\"article-body\",\n",
398 |     "                                         \"Contains: \\'\\(sda\\)\\'\":\"sda\",\n",
399 |     "                                         \"Contains: \\'\\\"scribble\\\"\\'\":\"scribble\",\n",
400 |     "                                         \"Contains: \\'basic-image--free-desktop\\'\":\"has-image\", \n",
401 |     "                                         \"Contains: \\'<iframe\\'\":\"iframe\", \n",
402 |     "                                         \"Contains: \\'layout-item--flex-desktop-4'\":\"normal-article-layout\", \n",
403 |     "                                         \"Contains: \\'\\(sda\\/\\'\":\"sda2\", \n",
404 |     "                                     })\n",
405 |     "dfCrawled['ID_Crawl'] = dfCrawled['Address'].str.extract('(?:.*id)([0-9]+)(?:.html)', expand=False)\n",
406 |     "dfCrawled = dfCrawled.drop(\"Status Code\",1)\n",
407 |     "dfCrawled = dfCrawled.drop(\"Status\",1)\n",
408 |     "dfCrawled = dfCrawled.drop(\"Content\",1)\n",
409 |     "dfCrawled.head(10)"
410 |    ]
411 |   },
412 |   {
413 |    "cell_type": "code",
414 |    "execution_count": 29,
415 |    "metadata": {},
416 |    "outputs": [
417 |     {
418 |      "data": {
419 |       "text/html": [
420 |        "<div>\n",
421 |        "<style scoped>\n",
422 |        "    .dataframe tbody tr th:only-of-type {\n",
423 |        "        vertical-align: middle;\n",
424 |        "    }\n",
425 |        "\n",
426 |        "    .dataframe tbody tr th {\n",
427 |        "        vertical-align: top;\n",
428 |        "    }\n",
429 |        "\n",
430 |        "    .dataframe thead th {\n",
431 |        "        text-align: right;\n",
432 |        "    }\n",
433 |        "</style>\n",
434 |        "<table border=\"1\" class=\"dataframe\">\n",
435 |        "  <thead>\n",
436 |        "    <tr style=\"text-align: right;\">\n",
437 |        "      <th></th>\n",
438 |        "      <th>ID_GA</th>\n",
439 |        "      <th>id count</th>\n",
440 |        "      <th>pageviews per id</th>\n",
441 |        "      <th>Address</th>\n",
442 |        "      <th>article-text-box</th>\n",
443 |        "      <th>video-player</th>\n",
444 |        "      <th>article-body</th>\n",
445 |        "      <th>sda</th>\n",
446 |        "      <th>scribble</th>\n",
447 |        "      <th>has-image</th>\n",
448 |        "      <th>iframe</th>\n",
449 |        "      <th>normal-article-layout</th>\n",
450 |        "      <th>sda2</th>\n",
451 |        "    </tr>\n",
452 |        "  </thead>\n",
453 |        "  <tbody>\n",
454 |        "    <tr>\n",
455 |        "      <th>34</th>\n",
456 |        "      <td>31000</td>\n",
457 |        "      <td>1.0</td>\n",
458 |        "      <td>3.0</td>\n",
459 |        "      <td>https://www.blick.ch/people-tv/international/e...</td>\n",
460 |        "      <td>0.0</td>\n",
461 |        "      <td>0.0</td>\n",
462 |        "      <td>1.0</td>\n",
463 |        "      <td>0.0</td>\n",
464 |        "      <td>2.0</td>\n",
465 |        "      <td>1.0</td>\n",
466 |        "      <td>1.0</td>\n",
467 |        "      <td>1.0</td>\n",
468 |        "      <td>2.0</td>\n",
469 |        "    </tr>\n",
470 |        "    <tr>\n",
471 |        "      <th>36</th>\n",
472 |        "      <td>31002</td>\n",
473 |        "      <td>1.0</td>\n",
474 |        "      <td>1.0</td>\n",
475 |        "      <td>https://www.blick.ch/news/ausland/afghanistan-...</td>\n",
476 |        "      <td>0.0</td>\n",
477 |        "      <td>0.0</td>\n",
478 |        "      <td>1.0</td>\n",
479 |        "      <td>3.0</td>\n",
480 |        "      <td>2.0</td>\n",
481 |        "      <td>0.0</td>\n",
482 |        "      <td>1.0</td>\n",
483 |        "      <td>1.0</td>\n",
484 |        "      <td>0.0</td>\n",
485 |        "    </tr>\n",
486 |        "    <tr>\n",
487 |        "      <th>37</th>\n",
488 |        "      <td>31004</td>\n",
489 |        "      <td>1.0</td>\n",
490 |        "      <td>5.0</td>\n",
491 |        "      <td>https://www.blick.ch/news/zwei-schweizer-in-ve...</td>\n",
492 |        "      <td>0.0</td>\n",
493 |        "      <td>0.0</td>\n",
494 |        "      <td>1.0</td>\n",
495 |        "      <td>0.0</td>\n",
496 |        "      <td>2.0</td>\n",
497 |        "      <td>0.0</td>\n",
498 |        "      <td>1.0</td>\n",
499 |        "      <td>1.0</td>\n",
500 |        "      <td>0.0</td>\n",
501 |        "    </tr>\n",
502 |        "    <tr>\n",
503 |        "      <th>51</th>\n",
504 |        "      <td>31275</td>\n",
505 |        "      <td>2.0</td>\n",
506 |        "      <td>6.0</td>\n",
507 |        "      <td>https://www.blick.ch/news/schweiz/zuerich/knas...</td>\n",
508 |        "      <td>0.0</td>\n",
509 |        "      <td>0.0</td>\n",
510 |        "      <td>1.0</td>\n",
511 |        "      <td>0.0</td>\n",
512 |        "      <td>2.0</td>\n",
513 |        "      <td>0.0</td>\n",
514 |        "      <td>1.0</td>\n",
515 |        "      <td>1.0</td>\n",
516 |        "      <td>0.0</td>\n",
517 |        "    </tr>\n",
518 |        "    <tr>\n",
519 |        "      <th>52</th>\n",
520 |        "      <td>31277</td>\n",
521 |        "      <td>1.0</td>\n",
522 |        "      <td>12.0</td>\n",
523 |        "      <td>https://www.blick.ch/news/ausland/steinmeier-p...</td>\n",
524 |        "      <td>0.0</td>\n",
525 |        "      <td>0.0</td>\n",
526 |        "      <td>1.0</td>\n",
527 |        "      <td>0.0</td>\n",
528 |        "      <td>2.0</td>\n",
529 |        "      <td>0.0</td>\n",
530 |        "      <td>1.0</td>\n",
531 |        "      <td>1.0</td>\n",
532 |        "      <td>0.0</td>\n",
533 |        "    </tr>\n",
534 |        "    <tr>\n",
535 |        "      <th>65</th>\n",
536 |        "      <td>31388</td>\n",
537 |        "      <td>1.0</td>\n",
538 |        "      <td>3.0</td>\n",
539 |        "      <td>https://www.blick.ch/news/schweiz/zentralschwe...</td>\n",
540 |        "      <td>0.0</td>\n",
541 |        "      <td>0.0</td>\n",
542 |        "      <td>5.0</td>\n",
543 |        "      <td>0.0</td>\n",
544 |        "      <td>2.0</td>\n",
545 |        "      <td>0.0</td>\n",
546 |        "      <td>1.0</td>\n",
547 |        "      <td>1.0</td>\n",
548 |        "      <td>0.0</td>\n",
549 |        "    </tr>\n",
550 |        "    <tr>\n",
551 |        "      <th>68</th>\n",
552 |        "      <td>31425</td>\n",
553 |        "      <td>1.0</td>\n",
554 |        "      <td>1.0</td>\n",
555 |        "      <td>https://www.blick.ch/people-tv/international/h...</td>\n",
556 |        "      <td>0.0</td>\n",
557 |        "      <td>0.0</td>\n",
558 |        "      <td>1.0</td>\n",
559 |        "      <td>0.0</td>\n",
560 |        "      <td>2.0</td>\n",
561 |        "      <td>0.0</td>\n",
562 |        "      <td>1.0</td>\n",
563 |        "      <td>1.0</td>\n",
564 |        "      <td>0.0</td>\n",
565 |        "    </tr>\n",
566 |        "    <tr>\n",
567 |        "      <th>69</th>\n",
568 |        "      <td>31426</td>\n",
569 |        "      <td>1.0</td>\n",
570 |        "      <td>1.0</td>\n",
571 |        "      <td>https://www.blick.ch/sport/eishockey/nla/serve...</td>\n",
572 |        "      <td>1.0</td>\n",
573 |        "      <td>0.0</td>\n",
574 |        "      <td>1.0</td>\n",
575 |        "      <td>0.0</td>\n",
576 |        "      <td>2.0</td>\n",
577 |        "      <td>0.0</td>\n",
578 |        "      <td>1.0</td>\n",
579 |        "      <td>1.0</td>\n",
580 |        "      <td>0.0</td>\n",
581 |        "    </tr>\n",
582 |        "    <tr>\n",
583 |        "      <th>70</th>\n",
584 |        "      <td>31427</td>\n",
585 |        "      <td>1.0</td>\n",
586 |        "      <td>1.0</td>\n",
587 |        "      <td>https://www.blick.ch/digital/games/news/colin-...</td>\n",
588 |        "      <td>0.0</td>\n",
589 |        "      <td>0.0</td>\n",
590 |        "      <td>3.0</td>\n",
591 |        "      <td>0.0</td>\n",
592 |        "      <td>2.0</td>\n",
593 |        "      <td>5.0</td>\n",
594 |        "      <td>1.0</td>\n",
595 |        "      <td>1.0</td>\n",
596 |        "      <td>0.0</td>\n",
597 |        "    </tr>\n",
598 |        "    <tr>\n",
599 |        "      <th>71</th>\n",
600 |        "      <td>31428</td>\n",
601 |        "      <td>1.0</td>\n",
602 |        "      <td>1.0</td>\n",
603 |        "      <td>https://www.blick.ch/digital/games/larry-magna...</td>\n",
604 |        "      <td>0.0</td>\n",
605 |        "      <td>0.0</td>\n",
606 |        "      <td>1.0</td>\n",
607 |        "      <td>0.0</td>\n",
608 |        "      <td>2.0</td>\n",
609 |        "      <td>0.0</td>\n",
610 |        "      <td>1.0</td>\n",
611 |        "      <td>1.0</td>\n",
612 |        "      <td>0.0</td>\n",
613 |        "    </tr>\n",
614 |        "  </tbody>\n",
615 |        "</table>\n",
616 |        "</div>"
617 |       ],
618 |       "text/plain": [
619 |        "    ID_GA  id count  pageviews per id  \\\n",
620 |        "34  31000       1.0               3.0   \n",
621 |        "36  31002       1.0               1.0   \n",
622 |        "37  31004       1.0               5.0   \n",
623 |        "51  31275       2.0               6.0   \n",
624 |        "52  31277       1.0              12.0   \n",
625 |        "65  31388       1.0               3.0   \n",
626 |        "68  31425       1.0               1.0   \n",
627 |        "69  31426       1.0               1.0   \n",
628 |        "70  31427       1.0               1.0   \n",
629 |        "71  31428       1.0               1.0   \n",
630 |        "\n",
631 |        "                                              Address  article-text-box  \\\n",
632 |        "34  https://www.blick.ch/people-tv/international/e...               0.0   \n",
633 |        "36  https://www.blick.ch/news/ausland/afghanistan-...               0.0   \n",
634 |        "37  https://www.blick.ch/news/zwei-schweizer-in-ve...               0.0   \n",
635 |        "51  https://www.blick.ch/news/schweiz/zuerich/knas...               0.0   \n",
636 |        "52  https://www.blick.ch/news/ausland/steinmeier-p...               0.0   \n",
637 |        "65  https://www.blick.ch/news/schweiz/zentralschwe...               0.0   \n",
638 |        "68  https://www.blick.ch/people-tv/international/h...               0.0   \n",
639 |        "69  https://www.blick.ch/sport/eishockey/nla/serve...               1.0   \n",
640 |        "70  https://www.blick.ch/digital/games/news/colin-...               0.0   \n",
641 |        "71  https://www.blick.ch/digital/games/larry-magna...               0.0   \n",
642 |        "\n",
643 |        "    video-player  article-body  sda  scribble  has-image  iframe  \\\n",
644 |        "34           0.0           1.0  0.0       2.0        1.0     1.0   \n",
645 |        "36           0.0           1.0  3.0       2.0        0.0     1.0   \n",
646 |        "37           0.0           1.0  0.0       2.0        0.0     1.0   \n",
647 |        "51           0.0           1.0  0.0       2.0        0.0     1.0   \n",
648 |        "52           0.0           1.0  0.0       2.0        0.0     1.0   \n",
649 |        "65           0.0           5.0  0.0       2.0        0.0     1.0   \n",
650 |        "68           0.0           1.0  0.0       2.0        0.0     1.0   \n",
651 |        "69           0.0           1.0  0.0       2.0        0.0     1.0   \n",
652 |        "70           0.0           3.0  0.0       2.0        5.0     1.0   \n",
653 |        "71           0.0           1.0  0.0       2.0        0.0     1.0   \n",
654 |        "\n",
655 |        "    normal-article-layout  sda2  \n",
656 |        "34                    1.0   2.0  \n",
657 |        "36                    1.0   0.0  \n",
658 |        "37                    1.0   0.0  \n",
659 |        "51                    1.0   0.0  \n",
660 |        "52                    1.0   0.0  \n",
661 |        "65                    1.0   0.0  \n",
662 |        "68                    1.0   0.0  \n",
663 |        "69                    1.0   0.0  \n",
664 |        "70                    1.0   0.0  \n",
665 |        "71                    1.0   0.0  "
666 |       ]
667 |      },
668 |      "execution_count": 29,
669 |      "metadata": {},
670 |      "output_type": "execute_result"
671 |     }
672 |    ],
673 |    "source": [
674 |     "dfMerged = pd.merge(dfTestCSV, dfCrawled, left_on='ID_GA', right_on='ID_Crawl', how='outer')\n",
675 |     "dfMerged = dfMerged[dfMerged['ID_Crawl'].notnull()]\n",
676 |     "dfMerged = dfMerged[dfMerged['ID_GA'].notnull()]\n",
677 |     "dfMerged = dfMerged.drop(\"ID_Crawl\",1)\n",
678 |     "dfMerged.head(10)"
679 |    ]
680 |   },
681 |   {
682 |    "cell_type": "code",
683 |    "execution_count": null,
684 |    "metadata": {},
685 |    "outputs": [],
686 |    "source": []
687 |   },
688 |   {
689 |    "cell_type": "code",
690 |    "execution_count": null,
691 |    "metadata": {},
692 |    "outputs": [],
693 |    "source": []
694 |   },
695 |   {
696 |    "cell_type": "code",
697 |    "execution_count": null,
698 |    "metadata": {},
699 |    "outputs": [],
700 |    "source": []
701 |   },
702 |   {
703 |    "cell_type": "code",
704 |    "execution_count": null,
705 |    "metadata": {},
706 |    "outputs": [],
707 |    "source": []
708 |   },
709 |   {
710 |    "cell_type": "code",
711 |    "execution_count": null,
712 |    "metadata": {},
713 |    "outputs": [],
714 |    "source": []
715 |   }
716 |  ],
717 |  "metadata": {
718 |   "kernelspec": {
719 |    "display_name": "Python 3",
720 |    "language": "python",
721 |    "name": "python3"
722 |   },
723 |   "language_info": {
724 |    "codemirror_mode": {
725 |     "name": "ipython",
726 |     "version": 3
727 |    },
728 |    "file_extension": ".py",
729 |    "mimetype": "text/x-python",
730 |    "name": "python",
731 |    "nbconvert_exporter": "python",
732 |    "pygments_lexer": "ipython3",
733 |    "version": "3.7.2"
734 |   }
735 |  },
736 |  "nbformat": 4,
737 |  "nbformat_minor": 2
738 | }
739 | 


--------------------------------------------------------------------------------
/#7 Filter and export in Python Pandas.ipynb:
--------------------------------------------------------------------------------
   1 | {
   2 |  "cells": [
   3 |   {
   4 |    "cell_type": "code",
   5 |    "execution_count": null,
   6 |    "metadata": {},
   7 |    "outputs": [],
   8 |    "source": [
   9 |     "# Video about this notebook: \n",
  10 |     "# https://youtu.be/54BwLbGWluc"
  11 |    ]
  12 |   },
  13 |   {
  14 |    "cell_type": "code",
  15 |    "execution_count": 2,
  16 |    "metadata": {},
  17 |    "outputs": [],
  18 |    "source": [
  19 |     "import pandas as pd"
  20 |    ]
  21 |   },
  22 |   {
  23 |    "cell_type": "code",
  24 |    "execution_count": 3,
  25 |    "metadata": {},
  26 |    "outputs": [],
  27 |    "source": [
  28 |     "dfTestCSV = pd.DataFrame(pd.read_csv('./dfanalyticstestdata1.csv', header=2, sep=\"\\t\", index_col=0))"
  29 |    ]
  30 |   },
  31 |   {
  32 |    "cell_type": "code",
  33 |    "execution_count": 4,
  34 |    "metadata": {},
  35 |    "outputs": [],
  36 |    "source": [
  37 |     "dfTestCSV['ID_GA'] = dfTestCSV['ga:pagePath'].str.extract('(?:.*id)([0-9]+)(?:.html)')"
  38 |    ]
  39 |   },
  40 |   {
  41 |    "cell_type": "code",
  42 |    "execution_count": 5,
  43 |    "metadata": {},
  44 |    "outputs": [
  45 |     {
  46 |      "data": {
  47 |       "text/html": [
  48 |        "<div>\n",
  49 |        "<style scoped>\n",
  50 |        "    .dataframe tbody tr th:only-of-type {\n",
  51 |        "        vertical-align: middle;\n",
  52 |        "    }\n",
  53 |        "\n",
  54 |        "    .dataframe tbody tr th {\n",
  55 |        "        vertical-align: top;\n",
  56 |        "    }\n",
  57 |        "\n",
  58 |        "    .dataframe thead th {\n",
  59 |        "        text-align: right;\n",
  60 |        "    }\n",
  61 |        "</style>\n",
  62 |        "<table border=\"1\" class=\"dataframe\">\n",
  63 |        "  <thead>\n",
  64 |        "    <tr style=\"text-align: right;\">\n",
  65 |        "      <th></th>\n",
  66 |        "      <th>ID_GA</th>\n",
  67 |        "      <th>id count</th>\n",
  68 |        "      <th>pageviews per id</th>\n",
  69 |        "    </tr>\n",
  70 |        "  </thead>\n",
  71 |        "  <tbody>\n",
  72 |        "    <tr>\n",
  73 |        "      <th>0</th>\n",
  74 |        "      <td>10061</td>\n",
  75 |        "      <td>2</td>\n",
  76 |        "      <td>8</td>\n",
  77 |        "    </tr>\n",
  78 |        "    <tr>\n",
  79 |        "      <th>1</th>\n",
  80 |        "      <td>1510256</td>\n",
  81 |        "      <td>1</td>\n",
  82 |        "      <td>1</td>\n",
  83 |        "    </tr>\n",
  84 |        "    <tr>\n",
  85 |        "      <th>2</th>\n",
  86 |        "      <td>151553</td>\n",
  87 |        "      <td>1</td>\n",
  88 |        "      <td>5</td>\n",
  89 |        "    </tr>\n",
  90 |        "    <tr>\n",
  91 |        "      <th>3</th>\n",
  92 |        "      <td>1742</td>\n",
  93 |        "      <td>1</td>\n",
  94 |        "      <td>1</td>\n",
  95 |        "    </tr>\n",
  96 |        "    <tr>\n",
  97 |        "      <th>4</th>\n",
  98 |        "      <td>19186</td>\n",
  99 |        "      <td>1</td>\n",
 100 |        "      <td>28</td>\n",
 101 |        "    </tr>\n",
 102 |        "    <tr>\n",
 103 |        "      <th>5</th>\n",
 104 |        "      <td>19969</td>\n",
 105 |        "      <td>1</td>\n",
 106 |        "      <td>2</td>\n",
 107 |        "    </tr>\n",
 108 |        "    <tr>\n",
 109 |        "      <th>6</th>\n",
 110 |        "      <td>2241980</td>\n",
 111 |        "      <td>1</td>\n",
 112 |        "      <td>1</td>\n",
 113 |        "    </tr>\n",
 114 |        "    <tr>\n",
 115 |        "      <th>7</th>\n",
 116 |        "      <td>22672</td>\n",
 117 |        "      <td>1</td>\n",
 118 |        "      <td>1</td>\n",
 119 |        "    </tr>\n",
 120 |        "    <tr>\n",
 121 |        "      <th>8</th>\n",
 122 |        "      <td>2483959</td>\n",
 123 |        "      <td>1</td>\n",
 124 |        "      <td>1</td>\n",
 125 |        "    </tr>\n",
 126 |        "    <tr>\n",
 127 |        "      <th>9</th>\n",
 128 |        "      <td>26545</td>\n",
 129 |        "      <td>1</td>\n",
 130 |        "      <td>1</td>\n",
 131 |        "    </tr>\n",
 132 |        "  </tbody>\n",
 133 |        "</table>\n",
 134 |        "</div>"
 135 |       ],
 136 |       "text/plain": [
 137 |        "     ID_GA  id count  pageviews per id\n",
 138 |        "0    10061         2                 8\n",
 139 |        "1  1510256         1                 1\n",
 140 |        "2   151553         1                 5\n",
 141 |        "3     1742         1                 1\n",
 142 |        "4    19186         1                28\n",
 143 |        "5    19969         1                 2\n",
 144 |        "6  2241980         1                 1\n",
 145 |        "7    22672         1                 1\n",
 146 |        "8  2483959         1                 1\n",
 147 |        "9    26545         1                 1"
 148 |       ]
 149 |      },
 150 |      "execution_count": 5,
 151 |      "metadata": {},
 152 |      "output_type": "execute_result"
 153 |     }
 154 |    ],
 155 |    "source": [
 156 |     "dfTestCSV = pd.pivot_table(dfTestCSV,index='ID_GA', values='ga:pageviews', aggfunc=['count', 'sum'])\n",
 157 |     "dfTestCSV.reset_index(inplace=True)\n",
 158 |     "dfTestCSV.columns = dfTestCSV.columns.droplevel(1)\n",
 159 |     "dfTestCSV = dfTestCSV.rename(columns={\"count\": \"id count\", \"sum\": \"pageviews per id\"})\n",
 160 |     "dfTestCSV.head(10)"
 161 |    ]
 162 |   },
 163 |   {
 164 |    "cell_type": "code",
 165 |    "execution_count": 6,
 166 |    "metadata": {},
 167 |    "outputs": [
 168 |     {
 169 |      "data": {
 170 |       "text/html": [
 171 |        "<div>\n",
 172 |        "<style scoped>\n",
 173 |        "    .dataframe tbody tr th:only-of-type {\n",
 174 |        "        vertical-align: middle;\n",
 175 |        "    }\n",
 176 |        "\n",
 177 |        "    .dataframe tbody tr th {\n",
 178 |        "        vertical-align: top;\n",
 179 |        "    }\n",
 180 |        "\n",
 181 |        "    .dataframe thead th {\n",
 182 |        "        text-align: right;\n",
 183 |        "    }\n",
 184 |        "</style>\n",
 185 |        "<table border=\"1\" class=\"dataframe\">\n",
 186 |        "  <thead>\n",
 187 |        "    <tr style=\"text-align: right;\">\n",
 188 |        "      <th></th>\n",
 189 |        "      <th>Address</th>\n",
 190 |        "      <th>article-text-box</th>\n",
 191 |        "      <th>video-player</th>\n",
 192 |        "      <th>article-body</th>\n",
 193 |        "      <th>sda</th>\n",
 194 |        "      <th>scribble</th>\n",
 195 |        "      <th>has-image</th>\n",
 196 |        "      <th>iframe</th>\n",
 197 |        "      <th>normal-article-layout</th>\n",
 198 |        "      <th>sda2</th>\n",
 199 |        "      <th>ID_Crawl</th>\n",
 200 |        "    </tr>\n",
 201 |        "  </thead>\n",
 202 |        "  <tbody>\n",
 203 |        "    <tr>\n",
 204 |        "      <th>0</th>\n",
 205 |        "      <td>https://www.blick.ch/people-tv/international/r...</td>\n",
 206 |        "      <td>0</td>\n",
 207 |        "      <td>0</td>\n",
 208 |        "      <td>0</td>\n",
 209 |        "      <td>0</td>\n",
 210 |        "      <td>0</td>\n",
 211 |        "      <td>0</td>\n",
 212 |        "      <td>0</td>\n",
 213 |        "      <td>0</td>\n",
 214 |        "      <td>0</td>\n",
 215 |        "      <td>4270807</td>\n",
 216 |        "    </tr>\n",
 217 |        "    <tr>\n",
 218 |        "      <th>1</th>\n",
 219 |        "      <td>https://www.blick.ch/sport/eishockey/overtime-...</td>\n",
 220 |        "      <td>0</td>\n",
 221 |        "      <td>0</td>\n",
 222 |        "      <td>0</td>\n",
 223 |        "      <td>0</td>\n",
 224 |        "      <td>0</td>\n",
 225 |        "      <td>0</td>\n",
 226 |        "      <td>0</td>\n",
 227 |        "      <td>0</td>\n",
 228 |        "      <td>0</td>\n",
 229 |        "      <td>106244</td>\n",
 230 |        "    </tr>\n",
 231 |        "    <tr>\n",
 232 |        "      <th>2</th>\n",
 233 |        "      <td>https://www.blick.ch/news/ausland/neben-berueh...</td>\n",
 234 |        "      <td>0</td>\n",
 235 |        "      <td>0</td>\n",
 236 |        "      <td>0</td>\n",
 237 |        "      <td>0</td>\n",
 238 |        "      <td>0</td>\n",
 239 |        "      <td>0</td>\n",
 240 |        "      <td>0</td>\n",
 241 |        "      <td>0</td>\n",
 242 |        "      <td>0</td>\n",
 243 |        "      <td>3896239</td>\n",
 244 |        "    </tr>\n",
 245 |        "    <tr>\n",
 246 |        "      <th>3</th>\n",
 247 |        "      <td>https://www.blick.ch/news/grausiger-fund-in-an...</td>\n",
 248 |        "      <td>0</td>\n",
 249 |        "      <td>0</td>\n",
 250 |        "      <td>0</td>\n",
 251 |        "      <td>0</td>\n",
 252 |        "      <td>0</td>\n",
 253 |        "      <td>0</td>\n",
 254 |        "      <td>0</td>\n",
 255 |        "      <td>0</td>\n",
 256 |        "      <td>0</td>\n",
 257 |        "      <td>2024473</td>\n",
 258 |        "    </tr>\n",
 259 |        "    <tr>\n",
 260 |        "      <th>4</th>\n",
 261 |        "      <td>https://www.blick.ch/news/mahamaya-bern-die-we...</td>\n",
 262 |        "      <td>0</td>\n",
 263 |        "      <td>0</td>\n",
 264 |        "      <td>0</td>\n",
 265 |        "      <td>0</td>\n",
 266 |        "      <td>0</td>\n",
 267 |        "      <td>0</td>\n",
 268 |        "      <td>0</td>\n",
 269 |        "      <td>0</td>\n",
 270 |        "      <td>0</td>\n",
 271 |        "      <td>110094</td>\n",
 272 |        "    </tr>\n",
 273 |        "    <tr>\n",
 274 |        "      <th>5</th>\n",
 275 |        "      <td>https://www.blick.ch/people-tv/musik/liechtens...</td>\n",
 276 |        "      <td>0</td>\n",
 277 |        "      <td>0</td>\n",
 278 |        "      <td>0</td>\n",
 279 |        "      <td>0</td>\n",
 280 |        "      <td>0</td>\n",
 281 |        "      <td>0</td>\n",
 282 |        "      <td>0</td>\n",
 283 |        "      <td>0</td>\n",
 284 |        "      <td>0</td>\n",
 285 |        "      <td>2131248</td>\n",
 286 |        "    </tr>\n",
 287 |        "    <tr>\n",
 288 |        "      <th>6</th>\n",
 289 |        "      <td>https://www.blick.ch/people-tv/tv/bachelorette...</td>\n",
 290 |        "      <td>0</td>\n",
 291 |        "      <td>0</td>\n",
 292 |        "      <td>0</td>\n",
 293 |        "      <td>0</td>\n",
 294 |        "      <td>0</td>\n",
 295 |        "      <td>0</td>\n",
 296 |        "      <td>0</td>\n",
 297 |        "      <td>0</td>\n",
 298 |        "      <td>0</td>\n",
 299 |        "      <td>3790667</td>\n",
 300 |        "    </tr>\n",
 301 |        "    <tr>\n",
 302 |        "      <th>7</th>\n",
 303 |        "      <td>https://www.blick.ch/news/das-ist-die-swiss-bo...</td>\n",
 304 |        "      <td>0</td>\n",
 305 |        "      <td>0</td>\n",
 306 |        "      <td>0</td>\n",
 307 |        "      <td>0</td>\n",
 308 |        "      <td>0</td>\n",
 309 |        "      <td>0</td>\n",
 310 |        "      <td>0</td>\n",
 311 |        "      <td>0</td>\n",
 312 |        "      <td>0</td>\n",
 313 |        "      <td>2238005</td>\n",
 314 |        "    </tr>\n",
 315 |        "    <tr>\n",
 316 |        "      <th>8</th>\n",
 317 |        "      <td>https://www.blick.ch/news/schweiz/vergangenhei...</td>\n",
 318 |        "      <td>0</td>\n",
 319 |        "      <td>0</td>\n",
 320 |        "      <td>0</td>\n",
 321 |        "      <td>0</td>\n",
 322 |        "      <td>0</td>\n",
 323 |        "      <td>0</td>\n",
 324 |        "      <td>0</td>\n",
 325 |        "      <td>0</td>\n",
 326 |        "      <td>0</td>\n",
 327 |        "      <td>5550392</td>\n",
 328 |        "    </tr>\n",
 329 |        "    <tr>\n",
 330 |        "      <th>9</th>\n",
 331 |        "      <td>https://www.blick.ch/news/wirtschaft/das-sind-...</td>\n",
 332 |        "      <td>0</td>\n",
 333 |        "      <td>0</td>\n",
 334 |        "      <td>0</td>\n",
 335 |        "      <td>0</td>\n",
 336 |        "      <td>0</td>\n",
 337 |        "      <td>0</td>\n",
 338 |        "      <td>0</td>\n",
 339 |        "      <td>0</td>\n",
 340 |        "      <td>0</td>\n",
 341 |        "      <td>4955860</td>\n",
 342 |        "    </tr>\n",
 343 |        "  </tbody>\n",
 344 |        "</table>\n",
 345 |        "</div>"
 346 |       ],
 347 |       "text/plain": [
 348 |        "                                             Address  article-text-box  \\\n",
 349 |        "0  https://www.blick.ch/people-tv/international/r...                 0   \n",
 350 |        "1  https://www.blick.ch/sport/eishockey/overtime-...                 0   \n",
 351 |        "2  https://www.blick.ch/news/ausland/neben-berueh...                 0   \n",
 352 |        "3  https://www.blick.ch/news/grausiger-fund-in-an...                 0   \n",
 353 |        "4  https://www.blick.ch/news/mahamaya-bern-die-we...                 0   \n",
 354 |        "5  https://www.blick.ch/people-tv/musik/liechtens...                 0   \n",
 355 |        "6  https://www.blick.ch/people-tv/tv/bachelorette...                 0   \n",
 356 |        "7  https://www.blick.ch/news/das-ist-die-swiss-bo...                 0   \n",
 357 |        "8  https://www.blick.ch/news/schweiz/vergangenhei...                 0   \n",
 358 |        "9  https://www.blick.ch/news/wirtschaft/das-sind-...                 0   \n",
 359 |        "\n",
 360 |        "   video-player  article-body  sda  scribble  has-image  iframe  \\\n",
 361 |        "0             0             0    0         0          0       0   \n",
 362 |        "1             0             0    0         0          0       0   \n",
 363 |        "2             0             0    0         0          0       0   \n",
 364 |        "3             0             0    0         0          0       0   \n",
 365 |        "4             0             0    0         0          0       0   \n",
 366 |        "5             0             0    0         0          0       0   \n",
 367 |        "6             0             0    0         0          0       0   \n",
 368 |        "7             0             0    0         0          0       0   \n",
 369 |        "8             0             0    0         0          0       0   \n",
 370 |        "9             0             0    0         0          0       0   \n",
 371 |        "\n",
 372 |        "   normal-article-layout  sda2 ID_Crawl  \n",
 373 |        "0                      0     0  4270807  \n",
 374 |        "1                      0     0   106244  \n",
 375 |        "2                      0     0  3896239  \n",
 376 |        "3                      0     0  2024473  \n",
 377 |        "4                      0     0   110094  \n",
 378 |        "5                      0     0  2131248  \n",
 379 |        "6                      0     0  3790667  \n",
 380 |        "7                      0     0  2238005  \n",
 381 |        "8                      0     0  5550392  \n",
 382 |        "9                      0     0  4955860  "
 383 |       ]
 384 |      },
 385 |      "execution_count": 6,
 386 |      "metadata": {},
 387 |      "output_type": "execute_result"
 388 |     }
 389 |    ],
 390 |    "source": [
 391 |     "dfCrawled = pd.DataFrame(pd.read_excel('./custom_search_all-test.xlsx', header=0, sheet_name='1 - All'))\n",
 392 |     "dfCrawled = dfCrawled.rename(columns={\"Contains: \\'\\\"article-text-box\\\"\\'\": \"article-text-box\", \n",
 393 |     "                                         \"Contains: \\'\\\"video-player\\\"\\'\":\"video-player\", \n",
 394 |     "                                         \"Contains: \\'\\\"article-body\\'\":\"article-body\",\n",
 395 |     "                                         \"Contains: \\'\\(sda\\)\\'\":\"sda\",\n",
 396 |     "                                         \"Contains: \\'\\\"scribble\\\"\\'\":\"scribble\",\n",
 397 |     "                                         \"Contains: \\'basic-image--free-desktop\\'\":\"has-image\", \n",
 398 |     "                                         \"Contains: \\'<iframe\\'\":\"iframe\", \n",
 399 |     "                                         \"Contains: \\'layout-item--flex-desktop-4'\":\"normal-article-layout\", \n",
 400 |     "                                         \"Contains: \\'\\(sda\\/\\'\":\"sda2\", \n",
 401 |     "                                     })\n",
 402 |     "dfCrawled['ID_Crawl'] = dfCrawled['Address'].str.extract('(?:.*id)([0-9]+)(?:.html)', expand=False)\n",
 403 |     "dfCrawled = dfCrawled.drop(\"Status Code\",1)\n",
 404 |     "dfCrawled = dfCrawled.drop(\"Status\",1)\n",
 405 |     "dfCrawled = dfCrawled.drop(\"Content\",1)\n",
 406 |     "dfCrawled.head(10)"
 407 |    ]
 408 |   },
 409 |   {
 410 |    "cell_type": "code",
 411 |    "execution_count": 7,
 412 |    "metadata": {},
 413 |    "outputs": [
 414 |     {
 415 |      "data": {
 416 |       "text/html": [
 417 |        "<div>\n",
 418 |        "<style scoped>\n",
 419 |        "    .dataframe tbody tr th:only-of-type {\n",
 420 |        "        vertical-align: middle;\n",
 421 |        "    }\n",
 422 |        "\n",
 423 |        "    .dataframe tbody tr th {\n",
 424 |        "        vertical-align: top;\n",
 425 |        "    }\n",
 426 |        "\n",
 427 |        "    .dataframe thead th {\n",
 428 |        "        text-align: right;\n",
 429 |        "    }\n",
 430 |        "</style>\n",
 431 |        "<table border=\"1\" class=\"dataframe\">\n",
 432 |        "  <thead>\n",
 433 |        "    <tr style=\"text-align: right;\">\n",
 434 |        "      <th></th>\n",
 435 |        "      <th>ID_GA</th>\n",
 436 |        "      <th>id count</th>\n",
 437 |        "      <th>pageviews per id</th>\n",
 438 |        "      <th>Address</th>\n",
 439 |        "      <th>article-text-box</th>\n",
 440 |        "      <th>video-player</th>\n",
 441 |        "      <th>article-body</th>\n",
 442 |        "      <th>sda</th>\n",
 443 |        "      <th>scribble</th>\n",
 444 |        "      <th>has-image</th>\n",
 445 |        "      <th>iframe</th>\n",
 446 |        "      <th>normal-article-layout</th>\n",
 447 |        "      <th>sda2</th>\n",
 448 |        "    </tr>\n",
 449 |        "  </thead>\n",
 450 |        "  <tbody>\n",
 451 |        "    <tr>\n",
 452 |        "      <th>34</th>\n",
 453 |        "      <td>31000</td>\n",
 454 |        "      <td>1.0</td>\n",
 455 |        "      <td>3.0</td>\n",
 456 |        "      <td>https://www.blick.ch/people-tv/international/e...</td>\n",
 457 |        "      <td>0.0</td>\n",
 458 |        "      <td>0.0</td>\n",
 459 |        "      <td>1.0</td>\n",
 460 |        "      <td>0.0</td>\n",
 461 |        "      <td>2.0</td>\n",
 462 |        "      <td>1.0</td>\n",
 463 |        "      <td>1.0</td>\n",
 464 |        "      <td>1.0</td>\n",
 465 |        "      <td>2.0</td>\n",
 466 |        "    </tr>\n",
 467 |        "    <tr>\n",
 468 |        "      <th>36</th>\n",
 469 |        "      <td>31002</td>\n",
 470 |        "      <td>1.0</td>\n",
 471 |        "      <td>1.0</td>\n",
 472 |        "      <td>https://www.blick.ch/news/ausland/afghanistan-...</td>\n",
 473 |        "      <td>0.0</td>\n",
 474 |        "      <td>0.0</td>\n",
 475 |        "      <td>1.0</td>\n",
 476 |        "      <td>3.0</td>\n",
 477 |        "      <td>2.0</td>\n",
 478 |        "      <td>0.0</td>\n",
 479 |        "      <td>1.0</td>\n",
 480 |        "      <td>1.0</td>\n",
 481 |        "      <td>0.0</td>\n",
 482 |        "    </tr>\n",
 483 |        "    <tr>\n",
 484 |        "      <th>37</th>\n",
 485 |        "      <td>31004</td>\n",
 486 |        "      <td>1.0</td>\n",
 487 |        "      <td>5.0</td>\n",
 488 |        "      <td>https://www.blick.ch/news/zwei-schweizer-in-ve...</td>\n",
 489 |        "      <td>0.0</td>\n",
 490 |        "      <td>0.0</td>\n",
 491 |        "      <td>1.0</td>\n",
 492 |        "      <td>0.0</td>\n",
 493 |        "      <td>2.0</td>\n",
 494 |        "      <td>0.0</td>\n",
 495 |        "      <td>1.0</td>\n",
 496 |        "      <td>1.0</td>\n",
 497 |        "      <td>0.0</td>\n",
 498 |        "    </tr>\n",
 499 |        "    <tr>\n",
 500 |        "      <th>51</th>\n",
 501 |        "      <td>31275</td>\n",
 502 |        "      <td>2.0</td>\n",
 503 |        "      <td>6.0</td>\n",
 504 |        "      <td>https://www.blick.ch/news/schweiz/zuerich/knas...</td>\n",
 505 |        "      <td>0.0</td>\n",
 506 |        "      <td>0.0</td>\n",
 507 |        "      <td>1.0</td>\n",
 508 |        "      <td>0.0</td>\n",
 509 |        "      <td>2.0</td>\n",
 510 |        "      <td>0.0</td>\n",
 511 |        "      <td>1.0</td>\n",
 512 |        "      <td>1.0</td>\n",
 513 |        "      <td>0.0</td>\n",
 514 |        "    </tr>\n",
 515 |        "    <tr>\n",
 516 |        "      <th>52</th>\n",
 517 |        "      <td>31277</td>\n",
 518 |        "      <td>1.0</td>\n",
 519 |        "      <td>12.0</td>\n",
 520 |        "      <td>https://www.blick.ch/news/ausland/steinmeier-p...</td>\n",
 521 |        "      <td>0.0</td>\n",
 522 |        "      <td>0.0</td>\n",
 523 |        "      <td>1.0</td>\n",
 524 |        "      <td>0.0</td>\n",
 525 |        "      <td>2.0</td>\n",
 526 |        "      <td>0.0</td>\n",
 527 |        "      <td>1.0</td>\n",
 528 |        "      <td>1.0</td>\n",
 529 |        "      <td>0.0</td>\n",
 530 |        "    </tr>\n",
 531 |        "    <tr>\n",
 532 |        "      <th>65</th>\n",
 533 |        "      <td>31388</td>\n",
 534 |        "      <td>1.0</td>\n",
 535 |        "      <td>3.0</td>\n",
 536 |        "      <td>https://www.blick.ch/news/schweiz/zentralschwe...</td>\n",
 537 |        "      <td>0.0</td>\n",
 538 |        "      <td>0.0</td>\n",
 539 |        "      <td>5.0</td>\n",
 540 |        "      <td>0.0</td>\n",
 541 |        "      <td>2.0</td>\n",
 542 |        "      <td>0.0</td>\n",
 543 |        "      <td>1.0</td>\n",
 544 |        "      <td>1.0</td>\n",
 545 |        "      <td>0.0</td>\n",
 546 |        "    </tr>\n",
 547 |        "    <tr>\n",
 548 |        "      <th>68</th>\n",
 549 |        "      <td>31425</td>\n",
 550 |        "      <td>1.0</td>\n",
 551 |        "      <td>1.0</td>\n",
 552 |        "      <td>https://www.blick.ch/people-tv/international/h...</td>\n",
 553 |        "      <td>0.0</td>\n",
 554 |        "      <td>0.0</td>\n",
 555 |        "      <td>1.0</td>\n",
 556 |        "      <td>0.0</td>\n",
 557 |        "      <td>2.0</td>\n",
 558 |        "      <td>0.0</td>\n",
 559 |        "      <td>1.0</td>\n",
 560 |        "      <td>1.0</td>\n",
 561 |        "      <td>0.0</td>\n",
 562 |        "    </tr>\n",
 563 |        "    <tr>\n",
 564 |        "      <th>69</th>\n",
 565 |        "      <td>31426</td>\n",
 566 |        "      <td>1.0</td>\n",
 567 |        "      <td>1.0</td>\n",
 568 |        "      <td>https://www.blick.ch/sport/eishockey/nla/serve...</td>\n",
 569 |        "      <td>1.0</td>\n",
 570 |        "      <td>0.0</td>\n",
 571 |        "      <td>1.0</td>\n",
 572 |        "      <td>0.0</td>\n",
 573 |        "      <td>2.0</td>\n",
 574 |        "      <td>0.0</td>\n",
 575 |        "      <td>1.0</td>\n",
 576 |        "      <td>1.0</td>\n",
 577 |        "      <td>0.0</td>\n",
 578 |        "    </tr>\n",
 579 |        "    <tr>\n",
 580 |        "      <th>70</th>\n",
 581 |        "      <td>31427</td>\n",
 582 |        "      <td>1.0</td>\n",
 583 |        "      <td>1.0</td>\n",
 584 |        "      <td>https://www.blick.ch/digital/games/news/colin-...</td>\n",
 585 |        "      <td>0.0</td>\n",
 586 |        "      <td>0.0</td>\n",
 587 |        "      <td>3.0</td>\n",
 588 |        "      <td>0.0</td>\n",
 589 |        "      <td>2.0</td>\n",
 590 |        "      <td>5.0</td>\n",
 591 |        "      <td>1.0</td>\n",
 592 |        "      <td>1.0</td>\n",
 593 |        "      <td>0.0</td>\n",
 594 |        "    </tr>\n",
 595 |        "    <tr>\n",
 596 |        "      <th>71</th>\n",
 597 |        "      <td>31428</td>\n",
 598 |        "      <td>1.0</td>\n",
 599 |        "      <td>1.0</td>\n",
 600 |        "      <td>https://www.blick.ch/digital/games/larry-magna...</td>\n",
 601 |        "      <td>0.0</td>\n",
 602 |        "      <td>0.0</td>\n",
 603 |        "      <td>1.0</td>\n",
 604 |        "      <td>0.0</td>\n",
 605 |        "      <td>2.0</td>\n",
 606 |        "      <td>0.0</td>\n",
 607 |        "      <td>1.0</td>\n",
 608 |        "      <td>1.0</td>\n",
 609 |        "      <td>0.0</td>\n",
 610 |        "    </tr>\n",
 611 |        "  </tbody>\n",
 612 |        "</table>\n",
 613 |        "</div>"
 614 |       ],
 615 |       "text/plain": [
 616 |        "    ID_GA  id count  pageviews per id  \\\n",
 617 |        "34  31000       1.0               3.0   \n",
 618 |        "36  31002       1.0               1.0   \n",
 619 |        "37  31004       1.0               5.0   \n",
 620 |        "51  31275       2.0               6.0   \n",
 621 |        "52  31277       1.0              12.0   \n",
 622 |        "65  31388       1.0               3.0   \n",
 623 |        "68  31425       1.0               1.0   \n",
 624 |        "69  31426       1.0               1.0   \n",
 625 |        "70  31427       1.0               1.0   \n",
 626 |        "71  31428       1.0               1.0   \n",
 627 |        "\n",
 628 |        "                                              Address  article-text-box  \\\n",
 629 |        "34  https://www.blick.ch/people-tv/international/e...               0.0   \n",
 630 |        "36  https://www.blick.ch/news/ausland/afghanistan-...               0.0   \n",
 631 |        "37  https://www.blick.ch/news/zwei-schweizer-in-ve...               0.0   \n",
 632 |        "51  https://www.blick.ch/news/schweiz/zuerich/knas...               0.0   \n",
 633 |        "52  https://www.blick.ch/news/ausland/steinmeier-p...               0.0   \n",
 634 |        "65  https://www.blick.ch/news/schweiz/zentralschwe...               0.0   \n",
 635 |        "68  https://www.blick.ch/people-tv/international/h...               0.0   \n",
 636 |        "69  https://www.blick.ch/sport/eishockey/nla/serve...               1.0   \n",
 637 |        "70  https://www.blick.ch/digital/games/news/colin-...               0.0   \n",
 638 |        "71  https://www.blick.ch/digital/games/larry-magna...               0.0   \n",
 639 |        "\n",
 640 |        "    video-player  article-body  sda  scribble  has-image  iframe  \\\n",
 641 |        "34           0.0           1.0  0.0       2.0        1.0     1.0   \n",
 642 |        "36           0.0           1.0  3.0       2.0        0.0     1.0   \n",
 643 |        "37           0.0           1.0  0.0       2.0        0.0     1.0   \n",
 644 |        "51           0.0           1.0  0.0       2.0        0.0     1.0   \n",
 645 |        "52           0.0           1.0  0.0       2.0        0.0     1.0   \n",
 646 |        "65           0.0           5.0  0.0       2.0        0.0     1.0   \n",
 647 |        "68           0.0           1.0  0.0       2.0        0.0     1.0   \n",
 648 |        "69           0.0           1.0  0.0       2.0        0.0     1.0   \n",
 649 |        "70           0.0           3.0  0.0       2.0        5.0     1.0   \n",
 650 |        "71           0.0           1.0  0.0       2.0        0.0     1.0   \n",
 651 |        "\n",
 652 |        "    normal-article-layout  sda2  \n",
 653 |        "34                    1.0   2.0  \n",
 654 |        "36                    1.0   0.0  \n",
 655 |        "37                    1.0   0.0  \n",
 656 |        "51                    1.0   0.0  \n",
 657 |        "52                    1.0   0.0  \n",
 658 |        "65                    1.0   0.0  \n",
 659 |        "68                    1.0   0.0  \n",
 660 |        "69                    1.0   0.0  \n",
 661 |        "70                    1.0   0.0  \n",
 662 |        "71                    1.0   0.0  "
 663 |       ]
 664 |      },
 665 |      "execution_count": 7,
 666 |      "metadata": {},
 667 |      "output_type": "execute_result"
 668 |     }
 669 |    ],
 670 |    "source": [
 671 |     "dfMerged = pd.merge(dfTestCSV, dfCrawled, left_on='ID_GA', right_on='ID_Crawl', how='outer')\n",
 672 |     "dfMerged = dfMerged[dfMerged['ID_Crawl'].notnull()]\n",
 673 |     "dfMerged = dfMerged[dfMerged['ID_GA'].notnull()]\n",
 674 |     "dfMerged = dfMerged.drop(\"ID_Crawl\",1)\n",
 675 |     "dfMerged.head(10)"
 676 |    ]
 677 |   },
 678 |   {
 679 |    "cell_type": "code",
 680 |    "execution_count": 14,
 681 |    "metadata": {},
 682 |    "outputs": [],
 683 |    "source": [
 684 |     "dfFilter = dfMerged"
 685 |    ]
 686 |   },
 687 |   {
 688 |    "cell_type": "code",
 689 |    "execution_count": 15,
 690 |    "metadata": {},
 691 |    "outputs": [
 692 |     {
 693 |      "data": {
 694 |       "text/html": [
 695 |        "<div>\n",
 696 |        "<style scoped>\n",
 697 |        "    .dataframe tbody tr th:only-of-type {\n",
 698 |        "        vertical-align: middle;\n",
 699 |        "    }\n",
 700 |        "\n",
 701 |        "    .dataframe tbody tr th {\n",
 702 |        "        vertical-align: top;\n",
 703 |        "    }\n",
 704 |        "\n",
 705 |        "    .dataframe thead th {\n",
 706 |        "        text-align: right;\n",
 707 |        "    }\n",
 708 |        "</style>\n",
 709 |        "<table border=\"1\" class=\"dataframe\">\n",
 710 |        "  <thead>\n",
 711 |        "    <tr style=\"text-align: right;\">\n",
 712 |        "      <th></th>\n",
 713 |        "      <th>ID_GA</th>\n",
 714 |        "      <th>id count</th>\n",
 715 |        "      <th>pageviews per id</th>\n",
 716 |        "      <th>Address</th>\n",
 717 |        "      <th>article-text-box</th>\n",
 718 |        "      <th>video-player</th>\n",
 719 |        "      <th>article-body</th>\n",
 720 |        "      <th>sda</th>\n",
 721 |        "      <th>scribble</th>\n",
 722 |        "      <th>has-image</th>\n",
 723 |        "      <th>iframe</th>\n",
 724 |        "      <th>normal-article-layout</th>\n",
 725 |        "      <th>sda2</th>\n",
 726 |        "    </tr>\n",
 727 |        "  </thead>\n",
 728 |        "  <tbody>\n",
 729 |        "    <tr>\n",
 730 |        "      <th>34</th>\n",
 731 |        "      <td>31000</td>\n",
 732 |        "      <td>1.0</td>\n",
 733 |        "      <td>3.0</td>\n",
 734 |        "      <td>https://www.blick.ch/people-tv/international/e...</td>\n",
 735 |        "      <td>0.0</td>\n",
 736 |        "      <td>0.0</td>\n",
 737 |        "      <td>1.0</td>\n",
 738 |        "      <td>0.0</td>\n",
 739 |        "      <td>2.0</td>\n",
 740 |        "      <td>1.0</td>\n",
 741 |        "      <td>1.0</td>\n",
 742 |        "      <td>1.0</td>\n",
 743 |        "      <td>2.0</td>\n",
 744 |        "    </tr>\n",
 745 |        "    <tr>\n",
 746 |        "      <th>36</th>\n",
 747 |        "      <td>31002</td>\n",
 748 |        "      <td>1.0</td>\n",
 749 |        "      <td>1.0</td>\n",
 750 |        "      <td>https://www.blick.ch/news/ausland/afghanistan-...</td>\n",
 751 |        "      <td>0.0</td>\n",
 752 |        "      <td>0.0</td>\n",
 753 |        "      <td>1.0</td>\n",
 754 |        "      <td>3.0</td>\n",
 755 |        "      <td>2.0</td>\n",
 756 |        "      <td>0.0</td>\n",
 757 |        "      <td>1.0</td>\n",
 758 |        "      <td>1.0</td>\n",
 759 |        "      <td>0.0</td>\n",
 760 |        "    </tr>\n",
 761 |        "    <tr>\n",
 762 |        "      <th>37</th>\n",
 763 |        "      <td>31004</td>\n",
 764 |        "      <td>1.0</td>\n",
 765 |        "      <td>5.0</td>\n",
 766 |        "      <td>https://www.blick.ch/news/zwei-schweizer-in-ve...</td>\n",
 767 |        "      <td>0.0</td>\n",
 768 |        "      <td>0.0</td>\n",
 769 |        "      <td>1.0</td>\n",
 770 |        "      <td>0.0</td>\n",
 771 |        "      <td>2.0</td>\n",
 772 |        "      <td>0.0</td>\n",
 773 |        "      <td>1.0</td>\n",
 774 |        "      <td>1.0</td>\n",
 775 |        "      <td>0.0</td>\n",
 776 |        "    </tr>\n",
 777 |        "    <tr>\n",
 778 |        "      <th>51</th>\n",
 779 |        "      <td>31275</td>\n",
 780 |        "      <td>2.0</td>\n",
 781 |        "      <td>6.0</td>\n",
 782 |        "      <td>https://www.blick.ch/news/schweiz/zuerich/knas...</td>\n",
 783 |        "      <td>0.0</td>\n",
 784 |        "      <td>0.0</td>\n",
 785 |        "      <td>1.0</td>\n",
 786 |        "      <td>0.0</td>\n",
 787 |        "      <td>2.0</td>\n",
 788 |        "      <td>0.0</td>\n",
 789 |        "      <td>1.0</td>\n",
 790 |        "      <td>1.0</td>\n",
 791 |        "      <td>0.0</td>\n",
 792 |        "    </tr>\n",
 793 |        "    <tr>\n",
 794 |        "      <th>52</th>\n",
 795 |        "      <td>31277</td>\n",
 796 |        "      <td>1.0</td>\n",
 797 |        "      <td>12.0</td>\n",
 798 |        "      <td>https://www.blick.ch/news/ausland/steinmeier-p...</td>\n",
 799 |        "      <td>0.0</td>\n",
 800 |        "      <td>0.0</td>\n",
 801 |        "      <td>1.0</td>\n",
 802 |        "      <td>0.0</td>\n",
 803 |        "      <td>2.0</td>\n",
 804 |        "      <td>0.0</td>\n",
 805 |        "      <td>1.0</td>\n",
 806 |        "      <td>1.0</td>\n",
 807 |        "      <td>0.0</td>\n",
 808 |        "    </tr>\n",
 809 |        "    <tr>\n",
 810 |        "      <th>65</th>\n",
 811 |        "      <td>31388</td>\n",
 812 |        "      <td>1.0</td>\n",
 813 |        "      <td>3.0</td>\n",
 814 |        "      <td>https://www.blick.ch/news/schweiz/zentralschwe...</td>\n",
 815 |        "      <td>0.0</td>\n",
 816 |        "      <td>0.0</td>\n",
 817 |        "      <td>5.0</td>\n",
 818 |        "      <td>0.0</td>\n",
 819 |        "      <td>2.0</td>\n",
 820 |        "      <td>0.0</td>\n",
 821 |        "      <td>1.0</td>\n",
 822 |        "      <td>1.0</td>\n",
 823 |        "      <td>0.0</td>\n",
 824 |        "    </tr>\n",
 825 |        "    <tr>\n",
 826 |        "      <th>68</th>\n",
 827 |        "      <td>31425</td>\n",
 828 |        "      <td>1.0</td>\n",
 829 |        "      <td>1.0</td>\n",
 830 |        "      <td>https://www.blick.ch/people-tv/international/h...</td>\n",
 831 |        "      <td>0.0</td>\n",
 832 |        "      <td>0.0</td>\n",
 833 |        "      <td>1.0</td>\n",
 834 |        "      <td>0.0</td>\n",
 835 |        "      <td>2.0</td>\n",
 836 |        "      <td>0.0</td>\n",
 837 |        "      <td>1.0</td>\n",
 838 |        "      <td>1.0</td>\n",
 839 |        "      <td>0.0</td>\n",
 840 |        "    </tr>\n",
 841 |        "    <tr>\n",
 842 |        "      <th>71</th>\n",
 843 |        "      <td>31428</td>\n",
 844 |        "      <td>1.0</td>\n",
 845 |        "      <td>1.0</td>\n",
 846 |        "      <td>https://www.blick.ch/digital/games/larry-magna...</td>\n",
 847 |        "      <td>0.0</td>\n",
 848 |        "      <td>0.0</td>\n",
 849 |        "      <td>1.0</td>\n",
 850 |        "      <td>0.0</td>\n",
 851 |        "      <td>2.0</td>\n",
 852 |        "      <td>0.0</td>\n",
 853 |        "      <td>1.0</td>\n",
 854 |        "      <td>1.0</td>\n",
 855 |        "      <td>0.0</td>\n",
 856 |        "    </tr>\n",
 857 |        "    <tr>\n",
 858 |        "      <th>72</th>\n",
 859 |        "      <td>31450</td>\n",
 860 |        "      <td>1.0</td>\n",
 861 |        "      <td>2.0</td>\n",
 862 |        "      <td>https://www.blick.ch/news/schweiz/zuerich/scho...</td>\n",
 863 |        "      <td>0.0</td>\n",
 864 |        "      <td>0.0</td>\n",
 865 |        "      <td>1.0</td>\n",
 866 |        "      <td>0.0</td>\n",
 867 |        "      <td>2.0</td>\n",
 868 |        "      <td>0.0</td>\n",
 869 |        "      <td>1.0</td>\n",
 870 |        "      <td>1.0</td>\n",
 871 |        "      <td>0.0</td>\n",
 872 |        "    </tr>\n",
 873 |        "    <tr>\n",
 874 |        "      <th>74</th>\n",
 875 |        "      <td>31463</td>\n",
 876 |        "      <td>1.0</td>\n",
 877 |        "      <td>8.0</td>\n",
 878 |        "      <td>https://www.blick.ch/news/schweiz/zuerich/zum-...</td>\n",
 879 |        "      <td>0.0</td>\n",
 880 |        "      <td>0.0</td>\n",
 881 |        "      <td>1.0</td>\n",
 882 |        "      <td>0.0</td>\n",
 883 |        "      <td>2.0</td>\n",
 884 |        "      <td>0.0</td>\n",
 885 |        "      <td>1.0</td>\n",
 886 |        "      <td>1.0</td>\n",
 887 |        "      <td>0.0</td>\n",
 888 |        "    </tr>\n",
 889 |        "    <tr>\n",
 890 |        "      <th>132</th>\n",
 891 |        "      <td>6260</td>\n",
 892 |        "      <td>1.0</td>\n",
 893 |        "      <td>1.0</td>\n",
 894 |        "      <td>https://www.blick.ch/news/schweiz/zentralschwe...</td>\n",
 895 |        "      <td>0.0</td>\n",
 896 |        "      <td>0.0</td>\n",
 897 |        "      <td>1.0</td>\n",
 898 |        "      <td>0.0</td>\n",
 899 |        "      <td>2.0</td>\n",
 900 |        "      <td>0.0</td>\n",
 901 |        "      <td>1.0</td>\n",
 902 |        "      <td>1.0</td>\n",
 903 |        "      <td>0.0</td>\n",
 904 |        "    </tr>\n",
 905 |        "    <tr>\n",
 906 |        "      <th>135</th>\n",
 907 |        "      <td>6328</td>\n",
 908 |        "      <td>1.0</td>\n",
 909 |        "      <td>1.0</td>\n",
 910 |        "      <td>https://www.blick.ch/news/beijing-fashion-week...</td>\n",
 911 |        "      <td>0.0</td>\n",
 912 |        "      <td>0.0</td>\n",
 913 |        "      <td>1.0</td>\n",
 914 |        "      <td>0.0</td>\n",
 915 |        "      <td>2.0</td>\n",
 916 |        "      <td>0.0</td>\n",
 917 |        "      <td>1.0</td>\n",
 918 |        "      <td>1.0</td>\n",
 919 |        "      <td>0.0</td>\n",
 920 |        "    </tr>\n",
 921 |        "  </tbody>\n",
 922 |        "</table>\n",
 923 |        "</div>"
 924 |       ],
 925 |       "text/plain": [
 926 |        "     ID_GA  id count  pageviews per id  \\\n",
 927 |        "34   31000       1.0               3.0   \n",
 928 |        "36   31002       1.0               1.0   \n",
 929 |        "37   31004       1.0               5.0   \n",
 930 |        "51   31275       2.0               6.0   \n",
 931 |        "52   31277       1.0              12.0   \n",
 932 |        "65   31388       1.0               3.0   \n",
 933 |        "68   31425       1.0               1.0   \n",
 934 |        "71   31428       1.0               1.0   \n",
 935 |        "72   31450       1.0               2.0   \n",
 936 |        "74   31463       1.0               8.0   \n",
 937 |        "132   6260       1.0               1.0   \n",
 938 |        "135   6328       1.0               1.0   \n",
 939 |        "\n",
 940 |        "                                               Address  article-text-box  \\\n",
 941 |        "34   https://www.blick.ch/people-tv/international/e...               0.0   \n",
 942 |        "36   https://www.blick.ch/news/ausland/afghanistan-...               0.0   \n",
 943 |        "37   https://www.blick.ch/news/zwei-schweizer-in-ve...               0.0   \n",
 944 |        "51   https://www.blick.ch/news/schweiz/zuerich/knas...               0.0   \n",
 945 |        "52   https://www.blick.ch/news/ausland/steinmeier-p...               0.0   \n",
 946 |        "65   https://www.blick.ch/news/schweiz/zentralschwe...               0.0   \n",
 947 |        "68   https://www.blick.ch/people-tv/international/h...               0.0   \n",
 948 |        "71   https://www.blick.ch/digital/games/larry-magna...               0.0   \n",
 949 |        "72   https://www.blick.ch/news/schweiz/zuerich/scho...               0.0   \n",
 950 |        "74   https://www.blick.ch/news/schweiz/zuerich/zum-...               0.0   \n",
 951 |        "132  https://www.blick.ch/news/schweiz/zentralschwe...               0.0   \n",
 952 |        "135  https://www.blick.ch/news/beijing-fashion-week...               0.0   \n",
 953 |        "\n",
 954 |        "     video-player  article-body  sda  scribble  has-image  iframe  \\\n",
 955 |        "34            0.0           1.0  0.0       2.0        1.0     1.0   \n",
 956 |        "36            0.0           1.0  3.0       2.0        0.0     1.0   \n",
 957 |        "37            0.0           1.0  0.0       2.0        0.0     1.0   \n",
 958 |        "51            0.0           1.0  0.0       2.0        0.0     1.0   \n",
 959 |        "52            0.0           1.0  0.0       2.0        0.0     1.0   \n",
 960 |        "65            0.0           5.0  0.0       2.0        0.0     1.0   \n",
 961 |        "68            0.0           1.0  0.0       2.0        0.0     1.0   \n",
 962 |        "71            0.0           1.0  0.0       2.0        0.0     1.0   \n",
 963 |        "72            0.0           1.0  0.0       2.0        0.0     1.0   \n",
 964 |        "74            0.0           1.0  0.0       2.0        0.0     1.0   \n",
 965 |        "132           0.0           1.0  0.0       2.0        0.0     1.0   \n",
 966 |        "135           0.0           1.0  0.0       2.0        0.0     1.0   \n",
 967 |        "\n",
 968 |        "     normal-article-layout  sda2  \n",
 969 |        "34                     1.0   2.0  \n",
 970 |        "36                     1.0   0.0  \n",
 971 |        "37                     1.0   0.0  \n",
 972 |        "51                     1.0   0.0  \n",
 973 |        "52                     1.0   0.0  \n",
 974 |        "65                     1.0   0.0  \n",
 975 |        "68                     1.0   0.0  \n",
 976 |        "71                     1.0   0.0  \n",
 977 |        "72                     1.0   0.0  \n",
 978 |        "74                     1.0   0.0  \n",
 979 |        "132                    1.0   0.0  \n",
 980 |        "135                    1.0   0.0  "
 981 |       ]
 982 |      },
 983 |      "execution_count": 15,
 984 |      "metadata": {},
 985 |      "output_type": "execute_result"
 986 |     }
 987 |    ],
 988 |    "source": [
 989 |     "dfFilter = dfFilter[(dfFilter['article-text-box'] == 0) &\n",
 990 |     "                    (dfFilter['has-image'] < 4)\n",
 991 |     "                   ]\n",
 992 |     "dfFilter"
 993 |    ]
 994 |   },
 995 |   {
 996 |    "cell_type": "code",
 997 |    "execution_count": 16,
 998 |    "metadata": {},
 999 |    "outputs": [
1000 |     {
1001 |      "data": {
1002 |       "text/html": [
1003 |        "<div>\n",
1004 |        "<style scoped>\n",
1005 |        "    .dataframe tbody tr th:only-of-type {\n",
1006 |        "        vertical-align: middle;\n",
1007 |        "    }\n",
1008 |        "\n",
1009 |        "    .dataframe tbody tr th {\n",
1010 |        "        vertical-align: top;\n",
1011 |        "    }\n",
1012 |        "\n",
1013 |        "    .dataframe thead th {\n",
1014 |        "        text-align: right;\n",
1015 |        "    }\n",
1016 |        "</style>\n",
1017 |        "<table border=\"1\" class=\"dataframe\">\n",
1018 |        "  <thead>\n",
1019 |        "    <tr style=\"text-align: right;\">\n",
1020 |        "      <th></th>\n",
1021 |        "      <th>ID_GA</th>\n",
1022 |        "      <th>id count</th>\n",
1023 |        "      <th>pageviews per id</th>\n",
1024 |        "      <th>Address</th>\n",
1025 |        "      <th>article-text-box</th>\n",
1026 |        "      <th>video-player</th>\n",
1027 |        "      <th>article-body</th>\n",
1028 |        "      <th>sda</th>\n",
1029 |        "      <th>scribble</th>\n",
1030 |        "      <th>has-image</th>\n",
1031 |        "      <th>iframe</th>\n",
1032 |        "      <th>normal-article-layout</th>\n",
1033 |        "      <th>sda2</th>\n",
1034 |        "    </tr>\n",
1035 |        "  </thead>\n",
1036 |        "  <tbody>\n",
1037 |        "    <tr>\n",
1038 |        "      <th>36</th>\n",
1039 |        "      <td>31002</td>\n",
1040 |        "      <td>1.0</td>\n",
1041 |        "      <td>1.0</td>\n",
1042 |        "      <td>https://www.blick.ch/news/ausland/afghanistan-...</td>\n",
1043 |        "      <td>0.0</td>\n",
1044 |        "      <td>0.0</td>\n",
1045 |        "      <td>1.0</td>\n",
1046 |        "      <td>3.0</td>\n",
1047 |        "      <td>2.0</td>\n",
1048 |        "      <td>0.0</td>\n",
1049 |        "      <td>1.0</td>\n",
1050 |        "      <td>1.0</td>\n",
1051 |        "      <td>0.0</td>\n",
1052 |        "    </tr>\n",
1053 |        "    <tr>\n",
1054 |        "      <th>37</th>\n",
1055 |        "      <td>31004</td>\n",
1056 |        "      <td>1.0</td>\n",
1057 |        "      <td>5.0</td>\n",
1058 |        "      <td>https://www.blick.ch/news/zwei-schweizer-in-ve...</td>\n",
1059 |        "      <td>0.0</td>\n",
1060 |        "      <td>0.0</td>\n",
1061 |        "      <td>1.0</td>\n",
1062 |        "      <td>0.0</td>\n",
1063 |        "      <td>2.0</td>\n",
1064 |        "      <td>0.0</td>\n",
1065 |        "      <td>1.0</td>\n",
1066 |        "      <td>1.0</td>\n",
1067 |        "      <td>0.0</td>\n",
1068 |        "    </tr>\n",
1069 |        "    <tr>\n",
1070 |        "      <th>51</th>\n",
1071 |        "      <td>31275</td>\n",
1072 |        "      <td>2.0</td>\n",
1073 |        "      <td>6.0</td>\n",
1074 |        "      <td>https://www.blick.ch/news/schweiz/zuerich/knas...</td>\n",
1075 |        "      <td>0.0</td>\n",
1076 |        "      <td>0.0</td>\n",
1077 |        "      <td>1.0</td>\n",
1078 |        "      <td>0.0</td>\n",
1079 |        "      <td>2.0</td>\n",
1080 |        "      <td>0.0</td>\n",
1081 |        "      <td>1.0</td>\n",
1082 |        "      <td>1.0</td>\n",
1083 |        "      <td>0.0</td>\n",
1084 |        "    </tr>\n",
1085 |        "    <tr>\n",
1086 |        "      <th>52</th>\n",
1087 |        "      <td>31277</td>\n",
1088 |        "      <td>1.0</td>\n",
1089 |        "      <td>12.0</td>\n",
1090 |        "      <td>https://www.blick.ch/news/ausland/steinmeier-p...</td>\n",
1091 |        "      <td>0.0</td>\n",
1092 |        "      <td>0.0</td>\n",
1093 |        "      <td>1.0</td>\n",
1094 |        "      <td>0.0</td>\n",
1095 |        "      <td>2.0</td>\n",
1096 |        "      <td>0.0</td>\n",
1097 |        "      <td>1.0</td>\n",
1098 |        "      <td>1.0</td>\n",
1099 |        "      <td>0.0</td>\n",
1100 |        "    </tr>\n",
1101 |        "    <tr>\n",
1102 |        "      <th>65</th>\n",
1103 |        "      <td>31388</td>\n",
1104 |        "      <td>1.0</td>\n",
1105 |        "      <td>3.0</td>\n",
1106 |        "      <td>https://www.blick.ch/news/schweiz/zentralschwe...</td>\n",
1107 |        "      <td>0.0</td>\n",
1108 |        "      <td>0.0</td>\n",
1109 |        "      <td>5.0</td>\n",
1110 |        "      <td>0.0</td>\n",
1111 |        "      <td>2.0</td>\n",
1112 |        "      <td>0.0</td>\n",
1113 |        "      <td>1.0</td>\n",
1114 |        "      <td>1.0</td>\n",
1115 |        "      <td>0.0</td>\n",
1116 |        "    </tr>\n",
1117 |        "    <tr>\n",
1118 |        "      <th>72</th>\n",
1119 |        "      <td>31450</td>\n",
1120 |        "      <td>1.0</td>\n",
1121 |        "      <td>2.0</td>\n",
1122 |        "      <td>https://www.blick.ch/news/schweiz/zuerich/scho...</td>\n",
1123 |        "      <td>0.0</td>\n",
1124 |        "      <td>0.0</td>\n",
1125 |        "      <td>1.0</td>\n",
1126 |        "      <td>0.0</td>\n",
1127 |        "      <td>2.0</td>\n",
1128 |        "      <td>0.0</td>\n",
1129 |        "      <td>1.0</td>\n",
1130 |        "      <td>1.0</td>\n",
1131 |        "      <td>0.0</td>\n",
1132 |        "    </tr>\n",
1133 |        "    <tr>\n",
1134 |        "      <th>74</th>\n",
1135 |        "      <td>31463</td>\n",
1136 |        "      <td>1.0</td>\n",
1137 |        "      <td>8.0</td>\n",
1138 |        "      <td>https://www.blick.ch/news/schweiz/zuerich/zum-...</td>\n",
1139 |        "      <td>0.0</td>\n",
1140 |        "      <td>0.0</td>\n",
1141 |        "      <td>1.0</td>\n",
1142 |        "      <td>0.0</td>\n",
1143 |        "      <td>2.0</td>\n",
1144 |        "      <td>0.0</td>\n",
1145 |        "      <td>1.0</td>\n",
1146 |        "      <td>1.0</td>\n",
1147 |        "      <td>0.0</td>\n",
1148 |        "    </tr>\n",
1149 |        "    <tr>\n",
1150 |        "      <th>132</th>\n",
1151 |        "      <td>6260</td>\n",
1152 |        "      <td>1.0</td>\n",
1153 |        "      <td>1.0</td>\n",
1154 |        "      <td>https://www.blick.ch/news/schweiz/zentralschwe...</td>\n",
1155 |        "      <td>0.0</td>\n",
1156 |        "      <td>0.0</td>\n",
1157 |        "      <td>1.0</td>\n",
1158 |        "      <td>0.0</td>\n",
1159 |        "      <td>2.0</td>\n",
1160 |        "      <td>0.0</td>\n",
1161 |        "      <td>1.0</td>\n",
1162 |        "      <td>1.0</td>\n",
1163 |        "      <td>0.0</td>\n",
1164 |        "    </tr>\n",
1165 |        "    <tr>\n",
1166 |        "      <th>135</th>\n",
1167 |        "      <td>6328</td>\n",
1168 |        "      <td>1.0</td>\n",
1169 |        "      <td>1.0</td>\n",
1170 |        "      <td>https://www.blick.ch/news/beijing-fashion-week...</td>\n",
1171 |        "      <td>0.0</td>\n",
1172 |        "      <td>0.0</td>\n",
1173 |        "      <td>1.0</td>\n",
1174 |        "      <td>0.0</td>\n",
1175 |        "      <td>2.0</td>\n",
1176 |        "      <td>0.0</td>\n",
1177 |        "      <td>1.0</td>\n",
1178 |        "      <td>1.0</td>\n",
1179 |        "      <td>0.0</td>\n",
1180 |        "    </tr>\n",
1181 |        "  </tbody>\n",
1182 |        "</table>\n",
1183 |        "</div>"
1184 |       ],
1185 |       "text/plain": [
1186 |        "     ID_GA  id count  pageviews per id  \\\n",
1187 |        "36   31002       1.0               1.0   \n",
1188 |        "37   31004       1.0               5.0   \n",
1189 |        "51   31275       2.0               6.0   \n",
1190 |        "52   31277       1.0              12.0   \n",
1191 |        "65   31388       1.0               3.0   \n",
1192 |        "72   31450       1.0               2.0   \n",
1193 |        "74   31463       1.0               8.0   \n",
1194 |        "132   6260       1.0               1.0   \n",
1195 |        "135   6328       1.0               1.0   \n",
1196 |        "\n",
1197 |        "                                               Address  article-text-box  \\\n",
1198 |        "36   https://www.blick.ch/news/ausland/afghanistan-...               0.0   \n",
1199 |        "37   https://www.blick.ch/news/zwei-schweizer-in-ve...               0.0   \n",
1200 |        "51   https://www.blick.ch/news/schweiz/zuerich/knas...               0.0   \n",
1201 |        "52   https://www.blick.ch/news/ausland/steinmeier-p...               0.0   \n",
1202 |        "65   https://www.blick.ch/news/schweiz/zentralschwe...               0.0   \n",
1203 |        "72   https://www.blick.ch/news/schweiz/zuerich/scho...               0.0   \n",
1204 |        "74   https://www.blick.ch/news/schweiz/zuerich/zum-...               0.0   \n",
1205 |        "132  https://www.blick.ch/news/schweiz/zentralschwe...               0.0   \n",
1206 |        "135  https://www.blick.ch/news/beijing-fashion-week...               0.0   \n",
1207 |        "\n",
1208 |        "     video-player  article-body  sda  scribble  has-image  iframe  \\\n",
1209 |        "36            0.0           1.0  3.0       2.0        0.0     1.0   \n",
1210 |        "37            0.0           1.0  0.0       2.0        0.0     1.0   \n",
1211 |        "51            0.0           1.0  0.0       2.0        0.0     1.0   \n",
1212 |        "52            0.0           1.0  0.0       2.0        0.0     1.0   \n",
1213 |        "65            0.0           5.0  0.0       2.0        0.0     1.0   \n",
1214 |        "72            0.0           1.0  0.0       2.0        0.0     1.0   \n",
1215 |        "74            0.0           1.0  0.0       2.0        0.0     1.0   \n",
1216 |        "132           0.0           1.0  0.0       2.0        0.0     1.0   \n",
1217 |        "135           0.0           1.0  0.0       2.0        0.0     1.0   \n",
1218 |        "\n",
1219 |        "     normal-article-layout  sda2  \n",
1220 |        "36                     1.0   0.0  \n",
1221 |        "37                     1.0   0.0  \n",
1222 |        "51                     1.0   0.0  \n",
1223 |        "52                     1.0   0.0  \n",
1224 |        "65                     1.0   0.0  \n",
1225 |        "72                     1.0   0.0  \n",
1226 |        "74                     1.0   0.0  \n",
1227 |        "132                    1.0   0.0  \n",
1228 |        "135                    1.0   0.0  "
1229 |       ]
1230 |      },
1231 |      "execution_count": 16,
1232 |      "metadata": {},
1233 |      "output_type": "execute_result"
1234 |     }
1235 |    ],
1236 |    "source": [
1237 |     "dfFilter = dfFilter[dfFilter[\"Address\"].str.contains(\"ch/news/|ch/wetter/\")]\n",
1238 |     "dfFilter"
1239 |    ]
1240 |   },
1241 |   {
1242 |    "cell_type": "code",
1243 |    "execution_count": null,
1244 |    "metadata": {},
1245 |    "outputs": [],
1246 |    "source": [
1247 |     "dfFilter.to_csv('dfFilter-Test.csv', sep='\\t')\n"
1248 |    ]
1249 |   },
1250 |   {
1251 |    "cell_type": "code",
1252 |    "execution_count": null,
1253 |    "metadata": {},
1254 |    "outputs": [],
1255 |    "source": []
1256 |   }
1257 |  ],
1258 |  "metadata": {
1259 |   "kernelspec": {
1260 |    "display_name": "Python 3",
1261 |    "language": "python",
1262 |    "name": "python3"
1263 |   },
1264 |   "language_info": {
1265 |    "codemirror_mode": {
1266 |     "name": "ipython",
1267 |     "version": 3
1268 |    },
1269 |    "file_extension": ".py",
1270 |    "mimetype": "text/x-python",
1271 |    "name": "python",
1272 |    "nbconvert_exporter": "python",
1273 |    "pygments_lexer": "ipython3",
1274 |    "version": "3.7.2"
1275 |   }
1276 |  },
1277 |  "nbformat": 4,
1278 |  "nbformat_minor": 2
1279 | }
1280 | 


--------------------------------------------------------------------------------
/Analytics Big Beach Spring Break.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 14,
  6 |    "metadata": {
  7 |     "scrolled": false
  8 |    },
  9 |    "outputs": [],
 10 |    "source": [
 11 |     "import pandas as pd"
 12 |    ]
 13 |   },
 14 |   {
 15 |    "cell_type": "code",
 16 |    "execution_count": 15,
 17 |    "metadata": {
 18 |     "scrolled": true
 19 |    },
 20 |    "outputs": [
 21 |     {
 22 |      "name": "stdout",
 23 |      "output_type": "stream",
 24 |      "text": [
 25 |       "len(rows): 9\n"
 26 |      ]
 27 |     }
 28 |    ],
 29 |    "source": [
 30 |     "from apiclient.discovery import build\n",
 31 |     "from oauth2client.service_account import ServiceAccountCredentials\n",
 32 |     "\n",
 33 |     "SCOPES = ['https://www.googleapis.com/auth/analytics.readonly']\n",
 34 |     "KEY_FILE_LOCATION = './Ga Test-1a2cc84a6ab1.json'\n",
 35 |     "VIEW_ID = '107822800'\n",
 36 |     "\n",
 37 |     "def initialize_analyticsreporting():\n",
 38 |     "  credentials = ServiceAccountCredentials.from_json_keyfile_name(\n",
 39 |     "      KEY_FILE_LOCATION, SCOPES)\n",
 40 |     "  analytics = build('analyticsreporting', 'v4', credentials=credentials)\n",
 41 |     "  return analytics\n",
 42 |     "\n",
 43 |     "#Get one report page\n",
 44 |     "def get_report(analytics, pageTokenVar):\n",
 45 |     "  return analytics.reports().batchGet(\n",
 46 |     "      body={\n",
 47 |     "        'reportRequests': [\n",
 48 |     "        {\n",
 49 |     "          'viewId': VIEW_ID,\n",
 50 |     "          'dateRanges': [{'startDate': '3daysAgo', 'endDate': 'yesterday'}],\n",
 51 |     "          'metrics': [{'expression': 'ga:pageviews'}],\n",
 52 |     "          'dimensions': [{'name': 'ga:pagePath'}],\n",
 53 |     "          'pageSize': 10000,\n",
 54 |     "          'pageToken': pageTokenVar,\n",
 55 |     "          'samplingLevel': 'LARGE'\n",
 56 |     "        }]\n",
 57 |     "      }\n",
 58 |     "  ).execute()\n",
 59 |     "    \n",
 60 |     "def handle_report(analytics,pagetoken,rows):  \n",
 61 |     "    response = get_report(analytics, pagetoken)\n",
 62 |     "\n",
 63 |     "    #Header, Dimentions Headers, Metric Headers \n",
 64 |     "    columnHeader = response.get(\"reports\")[0].get('columnHeader', {})\n",
 65 |     "    dimensionHeaders = columnHeader.get('dimensions', [])\n",
 66 |     "    metricHeaders = columnHeader.get('metricHeader', {}).get('metricHeaderEntries', [])\n",
 67 |     "\n",
 68 |     "    #Pagination\n",
 69 |     "    pagetoken = response.get(\"reports\")[0].get('nextPageToken', None)\n",
 70 |     "    \n",
 71 |     "    #Rows\n",
 72 |     "    rowsNew = response.get(\"reports\")[0].get('data', {}).get('rows', [])\n",
 73 |     "    rows = rows + rowsNew\n",
 74 |     "    print(\"len(rows): \" + str(len(rows)))\n",
 75 |     "\n",
 76 |     "    #Recursivly query next page\n",
 77 |     "    if pagetoken != None:\n",
 78 |     "        return handle_report(analytics,pagetoken,rows)\n",
 79 |     "    else:\n",
 80 |     "        #nicer results\n",
 81 |     "        nicerows=[]\n",
 82 |     "        for row in rows:\n",
 83 |     "            dic={}\n",
 84 |     "            dimensions = row.get('dimensions', [])\n",
 85 |     "            dateRangeValues = row.get('metrics', [])\n",
 86 |     "\n",
 87 |     "            for header, dimension in zip(dimensionHeaders, dimensions):\n",
 88 |     "                dic[header] = dimension\n",
 89 |     "\n",
 90 |     "            for i, values in enumerate(dateRangeValues):\n",
 91 |     "                for metric, value in zip(metricHeaders, values.get('values')):\n",
 92 |     "                    if ',' in value or ',' in value:\n",
 93 |     "                        dic[metric.get('name')] = float(value)\n",
 94 |     "                    else:\n",
 95 |     "                        dic[metric.get('name')] = int(value)\n",
 96 |     "            nicerows.append(dic)\n",
 97 |     "        return nicerows\n",
 98 |     "\n",
 99 |     "#Start\n",
100 |     "def main():    \n",
101 |     "    analytics = initialize_analyticsreporting()\n",
102 |     "    \n",
103 |     "    global dfanalytics\n",
104 |     "    dfanalytics = []\n",
105 |     "\n",
106 |     "    rows = []\n",
107 |     "    rows = handle_report(analytics,'0',rows)\n",
108 |     "\n",
109 |     "    dfanalytics = pd.DataFrame(list(rows))\n",
110 |     "\n",
111 |     "if __name__ == '__main__':\n",
112 |     "  main()"
113 |    ]
114 |   },
115 |   {
116 |    "cell_type": "code",
117 |    "execution_count": 16,
118 |    "metadata": {},
119 |    "outputs": [
120 |     {
121 |      "data": {
122 |       "text/html": [
123 |        "<div>\n",
124 |        "<style scoped>\n",
125 |        "    .dataframe tbody tr th:only-of-type {\n",
126 |        "        vertical-align: middle;\n",
127 |        "    }\n",
128 |        "\n",
129 |        "    .dataframe tbody tr th {\n",
130 |        "        vertical-align: top;\n",
131 |        "    }\n",
132 |        "\n",
133 |        "    .dataframe thead th {\n",
134 |        "        text-align: right;\n",
135 |        "    }\n",
136 |        "</style>\n",
137 |        "<table border=\"1\" class=\"dataframe\">\n",
138 |        "  <thead>\n",
139 |        "    <tr style=\"text-align: right;\">\n",
140 |        "      <th></th>\n",
141 |        "      <th>ga:pagePath</th>\n",
142 |        "      <th>ga:pageviews</th>\n",
143 |        "    </tr>\n",
144 |        "  </thead>\n",
145 |        "  <tbody>\n",
146 |        "    <tr>\n",
147 |        "      <th>0</th>\n",
148 |        "      <td>/</td>\n",
149 |        "      <td>16</td>\n",
150 |        "    </tr>\n",
151 |        "    <tr>\n",
152 |        "      <th>1</th>\n",
153 |        "      <td>/?s=Corona+</td>\n",
154 |        "      <td>1</td>\n",
155 |        "    </tr>\n",
156 |        "    <tr>\n",
157 |        "      <th>2</th>\n",
158 |        "      <td>/en/</td>\n",
159 |        "      <td>2</td>\n",
160 |        "    </tr>\n",
161 |        "    <tr>\n",
162 |        "      <th>3</th>\n",
163 |        "      <td>/en/program/</td>\n",
164 |        "      <td>1</td>\n",
165 |        "    </tr>\n",
166 |        "    <tr>\n",
167 |        "      <th>4</th>\n",
168 |        "      <td>/kontakt/</td>\n",
169 |        "      <td>2</td>\n",
170 |        "    </tr>\n",
171 |        "    <tr>\n",
172 |        "      <th>5</th>\n",
173 |        "      <td>/line-up/</td>\n",
174 |        "      <td>7</td>\n",
175 |        "    </tr>\n",
176 |        "    <tr>\n",
177 |        "      <th>6</th>\n",
178 |        "      <td>/pakete/</td>\n",
179 |        "      <td>3</td>\n",
180 |        "    </tr>\n",
181 |        "    <tr>\n",
182 |        "      <th>7</th>\n",
183 |        "      <td>/partyboot/</td>\n",
184 |        "      <td>2</td>\n",
185 |        "    </tr>\n",
186 |        "    <tr>\n",
187 |        "      <th>8</th>\n",
188 |        "      <td>/tickets/</td>\n",
189 |        "      <td>6</td>\n",
190 |        "    </tr>\n",
191 |        "  </tbody>\n",
192 |        "</table>\n",
193 |        "</div>"
194 |       ],
195 |       "text/plain": [
196 |        "    ga:pagePath  ga:pageviews\n",
197 |        "0             /            16\n",
198 |        "1   /?s=Corona+             1\n",
199 |        "2          /en/             2\n",
200 |        "3  /en/program/             1\n",
201 |        "4     /kontakt/             2\n",
202 |        "5     /line-up/             7\n",
203 |        "6      /pakete/             3\n",
204 |        "7   /partyboot/             2\n",
205 |        "8     /tickets/             6"
206 |       ]
207 |      },
208 |      "execution_count": 16,
209 |      "metadata": {},
210 |      "output_type": "execute_result"
211 |     }
212 |    ],
213 |    "source": [
214 |     "dfanalytics"
215 |    ]
216 |   },
217 |   {
218 |    "cell_type": "code",
219 |    "execution_count": null,
220 |    "metadata": {},
221 |    "outputs": [],
222 |    "source": []
223 |   },
224 |   {
225 |    "cell_type": "code",
226 |    "execution_count": null,
227 |    "metadata": {},
228 |    "outputs": [],
229 |    "source": []
230 |   }
231 |  ],
232 |  "metadata": {
233 |   "kernelspec": {
234 |    "display_name": "Python 3",
235 |    "language": "python",
236 |    "name": "python3"
237 |   },
238 |   "language_info": {
239 |    "codemirror_mode": {
240 |     "name": "ipython",
241 |     "version": 3
242 |    },
243 |    "file_extension": ".py",
244 |    "mimetype": "text/x-python",
245 |    "name": "python",
246 |    "nbconvert_exporter": "python",
247 |    "pygments_lexer": "ipython3",
248 |    "version": "3.7.2"
249 |   }
250 |  },
251 |  "nbformat": 4,
252 |  "nbformat_minor": 2
253 | }
254 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Python-Pandas-SEO-Videos
2 | 


--------------------------------------------------------------------------------
/Read News XML Sitemap with Python + Pandas.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 36,
  6 |    "metadata": {},
  7 |    "outputs": [
  8 |     {
  9 |      "data": {
 10 |       "text/html": [
 11 |        "<div>\n",
 12 |        "<style scoped>\n",
 13 |        "    .dataframe tbody tr th:only-of-type {\n",
 14 |        "        vertical-align: middle;\n",
 15 |        "    }\n",
 16 |        "\n",
 17 |        "    .dataframe tbody tr th {\n",
 18 |        "        vertical-align: top;\n",
 19 |        "    }\n",
 20 |        "\n",
 21 |        "    .dataframe thead th {\n",
 22 |        "        text-align: right;\n",
 23 |        "    }\n",
 24 |        "</style>\n",
 25 |        "<table border=\"1\" class=\"dataframe\">\n",
 26 |        "  <thead>\n",
 27 |        "    <tr style=\"text-align: right;\">\n",
 28 |        "      <th></th>\n",
 29 |        "      <th>loc</th>\n",
 30 |        "      <th>news:title</th>\n",
 31 |        "      <th>news:publication_date</th>\n",
 32 |        "    </tr>\n",
 33 |        "  </thead>\n",
 34 |        "  <tbody>\n",
 35 |        "    <tr>\n",
 36 |        "      <th>0</th>\n",
 37 |        "      <td>https://www.nzz.ch/feuilleton/in-zeiten-des-co...</td>\n",
 38 |        "      <td>Das virale Feuilleton-Tagebuch macht keine Ost...</td>\n",
 39 |        "      <td>2020-04-11T17:29:16+02:00</td>\n",
 40 |        "    </tr>\n",
 41 |        "    <tr>\n",
 42 |        "      <th>1</th>\n",
 43 |        "      <td>https://www.nzz.ch/international/coronavirus-i...</td>\n",
 44 |        "      <td>Bundespräsident ruft zu Geduld und Disziplin a...</td>\n",
 45 |        "      <td>2020-04-11T17:23:24+02:00</td>\n",
 46 |        "    </tr>\n",
 47 |        "    <tr>\n",
 48 |        "      <th>2</th>\n",
 49 |        "      <td>https://www.nzz.ch/zuerich/coronavirus-in-zuer...</td>\n",
 50 |        "      <td>Die Schliessung eines Spargelstands im Zürcher...</td>\n",
 51 |        "      <td>2020-04-11T16:25:00+02:00</td>\n",
 52 |        "    </tr>\n",
 53 |        "    <tr>\n",
 54 |        "      <th>3</th>\n",
 55 |        "      <td>https://www.nzz.ch/sport/verfahren-gegen-ex-fi...</td>\n",
 56 |        "      <td>Verfahren gegen Ex-Fifa-Präsident Sepp Blatter...</td>\n",
 57 |        "      <td>2020-04-11T15:25:36+02:00</td>\n",
 58 |        "    </tr>\n",
 59 |        "    <tr>\n",
 60 |        "      <th>4</th>\n",
 61 |        "      <td>https://www.nzz.ch/schweiz/coronavirus-in-der-...</td>\n",
 62 |        "      <td>Zahl der Neuinfizierten geht weiterhin zurück,...</td>\n",
 63 |        "      <td>2020-04-11T15:18:27+02:00</td>\n",
 64 |        "    </tr>\n",
 65 |        "    <tr>\n",
 66 |        "      <th>5</th>\n",
 67 |        "      <td>https://www.nzz.ch/panorama/die-wichtigsten-gr...</td>\n",
 68 |        "      <td>2700 Menschen sind bisher in Deutschland an Co...</td>\n",
 69 |        "      <td>2020-04-11T13:12:33+02:00</td>\n",
 70 |        "    </tr>\n",
 71 |        "    <tr>\n",
 72 |        "      <th>6</th>\n",
 73 |        "      <td>https://www.nzz.ch/international/coronavirus-w...</td>\n",
 74 |        "      <td>Die USA verzeichnen mehr als 2000 Todesfälle i...</td>\n",
 75 |        "      <td>2020-04-11T13:05:51+02:00</td>\n",
 76 |        "    </tr>\n",
 77 |        "    <tr>\n",
 78 |        "      <th>7</th>\n",
 79 |        "      <td>https://www.nzz.ch/panorama/radioaktiv-belaste...</td>\n",
 80 |        "      <td>Waldbrände bei Tschernobyl weiter nicht unter ...</td>\n",
 81 |        "      <td>2020-04-11T12:39:31+02:00</td>\n",
 82 |        "    </tr>\n",
 83 |        "    <tr>\n",
 84 |        "      <th>8</th>\n",
 85 |        "      <td>https://www.nzz.ch/zuerich/zuerich-philosophis...</td>\n",
 86 |        "      <td>Philosophie per Telefon während der Corona-Kri...</td>\n",
 87 |        "      <td>2020-04-11T12:00:00+02:00</td>\n",
 88 |        "    </tr>\n",
 89 |        "    <tr>\n",
 90 |        "      <th>9</th>\n",
 91 |        "      <td>https://www.nzz.ch/panorama/die-wichtigsten-gr...</td>\n",
 92 |        "      <td>Die Schweiz zählt 1000 Tote, in Genf wächst pr...</td>\n",
 93 |        "      <td>2020-04-11T11:31:00+02:00</td>\n",
 94 |        "    </tr>\n",
 95 |        "    <tr>\n",
 96 |        "      <th>10</th>\n",
 97 |        "      <td>https://www.nzz.ch/panorama/passant-findet-in-...</td>\n",
 98 |        "      <td>Passant findet in Köln Kriegsbombe am Rhein – ...</td>\n",
 99 |        "      <td>2020-04-11T11:26:07+02:00</td>\n",
100 |        "    </tr>\n",
101 |        "    <tr>\n",
102 |        "      <th>11</th>\n",
103 |        "      <td>https://www.nzz.ch/zuerich/coronavirus-in-zuer...</td>\n",
104 |        "      <td>Die Zürcher Regierung ist in der Krise mächtig...</td>\n",
105 |        "      <td>2020-04-11T10:30:00+02:00</td>\n",
106 |        "    </tr>\n",
107 |        "    <tr>\n",
108 |        "      <th>12</th>\n",
109 |        "      <td>https://www.nzz.ch/briefing/feiertagsbriefing-...</td>\n",
110 |        "      <td>Feiertagsbriefing</td>\n",
111 |        "      <td>2020-04-11T08:03:19+02:00</td>\n",
112 |        "    </tr>\n",
113 |        "    <tr>\n",
114 |        "      <th>13</th>\n",
115 |        "      <td>https://www.nzz.ch/briefing/feiertagsbriefing-...</td>\n",
116 |        "      <td>Feiertagsbriefing</td>\n",
117 |        "      <td>2020-04-11T07:56:12+02:00</td>\n",
118 |        "    </tr>\n",
119 |        "    <tr>\n",
120 |        "      <th>14</th>\n",
121 |        "      <td>https://www.nzz.ch/panorama/umweltschuetzer-be...</td>\n",
122 |        "      <td>Umweltschützer befürchten erneut schwere Waldb...</td>\n",
123 |        "      <td>2020-04-11T07:44:32+02:00</td>\n",
124 |        "    </tr>\n",
125 |        "    <tr>\n",
126 |        "      <th>15</th>\n",
127 |        "      <td>https://www.nzz.ch/international/pentagon-korr...</td>\n",
128 |        "      <td>Die USA versprechen bis zu zehn Millionen Doll...</td>\n",
129 |        "      <td>2020-04-11T07:13:32+02:00</td>\n",
130 |        "    </tr>\n",
131 |        "    <tr>\n",
132 |        "      <th>16</th>\n",
133 |        "      <td>https://www.nzz.ch/wirtschaft/coronavirus-bill...</td>\n",
134 |        "      <td>Bill Gates – der Mann, der die Pandemie kommen...</td>\n",
135 |        "      <td>2020-04-11T07:00:00+02:00</td>\n",
136 |        "    </tr>\n",
137 |        "    <tr>\n",
138 |        "      <th>17</th>\n",
139 |        "      <td>https://www.nzz.ch/sport/korruption-bei-der-wm...</td>\n",
140 |        "      <td>Korruption bei der WM-Doppelvergabe 2018 und 2...</td>\n",
141 |        "      <td>2020-04-11T06:30:00+02:00</td>\n",
142 |        "    </tr>\n",
143 |        "    <tr>\n",
144 |        "      <th>18</th>\n",
145 |        "      <td>https://www.nzz.ch/schweiz/coronavirus-gebaut-...</td>\n",
146 |        "      <td>Gebaut wird überall ungefähr gleich. Warum als...</td>\n",
147 |        "      <td>2020-04-11T06:00:00+02:00</td>\n",
148 |        "    </tr>\n",
149 |        "    <tr>\n",
150 |        "      <th>19</th>\n",
151 |        "      <td>https://www.nzz.ch/reisen/reisen-im-kopf-zu-be...</td>\n",
152 |        "      <td>Tränen auf Aurora</td>\n",
153 |        "      <td>2020-04-11T06:00:00+02:00</td>\n",
154 |        "    </tr>\n",
155 |        "    <tr>\n",
156 |        "      <th>20</th>\n",
157 |        "      <td>https://www.nzz.ch/reisen/lagerkoller-in-den-e...</td>\n",
158 |        "      <td>«Und dann sitzt du fest im Basislager, wartest...</td>\n",
159 |        "      <td>2020-04-11T06:00:00+02:00</td>\n",
160 |        "    </tr>\n",
161 |        "    <tr>\n",
162 |        "      <th>21</th>\n",
163 |        "      <td>https://www.nzz.ch/meinung/coronavirus-in-deut...</td>\n",
164 |        "      <td>Die Politik ist nun endgültig durchpädagogisie...</td>\n",
165 |        "      <td>2020-04-11T06:00:00+02:00</td>\n",
166 |        "    </tr>\n",
167 |        "    <tr>\n",
168 |        "      <th>22</th>\n",
169 |        "      <td>https://www.nzz.ch/wirtschaft/trump-putin-und-...</td>\n",
170 |        "      <td>Trump, Putin und der saudische Kronprinz bilde...</td>\n",
171 |        "      <td>2020-04-11T05:36:13+02:00</td>\n",
172 |        "    </tr>\n",
173 |        "    <tr>\n",
174 |        "      <th>23</th>\n",
175 |        "      <td>https://www.nzz.ch/wirtschaft/coronavirus-schw...</td>\n",
176 |        "      <td>Die Schweizer Banken bleiben krisenresistent</td>\n",
177 |        "      <td>2020-04-11T05:30:00+02:00</td>\n",
178 |        "    </tr>\n",
179 |        "    <tr>\n",
180 |        "      <th>24</th>\n",
181 |        "      <td>https://www.nzz.ch/sport/das-muessen-sie-frau-...</td>\n",
182 |        "      <td>«Das müssen Sie Frau Wang fragen» – Sky Sun we...</td>\n",
183 |        "      <td>2020-04-11T05:30:00+02:00</td>\n",
184 |        "    </tr>\n",
185 |        "    <tr>\n",
186 |        "      <th>25</th>\n",
187 |        "      <td>https://www.nzz.ch/finanzen/aktienmaerkte-so-e...</td>\n",
188 |        "      <td>So entwickeln sich Aktienmärkte nach wasserfal...</td>\n",
189 |        "      <td>2020-04-11T05:30:00+02:00</td>\n",
190 |        "    </tr>\n",
191 |        "    <tr>\n",
192 |        "      <th>26</th>\n",
193 |        "      <td>https://www.nzz.ch/wissenschaft/coronavirus-fu...</td>\n",
194 |        "      <td>Für die meisten Allergiker bedeutet das neue C...</td>\n",
195 |        "      <td>2020-04-11T05:30:00+02:00</td>\n",
196 |        "    </tr>\n",
197 |        "    <tr>\n",
198 |        "      <th>27</th>\n",
199 |        "      <td>https://www.nzz.ch/feuilleton/bei-t-c-boyle-is...</td>\n",
200 |        "      <td>Die Natur ist am Ende immer die Stärkere –T. C...</td>\n",
201 |        "      <td>2020-04-11T05:30:00+02:00</td>\n",
202 |        "    </tr>\n",
203 |        "    <tr>\n",
204 |        "      <th>28</th>\n",
205 |        "      <td>https://www.nzz.ch/feuilleton/wann-ist-jesus-a...</td>\n",
206 |        "      <td>Ohne Ostern gäbe es vielleicht keinen Computer...</td>\n",
207 |        "      <td>2020-04-11T05:30:00+02:00</td>\n",
208 |        "    </tr>\n",
209 |        "    <tr>\n",
210 |        "      <th>29</th>\n",
211 |        "      <td>https://www.nzz.ch/feuilleton/die-corona-krise...</td>\n",
212 |        "      <td>Die Corona-Krise offenbart auch eine Krise der...</td>\n",
213 |        "      <td>2020-04-11T05:30:00+02:00</td>\n",
214 |        "    </tr>\n",
215 |        "    <tr>\n",
216 |        "      <th>...</th>\n",
217 |        "      <td>...</td>\n",
218 |        "      <td>...</td>\n",
219 |        "      <td>...</td>\n",
220 |        "    </tr>\n",
221 |        "    <tr>\n",
222 |        "      <th>82</th>\n",
223 |        "      <td>https://www.nzz.ch/panorama/saengerin-rihanna-...</td>\n",
224 |        "      <td>Sängerin Rihanna und Twitter-Chef Dorsey spend...</td>\n",
225 |        "      <td>2020-04-10T07:15:40+02:00</td>\n",
226 |        "    </tr>\n",
227 |        "    <tr>\n",
228 |        "      <th>83</th>\n",
229 |        "      <td>https://www.nzz.ch/international/uno-sicherhei...</td>\n",
230 |        "      <td>Uno-Sicherheitsrat trifft sich virtuell: Guter...</td>\n",
231 |        "      <td>2020-04-10T06:37:43+02:00</td>\n",
232 |        "    </tr>\n",
233 |        "    <tr>\n",
234 |        "      <th>84</th>\n",
235 |        "      <td>https://www.nzz.ch/panorama/massensterben-von-...</td>\n",
236 |        "      <td>Massensterben von Zugvögeln in Griechenland du...</td>\n",
237 |        "      <td>2020-04-10T06:00:24+02:00</td>\n",
238 |        "    </tr>\n",
239 |        "    <tr>\n",
240 |        "      <th>85</th>\n",
241 |        "      <td>https://www.nzz.ch/zuerich/coronavirus-in-zuer...</td>\n",
242 |        "      <td>Ostern in der Isolation: 11 Ideen, damit einem...</td>\n",
243 |        "      <td>2020-04-10T06:00:00+02:00</td>\n",
244 |        "    </tr>\n",
245 |        "    <tr>\n",
246 |        "      <th>86</th>\n",
247 |        "      <td>https://www.nzz.ch/international/pakistan-in-d...</td>\n",
248 |        "      <td>Ziaullah Khan produziert in Pakistan Waffen fü...</td>\n",
249 |        "      <td>2020-04-10T06:00:00+02:00</td>\n",
250 |        "    </tr>\n",
251 |        "    <tr>\n",
252 |        "      <th>87</th>\n",
253 |        "      <td>https://www.nzz.ch/international/soldaten-toet...</td>\n",
254 |        "      <td>Terroreinsatz im Tschad: Soldaten töten rund 1...</td>\n",
255 |        "      <td>2020-04-10T05:49:27+02:00</td>\n",
256 |        "    </tr>\n",
257 |        "    <tr>\n",
258 |        "      <th>88</th>\n",
259 |        "      <td>https://www.nzz.ch/fotografie/die-corona-krise...</td>\n",
260 |        "      <td>Die Corona-Krise in Bildern: vier Fotografen, ...</td>\n",
261 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
262 |        "    </tr>\n",
263 |        "    <tr>\n",
264 |        "      <th>89</th>\n",
265 |        "      <td>https://www.nzz.ch/international/coronavirus-u...</td>\n",
266 |        "      <td>Auffällig viele Schwarze unter den Covid-19-Op...</td>\n",
267 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
268 |        "    </tr>\n",
269 |        "    <tr>\n",
270 |        "      <th>90</th>\n",
271 |        "      <td>https://www.nzz.ch/gesellschaft/corona-krise-j...</td>\n",
272 |        "      <td>Er ist der Albtraum jeder Schwiegermutter</td>\n",
273 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
274 |        "    </tr>\n",
275 |        "    <tr>\n",
276 |        "      <th>91</th>\n",
277 |        "      <td>https://www.nzz.ch/wissenschaft/coronavirus-wa...</td>\n",
278 |        "      <td>Die verseuchte Herzdame: Wie Viren von einem K...</td>\n",
279 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
280 |        "    </tr>\n",
281 |        "    <tr>\n",
282 |        "      <th>92</th>\n",
283 |        "      <td>https://www.nzz.ch/feuilleton/die-landschaft-v...</td>\n",
284 |        "      <td>Die Landschaft, von der Passstrasse her gesehe...</td>\n",
285 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
286 |        "    </tr>\n",
287 |        "    <tr>\n",
288 |        "      <th>93</th>\n",
289 |        "      <td>https://www.nzz.ch/feuilleton/trotz-coronaviru...</td>\n",
290 |        "      <td>In Berlin dürfen Bücher noch verkauft werden, ...</td>\n",
291 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
292 |        "    </tr>\n",
293 |        "    <tr>\n",
294 |        "      <th>94</th>\n",
295 |        "      <td>https://www.nzz.ch/feuilleton/postcorona-studi...</td>\n",
296 |        "      <td>Post-Corona-Studien: Das Virus wäre nichts ohn...</td>\n",
297 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
298 |        "    </tr>\n",
299 |        "    <tr>\n",
300 |        "      <th>95</th>\n",
301 |        "      <td>https://www.nzz.ch/technologie/wie-findet-die-...</td>\n",
302 |        "      <td>Wie findet die Swisscom unsere Standortdaten h...</td>\n",
303 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
304 |        "    </tr>\n",
305 |        "    <tr>\n",
306 |        "      <th>96</th>\n",
307 |        "      <td>https://www.nzz.ch/panorama/coronavirus-norweg...</td>\n",
308 |        "      <td>Wegen der Corona-Krise dürfen die Norweger nic...</td>\n",
309 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
310 |        "    </tr>\n",
311 |        "    <tr>\n",
312 |        "      <th>97</th>\n",
313 |        "      <td>https://www.nzz.ch/international/suedkoreaner-...</td>\n",
314 |        "      <td>Südkoreaner tauschen Daten gegen Bewegungsfrei...</td>\n",
315 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
316 |        "    </tr>\n",
317 |        "    <tr>\n",
318 |        "      <th>98</th>\n",
319 |        "      <td>https://www.nzz.ch/gesellschaft/corona-krise-z...</td>\n",
320 |        "      <td>Einer der reichsten Spanier lässt nun Wegwerfk...</td>\n",
321 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
322 |        "    </tr>\n",
323 |        "    <tr>\n",
324 |        "      <th>99</th>\n",
325 |        "      <td>https://www.nzz.ch/feuilleton/verschon-uns-got...</td>\n",
326 |        "      <td>«Verschon uns, Gott, mit Strafen»: Vielleicht ...</td>\n",
327 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
328 |        "    </tr>\n",
329 |        "    <tr>\n",
330 |        "      <th>100</th>\n",
331 |        "      <td>https://www.nzz.ch/sport/jakub-blaszczykowski-...</td>\n",
332 |        "      <td>Jakub Blaszczykowski erfreut die Fussball-Roma...</td>\n",
333 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
334 |        "    </tr>\n",
335 |        "    <tr>\n",
336 |        "      <th>101</th>\n",
337 |        "      <td>https://www.nzz.ch/feuilleton/corona-und-kunst...</td>\n",
338 |        "      <td>«Den Frühling können sie nicht absagen»</td>\n",
339 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
340 |        "    </tr>\n",
341 |        "    <tr>\n",
342 |        "      <th>102</th>\n",
343 |        "      <td>https://www.nzz.ch/feuilleton/coronavirus-zuku...</td>\n",
344 |        "      <td>Das Leben im ewigen Idyll ist zu Ende</td>\n",
345 |        "      <td>2020-04-10T05:30:00+02:00</td>\n",
346 |        "    </tr>\n",
347 |        "    <tr>\n",
348 |        "      <th>103</th>\n",
349 |        "      <td>https://www.nzz.ch/panorama/us-aktivistin-fuer...</td>\n",
350 |        "      <td>US-Aktivistin für Frauenrechte und Homo-Ehe mi...</td>\n",
351 |        "      <td>2020-04-10T05:01:53+02:00</td>\n",
352 |        "    </tr>\n",
353 |        "    <tr>\n",
354 |        "      <th>104</th>\n",
355 |        "      <td>https://www.nzz.ch/wissenschaft/coronavirus-di...</td>\n",
356 |        "      <td>Diese neuartigen Impfstoffe sollen das Coronav...</td>\n",
357 |        "      <td>2020-04-10T05:00:00+02:00</td>\n",
358 |        "    </tr>\n",
359 |        "    <tr>\n",
360 |        "      <th>105</th>\n",
361 |        "      <td>https://www.nzz.ch/zuerich/coronavirus-in-zuer...</td>\n",
362 |        "      <td>Den Startups trocknet in der Corona-Krise die ...</td>\n",
363 |        "      <td>2020-04-10T05:00:00+02:00</td>\n",
364 |        "    </tr>\n",
365 |        "    <tr>\n",
366 |        "      <th>106</th>\n",
367 |        "      <td>https://www.nzz.ch/panorama/nach-kanu-unfall-d...</td>\n",
368 |        "      <td>Nach Kanu-Unfall der Kennedy-Enkelin: Taucher ...</td>\n",
369 |        "      <td>2020-04-10T03:54:05+02:00</td>\n",
370 |        "    </tr>\n",
371 |        "    <tr>\n",
372 |        "      <th>107</th>\n",
373 |        "      <td>https://www.nzz.ch/wirtschaft/eu-finanzministe...</td>\n",
374 |        "      <td>Die EU-Finanzminister einigen sich auf ein 540...</td>\n",
375 |        "      <td>2020-04-10T01:57:21+02:00</td>\n",
376 |        "    </tr>\n",
377 |        "    <tr>\n",
378 |        "      <th>108</th>\n",
379 |        "      <td>https://www.nzz.ch/fotografie/bilder-des-tages...</td>\n",
380 |        "      <td>Bilder des Tages</td>\n",
381 |        "      <td>2020-04-10T00:11:23+02:00</td>\n",
382 |        "    </tr>\n",
383 |        "    <tr>\n",
384 |        "      <th>109</th>\n",
385 |        "      <td>https://www.nzz.ch/wirtschaft/noch-einmal-66-m...</td>\n",
386 |        "      <td>Weitere 6,6 Millionen Arbeitslose in den USA -...</td>\n",
387 |        "      <td>2020-04-09T23:43:41+02:00</td>\n",
388 |        "    </tr>\n",
389 |        "    <tr>\n",
390 |        "      <th>110</th>\n",
391 |        "      <td>https://www.nzz.ch/video/nzz-format/natur-pur-...</td>\n",
392 |        "      <td>Natur pur: Uruguays Rinder ohne Antibiotika un...</td>\n",
393 |        "      <td>2020-04-09T23:00:00+02:00</td>\n",
394 |        "    </tr>\n",
395 |        "    <tr>\n",
396 |        "      <th>111</th>\n",
397 |        "      <td>https://www.nzz.ch/meinung/zentralbanken-finan...</td>\n",
398 |        "      <td>Zentralbanken finanzieren Staatsausgaben: Das ...</td>\n",
399 |        "      <td>2020-04-09T20:24:24+02:00</td>\n",
400 |        "    </tr>\n",
401 |        "  </tbody>\n",
402 |        "</table>\n",
403 |        "<p>112 rows × 3 columns</p>\n",
404 |        "</div>"
405 |       ],
406 |       "text/plain": [
407 |        "                                                   loc  \\\n",
408 |        "0    https://www.nzz.ch/feuilleton/in-zeiten-des-co...   \n",
409 |        "1    https://www.nzz.ch/international/coronavirus-i...   \n",
410 |        "2    https://www.nzz.ch/zuerich/coronavirus-in-zuer...   \n",
411 |        "3    https://www.nzz.ch/sport/verfahren-gegen-ex-fi...   \n",
412 |        "4    https://www.nzz.ch/schweiz/coronavirus-in-der-...   \n",
413 |        "5    https://www.nzz.ch/panorama/die-wichtigsten-gr...   \n",
414 |        "6    https://www.nzz.ch/international/coronavirus-w...   \n",
415 |        "7    https://www.nzz.ch/panorama/radioaktiv-belaste...   \n",
416 |        "8    https://www.nzz.ch/zuerich/zuerich-philosophis...   \n",
417 |        "9    https://www.nzz.ch/panorama/die-wichtigsten-gr...   \n",
418 |        "10   https://www.nzz.ch/panorama/passant-findet-in-...   \n",
419 |        "11   https://www.nzz.ch/zuerich/coronavirus-in-zuer...   \n",
420 |        "12   https://www.nzz.ch/briefing/feiertagsbriefing-...   \n",
421 |        "13   https://www.nzz.ch/briefing/feiertagsbriefing-...   \n",
422 |        "14   https://www.nzz.ch/panorama/umweltschuetzer-be...   \n",
423 |        "15   https://www.nzz.ch/international/pentagon-korr...   \n",
424 |        "16   https://www.nzz.ch/wirtschaft/coronavirus-bill...   \n",
425 |        "17   https://www.nzz.ch/sport/korruption-bei-der-wm...   \n",
426 |        "18   https://www.nzz.ch/schweiz/coronavirus-gebaut-...   \n",
427 |        "19   https://www.nzz.ch/reisen/reisen-im-kopf-zu-be...   \n",
428 |        "20   https://www.nzz.ch/reisen/lagerkoller-in-den-e...   \n",
429 |        "21   https://www.nzz.ch/meinung/coronavirus-in-deut...   \n",
430 |        "22   https://www.nzz.ch/wirtschaft/trump-putin-und-...   \n",
431 |        "23   https://www.nzz.ch/wirtschaft/coronavirus-schw...   \n",
432 |        "24   https://www.nzz.ch/sport/das-muessen-sie-frau-...   \n",
433 |        "25   https://www.nzz.ch/finanzen/aktienmaerkte-so-e...   \n",
434 |        "26   https://www.nzz.ch/wissenschaft/coronavirus-fu...   \n",
435 |        "27   https://www.nzz.ch/feuilleton/bei-t-c-boyle-is...   \n",
436 |        "28   https://www.nzz.ch/feuilleton/wann-ist-jesus-a...   \n",
437 |        "29   https://www.nzz.ch/feuilleton/die-corona-krise...   \n",
438 |        "..                                                 ...   \n",
439 |        "82   https://www.nzz.ch/panorama/saengerin-rihanna-...   \n",
440 |        "83   https://www.nzz.ch/international/uno-sicherhei...   \n",
441 |        "84   https://www.nzz.ch/panorama/massensterben-von-...   \n",
442 |        "85   https://www.nzz.ch/zuerich/coronavirus-in-zuer...   \n",
443 |        "86   https://www.nzz.ch/international/pakistan-in-d...   \n",
444 |        "87   https://www.nzz.ch/international/soldaten-toet...   \n",
445 |        "88   https://www.nzz.ch/fotografie/die-corona-krise...   \n",
446 |        "89   https://www.nzz.ch/international/coronavirus-u...   \n",
447 |        "90   https://www.nzz.ch/gesellschaft/corona-krise-j...   \n",
448 |        "91   https://www.nzz.ch/wissenschaft/coronavirus-wa...   \n",
449 |        "92   https://www.nzz.ch/feuilleton/die-landschaft-v...   \n",
450 |        "93   https://www.nzz.ch/feuilleton/trotz-coronaviru...   \n",
451 |        "94   https://www.nzz.ch/feuilleton/postcorona-studi...   \n",
452 |        "95   https://www.nzz.ch/technologie/wie-findet-die-...   \n",
453 |        "96   https://www.nzz.ch/panorama/coronavirus-norweg...   \n",
454 |        "97   https://www.nzz.ch/international/suedkoreaner-...   \n",
455 |        "98   https://www.nzz.ch/gesellschaft/corona-krise-z...   \n",
456 |        "99   https://www.nzz.ch/feuilleton/verschon-uns-got...   \n",
457 |        "100  https://www.nzz.ch/sport/jakub-blaszczykowski-...   \n",
458 |        "101  https://www.nzz.ch/feuilleton/corona-und-kunst...   \n",
459 |        "102  https://www.nzz.ch/feuilleton/coronavirus-zuku...   \n",
460 |        "103  https://www.nzz.ch/panorama/us-aktivistin-fuer...   \n",
461 |        "104  https://www.nzz.ch/wissenschaft/coronavirus-di...   \n",
462 |        "105  https://www.nzz.ch/zuerich/coronavirus-in-zuer...   \n",
463 |        "106  https://www.nzz.ch/panorama/nach-kanu-unfall-d...   \n",
464 |        "107  https://www.nzz.ch/wirtschaft/eu-finanzministe...   \n",
465 |        "108  https://www.nzz.ch/fotografie/bilder-des-tages...   \n",
466 |        "109  https://www.nzz.ch/wirtschaft/noch-einmal-66-m...   \n",
467 |        "110  https://www.nzz.ch/video/nzz-format/natur-pur-...   \n",
468 |        "111  https://www.nzz.ch/meinung/zentralbanken-finan...   \n",
469 |        "\n",
470 |        "                                            news:title  \\\n",
471 |        "0    Das virale Feuilleton-Tagebuch macht keine Ost...   \n",
472 |        "1    Bundespräsident ruft zu Geduld und Disziplin a...   \n",
473 |        "2    Die Schliessung eines Spargelstands im Zürcher...   \n",
474 |        "3    Verfahren gegen Ex-Fifa-Präsident Sepp Blatter...   \n",
475 |        "4    Zahl der Neuinfizierten geht weiterhin zurück,...   \n",
476 |        "5    2700 Menschen sind bisher in Deutschland an Co...   \n",
477 |        "6    Die USA verzeichnen mehr als 2000 Todesfälle i...   \n",
478 |        "7    Waldbrände bei Tschernobyl weiter nicht unter ...   \n",
479 |        "8    Philosophie per Telefon während der Corona-Kri...   \n",
480 |        "9    Die Schweiz zählt 1000 Tote, in Genf wächst pr...   \n",
481 |        "10   Passant findet in Köln Kriegsbombe am Rhein – ...   \n",
482 |        "11   Die Zürcher Regierung ist in der Krise mächtig...   \n",
483 |        "12                                   Feiertagsbriefing   \n",
484 |        "13                                   Feiertagsbriefing   \n",
485 |        "14   Umweltschützer befürchten erneut schwere Waldb...   \n",
486 |        "15   Die USA versprechen bis zu zehn Millionen Doll...   \n",
487 |        "16   Bill Gates – der Mann, der die Pandemie kommen...   \n",
488 |        "17   Korruption bei der WM-Doppelvergabe 2018 und 2...   \n",
489 |        "18   Gebaut wird überall ungefähr gleich. Warum als...   \n",
490 |        "19                                   Tränen auf Aurora   \n",
491 |        "20   «Und dann sitzt du fest im Basislager, wartest...   \n",
492 |        "21   Die Politik ist nun endgültig durchpädagogisie...   \n",
493 |        "22   Trump, Putin und der saudische Kronprinz bilde...   \n",
494 |        "23        Die Schweizer Banken bleiben krisenresistent   \n",
495 |        "24   «Das müssen Sie Frau Wang fragen» – Sky Sun we...   \n",
496 |        "25   So entwickeln sich Aktienmärkte nach wasserfal...   \n",
497 |        "26   Für die meisten Allergiker bedeutet das neue C...   \n",
498 |        "27   Die Natur ist am Ende immer die Stärkere –T. C...   \n",
499 |        "28   Ohne Ostern gäbe es vielleicht keinen Computer...   \n",
500 |        "29   Die Corona-Krise offenbart auch eine Krise der...   \n",
501 |        "..                                                 ...   \n",
502 |        "82   Sängerin Rihanna und Twitter-Chef Dorsey spend...   \n",
503 |        "83   Uno-Sicherheitsrat trifft sich virtuell: Guter...   \n",
504 |        "84   Massensterben von Zugvögeln in Griechenland du...   \n",
505 |        "85   Ostern in der Isolation: 11 Ideen, damit einem...   \n",
506 |        "86   Ziaullah Khan produziert in Pakistan Waffen fü...   \n",
507 |        "87   Terroreinsatz im Tschad: Soldaten töten rund 1...   \n",
508 |        "88   Die Corona-Krise in Bildern: vier Fotografen, ...   \n",
509 |        "89   Auffällig viele Schwarze unter den Covid-19-Op...   \n",
510 |        "90           Er ist der Albtraum jeder Schwiegermutter   \n",
511 |        "91   Die verseuchte Herzdame: Wie Viren von einem K...   \n",
512 |        "92   Die Landschaft, von der Passstrasse her gesehe...   \n",
513 |        "93   In Berlin dürfen Bücher noch verkauft werden, ...   \n",
514 |        "94   Post-Corona-Studien: Das Virus wäre nichts ohn...   \n",
515 |        "95   Wie findet die Swisscom unsere Standortdaten h...   \n",
516 |        "96   Wegen der Corona-Krise dürfen die Norweger nic...   \n",
517 |        "97   Südkoreaner tauschen Daten gegen Bewegungsfrei...   \n",
518 |        "98   Einer der reichsten Spanier lässt nun Wegwerfk...   \n",
519 |        "99   «Verschon uns, Gott, mit Strafen»: Vielleicht ...   \n",
520 |        "100  Jakub Blaszczykowski erfreut die Fussball-Roma...   \n",
521 |        "101            «Den Frühling können sie nicht absagen»   \n",
522 |        "102              Das Leben im ewigen Idyll ist zu Ende   \n",
523 |        "103  US-Aktivistin für Frauenrechte und Homo-Ehe mi...   \n",
524 |        "104  Diese neuartigen Impfstoffe sollen das Coronav...   \n",
525 |        "105  Den Startups trocknet in der Corona-Krise die ...   \n",
526 |        "106  Nach Kanu-Unfall der Kennedy-Enkelin: Taucher ...   \n",
527 |        "107  Die EU-Finanzminister einigen sich auf ein 540...   \n",
528 |        "108                                   Bilder des Tages   \n",
529 |        "109  Weitere 6,6 Millionen Arbeitslose in den USA -...   \n",
530 |        "110  Natur pur: Uruguays Rinder ohne Antibiotika un...   \n",
531 |        "111  Zentralbanken finanzieren Staatsausgaben: Das ...   \n",
532 |        "\n",
533 |        "         news:publication_date  \n",
534 |        "0    2020-04-11T17:29:16+02:00  \n",
535 |        "1    2020-04-11T17:23:24+02:00  \n",
536 |        "2    2020-04-11T16:25:00+02:00  \n",
537 |        "3    2020-04-11T15:25:36+02:00  \n",
538 |        "4    2020-04-11T15:18:27+02:00  \n",
539 |        "5    2020-04-11T13:12:33+02:00  \n",
540 |        "6    2020-04-11T13:05:51+02:00  \n",
541 |        "7    2020-04-11T12:39:31+02:00  \n",
542 |        "8    2020-04-11T12:00:00+02:00  \n",
543 |        "9    2020-04-11T11:31:00+02:00  \n",
544 |        "10   2020-04-11T11:26:07+02:00  \n",
545 |        "11   2020-04-11T10:30:00+02:00  \n",
546 |        "12   2020-04-11T08:03:19+02:00  \n",
547 |        "13   2020-04-11T07:56:12+02:00  \n",
548 |        "14   2020-04-11T07:44:32+02:00  \n",
549 |        "15   2020-04-11T07:13:32+02:00  \n",
550 |        "16   2020-04-11T07:00:00+02:00  \n",
551 |        "17   2020-04-11T06:30:00+02:00  \n",
552 |        "18   2020-04-11T06:00:00+02:00  \n",
553 |        "19   2020-04-11T06:00:00+02:00  \n",
554 |        "20   2020-04-11T06:00:00+02:00  \n",
555 |        "21   2020-04-11T06:00:00+02:00  \n",
556 |        "22   2020-04-11T05:36:13+02:00  \n",
557 |        "23   2020-04-11T05:30:00+02:00  \n",
558 |        "24   2020-04-11T05:30:00+02:00  \n",
559 |        "25   2020-04-11T05:30:00+02:00  \n",
560 |        "26   2020-04-11T05:30:00+02:00  \n",
561 |        "27   2020-04-11T05:30:00+02:00  \n",
562 |        "28   2020-04-11T05:30:00+02:00  \n",
563 |        "29   2020-04-11T05:30:00+02:00  \n",
564 |        "..                         ...  \n",
565 |        "82   2020-04-10T07:15:40+02:00  \n",
566 |        "83   2020-04-10T06:37:43+02:00  \n",
567 |        "84   2020-04-10T06:00:24+02:00  \n",
568 |        "85   2020-04-10T06:00:00+02:00  \n",
569 |        "86   2020-04-10T06:00:00+02:00  \n",
570 |        "87   2020-04-10T05:49:27+02:00  \n",
571 |        "88   2020-04-10T05:30:00+02:00  \n",
572 |        "89   2020-04-10T05:30:00+02:00  \n",
573 |        "90   2020-04-10T05:30:00+02:00  \n",
574 |        "91   2020-04-10T05:30:00+02:00  \n",
575 |        "92   2020-04-10T05:30:00+02:00  \n",
576 |        "93   2020-04-10T05:30:00+02:00  \n",
577 |        "94   2020-04-10T05:30:00+02:00  \n",
578 |        "95   2020-04-10T05:30:00+02:00  \n",
579 |        "96   2020-04-10T05:30:00+02:00  \n",
580 |        "97   2020-04-10T05:30:00+02:00  \n",
581 |        "98   2020-04-10T05:30:00+02:00  \n",
582 |        "99   2020-04-10T05:30:00+02:00  \n",
583 |        "100  2020-04-10T05:30:00+02:00  \n",
584 |        "101  2020-04-10T05:30:00+02:00  \n",
585 |        "102  2020-04-10T05:30:00+02:00  \n",
586 |        "103  2020-04-10T05:01:53+02:00  \n",
587 |        "104  2020-04-10T05:00:00+02:00  \n",
588 |        "105  2020-04-10T05:00:00+02:00  \n",
589 |        "106  2020-04-10T03:54:05+02:00  \n",
590 |        "107  2020-04-10T01:57:21+02:00  \n",
591 |        "108  2020-04-10T00:11:23+02:00  \n",
592 |        "109  2020-04-09T23:43:41+02:00  \n",
593 |        "110  2020-04-09T23:00:00+02:00  \n",
594 |        "111  2020-04-09T20:24:24+02:00  \n",
595 |        "\n",
596 |        "[112 rows x 3 columns]"
597 |       ]
598 |      },
599 |      "execution_count": 36,
600 |      "metadata": {},
601 |      "output_type": "execute_result"
602 |     }
603 |    ],
604 |    "source": [
605 |     "import requests\n",
606 |     "import pandas as pd\n",
607 |     "import xmltodict\n",
608 |     "\n",
609 |     "url = \"https://www.nzz.ch/sitemap/news0.xml\t\"\n",
610 |     "res = requests.get(url)\n",
611 |     "raw = xmltodict.parse(res.text)\n",
612 |     "\n",
613 |     "data = [[r[\"loc\"], r[\"news:news\"][\"news:title\"], r[\"news:news\"][\"news:publication_date\"]] for r in raw[\"urlset\"][\"url\"]]\n",
614 |     "df = pd.DataFrame(data, columns=[\"loc\",\"news:title\", \"news:publication_date\"])\n",
615 |     "df"
616 |    ]
617 |   },
618 |   {
619 |    "cell_type": "code",
620 |    "execution_count": null,
621 |    "metadata": {},
622 |    "outputs": [],
623 |    "source": []
624 |   },
625 |   {
626 |    "cell_type": "code",
627 |    "execution_count": null,
628 |    "metadata": {},
629 |    "outputs": [],
630 |    "source": []
631 |   }
632 |  ],
633 |  "metadata": {
634 |   "kernelspec": {
635 |    "display_name": "Python 3",
636 |    "language": "python",
637 |    "name": "python3"
638 |   },
639 |   "language_info": {
640 |    "codemirror_mode": {
641 |     "name": "ipython",
642 |     "version": 3
643 |    },
644 |    "file_extension": ".py",
645 |    "mimetype": "text/x-python",
646 |    "name": "python",
647 |    "nbconvert_exporter": "python",
648 |    "pygments_lexer": "ipython3",
649 |    "version": "3.7.2"
650 |   }
651 |  },
652 |  "nbformat": 4,
653 |  "nbformat_minor": 2
654 | }
655 | 


--------------------------------------------------------------------------------