├── README.md ├── hacker_news.csv └── hacker_news.ipynb /README.md: -------------------------------------------------------------------------------- 1 | # Exploring-Hacker-News-Posts 2 | Hacker News is a site started by the startup incubator Y Combinator, where user-submitted stories (known as "posts") receive votes and comments, similar to reddit. Hacker News is extremely popular in technology and startup circles, and posts that make it to the top of the Hacker News listings can get hundreds of thousands of visitors as a result. 3 | -------------------------------------------------------------------------------- /hacker_news.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Reading CSV " 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [ 15 | { 16 | "data": { 17 | "text/plain": [ 18 | "[['id', 'title', 'url', 'num_points', 'num_comments', 'author', 'created_at'],\n", 19 | " ['12224879',\n", 20 | " 'Interactive Dynamic Video',\n", 21 | " 'http://www.interactivedynamicvideo.com/',\n", 22 | " '386',\n", 23 | " '52',\n", 24 | " 'ne0phyte',\n", 25 | " '8/4/2016 11:52'],\n", 26 | " ['10975351',\n", 27 | " 'How to Use Open Source and Shut the Fuck Up at the Same Time',\n", 28 | " 'http://hueniverse.com/2016/01/26/how-to-use-open-source-and-shut-the-fuck-up-at-the-same-time/',\n", 29 | " '39',\n", 30 | " '10',\n", 31 | " 'josep2',\n", 32 | " '1/26/2016 19:30'],\n", 33 | " ['11964716',\n", 34 | " \"Florida DJs May Face Felony for April Fools' Water Joke\",\n", 35 | " 'http://www.thewire.com/entertainment/2013/04/florida-djs-april-fools-water-joke/63798/',\n", 36 | " '2',\n", 37 | " '1',\n", 38 | " 'vezycash',\n", 39 | " '6/23/2016 22:20'],\n", 40 | " ['11919867',\n", 41 | " 'Technology ventures: From Idea to Enterprise',\n", 42 | " 'https://www.amazon.com/Technology-Ventures-Enterprise-Thomas-Byers/dp/0073523429',\n", 43 | " '3',\n", 44 | " '1',\n", 45 | " 'hswarna',\n", 46 | " '6/17/2016 0:01']]" 47 | ] 48 | }, 49 | "execution_count": 1, 50 | "metadata": {}, 51 | "output_type": "execute_result" 52 | } 53 | ], 54 | "source": [ 55 | "from csv import reader\n", 56 | "hn = open(\"hacker_news.csv\")\n", 57 | "hn = list(reader(hn))\n", 58 | "hn[:5]" 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "metadata": {}, 64 | "source": [ 65 | "# Seperating Headers and Values" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 2, 71 | "metadata": {}, 72 | "outputs": [ 73 | { 74 | "name": "stdout", 75 | "output_type": "stream", 76 | "text": [ 77 | "[['id', 'title', 'url', 'num_points', 'num_comments', 'author', 'created_at']]\n" 78 | ] 79 | } 80 | ], 81 | "source": [ 82 | "headers = hn[:1]\n", 83 | "hn = hn[1:]\n", 84 | "print(headers)" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 3, 90 | "metadata": {}, 91 | "outputs": [ 92 | { 93 | "data": { 94 | "text/plain": [ 95 | "[['12224879',\n", 96 | " 'Interactive Dynamic Video',\n", 97 | " 'http://www.interactivedynamicvideo.com/',\n", 98 | " '386',\n", 99 | " '52',\n", 100 | " 'ne0phyte',\n", 101 | " '8/4/2016 11:52'],\n", 102 | " ['10975351',\n", 103 | " 'How to Use Open Source and Shut the Fuck Up at the Same Time',\n", 104 | " 'http://hueniverse.com/2016/01/26/how-to-use-open-source-and-shut-the-fuck-up-at-the-same-time/',\n", 105 | " '39',\n", 106 | " '10',\n", 107 | " 'josep2',\n", 108 | " '1/26/2016 19:30'],\n", 109 | " ['11964716',\n", 110 | " \"Florida DJs May Face Felony for April Fools' Water Joke\",\n", 111 | " 'http://www.thewire.com/entertainment/2013/04/florida-djs-april-fools-water-joke/63798/',\n", 112 | " '2',\n", 113 | " '1',\n", 114 | " 'vezycash',\n", 115 | " '6/23/2016 22:20'],\n", 116 | " ['11919867',\n", 117 | " 'Technology ventures: From Idea to Enterprise',\n", 118 | " 'https://www.amazon.com/Technology-Ventures-Enterprise-Thomas-Byers/dp/0073523429',\n", 119 | " '3',\n", 120 | " '1',\n", 121 | " 'hswarna',\n", 122 | " '6/17/2016 0:01'],\n", 123 | " ['10301696',\n", 124 | " 'Note by Note: The Making of Steinway L1037 (2007)',\n", 125 | " 'http://www.nytimes.com/2007/11/07/movies/07stein.html?_r=0',\n", 126 | " '8',\n", 127 | " '2',\n", 128 | " 'walterbell',\n", 129 | " '9/30/2015 4:12']]" 130 | ] 131 | }, 132 | "execution_count": 3, 133 | "metadata": {}, 134 | "output_type": "execute_result" 135 | } 136 | ], 137 | "source": [ 138 | "hn[:5]" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "metadata": {}, 144 | "source": [ 145 | "# Exploring Ask Posts and Show Posts" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": 4, 151 | "metadata": {}, 152 | "outputs": [ 153 | { 154 | "name": "stdout", 155 | "output_type": "stream", 156 | "text": [ 157 | "No of posts on ask_posts : 1744\n", 158 | "No of posts on show_posts : 1162\n", 159 | "No of posts on other_posts : 17194\n" 160 | ] 161 | } 162 | ], 163 | "source": [ 164 | "ask_posts = []\n", 165 | "show_posts = []\n", 166 | "other_posts = []\n", 167 | "\n", 168 | "for row in hn:\n", 169 | " title = row[1]\n", 170 | " if title.lower().startswith(\"ask hn\"):\n", 171 | " ask_posts.append(row)\n", 172 | " elif title.lower().startswith(\"show hn\"):\n", 173 | " show_posts.append(row)\n", 174 | " \n", 175 | " else:\n", 176 | " other_posts.append(row)\n", 177 | " \n", 178 | "print(\"No of posts on ask_posts : \",len(ask_posts))\n", 179 | "print(\"No of posts on show_posts : \",len(show_posts))\n", 180 | "print(\"No of posts on other_posts : \",len(other_posts))" 181 | ] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": {}, 186 | "source": [ 187 | "First Five rows of Ask Posts" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": 5, 193 | "metadata": {}, 194 | "outputs": [ 195 | { 196 | "name": "stdout", 197 | "output_type": "stream", 198 | "text": [ 199 | "[['12296411', 'Ask HN: How to improve my personal website?', '', '2', '6', 'ahmedbaracat', '8/16/2016 9:55'], ['10610020', 'Ask HN: Am I the only one outraged by Twitter shutting down share counts?', '', '28', '29', 'tkfx', '11/22/2015 13:43'], ['11610310', 'Ask HN: Aby recent changes to CSS that broke mobile?', '', '1', '1', 'polskibus', '5/2/2016 10:14'], ['12210105', 'Ask HN: Looking for Employee #3 How do I do it?', '', '1', '3', 'sph130', '8/2/2016 14:20'], ['10394168', 'Ask HN: Someone offered to buy my browser extension from me. What now?', '', '28', '17', 'roykolak', '10/15/2015 16:38']]\n" 200 | ] 201 | } 202 | ], 203 | "source": [ 204 | "print(ask_posts[:5])" 205 | ] 206 | }, 207 | { 208 | "cell_type": "markdown", 209 | "metadata": {}, 210 | "source": [ 211 | "First Five rows of Send Posts" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 6, 217 | "metadata": {}, 218 | "outputs": [ 219 | { 220 | "name": "stdout", 221 | "output_type": "stream", 222 | "text": [ 223 | "[['10627194', 'Show HN: Wio Link ESP8266 Based Web of Things Hardware Development Platform', 'https://iot.seeed.cc', '26', '22', 'kfihihc', '11/25/2015 14:03'], ['10646440', 'Show HN: Something pointless I made', 'http://dn.ht/picklecat/', '747', '102', 'dhotson', '11/29/2015 22:46'], ['11590768', 'Show HN: Shanhu.io, a programming playground powered by e8vm', 'https://shanhu.io', '1', '1', 'h8liu', '4/28/2016 18:05'], ['12178806', 'Show HN: Webscope Easy way for web developers to communicate with Clients', 'http://webscopeapp.com', '3', '3', 'fastbrick', '7/28/2016 7:11'], ['10872799', 'Show HN: GeoScreenshot Easily test Geo-IP based web pages', 'https://www.geoscreenshot.com/', '1', '9', 'kpsychwave', '1/9/2016 20:45']]\n" 224 | ] 225 | } 226 | ], 227 | "source": [ 228 | "print(show_posts[:5])" 229 | ] 230 | }, 231 | { 232 | "cell_type": "markdown", 233 | "metadata": {}, 234 | "source": [ 235 | "let's determine if ask posts or show posts receive more comments on average." 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": 7, 241 | "metadata": {}, 242 | "outputs": [ 243 | { 244 | "name": "stdout", 245 | "output_type": "stream", 246 | "text": [ 247 | "14.038417431192661\n" 248 | ] 249 | } 250 | ], 251 | "source": [ 252 | "total_ask_comments = 0\n", 253 | "\n", 254 | "for row in ask_posts:\n", 255 | " num_comments = int(row[4])\n", 256 | " total_ask_comments += (num_comments)\n", 257 | " \n", 258 | "avg_ask_comments = total_ask_comments / len(ask_posts)\n", 259 | "print(avg_ask_comments)" 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": 8, 265 | "metadata": {}, 266 | "outputs": [ 267 | { 268 | "name": "stdout", 269 | "output_type": "stream", 270 | "text": [ 271 | "10.31669535283993\n" 272 | ] 273 | } 274 | ], 275 | "source": [ 276 | "total_show_comments = 0\n", 277 | "\n", 278 | "for row in show_posts:\n", 279 | " num_comments = int(row[4])\n", 280 | " total_show_comments += num_comments\n", 281 | " \n", 282 | "avg_show_comments = total_show_comments / len(show_posts)\n", 283 | "print(avg_show_comments)\n", 284 | " " 285 | ] 286 | }, 287 | { 288 | "cell_type": "markdown", 289 | "metadata": {}, 290 | "source": [ 291 | "In the above calculations we can clearly see that ask posts receieved more comments on average." 292 | ] 293 | }, 294 | { 295 | "cell_type": "markdown", 296 | "metadata": {}, 297 | "source": [ 298 | "Since ask posts are more likely to receive comments, we'll focus our remaining analysis just on these posts." 299 | ] 300 | }, 301 | { 302 | "cell_type": "markdown", 303 | "metadata": {}, 304 | "source": [ 305 | " we'll determine if ask posts created at a certain time are more likely to attract comments. We'll use the following steps to perform this analysis:" 306 | ] 307 | }, 308 | { 309 | "cell_type": "markdown", 310 | "metadata": {}, 311 | "source": [ 312 | "1. Calculate the number of ask posts created in each hour of the day, along with the number of comments received.\n", 313 | "\n", 314 | "2. Calculate the average number of comments ask posts receive by hour created." 315 | ] 316 | }, 317 | { 318 | "cell_type": "markdown", 319 | "metadata": {}, 320 | "source": [ 321 | "# Importing datetime module" 322 | ] 323 | }, 324 | { 325 | "cell_type": "code", 326 | "execution_count": 9, 327 | "metadata": {}, 328 | "outputs": [], 329 | "source": [ 330 | "import datetime as dt" 331 | ] 332 | }, 333 | { 334 | "cell_type": "code", 335 | "execution_count": 10, 336 | "metadata": {}, 337 | "outputs": [], 338 | "source": [ 339 | "result_list = []\n", 340 | "\n", 341 | "for row in ask_posts:\n", 342 | " created_at = row[6]\n", 343 | " no_comments = row[4]\n", 344 | " result_list.append([created_at,no_comments])\n", 345 | "#result_list" 346 | ] 347 | }, 348 | { 349 | "cell_type": "markdown", 350 | "metadata": {}, 351 | "source": [ 352 | "# Calculating Comments per hour" 353 | ] 354 | }, 355 | { 356 | "cell_type": "code", 357 | "execution_count": 11, 358 | "metadata": {}, 359 | "outputs": [], 360 | "source": [ 361 | "counts_by_hour = {}\n", 362 | "comments_by_hour = {}\n", 363 | "\n", 364 | "for row in result_list:\n", 365 | " #8/16/2016 9:55\n", 366 | " hour = dt.datetime.strptime(row[0],\"%m/%d/%Y %H:%M\")\n", 367 | " hour = hour.hour\n", 368 | " comments = int(row[1])\n", 369 | " \n", 370 | " if hour not in counts_by_hour:\n", 371 | " \n", 372 | " counts_by_hour[hour] = 1\n", 373 | " comments_by_hour[hour] = comments\n", 374 | " \n", 375 | " else:\n", 376 | " counts_by_hour[hour] += 1\n", 377 | " comments_by_hour[hour] += comments\n", 378 | " " 379 | ] 380 | }, 381 | { 382 | "cell_type": "markdown", 383 | "metadata": {}, 384 | "source": [ 385 | "Counts By Hour" 386 | ] 387 | }, 388 | { 389 | "cell_type": "code", 390 | "execution_count": 12, 391 | "metadata": {}, 392 | "outputs": [ 393 | { 394 | "name": "stdout", 395 | "output_type": "stream", 396 | "text": [ 397 | "{9: 45, 13: 85, 10: 59, 14: 107, 16: 108, 23: 68, 12: 73, 17: 100, 15: 116, 21: 109, 20: 80, 2: 58, 18: 109, 3: 54, 5: 46, 19: 110, 1: 60, 22: 71, 8: 48, 4: 47, 0: 55, 6: 44, 7: 34, 11: 58}\n" 398 | ] 399 | } 400 | ], 401 | "source": [ 402 | "print(counts_by_hour)" 403 | ] 404 | }, 405 | { 406 | "cell_type": "markdown", 407 | "metadata": {}, 408 | "source": [ 409 | "Comments By Hour" 410 | ] 411 | }, 412 | { 413 | "cell_type": "code", 414 | "execution_count": 13, 415 | "metadata": {}, 416 | "outputs": [ 417 | { 418 | "name": "stdout", 419 | "output_type": "stream", 420 | "text": [ 421 | "{9: 251, 13: 1253, 10: 793, 14: 1416, 16: 1814, 23: 543, 12: 687, 17: 1146, 15: 4477, 21: 1745, 20: 1722, 2: 1381, 18: 1439, 3: 421, 5: 464, 19: 1188, 1: 683, 22: 479, 8: 492, 4: 337, 0: 447, 6: 397, 7: 267, 11: 641}\n" 422 | ] 423 | } 424 | ], 425 | "source": [ 426 | "print(comments_by_hour)" 427 | ] 428 | }, 429 | { 430 | "cell_type": "markdown", 431 | "metadata": {}, 432 | "source": [ 433 | "calculating the average number of comments per post for posts created during each hour of the da" 434 | ] 435 | }, 436 | { 437 | "cell_type": "code", 438 | "execution_count": 14, 439 | "metadata": {}, 440 | "outputs": [ 441 | { 442 | "data": { 443 | "text/plain": [ 444 | "[[9, 5.5777777777777775],\n", 445 | " [13, 14.741176470588234],\n", 446 | " [10, 13.440677966101696],\n", 447 | " [14, 13.233644859813085],\n", 448 | " [16, 16.796296296296298],\n", 449 | " [23, 7.985294117647059],\n", 450 | " [12, 9.41095890410959],\n", 451 | " [17, 11.46],\n", 452 | " [15, 38.5948275862069],\n", 453 | " [21, 16.009174311926607],\n", 454 | " [20, 21.525],\n", 455 | " [2, 23.810344827586206],\n", 456 | " [18, 13.20183486238532],\n", 457 | " [3, 7.796296296296297],\n", 458 | " [5, 10.08695652173913],\n", 459 | " [19, 10.8],\n", 460 | " [1, 11.383333333333333],\n", 461 | " [22, 6.746478873239437],\n", 462 | " [8, 10.25],\n", 463 | " [4, 7.170212765957447],\n", 464 | " [0, 8.127272727272727],\n", 465 | " [6, 9.022727272727273],\n", 466 | " [7, 7.852941176470588],\n", 467 | " [11, 11.051724137931034]]" 468 | ] 469 | }, 470 | "execution_count": 14, 471 | "metadata": {}, 472 | "output_type": "execute_result" 473 | } 474 | ], 475 | "source": [ 476 | "avg_by_hour = []\n", 477 | "\n", 478 | "for posts in counts_by_hour:\n", 479 | " avg_by_hour.append([posts,comments_by_hour[posts]/counts_by_hour[posts]])\n", 480 | " \n", 481 | "avg_by_hour" 482 | ] 483 | }, 484 | { 485 | "cell_type": "code", 486 | "execution_count": 15, 487 | "metadata": {}, 488 | "outputs": [ 489 | { 490 | "name": "stdout", 491 | "output_type": "stream", 492 | "text": [ 493 | "[[5.5777777777777775, 9], [14.741176470588234, 13], [13.440677966101696, 10], [13.233644859813085, 14], [16.796296296296298, 16], [7.985294117647059, 23], [9.41095890410959, 12], [11.46, 17], [38.5948275862069, 15], [16.009174311926607, 21], [21.525, 20], [23.810344827586206, 2], [13.20183486238532, 18], [7.796296296296297, 3], [10.08695652173913, 5], [10.8, 19], [11.383333333333333, 1], [6.746478873239437, 22], [10.25, 8], [7.170212765957447, 4], [8.127272727272727, 0], [9.022727272727273, 6], [7.852941176470588, 7], [11.051724137931034, 11]]\n" 494 | ] 495 | } 496 | ], 497 | "source": [ 498 | "swap_avg_by_hour = []\n", 499 | "\n", 500 | "for row in avg_by_hour:\n", 501 | " swap_avg_by_hour.append([row[1],row[0]])\n", 502 | " \n", 503 | "print(swap_avg_by_hour)" 504 | ] 505 | }, 506 | { 507 | "cell_type": "code", 508 | "execution_count": 16, 509 | "metadata": {}, 510 | "outputs": [ 511 | { 512 | "name": "stdout", 513 | "output_type": "stream", 514 | "text": [ 515 | "[[38.5948275862069, 15], [23.810344827586206, 2], [21.525, 20], [16.796296296296298, 16], [16.009174311926607, 21], [14.741176470588234, 13], [13.440677966101696, 10], [13.233644859813085, 14], [13.20183486238532, 18], [11.46, 17], [11.383333333333333, 1], [11.051724137931034, 11], [10.8, 19], [10.25, 8], [10.08695652173913, 5], [9.41095890410959, 12], [9.022727272727273, 6], [8.127272727272727, 0], [7.985294117647059, 23], [7.852941176470588, 7], [7.796296296296297, 3], [7.170212765957447, 4], [6.746478873239437, 22], [5.5777777777777775, 9]]\n" 516 | ] 517 | } 518 | ], 519 | "source": [ 520 | "sorted_swap = sorted(swap_avg_by_hour,reverse= True)\n", 521 | "print(sorted_swap)" 522 | ] 523 | }, 524 | { 525 | "cell_type": "markdown", 526 | "metadata": {}, 527 | "source": [ 528 | "# Top 5 Hours for Ask Posts Comments" 529 | ] 530 | }, 531 | { 532 | "cell_type": "code", 533 | "execution_count": 17, 534 | "metadata": {}, 535 | "outputs": [ 536 | { 537 | "name": "stdout", 538 | "output_type": "stream", 539 | "text": [ 540 | "15 :00 38.59 average comments per post\n", 541 | "2 :00 23.81 average comments per post\n", 542 | "20 :00 21.52 average comments per post\n", 543 | "16 :00 16.80 average comments per post\n", 544 | "21 :00 16.01 average comments per post\n", 545 | "13 :00 14.74 average comments per post\n", 546 | "10 :00 13.44 average comments per post\n", 547 | "14 :00 13.23 average comments per post\n", 548 | "18 :00 13.20 average comments per post\n", 549 | "17 :00 11.46 average comments per post\n", 550 | "1 :00 11.38 average comments per post\n", 551 | "11 :00 11.05 average comments per post\n", 552 | "19 :00 10.80 average comments per post\n", 553 | "8 :00 10.25 average comments per post\n", 554 | "5 :00 10.09 average comments per post\n", 555 | "12 :00 9.41 average comments per post\n", 556 | "6 :00 9.02 average comments per post\n", 557 | "0 :00 8.13 average comments per post\n", 558 | "23 :00 7.99 average comments per post\n", 559 | "7 :00 7.85 average comments per post\n", 560 | "3 :00 7.80 average comments per post\n", 561 | "4 :00 7.17 average comments per post\n", 562 | "22 :00 6.75 average comments per post\n", 563 | "9 :00 5.58 average comments per post\n" 564 | ] 565 | } 566 | ], 567 | "source": [ 568 | "for row in sorted_swap:\n", 569 | " hour = dt.datetime.strptime(str(row[1]),\"%H\")\n", 570 | " hour = hour.hour\n", 571 | " print(hour,\":00\",\"{:.2f} average comments per post\".format(row[0]))" 572 | ] 573 | }, 574 | { 575 | "cell_type": "code", 576 | "execution_count": null, 577 | "metadata": {}, 578 | "outputs": [], 579 | "source": [] 580 | } 581 | ], 582 | "metadata": { 583 | "kernelspec": { 584 | "display_name": "Python 3", 585 | "language": "python", 586 | "name": "python3" 587 | }, 588 | "language_info": { 589 | "codemirror_mode": { 590 | "name": "ipython", 591 | "version": 3 592 | }, 593 | "file_extension": ".py", 594 | "mimetype": "text/x-python", 595 | "name": "python", 596 | "nbconvert_exporter": "python", 597 | "pygments_lexer": "ipython3", 598 | "version": "3.8.2" 599 | } 600 | }, 601 | "nbformat": 4, 602 | "nbformat_minor": 2 603 | } 604 | --------------------------------------------------------------------------------