![](\"https://i.imgur.com/0s6Iiu3.png\")
\n",
18 | "\n",
19 | "**透過動態網站爬蟲會看到的圖片是**\n",
20 | "\n",
21 | "![](\"https://i.imgur.com/db3tGBG.png\")
"
22 | ]
23 | },
24 | {
25 | "cell_type": "code",
26 | "execution_count": 1,
27 | "metadata": {
28 | "collapsed": true
29 | },
30 | "outputs": [],
31 | "source": [
32 | "import os\n",
33 | "import requests\n",
34 | "import re\n",
35 | "\n",
36 | "from bs4 import BeautifulSoup\n",
37 | "from selenium import webdriver\n",
38 | "from selenium.webdriver.common.by import By\n",
39 | "from fake_useragent import UserAgent\n",
40 | "from pprint import pprint\n",
41 | "\n",
42 | "url = 'https://afuntw.github.io/Test-Crawling-Website/pages/gallery/index.html'\n",
43 | "fu = UserAgent()"
44 | ]
45 | },
46 | {
47 | "cell_type": "markdown",
48 | "metadata": {},
49 | "source": [
50 | "# 使用 requests 做靜態爬蟲"
51 | ]
52 | },
53 | {
54 | "cell_type": "code",
55 | "execution_count": 2,
56 | "metadata": {},
57 | "outputs": [
58 | {
59 | "name": "stdout",
60 | "output_type": "stream",
61 | "text": [
62 | "['https://i.imgur.com/0s6Iiu3.png']\n"
63 | ]
64 | }
65 | ],
66 | "source": [
67 | "resp = requests.get(url)\n",
68 | "soup = BeautifulSoup(resp.text, 'lxml')\n",
69 | "imgs = soup.find_all('img', class_=re.compile('.*img-change'))\n",
70 | "imgs = [i['src'] for i in imgs]\n",
71 | "imgs = list(set(imgs))\n",
72 | "pprint(imgs)"
73 | ]
74 | },
75 | {
76 | "cell_type": "markdown",
77 | "metadata": {},
78 | "source": [
79 | "# 使用 Selenium 做動態爬蟲"
80 | ]
81 | },
82 | {
83 | "cell_type": "code",
84 | "execution_count": 3,
85 | "metadata": {},
86 | "outputs": [
87 | {
88 | "name": "stdout",
89 | "output_type": "stream",
90 | "text": [
91 | "['https://i.imgur.com/db3tGBG.png']\n",
92 | "catch - db3tGBG.png\n",
93 | "save - /home/dirl/github/Python-Crawling-Tutorial/results/db3tGBG.png\n"
94 | ]
95 | }
96 | ],
97 | "source": [
98 | "driver = webdriver.Chrome()\n",
99 | "results = os.path.abspath('../results')\n",
100 | "if not os.path.exists(results):\n",
101 | " os.makedirs(results)\n",
102 | "\n",
103 | "try:\n",
104 | " # webdriver setting\n",
105 | " driver.get(url)\n",
106 | " driver.maximize_window()\n",
107 | " driver.implicitly_wait(10)\n",
108 | " \n",
109 | " # xpath\n",
110 | " imgs = driver.find_elements(By.XPATH, '/html/body/div/div/div/a/img')\n",
111 | " imgs = [i.get_attribute('src') for i in imgs]\n",
112 | " imgs = list(set(imgs))\n",
113 | " print(imgs)\n",
114 | " \n",
115 | " # download\n",
116 | " for img in imgs:\n",
117 | " headers = {'User-Agent': fu.random}\n",
118 | " img_resp = requests.get(img, stream=True, headers=headers)\n",
119 | " \n",
120 | " filename = os.path.basename(img)\n",
121 | " print('catch - {}'.format(filename))\n",
122 | " filename = os.path.join(results, filename)\n",
123 | " \n",
124 | " with open(filename, 'wb') as f:\n",
125 | " for chunk in img_resp.iter_content(2048):\n",
126 | " f.write(chunk)\n",
127 | " print('save - {}'.format(filename))\n",
128 | " \n",
129 | "except Exception as e:\n",
130 | " print(e)\n",
131 | "finally:\n",
132 | " driver.quit()"
133 | ]
134 | }
135 | ],
136 | "metadata": {
137 | "kernelspec": {
138 | "display_name": "Python 3",
139 | "language": "python",
140 | "name": "python3"
141 | },
142 | "language_info": {
143 | "codemirror_mode": {
144 | "name": "ipython",
145 | "version": 3
146 | },
147 | "file_extension": ".py",
148 | "mimetype": "text/x-python",
149 | "name": "python",
150 | "nbconvert_exporter": "python",
151 | "pygments_lexer": "ipython3",
152 | "version": "3.5.2"
153 | }
154 | },
155 | "nbformat": 4,
156 | "nbformat_minor": 2
157 | }
158 |
--------------------------------------------------------------------------------
/02_selenium/01_pchome_crawling_item.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# 練習\n",
8 | "\n",
9 | "- 觀察 http://24h.pchome.com.tw/region/DHBE 並撰寫爬蟲程式\n",
10 | "- 判斷是否為 JavaScript rendered website\n",
11 | "- 設定 Implicit Wait\n",
12 | "- 透過 XPath 定位\n",
13 | "- 抓取商品的名稱與價格"
14 | ]
15 | },
16 | {
17 | "cell_type": "code",
18 | "execution_count": 1,
19 | "metadata": {
20 | "collapsed": true
21 | },
22 | "outputs": [],
23 | "source": [
24 | "import os\n",
25 | "\n",
26 | "from selenium import webdriver\n",
27 | "from selenium.webdriver.common.by import By\n",
28 | "\n",
29 | "url = 'http://24h.pchome.com.tw/region/DHBE'"
30 | ]
31 | },
32 | {
33 | "cell_type": "code",
34 | "execution_count": 2,
35 | "metadata": {},
36 | "outputs": [
37 | {
38 | "name": "stdout",
39 | "output_type": "stream",
40 | "text": [
41 | "ASUS B9440UA-0251A7200U (i5-7200/8G/256G SSD/W10P) - 32900\n",
42 | "(商)Lenovo ThinkPad T470 20HDA00STW(i5-7200U/940MX-2G/1TB/W10P) - 39900\n",
43 | "(商)HP X360 1030 G2(i7-7600U/8G*2/512GB SSD/FHD/W10Pro) - 68900\n",
44 | "(商)HP 240 G6 (i5-7200U/14/4G/500GB/Win10) - 18900\n",
45 | "DELL M7520(i7-7820HQ/Nvidia Quadro M2200M-4G/1TB+256GB/W10P/FHD)繪圖工作站筆電 - 89990\n",
46 | "ASUS P2430UJ-0321A6200U (i5-6200U/4G/500G/920M-2G/Win10) - 22900\n",
47 | "(商)Lenovo ThinkPad X1c 20HRA010TW(i7-7500U/256G SSD/W10P/FHD) - 59900\n",
48 | "(商) HP Probook 430 G4(i5-7200U/4G DDR4/500GB/Win10) - 25900\n",
49 | "(商)HP 240 G6 (i5-7200U/14/4G/500GB/Win10pro) - 23900\n",
50 | "DELL M5520(i7-7820HQ/M1200M/512GB SSD/Win10 Pro/UHD)繪圖工作站筆電 - 98990\n",
51 | "ASUS P2530UJ-0461A6200U (i5-6200U/8G/1TB/GeForce 920M-2G/W10P) - 23900\n",
52 | "(商)Lenovo ThinkPad T470s 20HFA00ETW(i7-7600U/512G SSD/W10P) - 63900\n",
53 | "(商)HP Probook 650 G3(i7-7600U/512GB SSD/AMD Radeon R7 M465 2GB/W10P) - 50900\n",
54 | "(商)HP 240 G6 (i3-6006U/14/UMA/500G/W10DW7) - 17900\n",
55 | "ASUS B9440UA-0251A7200U (i5-7200/8G/256G SSD/W10P) - 27900\n",
56 | "ASUS P2530UJ-0271A6500U (i7-6500U/8G/1TB/920M-2G/FHD/Win10P) - 32900\n",
57 | "(商)Lenovo ThinkPad Edge15 E570 20H5A037TW(i7-7500U/GTX 950M-2G/1TB/W10/FHD) - 34900\n",
58 | "ACER TravelMate TMP238-M-77JQ (i7-6500U/8GB/256GB SSD/W7P+W10P) - 33800\n",
59 | "ACER TravelMate TMX349-G2-M-53L8.(i5-7200U/8GB/256GB SSD/W10P) - 30500\n",
60 | "DELL M7520(i7-7820HQ/Nvidia Quadro M1200M-4G/1TB/W10P/FHD)繪圖工作站筆電 - 76900\n",
61 | "ASUS A550V-0203J6700HQ (i7-6700HQ/500G/GTX950M 2G獨顯/W10P) - 28900\n",
62 | "(商)Lenovo ThinkPad X260 20F6A07QTW(i5-6200U/1TB/W10P) - 38900\n",
63 | "ACER TravelMate TMP249-M-C1DV.(CM3855U/4GB DDR4/500GB/W10P) - 14990\n",
64 | "ACER TravelMate TMP249-M-3142.(i3-6100U/4GB/500GB/W7P+W10P) - 17990\n",
65 | "DELL M7510(i7-6820HQ/Nvidia Quadro M1000M-2G/1TB/W7P/FHD)繪圖工作站筆電 - 69900\n",
66 | "ASUS B8230UA-0061A6500U (i7-6500U/512G SSD/W7P) - 44900\n",
67 | "(商)Lenovo ThinkPad X270 20HNA00RTW (i5-7200U/1TB/W10P) - 41900\n",
68 | "ACER TravelMate TMP446-M-54S0.(i5-5200U/4GB/500G/W7P+W10P) - 19900\n",
69 | "ACER TravelMate TMP259-M-5726(i5-6200U/4GB/128GB SSD/W7P+W10P) - 26900\n",
70 | "DELL Vostro 14 5000 (i5-7200U/4G/1TB/940MX-2G/W10/HD/Jingle Gold) - 23999\n"
71 | ]
72 | }
73 | ],
74 | "source": [
75 | "try:\n",
76 | " driver = webdriver.Chrome()\n",
77 | " driver.get(url)\n",
78 | " driver.maximize_window()\n",
79 | " driver.implicitly_wait(10)\n",
80 | " \n",
81 | " # get web elements\n",
82 | " items = driver.find_elements(By.XPATH, '//dl[@id=\"Block12Container50\"]/dd')\n",
83 | " \n",
84 | " for item in items:\n",
85 | " title = item.find_element(By.XPATH, './div/h5/a').text\n",
86 | " price = item.find_element(By.XPATH, './div/ul/li/span/span').text\n",
87 | " if title and price:\n",
88 | " print('{} - {}'.format(title, price))\n",
89 | "\n",
90 | "except Exception as e:\n",
91 | " print(e)\n",
92 | "finally:\n",
93 | " driver.quit()"
94 | ]
95 | }
96 | ],
97 | "metadata": {
98 | "kernelspec": {
99 | "display_name": "Python 3",
100 | "language": "python",
101 | "name": "python3"
102 | },
103 | "language_info": {
104 | "codemirror_mode": {
105 | "name": "ipython",
106 | "version": 3
107 | },
108 | "file_extension": ".py",
109 | "mimetype": "text/x-python",
110 | "name": "python",
111 | "nbconvert_exporter": "python",
112 | "pygments_lexer": "ipython3",
113 | "version": "3.5.2"
114 | }
115 | },
116 | "nbformat": 4,
117 | "nbformat_minor": 2
118 | }
119 |
--------------------------------------------------------------------------------
/02_selenium/02_selenium_google_search.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# 練習\n",
8 | "\n",
9 | "- 模擬 google search 流程\n",
10 | "- https://www.google.com.tw/\n",
11 | "- 搜尋「人工智慧」\n",
12 | "- 紀錄前兩頁搜尋結果的連結"
13 | ]
14 | },
15 | {
16 | "cell_type": "code",
17 | "execution_count": 1,
18 | "metadata": {},
19 | "outputs": [],
20 | "source": [
21 | "import os\n",
22 | "\n",
23 | "from selenium import webdriver\n",
24 | "from selenium.webdriver.common.keys import Keys\n",
25 | "from selenium.webdriver.common.by import By\n",
26 | "from pprint import pprint\n",
27 | "\n",
28 | "url = 'https://google.com'"
29 | ]
30 | },
31 | {
32 | "cell_type": "code",
33 | "execution_count": 2,
34 | "metadata": {},
35 | "outputs": [
36 | {
37 | "name": "stdout",
38 | "output_type": "stream",
39 | "text": [
40 | "======================================================================================= Page 0\n",
41 | "title: 人工智能- 维基百科，自由的百科全书\n",
42 | "url: https://zh.wikipedia.org/zh-tw/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD\n",
43 | "---\n",
44 | "title: 人工智慧三大關鍵技術 - 數位時代\n",
45 | "url: https://www.bnext.com.tw/article/41534/3-key-techniques-of-ai\n",
46 | "---\n",
47 | "title: AI 人工智慧| TechNews 科技新報\n",
48 | "url: https://technews.tw/category/cutting-edge/ai/\n",
49 | "---\n",
50 | "title: 人工智慧不可能超越人類，原因居然是這樣的……（上） ｜智慧機器人網 ...\n",
51 | "url: https://www.limitlessiq.com/news/post/view/id/3596/\n",
52 | "---\n",
53 | "title: 人工智慧AI – CASE報科學 - 國立臺灣大學科學教育發展中心\n",
54 | "url: https://case.ntu.edu.tw/blog/?cat=3772\n",
55 | "---\n",
56 | "title: AI人工智慧來了! 你的未來在哪裡？｜深度專題｜天下雜誌\n",
57 | "url: https://www.cw.com.tw/special/2073\n",
58 | "---\n",
59 | "title: 台灣人工智慧學校| Taiwan AI Academy\n",
60 | "url: http://aiacademy.tw/\n",
61 | "---\n",
62 | "title: 人工智慧- MBA智库百科\n",
63 | "url: https://wiki.mbalib.com/zh-tw/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD\n",
64 | "---\n",
65 | "======================================================================================= Page 1\n",
66 | "title: 什麼是人工智慧？︱《三分鐘財經教室》#01 - YouTube\n",
67 | "url: https://www.youtube.com/watch?v=nKcsu4JierI\n",
68 | "---\n",
69 | "title: 『AI人工智慧！機器學習& 突如其來的危機』芬特克FinTech EP3 - YouTube\n",
70 | "url: https://www.youtube.com/watch?v=i0UxYDqlX6o\n",
71 | "---\n",
72 | "title: 人工智慧：搜尋方法與邏輯推論(Artificial Intelligence - Search & Logic ...\n",
73 | "url: https://www.coursera.org/learn/rengong-zhineng\n",
74 | "---\n",
75 | "title: 博客來-中文書>電腦資訊>概論/科技趨勢>人工智慧/機器學習\n",
76 | "url: https://www.books.com.tw/web/sys_bbotm/books/190102\n",
77 | "---\n",
78 | "title: 人工智能| 大紀元\n",
79 | "url: http://www.epochtimes.com/b5/tag/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD.html\n",
80 | "---\n",
81 | "title: 如何對面AI時代的孩子？人工智慧博士教年輕父母強化個人實力- Yahoo ...\n",
82 | "url: https://tw.news.yahoo.com/%E5%A6%82%E4%BD%95%E5%B0%8D%E9%9D%A2ai%E6%99%82%E4%BB%A3%E7%9A%84%E5%AD%A9%E5%AD%90-%E4%BA%BA%E5%B7%A5%E6%99%BA%E6%85%A7%E5%8D%9A%E5%A3%AB%E6%95%99%E5%B9%B4%E8%BC%95%E7%88%B6%E6%AF%8D%E5%BC%B7%E5%8C%96%E5%80%8B%E4%BA%BA%E5%AF%A6%E5%8A%9B-010012301.html\n",
83 | "---\n",
84 | "title: 人工智慧對勞動就業的影響- STPI Research Portal - 科技政策觀點\n",
85 | "url: https://portal.stpi.narl.org.tw/index/article/10401\n",
86 | "---\n",
87 | "title: AI人工智慧時代來臨- 中時電子報\n",
88 | "url: https://www.chinatimes.com/newspapers/20180907000541-260204\n",
89 | "---\n",
90 | "title: 人工智慧應用新趨勢與展望—學生與機器人共同學習-臺北產經資訊網\n",
91 | "url: https://www.taipeiecon.taipei/article_cont.aspx?MmmID=1201&MSid=1001302007727155764\n",
92 | "---\n",
93 | "title: 人工智慧技術的下一波研發核心 - Digitimes\n",
94 | "url: https://www.digitimes.com.tw/col/article.asp?id=944\n",
95 | "---\n"
96 | ]
97 | }
98 | ],
99 | "source": [
100 | "try:\n",
101 | " driver = webdriver.Chrome('/home/afun/Downloads/chromedriver')\n",
102 | " driver.get(url)\n",
103 | " driver.maximize_window()\n",
104 | " driver.implicitly_wait(10)\n",
105 | " \n",
106 | " search_input = driver.find_element(By.ID, 'lst-ib')\n",
107 | " search_input.send_keys(u'人工智慧')\n",
108 | " search_input.send_keys(Keys.ENTER)\n",
109 | " \n",
110 | " for i in range(2):\n",
111 | " print('='*87, 'Page {}'.format(i))\n",
112 | "\n",
113 | " links = driver.find_elements(By.XPATH, '//div[@class=\"r\"]/a[@href]')\n",
114 | "\n",
115 | " for link in links:\n",
116 | " page_title = link.find_element(By.TAG_NAME, 'h3').text\n",
117 | " page_url = ''\n",
118 | "\n",
119 | " if link.get_attribute('href'):\n",
120 | " page_url = link.get_attribute('href')\n",
121 | "\n",
122 | " print('title: {}\\nurl: {}\\n---'.format(page_title, page_url))\n",
123 | "\n",
124 | " next_page = driver.find_element(By.XPATH, '//*[@id=\"pnnext\"]/span[2]').click()\n",
125 | "\n",
126 | "except Exception as e:\n",
127 | " print(e)\n",
128 | "finally:\n",
129 | " driver.quit()"
130 | ]
131 | }
132 | ],
133 | "metadata": {
134 | "kernelspec": {
135 | "display_name": "Python 3",
136 | "language": "python",
137 | "name": "python3"
138 | },
139 | "language_info": {
140 | "codemirror_mode": {
141 | "name": "ipython",
142 | "version": 3
143 | },
144 | "file_extension": ".py",
145 | "mimetype": "text/x-python",
146 | "name": "python",
147 | "nbconvert_exporter": "python",
148 | "pygments_lexer": "ipython3",
149 | "version": "3.6.6"
150 | }
151 | },
152 | "nbformat": 4,
153 | "nbformat_minor": 2
154 | }
155 |
--------------------------------------------------------------------------------
/02_selenium/03_crawling_reCAPTCHA_image.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# 練習\n",
8 | "\n",
9 | "- https://www.google.com/recaptcha/demo/recaptcha\n",
10 | "- 透過 google reCAPTCHA demo 生成圖片\n",
11 | "- 將 reCAPTCHA 的圖片抓下來"
12 | ]
13 | },
14 | {
15 | "cell_type": "code",
16 | "execution_count": 1,
17 | "metadata": {
18 | "collapsed": true
19 | },
20 | "outputs": [],
21 | "source": [
22 | "import os\n",
23 | "import hashlib\n",
24 | "import requests\n",
25 | "import time\n",
26 | "\n",
27 | "from selenium import webdriver\n",
28 | "from selenium.webdriver.common.keys import Keys\n",
29 | "from selenium.webdriver.common.by import By\n",
30 | "\n",
31 | "from fake_useragent import UserAgent\n",
32 | "from PIL import Image\n",
33 | "\n",
34 | "url = 'https://www.google.com/recaptcha/demo/recaptcha'\n",
35 | "fu = UserAgent()"
36 | ]
37 | },
38 | {
39 | "cell_type": "code",
40 | "execution_count": 2,
41 | "metadata": {},
42 | "outputs": [
43 | {
44 | "name": "stdout",
45 | "output_type": "stream",
46 | "text": [
47 | "Save img - /home/dirl/github/Python-Crawling-Tutorial/results/ceecac6a5a9677750a69c80a87f26080.JPEG\n",
48 | "Save img - /home/dirl/github/Python-Crawling-Tutorial/results/ceecac6a5a9677750a69c80a87f26080.JPEG\n",
49 | "Save img - /home/dirl/github/Python-Crawling-Tutorial/results/4cce70c2cdde67af52e27920693da213.JPEG\n",
50 | "Save img - /home/dirl/github/Python-Crawling-Tutorial/results/1682c3490f1ec9df1da4a43407f890b7.JPEG\n",
51 | "Save img - /home/dirl/github/Python-Crawling-Tutorial/results/aa6a4d1bfa181fc53636a341562fb2ea.PNG\n"
52 | ]
53 | }
54 | ],
55 | "source": [
56 | "results = os.path.abspath('../results')\n",
57 | "if not os.path.exists(results):\n",
58 | " os.makedirs(results)\n",
59 | "\n",
60 | "try:\n",
61 | " driver = webdriver.Chrome()\n",
62 | " driver.get(url)\n",
63 | " driver.maximize_window()\n",
64 | " driver.implicitly_wait(10)\n",
65 | " compare_url = ''\n",
66 | " \n",
67 | " for i in range(5):\n",
68 | " # get image\n",
69 | " img_el = driver.find_element(By.XPATH, '//div[@id=\"recaptcha_image\"]/img')\n",
70 | " img_url = img_el.get_attribute('src')\n",
71 | " img_filename = hashlib.md5(img_url.encode('utf-8')).hexdigest()\n",
72 | " compare_url = img_url\n",
73 | "\n",
74 | " headers = {'User-Agent': fu.random}\n",
75 | " img_resp = requests.get(img_url, stream=True, headers=headers)\n",
76 | " img = Image.open(img_resp.raw)\n",
77 | " img_filename = '{}.{}'.format(img_filename, img.format)\n",
78 | " img_filename = os.path.join(results, img_filename)\n",
79 | " img.save(img_filename)\n",
80 | " print('Save img - {}'.format(img_filename))\n",
81 | " \n",
82 | " # re-generate image\n",
83 | " btn_refresh = driver.find_element(By.XPATH, '//*[@id=\"recaptcha_reload_btn\"]').click()\n",
84 | " time.sleep(2)\n",
85 | " \n",
86 | "\n",
87 | "except Exception as e:\n",
88 | " print(e)\n",
89 | "finally:\n",
90 | " driver.quit()"
91 | ]
92 | }
93 | ],
94 | "metadata": {
95 | "kernelspec": {
96 | "display_name": "Python 3",
97 | "language": "python",
98 | "name": "python3"
99 | },
100 | "language_info": {
101 | "codemirror_mode": {
102 | "name": "ipython",
103 | "version": 3
104 | },
105 | "file_extension": ".py",
106 | "mimetype": "text/x-python",
107 | "name": "python",
108 | "nbconvert_exporter": "python",
109 | "pygments_lexer": "ipython3",
110 | "version": "3.5.2"
111 | }
112 | },
113 | "nbformat": 4,
114 | "nbformat_minor": 2
115 | }
116 |
--------------------------------------------------------------------------------
/03_graph_api/00_facebook_crawling_article_comments.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# 練習\n",
8 | "\n",
9 | "- 取得 FB 文章底下所有留言\n",
10 | "- 使用 [Graph API](https://developers.facebook.com/tools/explorer/)\n",
11 | "- https://www.facebook.com/DoctorKoWJ/videos/1213927345375910/\n",
12 | "- 輸出成 CSV"
13 | ]
14 | },
15 | {
16 | "cell_type": "code",
17 | "execution_count": 1,
18 | "metadata": {
19 | "collapsed": true
20 | },
21 | "outputs": [],
22 | "source": [
23 | "import os\n",
24 | "import requests\n",
25 | "import pandas as pd\n",
26 | "\n",
27 | "from datetime import datetime"
28 | ]
29 | },
30 | {
31 | "cell_type": "code",
32 | "execution_count": 2,
33 | "metadata": {
34 | "collapsed": true
35 | },
36 | "outputs": [],
37 | "source": [
38 | "# 透過 Graph API 觀察文章 ID 與 token\n",
39 | "article_id = '1213927345375910'\n",
40 | "token = ''"
41 | ]
42 | },
43 | {
44 | "cell_type": "code",
45 | "execution_count": 3,
46 | "metadata": {},
47 | "outputs": [
48 | {
49 | "name": "stdout",
50 | "output_type": "stream",
51 | "text": [
52 | "pages 1\n",
53 | "pages 2\n",
54 | "pages 3\n",
55 | "pages 4\n",
56 | "EOF\n",
57 | "comment length = 431\n"
58 | ]
59 | }
60 | ],
61 | "source": [
62 | "comments = []\n",
63 | "pages = 0\n",
64 | "\n",
65 | "url = 'https://graph.facebook.com/v2.11/{}/comments?pretty=0&limit={}&access_token={}'.format(\n",
66 | " article_id, 100, token\n",
67 | ")\n",
68 | "\n",
69 | "while True:\n",
70 | " pages += 1\n",
71 | " resp = requests.get(url)\n",
72 | " data = resp.json()\n",
73 | " comments += data['data']\n",
74 | " \n",
75 | " if 'next' not in data['paging']:\n",
76 | " print('EOF')\n",
77 | " break\n",
78 | " else:\n",
79 | " url = data['paging']['next']\n",
80 | " print('pages {}'.format(pages))\n",
81 | " \n",
82 | "print('comment length = {}'.format(len(comments)))"
83 | ]
84 | },
85 | {
86 | "cell_type": "code",
87 | "execution_count": 4,
88 | "metadata": {},
89 | "outputs": [
90 | {
91 | "data": {
92 | "text/html": [
93 | "推 bobobola: 中央不缺商阿 42.75.76.1 09/25 11:54\n",
240 | "
→ nikewang: 北中和中興合併不就好了121.157.204.247 09/25 11:54\n",
241 | "
→ nikewang: 北大121.157.204.247 09/25 11:54\n",
242 | "
推 qqq1234: 北大好不容易才脫離中興獨立 怎可能去併 117.56.55.46 09/25 11:59\n",
243 | "
→ Lakland: 北大跟北科合作一陣子了，中央找體大吧 114.24.29.42 09/25 11:59\n",
244 | "
推 atlaswhz: 中央找體大和警大組成桃聯大好了 1.34.181.133 09/25 12:07\n",
245 | "
推 sooppp: 體大的聽的懂中央上課在教什麼嗎？223.140.169.234 09/25 12:15\n",
246 | "
推 homepark: 中央缺醫學喇223.137.74.137 09/25 12:16\n",
247 | "
→ lee457088: 197.252是哪棟140.115.216.209 09/25 12:18\n",
248 | "
不知道\n",
249 | "推 mecca: 當年有文法商理工醫農學院 現在洗洗睡吧210.64.134.103 09/25 12:40\n",
250 | "
※ 編輯: R101 (140.115.130.200), 09/25/2018 15:41:01\n",
251 | "