├── .gitignore
├── 10 selenium 套件.ipynb
├── 11 selenium 瀏覽器自動化測試.ipynb
├── 12 selenium 等待.ipynb
├── 13 ActionChains.ipynb
├── 2 正規表達式.ipynb
├── 5 requests 套件.ipynb
├── 7 beautifulsoup 套件.ipynb
├── 8 cookie 用於 requests.ipynb
├── 9 PTT_NBA_看板主頁與內頁.ipynb
├── GPT2-Chinese.md
├── README.md
├── cases
├── .gitignore
├── PyAutoGUI.ipynb
├── archived
│ ├── JCC_reject.ipynb
│ ├── crowdfunding.ipynb
│ ├── fb_video.ipynb
│ ├── ig.ipynb
│ ├── journals
│ │ ├── EBSCO.ipynb
│ │ ├── WebOfScience_custom_search.ipynb
│ │ └── WebOfScience_journal_title.ipynb
│ ├── lm_studio
│ │ ├── README.md
│ │ ├── example01.jpg
│ │ ├── templates
│ │ │ └── index.html
│ │ ├── web_api.py
│ │ └── web_api_embedding.py
│ ├── network.ipynb
│ ├── open_access
│ │ ├── deprecated
│ │ │ ├── (deprecated)臺灣大學學術期刊資料庫.ipynb
│ │ │ ├── 中國學術年刊.ipynb
│ │ │ ├── 中正漢學研究.ipynb
│ │ │ ├── 政大中文學報.ipynb
│ │ │ └── 東吳大學政治學報.ipynb
│ │ ├── 中國文哲研究集刊.ipynb
│ │ ├── 中研院法學期刊.ipynb
│ │ ├── 台大日本語文研究.ipynb
│ │ ├── 同心圓：文學與文化研究.ipynb
│ │ ├── 國立中山大學_中國文學系_文與哲.ipynb
│ │ ├── 國立臺灣大學美術史研究集刊.ipynb
│ │ ├── 成大歷史學報.ipynb
│ │ ├── 戲劇學刊.ipynb
│ │ ├── 東吳中文學報.ipynb
│ │ ├── 清華中文學報.ipynb
│ │ └── 臺大中文學報.ipynb
│ ├── tika.ipynb
│ ├── twitter
│ │ ├── README.md
│ │ ├── basic.py
│ │ └── twint_run.py
│ ├── vector_index
│ │ ├── README.md
│ │ ├── make_index.py
│ │ └── query.py
│ ├── wikiart.ipynb
│ └── 綜合.ipynb
├── digital_archives
│ ├── taco-ith.ipynb
│ ├── wikisource.csv
│ └── wikisource.ipynb
├── download_captcha_images.ipynb
├── excel.ipynb
├── free-proxy.ipynb
├── hetubook_jinyong_requests.ipynb
├── image_gen
│ ├── ComfyUI.md
│ ├── civitai.md
│ ├── colab_diffusers_pipeline.ipynb
│ ├── microsoft_bing_chat.md
│ └── run_diffusers_flux_pipeline.py
├── ixdzs_jinyong_post_requests.ipynb
├── jinyong_requests.ipynb
├── jinyong_selenium.ipynb
├── leaflet
│ ├── README.md
│ ├── templates
│ │ └── index_cafe.html
│ └── web_api.py
├── line-stickers.ipynb
├── mouse_XY_colors.py
├── nidss.ipynb
├── pymysql.ipynb
├── read.xlsx
├── sound
│ ├── ebook
│ │ ├── css
│ │ │ └── jquery.highlight-within-textarea.css
│ │ ├── js
│ │ │ └── jquery.highlight-within-textarea.js
│ │ ├── templates
│ │ │ └── fetch.html
│ │ ├── tmp
│ │ │ └── .gitignore
│ │ └── web_api.py
│ └── google_lady.py
├── sqlite.ipynb
├── tabs.ipynb
├── twse.ipynb
├── weather.ipynb
└── youtube.ipynb
├── html
├── HTML_CSS.zip
├── README.md
├── images
│ ├── .DS_Store
│ ├── Logo.svg
│ ├── Shop.png
│ ├── banner.png
│ ├── p1.png
│ ├── p2.png
│ └── p3.png
├── index.html
└── style.css
├── python_web_scraping.docx
├── python_web_scraping.pdf
├── turingcerts.jpg
└── yt-dlp_and_ffmpeg.docx
/.gitignore:
--------------------------------------------------------------------------------
1 | .ipynb_checkpoints/
2 | dev/
3 | chromedriver*
4 | test*
--------------------------------------------------------------------------------
/10 selenium 套件.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "'''\n",
10 | "參考網頁:\n",
11 | "[1] 下載 Chrome Web Driver\n",
12 | "https://chromedriver.chromium.org/downloads\n",
13 | "'''\n",
14 | "\n",
15 | "# 操作 browser 的 API\n",
16 | "# from selenium.webdriver.chrome.service import Service\n",
17 | "from selenium import webdriver\n",
18 | "\n",
19 | "# 匯入套件\n",
20 | "from bs4 import BeautifulSoup as bs\n",
21 | "\n",
22 | "# 強制等待 (執行期間休息一下)\n",
23 | "from time import sleep\n",
24 | "\n",
25 | "# 使用 Chrome 的 WebDriver\n",
26 | "'''\n",
27 | "my_service = Service(executable_path=\"./chromedriver.exe\")\n",
28 | "driver = webdriver.Chrome(service=my_service)\n",
29 | "'''\n",
30 | "\n",
31 | "# 補充: 若沒有特別設定，只要電腦有安裝 Chrome，就可以直接使用\n",
32 | "driver = webdriver.Chrome()\n",
33 | "\n",
34 | "# 開啟 104人力行銀 首頁\n",
35 | "driver.get(\"https://www.104.com.tw/jobs/main/\")\n",
36 | "\n",
37 | "# 取得檢視原始碼的內容 (page_source 取得的 html，是動態的、使用者操作過後的結果)\n",
38 | "html = driver.page_source\n",
39 | "\n",
40 | "# 印出 html (也可以跟 Beautifulsoup 整合)\n",
41 | "# print(html)\n",
42 | "\n",
43 | "# 指定 lxml 作為解析器\n",
44 | "soup = bs(html, \"lxml\")\n",
45 | "\n",
46 | "# 取得元素\n",
47 | "div = soup.select('div.header__container')[0]\n",
48 | "\n",
49 | "# 顯示內文\n",
50 | "print(div.get_text())\n",
51 | "\n",
52 | "# 關閉瀏覽器\n",
53 | "driver.quit()"
54 | ]
55 | }
56 | ],
57 | "metadata": {
58 | "kernelspec": {
59 | "display_name": "Python 3 (ipykernel)",
60 | "language": "python",
61 | "name": "python3"
62 | },
63 | "language_info": {
64 | "codemirror_mode": {
65 | "name": "ipython",
66 | "version": 3
67 | },
68 | "file_extension": ".py",
69 | "mimetype": "text/x-python",
70 | "name": "python",
71 | "nbconvert_exporter": "python",
72 | "pygments_lexer": "ipython3",
73 | "version": "3.10.14"
74 | }
75 | },
76 | "nbformat": 4,
77 | "nbformat_minor": 4
78 | }
79 |
--------------------------------------------------------------------------------
/11 selenium 瀏覽器自動化測試.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# 匯入自動測試工具相關套件"
8 | ]
9 | },
10 | {
11 | "cell_type": "code",
12 | "execution_count": null,
13 | "metadata": {},
14 | "outputs": [],
15 | "source": [
16 | "'''\n",
17 | "匯入套件\n",
18 | "'''\n",
19 | "# 操作 browser 的 API\n",
20 | "from selenium.webdriver.chrome.service import Service\n",
21 | "from selenium import webdriver\n",
22 | "\n",
23 | "# 處理逾時例外的工具\n",
24 | "from selenium.common.exceptions import TimeoutException\n",
25 | "\n",
26 | "# 面對動態網頁，等待某個元素出現的工具，通常與 exptected_conditions 搭配\n",
27 | "from selenium.webdriver.support.ui import WebDriverWait\n",
28 | "\n",
29 | "# 搭配 WebDriverWait 使用，對元素狀態的一種期待條件，若條件發生，則等待結束，往下一行執行\n",
30 | "from selenium.webdriver.support import expected_conditions as EC\n",
31 | "\n",
32 | "# 期待元素出現要透過什麼方式指定，通常與 EC、WebDriverWait 一起使用\n",
33 | "from selenium.webdriver.common.by import By\n",
34 | "\n",
35 | "# 強制等待 (執行期間休息一下)\n",
36 | "from time import sleep\n",
37 | "\n",
38 | "'''\n",
39 | "selenium 啓動 Chrome 的進階配置參數\n",
40 | "參考網址：https://stackoverflow.max-everyday.com/2019/12/selenium-chrome-options/\n",
41 | "'''\n",
42 | "# 啟動瀏覽器工具的選項\n",
43 | "my_options = webdriver.ChromeOptions()\n",
44 | "# my_options.add_argument(\"--headless\") #不開啟實體瀏覽器背景執行\n",
45 | "my_options.add_argument(\"--start-maximized\") #最大化視窗\n",
46 | "my_options.add_argument(\"--incognito\") #開啟無痕模式\n",
47 | "my_options.add_argument(\"--disable-popup-blocking\") #禁用彈出攔截\n",
48 | "my_options.add_argument(\"--disable-notifications\") #取消 chrome 推播通知\n",
49 | "my_options.add_argument(\"--lang=zh-TW\") #設定為正體中文\n",
50 | "\n",
51 | "\n",
52 | "# 使用 Chrome 的 WebDriver\n",
53 | "# my_service = Service(executable_path=\"./chromedriver.exe\")\n",
54 | "driver = webdriver.Chrome(\n",
55 | " options = my_options,\n",
56 | "# service = my_service\n",
57 | ")"
58 | ]
59 | },
60 | {
61 | "cell_type": "markdown",
62 | "metadata": {},
63 | "source": [
64 | "# 在瀏覽器中執行自訂 JavaScript 程式"
65 | ]
66 | },
67 | {
68 | "cell_type": "code",
69 | "execution_count": null,
70 | "metadata": {},
71 | "outputs": [],
72 | "source": [
73 | "# 開啟網頁\n",
74 | "driver.get(\"https://crptransfer.moe.gov.tw/\")\n",
75 | "\n",
76 | "# 跳出 alert 視窗 (在 chrome 裡面執行 javascript 語法)\n",
77 | "driver.execute_script(\"window.alert('這是我們自訂的彈跳視窗');\")\n",
78 | "\n",
79 | "# 等個幾秒\n",
80 | "sleep(3)\n",
81 | "\n",
82 | "# 點選彈出裡面的確定按鈕\n",
83 | "driver.switch_to.alert.accept()"
84 | ]
85 | },
86 | {
87 | "cell_type": "markdown",
88 | "metadata": {},
89 | "source": [
90 | "# 輸入文字，送出表單"
91 | ]
92 | },
93 | {
94 | "cell_type": "code",
95 | "execution_count": null,
96 | "metadata": {},
97 | "outputs": [],
98 | "source": [
99 | "# 開啟網頁\n",
100 | "driver.get(\"https://crptransfer.moe.gov.tw/\")\n",
101 | "\n",
102 | "# 尋找網頁中的搜尋框\n",
103 | "inputElement = driver.find_element(\n",
104 | " By.CSS_SELECTOR, 'input#SN'\n",
105 | ")\n",
106 | "\n",
107 | "# 在搜尋框中輸入文字\n",
108 | "inputElement.send_keys(\"人帥真好\")\n",
109 | "\n",
110 | "# 睡個幾秒\n",
111 | "sleep(2)\n",
112 | "\n",
113 | "# 送出搜尋\n",
114 | "inputElement.submit()\n",
115 | "\n",
116 | "# 搜尋結果的 CSS Selector\n",
117 | "cssSelector = \"body > table > tbody > tr:nth-child(1) > td > main > article > div > table > tbody > tr:nth-child(2) > td\"\n",
118 | "\n",
119 | "try:\n",
120 | " # 等待網頁搜尋結果\n",
121 | " WebDriverWait(driver, 10).until(\n",
122 | " EC.presence_of_element_located(\n",
123 | " (By.CSS_SELECTOR, cssSelector)\n",
124 | " )\n",
125 | " )\n",
126 | " \n",
127 | " # 取得第一頁搜尋結果\n",
128 | " element = driver.find_element(\n",
129 | " By.CSS_SELECTOR, cssSelector\n",
130 | " )\n",
131 | " \n",
132 | " # 輸出想要爬取的文字\n",
133 | " print(element.text) \n",
134 | " print(element.get_attribute('innerText')) # 另一種寫法\n",
135 | "except TimeoutException:\n",
136 | " print('等待逾時！')"
137 | ]
138 | },
139 | {
140 | "cell_type": "markdown",
141 | "metadata": {},
142 | "source": [
143 | "# 輸入文字，按下送出鈕"
144 | ]
145 | },
146 | {
147 | "cell_type": "code",
148 | "execution_count": null,
149 | "metadata": {},
150 | "outputs": [],
151 | "source": [
152 | "# 開啟網頁\n",
153 | "driver.get(\"https://www.104.com.tw/jobs/main/\")\n",
154 | "\n",
155 | "# 尋找網頁中的搜尋框\n",
156 | "inputElement = driver.find_element(\n",
157 | " By.CSS_SELECTOR, 'input[data-gtm-index^=\"搜尋欄位\"]'\n",
158 | ")\n",
159 | "\n",
160 | "# 在搜尋框中輸入文字\n",
161 | "inputElement.send_keys(\"python\")\n",
162 | "\n",
163 | "# 睡個幾秒\n",
164 | "sleep(3)\n",
165 | "\n",
166 | "# 按鈕選擇器\n",
167 | "cssSelectorBtn = 'button.btn[type=\"submit\"][data-gtm-index^=\"搜尋欄位\"]'\n",
168 | "\n",
169 | "try:\n",
170 | " # 等待元素\n",
171 | " WebDriverWait(driver, 10).until(\n",
172 | " EC.presence_of_element_located(\n",
173 | " (By.CSS_SELECTOR, cssSelectorBtn)\n",
174 | " )\n",
175 | " )\n",
176 | " \n",
177 | " # 取得按鈕元素\n",
178 | " btn = driver.find_element(\n",
179 | " By.CSS_SELECTOR, cssSelectorBtn\n",
180 | " )\n",
181 | " \n",
182 | " # 按下按鈕\n",
183 | " btn.click()\n",
184 | "except TimeoutException:\n",
185 | " print('等待逾時！')"
186 | ]
187 | },
188 | {
189 | "cell_type": "markdown",
190 | "metadata": {},
191 | "source": [
192 | "# 刷新頁面 (類似 F5 或 Ctrl + R)"
193 | ]
194 | },
195 | {
196 | "cell_type": "code",
197 | "execution_count": null,
198 | "metadata": {},
199 | "outputs": [],
200 | "source": [
201 | "# 開啟網頁\n",
202 | "driver.get(\"https://reurl.cc/jR725D\")\n",
203 | "\n",
204 | "# 睡個幾秒\n",
205 | "sleep(3)\n",
206 | "\n",
207 | "# 刷新頁面\n",
208 | "driver.refresh()\n",
209 | "\n",
210 | "# 睡個幾秒\n",
211 | "sleep(3)\n",
212 | "\n",
213 | "# 刷新頁面\n",
214 | "driver.refresh()\n",
215 | "\n",
216 | "# 睡個幾秒\n",
217 | "sleep(3)\n",
218 | "\n",
219 | "# 關閉瀏覽器\n",
220 | "driver.quit()"
221 | ]
222 | }
223 | ],
224 | "metadata": {
225 | "kernelspec": {
226 | "display_name": "Python 3 (ipykernel)",
227 | "language": "python",
228 | "name": "python3"
229 | },
230 | "language_info": {
231 | "codemirror_mode": {
232 | "name": "ipython",
233 | "version": 3
234 | },
235 | "file_extension": ".py",
236 | "mimetype": "text/x-python",
237 | "name": "python",
238 | "nbconvert_exporter": "python",
239 | "pygments_lexer": "ipython3",
240 | "version": "3.10.14"
241 | },
242 | "vscode": {
243 | "interpreter": {
244 | "hash": "585a938ec471c889bf0cce0aed741a99eaf47ca09c0fa8393793bc5bfe77ba11"
245 | }
246 | }
247 | },
248 | "nbformat": 4,
249 | "nbformat_minor": 4
250 | }
251 |
--------------------------------------------------------------------------------
/12 selenium 等待.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# 匯入自動測試工具相關套件"
8 | ]
9 | },
10 | {
11 | "cell_type": "code",
12 | "execution_count": null,
13 | "metadata": {},
14 | "outputs": [],
15 | "source": [
16 | "'''\n",
17 | "參考網址:\n",
18 | "[1] Webdriver Manager for Python\n",
19 | "https://pypi.org/project/webdriver-manager/\n",
20 | "'''\n",
21 | "\n",
22 | "# 匯入套件\n",
23 | "from selenium import webdriver\n",
24 | "from selenium.webdriver.chrome.service import Service\n",
25 | "from webdriver_manager.chrome import ChromeDriverManager\n",
26 | "from selenium.common.exceptions import TimeoutException, NoSuchElementException\n",
27 | "from selenium.webdriver.support.ui import WebDriverWait\n",
28 | "from selenium.webdriver.support import expected_conditions as EC\n",
29 | "from selenium.webdriver.common.by import By\n",
30 | "from time import sleep"
31 | ]
32 | },
33 | {
34 | "cell_type": "code",
35 | "execution_count": null,
36 | "metadata": {},
37 | "outputs": [],
38 | "source": [
39 | "# (Optional) 開啟用於自動控制的瀏覽器 (自動取得 Chrome 的 WebDriver)\n",
40 | "driver = webdriver.Chrome(\n",
41 | " service = Service(ChromeDriverManager().install())\n",
42 | ")"
43 | ]
44 | },
45 | {
46 | "cell_type": "markdown",
47 | "metadata": {},
48 | "source": [
49 | "# 強制等待"
50 | ]
51 | },
52 | {
53 | "cell_type": "code",
54 | "execution_count": null,
55 | "metadata": {},
56 | "outputs": [],
57 | "source": [
58 | "'''\n",
59 | "強制等待\n",
60 | "'''\n",
61 | "# 開啟用於自動控制的瀏覽器\n",
62 | "driver = webdriver.Chrome()\n",
63 | "\n",
64 | "try:\n",
65 | " # 走訪網址\n",
66 | " driver.get('https://tw.yahoo.com/')\n",
67 | " \n",
68 | " # 強制等待 3 秒\n",
69 | " sleep(3)\n",
70 | " \n",
71 | " # 印出網址\n",
72 | " print(driver.current_url)\n",
73 | "except:\n",
74 | " print(\"程式出錯!\")\n",
75 | "finally:\n",
76 | " # 關閉瀏覽器\n",
77 | " driver.quit()"
78 | ]
79 | },
80 | {
81 | "cell_type": "markdown",
82 | "metadata": {},
83 | "source": [
84 | "# 隱性等待"
85 | ]
86 | },
87 | {
88 | "cell_type": "code",
89 | "execution_count": null,
90 | "metadata": {},
91 | "outputs": [],
92 | "source": [
93 | "'''\n",
94 | "隱性等待\n",
95 | "'''\n",
96 | "# 開啟用於自動控制的瀏覽器\n",
97 | "driver = webdriver.Chrome()\n",
98 | "\n",
99 | "try:\n",
100 | " # 最多等 15 秒\n",
101 | " driver.implicitly_wait(15)\n",
102 | " \n",
103 | " # 走訪網址\n",
104 | " driver.get('https://tw.yahoo.com/')\n",
105 | " \n",
106 | " # 取得元素\n",
107 | " element = driver.find_element(\n",
108 | " By.CSS_SELECTOR, \n",
109 | " 'a#header-logo'\n",
110 | " )\n",
111 | "\n",
112 | " # 印出超連結 ( 透過 .get_attribute('屬性') 來取得屬性的值 )\n",
113 | " print(element.get_attribute('href'))\n",
114 | "except NoSuchElementException:\n",
115 | " print(\"找不到元素!\")\n",
116 | "finally:\n",
117 | " # 關閉瀏覽器\n",
118 | " driver.quit()"
119 | ]
120 | },
121 | {
122 | "cell_type": "markdown",
123 | "metadata": {},
124 | "source": [
125 | "# 顯性等待"
126 | ]
127 | },
128 | {
129 | "cell_type": "code",
130 | "execution_count": null,
131 | "metadata": {},
132 | "outputs": [],
133 | "source": [
134 | "'''\n",
135 | "顯性等待\n",
136 | "'''\n",
137 | "# 開啟用於自動控制的瀏覽器\n",
138 | "driver = webdriver.Chrome()\n",
139 | "\n",
140 | "try:\n",
141 | " # 走訪網址\n",
142 | " driver.get('https://www.youtube.com/?gl=TW')\n",
143 | "\n",
144 | " # 滿足條件（10秒內找到元素），則往下一步\n",
145 | " WebDriverWait(driver, 10).until(\n",
146 | " EC.presence_of_element_located( \n",
147 | " (By.LINK_TEXT, '首頁') \n",
148 | " )\n",
149 | " )\n",
150 | " \n",
151 | " # 印出首頁連結\n",
152 | " link = driver.find_element(\n",
153 | " By.LINK_TEXT, '首頁'\n",
154 | " ).get_attribute('href')\n",
155 | " print(link)\n",
156 | " \n",
157 | "except TimeoutException:\n",
158 | " print('等待逾時！')\n",
159 | "finally:\n",
160 | " # 關閉瀏覽器\n",
161 | " driver.quit()"
162 | ]
163 | },
164 | {
165 | "cell_type": "markdown",
166 | "metadata": {},
167 | "source": [
168 | "# 參考連結\n",
169 | "- [Ubuntu安装运行无头Selenium Chrome](https://cloud.tencent.com/developer/article/1966470)"
170 | ]
171 | }
172 | ],
173 | "metadata": {
174 | "kernelspec": {
175 | "display_name": "Python 3 (ipykernel)",
176 | "language": "python",
177 | "name": "python3"
178 | },
179 | "language_info": {
180 | "codemirror_mode": {
181 | "name": "ipython",
182 | "version": 3
183 | },
184 | "file_extension": ".py",
185 | "mimetype": "text/x-python",
186 | "name": "python",
187 | "nbconvert_exporter": "python",
188 | "pygments_lexer": "ipython3",
189 | "version": "3.10.14"
190 | }
191 | },
192 | "nbformat": 4,
193 | "nbformat_minor": 4
194 | }
195 |
--------------------------------------------------------------------------------
/2 正規表達式.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# 正規表達式 Regular Expression"
8 | ]
9 | },
10 | {
11 | "cell_type": "code",
12 | "execution_count": null,
13 | "metadata": {},
14 | "outputs": [],
15 | "source": [
16 | "# 匯入 regex 套件\n",
17 | "import re"
18 | ]
19 | },
20 | {
21 | "cell_type": "code",
22 | "execution_count": null,
23 | "metadata": {},
24 | "outputs": [],
25 | "source": [
26 | "# match\n",
27 | "'''\n",
28 | "說明\n",
29 | "re.match 會從字串的「開頭」開始比對，\n",
30 | "比對不到，則回傳 None\n",
31 | "'''\n",
32 | "regex01 = r'2[0-9]{3}\\/[0-1]?[0-9]{1}\\/([0-3]?[0-9])'\n",
33 | "string01 = \"2024/09/18\"\n",
34 | "match01 = re.match(regex01, string01)\n",
35 | "print(match01)\n",
36 | "print(match01[0])\n",
37 | "print(match01[1])\n",
38 | "\n",
39 | "'''\n",
40 | "補充:\n",
41 | "match.group() 或 match.group(0) 是regex所代表的整個完整比對的字串，\n",
42 | "match.group(1)是第一組()中的內容，\n",
43 | "match.group(2)是第二組()中的內容...\n",
44 | "'''\n",
45 | "print(match01.group(0))\n",
46 | "print(match01.group(1))"
47 | ]
48 | },
49 | {
50 | "cell_type": "code",
51 | "execution_count": null,
52 | "metadata": {},
53 | "outputs": [],
54 | "source": [
55 | "# findall\n",
56 | "'''\n",
57 | "說明\n",
58 | "re.findall 會將所有配對到的字串\n",
59 | "回傳成一個 list\n",
60 | "'''\n",
61 | "regex02 = r'[0-9]+'\n",
62 | "string02 = \"0911111111, 0922222222, 0933333333\"\n",
63 | "listMatch02 = re.findall(regex02, string02)\n",
64 | "print(listMatch02)\n",
65 | "print(listMatch02[0])\n",
66 | "print(listMatch02[2])"
67 | ]
68 | },
69 | {
70 | "cell_type": "code",
71 | "execution_count": null,
72 | "metadata": {},
73 | "outputs": [],
74 | "source": [
75 | "# finditer\n",
76 | "'''\n",
77 | "說明\n",
78 | "re.finditer 會將所有配對到的字串\n",
79 | "以迭代的方式呈現，若沒有配對到，則回傳 None\n",
80 | "'''\n",
81 | "regex03 = r'[0-9]+'\n",
82 | "string03 = \"0911111111, 0922222222, 0933333333\"\n",
83 | "iterableMatch03 = re.finditer(regex03, string03)\n",
84 | "if iterableMatch03 != None:\n",
85 | " for match in iterableMatch03:\n",
86 | " print(match[0])"
87 | ]
88 | },
89 | {
90 | "cell_type": "code",
91 | "execution_count": null,
92 | "metadata": {},
93 | "outputs": [],
94 | "source": [
95 | "# search\n",
96 | "'''\n",
97 | "說明\n",
98 | "re.search 會將整個字串進行搜尋，\n",
99 | "但只會比對到第一組，\n",
100 | "比對不到，則回傳 None\n",
101 | "'''\n",
102 | "regex04 = r'[a-zA-Z]([12])\\d{8}'\n",
103 | "string04 = \"A123456789, S299888777\"\n",
104 | "match04 = re.search(regex04, string04)\n",
105 | "print(match04)\n",
106 | "print(match04[0])\n",
107 | "print(match04[1])"
108 | ]
109 | },
110 | {
111 | "cell_type": "code",
112 | "execution_count": null,
113 | "metadata": {},
114 | "outputs": [],
115 | "source": [
116 | "# split\n",
117 | "'''\n",
118 | "說明\n",
119 | "re.split 類似 string.split('separator')，\n",
120 | "只是用正規表達式來作為 separator，\n",
121 | "並回傳 list\n",
122 | "'''\n",
123 | "regex05 = r'\\d'\n",
124 | "string05 = \"One1Two2Three3Four4\"\n",
125 | "listMatch05 = re.split(regex05, string05)\n",
126 | "print(listMatch05)"
127 | ]
128 | },
129 | {
130 | "cell_type": "code",
131 | "execution_count": null,
132 | "metadata": {},
133 | "outputs": [],
134 | "source": [
135 | "# sub\n",
136 | "'''\n",
137 | "說明\n",
138 | "re.sub(regex, replace_string, test_string)\n",
139 | "將 regex 所代表的文字，改成 replace_string，文字來源是 test_string\n",
140 | "'''\n",
141 | "regex06 = r\"\\D\"\n",
142 | "string06 = \"5-20 #1314\"\n",
143 | "strResult = re.sub(regex06, \"\", string06)\n",
144 | "print(strResult)"
145 | ]
146 | },
147 | {
148 | "cell_type": "markdown",
149 | "metadata": {},
150 | "source": [
151 | "# 環視\n",
152 | "| 名稱 | 語法 | 說明 |\n",
153 | "|:---:|:---:|:---:|\n",
154 | "| 正向環視 | (?=) | 這位置右邊要出現什麼 |\n",
155 | "| 正向環視否定 | (?!) | 這位置右邊不能出現什麼 |\n",
156 | "| 反向環視 | (?<=) | 這位置左邊要出現什麼 |\n",
157 | "| 反向環視否定 | (?[12])\\d{8}'\n",
207 | "string09 = \"A100000001\"\n",
208 | "match09 = re.match(regex09, string09)\n",
209 | "\n",
210 | "# 完整配對的文字\n",
211 | "print(match09[0])\n",
212 | "print(match09.group(0))\n",
213 | "print(match09.group())\n",
214 | "\n",
215 | "# 具名(類似key)所代表的值，也可以用索引代號來取得\n",
216 | "print(match09.group('gender'))\n",
217 | "print(match09['gender'])\n",
218 | "print(match09[1])"
219 | ]
220 | },
221 | {
222 | "cell_type": "markdown",
223 | "metadata": {},
224 | "source": [
225 | "# 參考資料\n",
226 | "1. [Python3 正则表达式](https://www.runoob.com/python3/python3-reg-expressions.html \"Python3 正则表达式\")\n",
227 | "2. [正則表達式-全型英數中文字、常用符號unicode對照表](https://blog.typeart.cc/%E6%AD%A3%E5%89%87%E8%A1%A8%E9%81%94%E5%BC%8F-%E5%85%A8%E5%9E%8B%E8%8B%B1%E6%95%B8%E4%B8%AD%E6%96%87%E5%AD%97%E3%80%81%E5%B8%B8%E7%94%A8%E7%AC%A6%E8%99%9Funicode%E5%B0%8D%E7%85%A7%E8%A1%A8/ \"正則表達式-全型英數中文字、常用符號unicode對照表\")\n",
228 | "3. [匹配中文字符的正則表達式： [/u4e00-/u9fa5]](https://www.itread01.com/content/1513168876.html \"匹配中文字符的正則表達式： [/u4e00-/u9fa5]\")\n",
229 | "4. [【Regular Expression】正向環視、反向環視](https://toyo0103.blogspot.com/2017/01/regular-expression.html \"【Regular Expression】正向環視、反向環視\")"
230 | ]
231 | }
232 | ],
233 | "metadata": {
234 | "kernelspec": {
235 | "display_name": "Python 3 (ipykernel)",
236 | "language": "python",
237 | "name": "python3"
238 | },
239 | "language_info": {
240 | "codemirror_mode": {
241 | "name": "ipython",
242 | "version": 3
243 | },
244 | "file_extension": ".py",
245 | "mimetype": "text/x-python",
246 | "name": "python",
247 | "nbconvert_exporter": "python",
248 | "pygments_lexer": "ipython3",
249 | "version": "3.10.11"
250 | },
251 | "vscode": {
252 | "interpreter": {
253 | "hash": "585a938ec471c889bf0cce0aed741a99eaf47ca09c0fa8393793bc5bfe77ba11"
254 | }
255 | }
256 | },
257 | "nbformat": 4,
258 | "nbformat_minor": 4
259 | }
260 |
--------------------------------------------------------------------------------
/7 beautifulsoup 套件.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "id": "5a441807",
7 | "metadata": {},
8 | "outputs": [],
9 | "source": [
10 | "'''\n",
11 | "參考網頁\n",
12 | "[1] Python 使用 Beautiful Soup 抓取與解析網頁資料，開發網路爬蟲教學\n",
13 | "https://blog.gtwang.org/programming/python-beautiful-soup-module-scrape-web-pages-tutorial/2/\n",
14 | "'''\n",
15 | "\n",
16 | "import requests as req\n",
17 | "from bs4 import BeautifulSoup as bs\n",
18 | "\n",
19 | " # PTT NBA 板\n",
20 | "url = \"https://www.ptt.cc/bbs/NBA/index.html\"\n",
21 | "\n",
22 | "# 用 requests 的 get 方法把網頁抓下來\n",
23 | "res = req.get(url) \n",
24 | "\n",
25 | "# 指定 lxml 作為解析器\n",
26 | "soup = bs(res.text, \"lxml\") "
27 | ]
28 | },
29 | {
30 | "cell_type": "code",
31 | "execution_count": null,
32 | "id": "33639bed",
33 | "metadata": {},
34 | "outputs": [],
35 | "source": [
36 | "# 第一個 \n",
37 | "print(soup.find(\"a\")) \n",
38 | "\n",
39 | "# 全部 ，此時回傳 list\n",
40 | "print(soup.find_all(\"a\")) \n",
41 | "\n",
42 | "# 指定 list 某個元素的 html\n",
43 | "print(soup.find_all(\"a\")[2]) "
44 | ]
45 | },
46 | {
47 | "cell_type": "code",
48 | "execution_count": null,
49 | "id": "532d85f1",
50 | "metadata": {},
51 | "outputs": [],
52 | "source": [
53 | "# 取得 id 為 logo 的元素\n",
54 | "a = soup.find(id = \"logo\")\n",
55 | "print(a)\n",
56 | "\n",
57 | "# 取得所有 div，類別名稱為 r-ent，回傳為 list\n",
58 | "divs = soup.find_all(\"div\", class_ = \"r-ent\")\n",
59 | "print(divs)"
60 | ]
61 | },
62 | {
63 | "cell_type": "code",
64 | "execution_count": null,
65 | "id": "9c1ddf9a",
66 | "metadata": {},
67 | "outputs": [],
68 | "source": [
69 | "'''\n",
70 | "以下透過 CSS selector 取得元素，\n",
71 | "回傳格式為 list\n",
72 | "'''\n",
73 | "# 輸出 title\n",
74 | "print(soup.select_one('title'))\n",
75 | "\n",
76 | "# 輸出 a\n",
77 | "print(soup.select('a'))\n",
78 | "\n",
79 | "# 透過 class 名稱取得元素\n",
80 | "print(soup.select(\"a.board\"))\n",
81 | "\n",
82 | "# 透過 id 名稱取得元素\n",
83 | "print(soup.select_one(\"#logo\"))\n",
84 | "\n",
85 | "# 透過 attribute 取得元素\n",
86 | "print(soup.select('a[class=\"board\"]'))"
87 | ]
88 | },
89 | {
90 | "cell_type": "code",
91 | "execution_count": null,
92 | "id": "77efcd15",
93 | "metadata": {},
94 | "outputs": [],
95 | "source": [
96 | "# 取得單一節點的文字內容 (select_one 會回傳單一 bs element 物件，select 會回傳 list)\n",
97 | "print(soup.select_one('title').get_text())\n",
98 | "print(soup.select('a')[0].get_text())"
99 | ]
100 | },
101 | {
102 | "cell_type": "code",
103 | "execution_count": null,
104 | "id": "59d702d3",
105 | "metadata": {},
106 | "outputs": [],
107 | "source": [
108 | "# 透過迭代取得所有 a 的文字內容\n",
109 | "for a in soup.select('a'):\n",
110 | " print(a.get_text())"
111 | ]
112 | },
113 | {
114 | "cell_type": "code",
115 | "execution_count": null,
116 | "id": "8b5ccf58",
117 | "metadata": {},
118 | "outputs": [],
119 | "source": [
120 | "# 透過迭代取得所有 a 的屬性 href\n",
121 | "for a in soup.select('a'):\n",
122 | " if a.has_attr('href'):\n",
123 | " print(a['href']) # a.get(\"href\")\n",
124 | " else:\n",
125 | " print(\"=\" * 50)\n",
126 | " print(f\"連結[{a.get_text()}] 沒有 href 屬性\")\n",
127 | " print(\"=\" * 50)"
128 | ]
129 | }
130 | ],
131 | "metadata": {
132 | "kernelspec": {
133 | "display_name": "Python 3 (ipykernel)",
134 | "language": "python",
135 | "name": "python3"
136 | },
137 | "language_info": {
138 | "codemirror_mode": {
139 | "name": "ipython",
140 | "version": 3
141 | },
142 | "file_extension": ".py",
143 | "mimetype": "text/x-python",
144 | "name": "python",
145 | "nbconvert_exporter": "python",
146 | "pygments_lexer": "ipython3",
147 | "version": "3.10.11"
148 | },
149 | "vscode": {
150 | "interpreter": {
151 | "hash": "585a938ec471c889bf0cce0aed741a99eaf47ca09c0fa8393793bc5bfe77ba11"
152 | }
153 | }
154 | },
155 | "nbformat": 4,
156 | "nbformat_minor": 5
157 | }
158 |
--------------------------------------------------------------------------------
/8 cookie 用於 requests.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "id": "af7c99e7",
6 | "metadata": {},
7 | "source": [
8 | "# 以 PTT Gossiiping (八卦版) 為例"
9 | ]
10 | },
11 | {
12 | "cell_type": "code",
13 | "execution_count": null,
14 | "id": "cd2ac08a",
15 | "metadata": {},
16 | "outputs": [],
17 | "source": [
18 | "import requests as req\n",
19 | "from bs4 import BeautifulSoup as bs\n",
20 | "\n",
21 | " # PTT Gossiiping (八卦版) \n",
22 | "url = \"https://www.ptt.cc/bbs/Gossiping/index.html\"\n",
23 | "\n",
24 | "# 首頁網址\n",
25 | "prefix = 'https://www.ptt.cc'\n",
26 | "\n",
27 | "# 設定 cookie\n",
28 | "my_cookies = {\n",
29 | " \"over18\": \"1\"\n",
30 | "}\n",
31 | "\n",
32 | "# 用 requests 的 get 方法把網頁抓下來\n",
33 | "res = req.get(url, cookies = my_cookies) \n",
34 | "\n",
35 | "# 指定 lxml 作為解析器\n",
36 | "soup = bs(res.text, \"lxml\")\n",
37 | "\n",
38 | "# 顯示連結列表\n",
39 | "for a in soup.select('div.r-ent > div.title > a'):\n",
40 | " print(a.get_text())\n",
41 | " print(prefix + a['href'])"
42 | ]
43 | }
44 | ],
45 | "metadata": {
46 | "kernelspec": {
47 | "display_name": "Python 3 (ipykernel)",
48 | "language": "python",
49 | "name": "python3"
50 | },
51 | "language_info": {
52 | "codemirror_mode": {
53 | "name": "ipython",
54 | "version": 3
55 | },
56 | "file_extension": ".py",
57 | "mimetype": "text/x-python",
58 | "name": "python",
59 | "nbconvert_exporter": "python",
60 | "pygments_lexer": "ipython3",
61 | "version": "3.10.13"
62 | }
63 | },
64 | "nbformat": 4,
65 | "nbformat_minor": 5
66 | }
67 |
--------------------------------------------------------------------------------
/9 PTT_NBA_看板主頁與內頁.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "# 匯入套件\n",
10 | "from bs4 import BeautifulSoup as bs\n",
11 | "import requests as req\n",
12 | "from pprint import pprint"
13 | ]
14 | },
15 | {
16 | "cell_type": "markdown",
17 | "metadata": {},
18 | "source": [
19 | "# 一般用法\n",
20 | "---"
21 | ]
22 | },
23 | {
24 | "cell_type": "code",
25 | "execution_count": null,
26 | "metadata": {},
27 | "outputs": [],
28 | "source": [
29 | "# 取得新聞列表\n",
30 | "url = \"https://www.ptt.cc/bbs/NBA/index.html\" \n",
31 | "\n",
32 | "# 用 requests 的 get 方法把網頁抓下來\n",
33 | "res = req.get(url) \n",
34 | "\n",
35 | "# 指定 lxml 作為解析器\n",
36 | "soup = bs(res.text, \"lxml\") \n",
37 | "\n",
38 | "# 建立 list 來放置列表資訊\n",
39 | "list_posts = []"
40 | ]
41 | },
42 | {
43 | "cell_type": "code",
44 | "execution_count": null,
45 | "metadata": {},
46 | "outputs": [],
47 | "source": [
48 | "# 清空放置列表資訊的變數\n",
49 | "list_posts.clear()\n",
50 | "\n",
51 | "# 取得 列表 的文字與超連結\n",
52 | "for a in soup.select('div.r-ent div.title a[href]'):\n",
53 | " print(a.get_text())\n",
54 | " print(a['href']) # 或是 a.get('href') \n",
55 | " \n",
56 | " # 加入列表資訊\n",
57 | " list_posts.append({\n",
58 | " 'title': a.get_text(),\n",
59 | " 'link': 'https://www.ptt.cc' + a['href']\n",
60 | " })"
61 | ]
62 | },
63 | {
64 | "cell_type": "code",
65 | "execution_count": null,
66 | "metadata": {},
67 | "outputs": [],
68 | "source": [
69 | "# 走訪每一個 a link，整合網頁內文\n",
70 | "for index, obj in enumerate(list_posts):\n",
71 | " res_ = req.get(obj['link'])\n",
72 | " soup_ = bs(res_.text, \"lxml\")\n",
73 | " \n",
74 | " # 去掉 div.article-metaline (作者、標題、時間…等)\n",
75 | " for div in soup_.select('div[class^=\"article-metaline\"]'):\n",
76 | " div.decompose()\n",
77 | " \n",
78 | " # 去掉 div.push (推文: 推、→、噓) (判斷元素是否存在)\n",
79 | " if len( soup_.select('div.push') ) > 0:\n",
80 | " for div in soup_.select('div.push'):\n",
81 | " div.decompose()\n",
82 | " \n",
83 | " # 取得實際需要的內容 (類似 JavaScript 的 innerHTML)\n",
84 | " html = soup_.select_one('div#main-content').decode_contents()\n",
85 | "\n",
86 | " # 類似 JavaScript outerHTML\n",
87 | " # html = str(soup_.select_one('div#main-content')) \n",
88 | " \n",
89 | " # 整合到列表資訊的變數當中\n",
90 | " list_posts[index]['html'] = html\n",
91 | "\n",
92 | "\n",
93 | "# 預覽所有結果\n",
94 | "pprint(list_posts)"
95 | ]
96 | },
97 | {
98 | "cell_type": "markdown",
99 | "metadata": {},
100 | "source": [
101 | "# 思考\n",
102 | "- 如何取得**多個分頁**的內容?\n",
103 | " - 觀察分頁數字在網址的呈現方式\n",
104 | " - 將觀察到的分頁數字嵌入對應的網址當中"
105 | ]
106 | },
107 | {
108 | "cell_type": "code",
109 | "execution_count": null,
110 | "metadata": {},
111 | "outputs": [],
112 | "source": [
113 | "# 清空放置列表資訊的變數\n",
114 | "list_posts.clear()\n",
115 | "\n",
116 | "# 起始頁數\n",
117 | "init_page = 6503\n",
118 | "\n",
119 | "# 最新頁數\n",
120 | "latest_page = 6504\n",
121 | "\n",
122 | "# 在已經知道分頁數的情況下\n",
123 | "for page in range(init_page, latest_page + 1):\n",
124 | " \n",
125 | " # 取得新聞列表\n",
126 | " url = f\"https://www.ptt.cc/bbs/NBA/index{page}.html\" \n",
127 | "\n",
128 | " # 用 requests 的 get 方法把網頁抓下來\n",
129 | " res = req.get(url) \n",
130 | "\n",
131 | " # 指定 lxml 作為解析器\n",
132 | " soup = bs(res.text, \"lxml\") \n",
133 | " \n",
134 | " # 取得 列表 的文字與超連結\n",
135 | " for a in soup.select('div.r-ent div.title a[href]'):\n",
136 | " # 加入列表資訊\n",
137 | " list_posts.append({\n",
138 | " 'title': a.get_text(),\n",
139 | " 'link': 'https://www.ptt.cc' + a['href']\n",
140 | " })\n",
141 | " \n",
142 | "# 走訪每一個 a link，整合網頁內文\n",
143 | "for index, obj in enumerate(list_posts):\n",
144 | " res_ = req.get(obj['link'])\n",
145 | " soup_ = bs(res_.text, \"lxml\")\n",
146 | "\n",
147 | " # 去掉 div.article-metaline (作者、標題、時間…等)\n",
148 | " for div in soup_.select('div[class^=\"article-metaline\"]'):\n",
149 | " div.decompose()\n",
150 | " \n",
151 | " # 去掉 div.push (推文: 推、→、噓) (判斷去掉元素是否存在)\n",
152 | " if len( soup_.select('div.push') ) > 0:\n",
153 | " for div in soup_.select('div.push'):\n",
154 | " div.decompose()\n",
155 | "\n",
156 | " # 取得實際需要的內容 (類似 JavaScript 的 innerHTML)\n",
157 | " html = soup_.select_one('div#main-content').decode_contents()\n",
158 | "\n",
159 | " # 整合到列表資訊的變數當中\n",
160 | " list_posts[index]['html'] = html\n",
161 | "\n",
162 | " \n",
163 | "# 預覽所有結果\n",
164 | "pprint(list_posts)"
165 | ]
166 | }
167 | ],
168 | "metadata": {
169 | "kernelspec": {
170 | "display_name": "Python 3 (ipykernel)",
171 | "language": "python",
172 | "name": "python3"
173 | },
174 | "language_info": {
175 | "codemirror_mode": {
176 | "name": "ipython",
177 | "version": 3
178 | },
179 | "file_extension": ".py",
180 | "mimetype": "text/x-python",
181 | "name": "python",
182 | "nbconvert_exporter": "python",
183 | "pygments_lexer": "ipython3",
184 | "version": "3.10.13"
185 | },
186 | "vscode": {
187 | "interpreter": {
188 | "hash": "585a938ec471c889bf0cce0aed741a99eaf47ca09c0fa8393793bc5bfe77ba11"
189 | }
190 | }
191 | },
192 | "nbformat": 4,
193 | "nbformat_minor": 4
194 | }
195 |
--------------------------------------------------------------------------------
/GPT2-Chinese.md:
--------------------------------------------------------------------------------
1 | # GPT2-Chinese
2 |
3 | ## 一、簡介
4 | 自然語言生成（natural language generation，NLG）是自然語言處理（natural language processing，NLP）重要的一部分。它可以降低人類與機器之間資訊交流的鴻溝，也試著將大量的非結構性資料，轉換成人類能夠理解的呈現方式。自然語言生成的系統，常用於協助人類作家撰寫日常的文件，包括商業書信、天氣報告，也被用作互動式解說工具，以易於理解的方式，與非專業使用者進行溝通交流；在進行決策時，儘管最常用的方法是以圖形方式顯示數據，但已證明文本摘要可以改善決策制定。
5 | ***
6 | 由於近年人工智慧（artificial intelligence，AI）議題的興起，透過 AI 即時處理並應用大量數據的特性，延伸出許多 NLG 的應用，例如聊天機器人（chatbot）、線上客戶（online customer service）等對話系統，其對話的內容與訊息，是透過 NLG 來產生。
7 | ***
8 | 在過去，對話生成通常使用循環神經網路（recurrent neural network）、長短期記憶（long-short term memory，LSTM）、閘道循環單位（gated recurrent unit，GRU）等來建立語言模型（language model），以考慮前一個文字，來對下一個可能生成的文字進行預測，然而 RNN 在長時間的記憶表現不好，LSTM 與 GRU 雖然改善了 RNN 架構，卻又有耗費系統資源的問題。在 2019 年，Open AI 推出了 GPT-2（generative pre-trained 2），在對話生成上，有著優異的表現。GPT-2 是一個大型、以 transformer 為基礎的語言模型，其語言建模對大約 40 GB 的超大語料庫進行了預訓練。GPT-2 的模型訓練效果不僅比 RNN、LSTM、GRU 好，且對上下文的文句預測精準度更佳，徹底改變了自然語言處理的生態；現今主流自然語言生成領域的機器學習模型基礎，大多建立在 transformer 之上。
9 | ***
10 | 近年 GPT-2 應用的範圍很多，例如透過生成專利範圍內容，用來協助發明人解決對於專利範圍無法掌握撰寫要領的問題；在收集多樣化的問題過後，透過人、事、時、地、物加以分類，再透過生成完整的問題句子，來解決將相同分類中相似的問題進行替換後，問句不完整的情形；給定風格、關鍵句子與隨機參數，進行歌詞生成，根據風格條件生成具有結構性、押韻性、原創性的歌詞等等。
11 |
12 | ## 二、本例練習目的
13 | 1. 透過 GPT2-Chinese 訓練自行整理的語料。
14 | 2. 套用訓練完成的語言模型，透過自訂的前導文字，來進行後續的文字生成。
15 |
16 | ## 三、說明
17 | - 作業環境:
18 | - Windows 10 or Linux Ubuntu 18.04+
19 | - Anaconda (Python 3.7+)
20 | - 專案連結:
21 | - [GPT2-Chinese 專案連結](https://github.com/Morizeyao/GPT2-Chinese/tree/old_gpt_2_chinese_before_2021_4_22 "GPT2-Chinese 專案連結")
22 | - 下載方式:
23 | - Git 下載指令：```git clone https://github.com/Morizeyao/GPT2-Chinese.git```
24 | - 手動下載：專案連結頁面 -> Code -> Download ZIP
25 | - 開始前的準備與流程
26 | - 說明:
27 | - 以 電腦 / 筆電 有 GPU 的 Windows 環境為例
28 | - 請注意 nvidia driver 與 CUDA、CUDA 與 CuDNN 之間的相依問題
29 | - 流程一: 先確認 nVIDIA driver 是否安裝，如果不是進階使用者，建議用最新版 [NVIDIA 驅動程式下載](https://www.nvidia.com.tw/Download/index.aspx?lang=tw "NVIDIA 驅動程式下載")
30 | - 下載方式分為「SD」與「GRD」
31 | - 如果你需要對最新遊戲、DLC 提供最即時支援的玩家，請選擇 Game Ready 驅動程式。
32 | - 如果你需要對影片編輯、動畫、攝影、繪圖設計和直播等創作流程提供穩定的品質，請選擇 Studio 驅動程式。
33 | - 流程二: 以 GPT-2 所使用的 Transformer 為例
34 | - 它使用 PyTorch 框架，所以要先了解 PyTorch 支援的 CUDA 版本: [INSTALL PYTORCH](https://pytorch.org/ "INSTALL PYTORCH")
35 | - 下載 CUDA 前，請先至 PyTorch 網站，了解目前支援 CUDA 的版本，下載 cuDNN 亦同。
36 | - 
37 | - 流程三: 下載: [CUDA Toolkit Archive](https://developer.nvidia.com/cuda-toolkit-archive "CUDA Toolkit Archive")
38 | - 流程四: 下載: [NVIDIA cuDNN](https://developer.nvidia.com/cudnn "NVIDIA cuDNN")
39 | - 需要先申請帳號密碼，才能進入下載頁面
40 | - 中間可能會請你填寫問卷，依實際情況填寫即可
41 | - 目前個人電腦環境所配對的 CUDA / cuDNN 下載連結
42 | - CUDA: [CUDA Toolkit 11.1 Update 1 Downloads](https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_456.81_win10.exe "cuda_11.1.1_456.81_win10.exe")
43 | - cuDNN: [cuDNN Library for Windows (x86)](https://developer.nvidia.com/compute/machine-learning/cudnn/secure/8.0.5/11.1_20201106/cudnn-11.1-windows-x64-v8.0.5.39.zip "cudnn-11.1-windows-x64-v8.0.5.39.zip")
44 | - 安裝套件:
45 | - 進入專案資料夾後輸入 ```pip install -r requirements.txt```
46 | - 檢測目前電腦有沒有支援 CUDA 的程式碼
47 | ```
48 | import torch
49 | print(torch.cuda.is_available())
50 | print(torch.cuda.current_device())
51 | print(torch.cuda.device(0))
52 | ```
53 | ```
54 | 若有支援，應該輸出類似下方的字樣:
55 | True
56 | 0
57 |
58 | ```
59 | - 安裝參考連結:
60 | - [Table 3. CUDA Toolkit and Corresponding Driver Versions](https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/index.html "Table 3. CUDA Toolkit and Corresponding Driver Versions")
61 | - [CUDA 與 CuDNN 安裝及環境變數設定](https://qqmanlin.medium.com/cuda-%E8%88%87-cudnn-%E5%AE%89%E8%A3%9D-e982d92162af "CUDA 與 CuDNN 安裝及環境變數設定")
62 | - [Win10環境下配置CUDA與cuDNN](https://dotblogs.com.tw/CYLcode/2018/09/20/163005 "Win10環境下配置CUDA與cuDNN")
63 | - 確認 CUDA 是否正確安裝的指令:
64 | - 安裝完 nVIDIA driver 後可用的指令（GPU 當前狀態）: ```nvidia-smi```
65 | - 安裝完 CUDA 後可用的指令（觀看目前安裝的 CUDA 版本）: ```nvcc -V```
66 | - 訓練資料結構:
67 | - [連結](https://github.com/Morizeyao/GPT2-Chinese/blob/old_gpt_2_chinese_before_2021_4_22/train.json "連結")
68 | - 
69 | - 參考網頁:
70 | - [直觀理解 GPT-2 語言模型並生成金庸武俠小說](https://leemeng.tw/gpt2-language-model-generate-chinese-jing-yong-novels.html "直觀理解 GPT-2 語言模型並生成金庸武俠小說")
71 | - [使用GPT2-Chinese生成中文小說](https://www.cc.ntu.edu.tw/chinese/epaper/0058/20210920_5808.html "作者：楊德倫 / 資策會數位教育研究所數位人才培育中心講師")
72 |
73 | - 金庸小說訓練資料的爬取教學
74 | - [selenium 取得金庸小說的內容，並存成txt與json檔](https://www.youtube.com/watch?v=jJzZcMjZsKM "[selenium]取得金庸小說的內容，並存成txt與json檔")
75 | - [requests 取得金庸小說的內容，並存成txt與json檔](https://www.youtube.com/watch?v=JsmLtMC43Lc "[requests]取得金庸小說的內容，並存成txt與json檔")
76 |
77 | ## 四、在 Windows 的基本使用方式
78 |
79 | ### （一）win10 安裝pytorch gpu 及 解決報錯“OSError: [WinError 126] 找不到指定的模組
80 | [WinError 126 找不到指定的模組](https://www.mdeditor.tw/pl/pndo/zh-tw "WinError 126 找不到指定的模組")
81 |
82 | ### （二）[WinError 126] VC-redist 安裝檔
83 | [WinError 126 VC-redist 安裝檔](https://download.visualstudio.microsoft.com/download/pr/89a3b9df-4a09-492e-8474-8f92c115c51d/B1A32C71A6B7D5978904FB223763263EA5A7EB23B2C44A0D60E90D234AD99178/VC_redist.x64.exe "WinError 126 VC-redist 安裝檔")
84 |
85 | ### （三）訓練文章的指令
86 | ```
87 | python train.py --device=0 --epochs=1 --batch_size=1 --min_length=10 --raw_data_path=data/jinyong.json --output_dir=model/ --raw
88 | ```
89 | | 參數 | 說明 |
90 | | ------ | ------ |
91 | | train.py | 訓練用主程式 |
92 | | device | 指定用哪一個 GPU (沒 GPU，預設 CPU) |
93 | | epochs | 訓練幾回 |
94 | | batch_size | 每次拿幾個樣本進行訓練。常見的是 2 的 n 次方 |
95 | | min_length | 每個樣本至少需要多少長度才拿來訓練 |
96 | | raw_data_path | 訓練資料 JSON 檔案路徑 |
97 | | output_dir | 訓練完的語言模型存放資料夾 |
98 | | raw | 設定此參數，會將樣本進行 tokenize |
99 |
100 | ### （四）生成文章的指令
101 | ```
102 | python generate.py --length=250 --nsamples=3 --prefix="張無忌見三名老僧在片刻間連斃崑崙派四位高手，" --temperature=0.7 --model_path=model/model_epoch100_jinyong/ --save_samples --save_samples_path=output/
103 | ```
104 | | 參數 | 說明 |
105 | | ------ | ------ |
106 | | generate.py | 生成文字用主程式 |
107 | | length | 生成文字的長度 |
108 | | nsamples | 生成幾個文章範本 |
109 | | prefix | 生成文章的前導文字，會影響生成的發展 |
110 | | temperature | 生成溫度溫度越高，模型產生出來的結果越隨機、越不可預測；換言之，使得原先容易被選到的字，抽出的機會變小，平常較少出現的字，被選到的機會稍微增加 |
111 | | model_path | 生成文字所使用的語言模型資料夾路徑 |
112 | | save_samples | 有設定的話，會儲存生成文章的範本 |
113 | | save_samples_path | 生成文章範本的儲存路徑 |
114 |
115 | 有關溫度（temperature）的參考資料
116 | [Deep learning with Python 學習筆記（10）生成式深度學習](https://www.cnblogs.com/zhhfan/p/10335907.html "Deep learning with Python 學習筆記（10）生成式深度學習")
117 | 
118 |
119 | ## 五、在 Linux、MacOS 的基本使用方式
120 |
121 | ### （一）訓練文章的指令
122 | ```
123 | time python3.7 train.py \
124 | --device=0 \
125 | --epochs=1 \
126 | --num_pieces=100 \
127 | --batch_size=32 \
128 | --min_length=10 \
129 | --raw_data_path=data/jinyong.json \
130 | --output_dir=model/ \
131 | --raw
132 | ```
133 |
134 | ### （二）生成文章的指令
135 | ```
136 | time python3 generate.py \
137 | --length=250 \
138 | --nsamples=3 \
139 | --prefix="張無忌見三名老僧在片刻間連斃崑崙派四位高手，" \
140 | --temperature=0.7 \
141 | --model_path=model/model_epoch100_jinyong/ \
142 | --save_samples \
143 | --save_samples_path=output/
144 | ```
145 |
146 | ## 六、擷圖
147 | 
148 |
149 | 圖 1 訓練過程中，每個 step 輸出 log 的畫面
150 |
151 | 
152 |
153 | 圖 2 每一回合訓練的 model，會各別輸出儲存
154 |
155 | 
156 |
157 | 圖 3 每訓練完 1 個 epoch，會計算與顯示當前回合的訓練時間長(圖片以 epoch 1 為例)
158 |
159 | 
160 |
161 | 圖 4 以訓練第 100 回合的 model 來進行生成測試
162 |
163 | 
164 |
165 | 圖 5 訓練 100 epochs 後，訓練程式結束的畫面
166 |
167 | ## 七、預訓練模型與相關檔案連結
168 | [Google 雲端硬碟](https://drive.google.com/drive/folders/1EmqZsb3Lp_M7ftSiKVgHC6xIiWQVmDBe?usp=sharing "Google 雲端硬碟")
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # python_web_scraping
2 | Python 網路爬蟲講義與範例程式碼
3 |
4 |
5 | ## 提問
6 | - 通則
7 | - 「結業前」可提問、討論，要把多餘時間和資源，留給當前上課的學員。
8 | - 寫信
9 | - E-mail: `darren@darreninfo.cc`
10 | - 信件標題寫上你的**班別和姓名**，或是在哪裡參與我的課程，例如 `[資展 BDSEXX / 臺大計中 / 聯成]` 你的主旨 ○○○。
11 | - 提問的內容要與本專案有關，**其它課程的部分，去請益原本授課的老師**。
12 | - **不要把程式碼寄給我**，可能沒時間看，討論儘量以解決問題的方向為主。
13 | - 不符合以上幾點，將**直接刪除**，敬請見諒。
14 |
15 |
16 | ## 作業
17 | - 僅限授課學員。
18 | - 同學之間可以互相討論，但千萬不要抄襲。
19 | - 使用 `requests` 和 `BeautifulSoup`，或是 `selenium` 來爬取網站資料。
20 | - [Project Gutenberg](https://www.gutenberg.org/)
21 | - 爬取 [中文](https://www.gutenberg.org/browse/languages/zh) 書籍資料 (注意: **只要取得中文字，不要英文字**。)
22 | - 可使用選擇器 `li.pgdbetext > a[href]` 來檢視相關連結數量。
23 | - 取得中文字的正規表示式: [正則表達式-全型英數中文字、常用符號unicode對照表](https://blog.typeart.cc/正則表達式-全型英數中文字、常用符號unicode對照表/)
24 | - `80` 分條件
25 | - 新增 `project_gutenberg` 資料夾，並將每一本書的中文內容存入 txt 檔，txt 的檔名是超連結名稱，例如 `豆棚閒話.txt`。
26 | - 注意：每一個 txt 都會被存在 `project_gutenberg` 資料夾內。
27 | - `錄製`執行過程，並提供`影片連結`，可以放在 `YouTube` 或是 `Google Drive`，影片當中要`隨機打開 3 個 .txt`，驗證內容是否是純中文字 (不要英文字)。
28 | - 至少要有 `200` 本，少 1 本扣 1 分，要在影片中`顯示 .txt 的數量`，例如在檔案總管的某一個角落，有寫著檔案總數。
29 | - 不用給我看程式碼，也不用邊寫邊執行，錄製的時候直接執行程式、直接爬取資料到 `project_gutenberg` 即可。
30 | - 參考影片: [古騰堡計劃（Project Gutenberg）中文電子書爬取](https://www.youtube.com/watch?v=gKDBiVvzMfk)
31 | - `100` 分條件 (基於 `80` 分條件)
32 | - 使用 `GitHub` 平台來提交作業，並且將 `github repo 連結` 以及 `影片連結` 連結寄給我。
33 | - Git 與 GitHub 使用教學: [程式與網頁開發者必備技能！Git 和 GitHub 零基礎快速上手，輕鬆掌握版本控制的要訣！](https://www.youtube.com/watch?v=FKXRiAiQFiY)
34 | - Markdown 語法: [如何使用 Markdown 語言撰寫技術文件](https://experienceleague.adobe.com/zh-hant/docs/contributor/contributor-guide/writing-essentials/markdown)
35 | - `repository` 裡面至少要有 `project_gutenberg` 資料夾，還有你的 `.py` 或 `.ipynb` 檔案，以及 `README.md`。
36 | ```
37 | project_gutenberg/
38 | project_gutenberg.ipynb (或 .py)
39 | README.md
40 | ```
41 | - `README.md` 要有說明 (用 `.py` 執行要額外說明執行指令或方法)，例如:
42 | ```markdown
43 | # Project Gutenberg
44 | 爬取中文書籍，共 xxx 本。
45 |
46 | ## 安裝套件
47 | - requests (版本號)
48 | - beautifulsoup4 (版本號)
49 | - selenium (版本號)
50 | ...
51 | (版本號可用 pip list，或是 conda list 來檢視)
52 | ...
53 |
54 | ## 成果
55 | 
56 | ...
57 | [影片名稱或其它標題](你的影片連結)
58 | ...
59 |
60 | ## 其它你想要補充標題和內容
61 | ...
62 | ...
63 | ```
64 | - 可以參考以前學長的 README 撰寫方式: [FaceBook FanPage Scraper with selenium](https://github.com/nana89823/facebook_scraper)
65 | - 沒交：`0` 分。
66 | - 繳交時間
67 | - 原則上最後一堂課結束後 2 週內，準確時間上課說明。
68 |
69 |
70 | ## 教學參考影片
71 | - [資展國際-OJTP01-18小時-Python網路爬蟲](https://www.youtube.com/playlist?list=PLV4FeK54eNbzgcKtC5s3u7Tv2dZ0BnVsW "資展國際-OJTP01-18小時-Python網路爬蟲")
72 | - [資展國際-BDSE33-Python網路爬蟲 - 12/2, 12/3, 12/9, 12/10](https://www.youtube.com/playlist?list=PLV4FeK54eNbxprT9Sn6FWlcb63u8t0HKt "資展國際-BDSE33-Python網路爬蟲")
73 | - [臺大計算機中心 - Python 網路爬蟲 - 2022/06/20 開班](https://www.youtube.com/playlist?list=PLV4FeK54eNbyZ_rvAAkCICYufOtuQZtTI)
74 | - [資展國際 - 養成班 BDSE22 - Python網路爬蟲](https://www.youtube.com/playlist?list=PLV4FeK54eNbwOKHOH4aWR95fo0cU4wH3O "Python網路爬蟲")
75 | - [臺大計算機中心 - Python 網路爬蟲 - 2021/10/01 開班](https://www.youtube.com/playlist?list=PLV4FeK54eNby0rK-Xpex6baRXE3DG-leg "Python網路爬蟲")
76 | - [資展國際 - 在職班 20210925 至 20210926 - Python 網路爬蟲](https://www.youtube.com/playlist?list=PLV4FeK54eNbwqSdrLfXitmfb4HhB51yOM "Python網路爬蟲")
77 |
78 |
79 | ## 延伸應用
80 | - [Leaflet.js - Web 互動式地圖](https://www.youtube.com/playlist?list=PLV4FeK54eNbwNaCoJomI1jhvgm-A-vOsz)
81 | - [GPT2-Chinese old branch 中文語言模型訓練與生成](https://youtu.be/c3fHRQonqlM)
82 | - [臺大計中電子報 - 第0062期‧2022-09-20 發行 - 使用PyAutoGUI開發桌面自動化程式](https://www.cc.ntu.edu.tw/chinese/epaper/home/20220920_006203.html "臺大計中電子報 - 第0062期‧2022-09-20 發行 - 使用PyAutoGUI開發桌面自動化程式")
83 | - [臺大計中電子報 - 第0059期‧2021.12.20 發行 - 使用GPT2-Chinese生成具有情感的中文對話文字](https://www.cc.ntu.edu.tw/chinese/epaper/0059/20211220_5908.html "臺大計中電子報 - 第0059期‧2021.12.20 發行 - 使用GPT2-Chinese生成具有情感的中文對話文字")
84 | - [臺大計中電子報 - 第0058期‧2021.09.20 發行 - 使用GPT2-Chinese生成中文小說](https://www.cc.ntu.edu.tw/chinese/epaper/0058/20210920_5808.html "臺大計中電子報 - 第0058期‧2021.09.20 發行 - 使用GPT2-Chinese生成中文小說")
--------------------------------------------------------------------------------
/cases/.gitignore:
--------------------------------------------------------------------------------
1 | lawbank*
2 |
--------------------------------------------------------------------------------
/cases/archived/JCC_reject.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 2,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "'''\n",
10 | "司法院大法官 -> 解釋及不受理決議 -> 不受理決議\n",
11 | "\n",
12 | "範例連結: https://cons.judicial.gov.tw/jcc/zh-tw/rrp04?page=1\n",
13 | "'''\n",
14 | "import requests, json\n",
15 | "from bs4 import BeautifulSoup\n",
16 | "\n",
17 | "# 整理所有資料的變數\n",
18 | "listData = []\n",
19 | "\n",
20 | "# 走訪總頁數\n",
21 | "pages = 1\n",
22 | "\n",
23 | "# 取得 table 中的列表連結\n",
24 | "def getListItems():\n",
25 | " for page in range(1, pages + 1):\n",
26 | " response = requests.get(f\"https://cons.judicial.gov.tw/jcc/zh-tw/rrp04?page={page}\")\n",
27 | " soup = BeautifulSoup(response.text, 'lxml')\n",
28 | " a_elms = soup.select(\"table.blocky_body.form_table.form_table_second.sm-responsive tbody tr td[data-head='案號'] a\")\n",
29 | " for a in a_elms:\n",
30 | " listData.append({\n",
31 | " \"title\": a.get_text(),\n",
32 | " \"link\": \"https://cons.judicial.gov.tw\" + a[\"href\"]\n",
33 | " })\n",
34 | "\n",
35 | "# 根據先前儲存的列表連結，爬出需要的資訊\n",
36 | "def getItemDetail():\n",
37 | " for index, _dict in enumerate(listData):\n",
38 | " response = requests.get(_dict['link'])\n",
39 | " soup = BeautifulSoup(response.text, 'lxml')\n",
40 | " pre_elms = soup.select('div.item.title-w-8 pre.content.pure_text')\n",
41 | " listData[index]['公布院令'] = pre_elms[0].get_text()\n",
42 | " listData[index]['會次'] = pre_elms[1].get_text()\n",
43 | " listData[index]['日期'] = pre_elms[2].get_text()\n",
44 | " listData[index]['案號'] = pre_elms[3].get_text()\n",
45 | " listData[index]['聲請人'] = pre_elms[4].get_text()\n",
46 | " listData[index]['案由'] = pre_elms[5].get_text()\n",
47 | " listData[index]['決議'] = pre_elms[6].get_text()\n",
48 | "\n",
49 | "# 將所有資訊轉成 JSON 檔\n",
50 | "def saveJson():\n",
51 | " with open(\"JCC_reject.json\", \"w\", encoding=\"utf-8\") as file:\n",
52 | " file.write( json.dumps(listData, ensure_ascii=False, indent=4) )\n",
53 | "\n",
54 | "# 主程式區段\n",
55 | "if __name__ == \"__main__\":\n",
56 | " getListItems()\n",
57 | " getItemDetail()\n",
58 | " saveJson()\n",
59 | " "
60 | ]
61 | }
62 | ],
63 | "metadata": {
64 | "kernelspec": {
65 | "display_name": "Python 3 (ipykernel)",
66 | "language": "python",
67 | "name": "python3"
68 | },
69 | "language_info": {
70 | "codemirror_mode": {
71 | "name": "ipython",
72 | "version": 3
73 | },
74 | "file_extension": ".py",
75 | "mimetype": "text/x-python",
76 | "name": "python",
77 | "nbconvert_exporter": "python",
78 | "pygments_lexer": "ipython3",
79 | "version": "3.8.8"
80 | }
81 | },
82 | "nbformat": 4,
83 | "nbformat_minor": 4
84 | }
85 |
--------------------------------------------------------------------------------
/cases/archived/crowdfunding.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "id": "d2dc61bd",
7 | "metadata": {},
8 | "outputs": [],
9 | "source": [
10 | "# 匯入套件\n",
11 | "import requests as req\n",
12 | "from bs4 import BeautifulSoup as bs\n",
13 | "import re, json\n",
14 | "from time import sleep\n",
15 | "from random import randint\n",
16 | "\n",
17 | "# 隨機取得 User-Agent\n",
18 | "'''\n",
19 | "# 從外部資料來取得清單，清單預設儲存路徑: /tmp\n",
20 | "ua = UserAgent(use_external_data=True)\n",
21 | "# 從外部資料來取得清單，儲存在指定路徑\n",
22 | "ua = UserAgent(use_external_data=True, cache_path=/home/fake_useragent.json)\n",
23 | "\n",
24 | "更詳細的說明，請見以下網頁:\n",
25 | "https://pypi.org/project/fake-useragent/\n",
26 | "'''\n",
27 | "from fake_useragent import UserAgent\n",
28 | "ua = UserAgent(use_external_data=True)\n",
29 | "\n",
30 | "# 設定請求標頭\n",
31 | "my_headers = {\n",
32 | " 'user-agent': ua.random\n",
33 | "}\n",
34 | "\n",
35 | "# 設定 cookie\n",
36 | "my_cookies = {\n",
37 | " 'age_checked_for': \"12190\"\n",
38 | "}\n",
39 | "\n",
40 | "# 整理所有取得資料的變數\n",
41 | "listData = []\n",
42 | "\n",
43 | "# 欲抓取資料的網址\n",
44 | "domainName = 'https://www.zeczec.com/'\n",
45 | "\n",
46 | "# 指定最後頁數\n",
47 | "pages = 1\n",
48 | "\n",
49 | "# 取得首頁列表資訊\n",
50 | "def getMainData():\n",
51 | " for page in range(1, pages + 1):\n",
52 | " # 取得回應\n",
53 | " res = req.get(url = f'{domainName}categories?page={page}', headers = my_headers)\n",
54 | "\n",
55 | " # 初始化 soup 物件\n",
56 | " soup = bs(res.text, 'lxml')\n",
57 | "\n",
58 | " # 取得所有超連結\n",
59 | " for a in soup.select('div.flex.gutter3-l a.db'):\n",
60 | " # 取得圖片連結\n",
61 | " strStyle = a.select_one('div.aspect-ratio-project-cover')['data-bg']\n",
62 | " regexImg = r\"https:\\/\\/assets\\.zeczec\\.com\\/asset_\\d+_image_big\\.(jpe?g|png)\"\n",
63 | " matchImg = re.match(regexImg, strStyle)\n",
64 | " strImg = matchImg[0]\n",
65 | "\n",
66 | " # 取得超連結\n",
67 | " strLink = domainName + a['href']\n",
68 | "\n",
69 | " # 取得標題文字\n",
70 | " strTitle = a.select_one('h3.b').get_text()\n",
71 | "\n",
72 | " # 整理首頁資料\n",
73 | " listData.append({\n",
74 | " \"cover\": strImg,\n",
75 | " \"link\": strLink,\n",
76 | " \"title\": strTitle\n",
77 | " })\n",
78 | "\n",
79 | "# 取得詳細頁面資訊\n",
80 | "def getDetailData():\n",
81 | " # 走訪每一個頁面\n",
82 | " for index, _dict in enumerate(listData): \n",
83 | " # 輸出網址，以便於 debug\n",
84 | " print(_dict['link'])\n",
85 | " \n",
86 | " # 取得回應\n",
87 | " res = req.get(url = _dict['link'], headers = my_headers, cookies = my_cookies)\n",
88 | "\n",
89 | " # 初始化 soup 物件\n",
90 | " soup = bs(res.text, 'lxml')\n",
91 | " \n",
92 | " # 取得價格\n",
93 | " strPrice = soup.select_one('div.js-sum-raised').get_text()\n",
94 | " regexPrice = r\"\\D\"\n",
95 | " strPrice = re.sub(regexPrice, \"\", strPrice)\n",
96 | " \n",
97 | " # 取得贊助人數\n",
98 | " strBacker = soup.select_one('span.js-backers-count').get_text()\n",
99 | " \n",
100 | " # 取得剩餘時間\n",
101 | " strTime = \"longterm\"\n",
102 | " if soup.select_one('span.js-time-left') != None:\n",
103 | " strTime = soup.select_one('span.js-time-left').get_text()\n",
104 | " regexTime = r\"\\d+\"\n",
105 | " strTime = re.search(regexTime, strTime)[0]\n",
106 | " \n",
107 | " # 取得持續時間\n",
108 | " dictDuration = {\"begin\": \"\", \"end\": \"\"}\n",
109 | " strDuration = soup.select_one('div.mb2.f7').get_text()\n",
110 | " regexDuration = r\"(\\d{4}\\/\\d{2}\\/\\d{2}\\s\\d{2}:\\d{2})(\\s–\\s(\\d{4}\\/\\d{2}\\/\\d{2}\\s\\d{2}:\\d{2}))?\"\n",
111 | " matchDuration = re.search(regexDuration, strDuration)\n",
112 | " dictDuration['begin'] = matchDuration[1]\n",
113 | " if matchDuration[3] != None:\n",
114 | " dictDuration['end'] = matchDuration[3]\n",
115 | " \n",
116 | " # 整理詳細頁面資料\n",
117 | " listData[index]['price'] = strPrice\n",
118 | " listData[index]['backer'] = strBacker\n",
119 | " listData[index]['time'] = strTime\n",
120 | " listData[index]['duration'] = dictDuration\n",
121 | " \n",
122 | " # 隨機等待\n",
123 | " sleep(randint(5,10))\n",
124 | "\n",
125 | "# 儲存成 json 檔案\n",
126 | "def saveJson():\n",
127 | " with open(\"crowdfunding.json\", \"w\", encoding=\"utf-8\") as file:\n",
128 | " file.write( json.dumps(listData, ensure_ascii=False, indent=4) )"
129 | ]
130 | },
131 | {
132 | "cell_type": "code",
133 | "execution_count": null,
134 | "id": "c7ef6fb5",
135 | "metadata": {},
136 | "outputs": [],
137 | "source": [
138 | "# 主程式\n",
139 | "if __name__ == \"__main__\":\n",
140 | " getMainData()\n",
141 | " getDetailData()\n",
142 | " saveJson()"
143 | ]
144 | }
145 | ],
146 | "metadata": {
147 | "kernelspec": {
148 | "display_name": "Python 3.9.13 ('base')",
149 | "language": "python",
150 | "name": "python3"
151 | },
152 | "language_info": {
153 | "codemirror_mode": {
154 | "name": "ipython",
155 | "version": 3
156 | },
157 | "file_extension": ".py",
158 | "mimetype": "text/x-python",
159 | "name": "python",
160 | "nbconvert_exporter": "python",
161 | "pygments_lexer": "ipython3",
162 | "version": "3.9.13"
163 | },
164 | "vscode": {
165 | "interpreter": {
166 | "hash": "585a938ec471c889bf0cce0aed741a99eaf47ca09c0fa8393793bc5bfe77ba11"
167 | }
168 | }
169 | },
170 | "nbformat": 4,
171 | "nbformat_minor": 5
172 | }
173 |
--------------------------------------------------------------------------------
/cases/archived/lm_studio/example01.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/telunyang/python_web_scraping/22e2408b9c7d745e9fd5a6ccf8a2f76c2aa68df1/cases/archived/lm_studio/example01.jpg
--------------------------------------------------------------------------------
/cases/archived/lm_studio/templates/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | 測試 LLM 串流傳輸效果
7 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
改成 換行符號\n",
106 | " strContent = re.sub(r\"\", '\\n', str(res.text))\n",
107 | " \n",
108 | " # 建立 soup 物件\n",
109 | " soup = bs(strContent, \"lxml\")\n",
110 | "\n",
111 | " # 去掉內文標題後，取得內文\n",
112 | " div = soup.select_one('div#html > div')\n",
113 | " strContent = div.get_text()\n",
114 | " \n",
115 | " # 去除不必要的文字\n",
116 | " # strContent = re.sub(r\" |\\r|\\n| |\\s\", '', strContent)\n",
117 | "\n",
118 | " # 決定 txt 的檔案名稱\n",
119 | " fileName = f\"{listResult[i]['title']}_{listResult[i]['sub'][j]['sub_title']}.txt\"\n",
120 | " \n",
121 | " # 將小說內容存到 txt 中\n",
122 | " with open(f\"{folderPath}/{fileName}\", \"w\", encoding=\"utf-8\") as file:\n",
123 | " file.write(strContent)\n",
124 | " \n",
125 | " # 去除不必要的文字\n",
126 | " strContent = re.sub(r\" |\\r|\\n| |\\s\", '', strContent)\n",
127 | "\n",
128 | " # 額外將小說內容放到 list 當中，建立 train.json\n",
129 | " listContent.append(strContent)\n",
130 | "\n",
131 | " # 延伸之後的教學，在此建立訓練資料\n",
132 | " with open(f\"{folderPath}/train.json\", \"w\", encoding=\"utf-8\") as file:\n",
133 | " file.write( json.dumps(listContent, ensure_ascii=False, indent=4) )"
134 | ]
135 | },
136 | {
137 | "cell_type": "code",
138 | "execution_count": null,
139 | "id": "917b963a",
140 | "metadata": {},
141 | "outputs": [],
142 | "source": [
143 | "# 主程式\n",
144 | "if __name__ == \"__main__\":\n",
145 | " time1 = time.time()\n",
146 | " getMainLinks()\n",
147 | " getSubLinks()\n",
148 | " saveJson()\n",
149 | " writeTxt()\n",
150 | " print(f\"執行總花費時間: {time.time() - time1}\")"
151 | ]
152 | }
153 | ],
154 | "metadata": {
155 | "kernelspec": {
156 | "display_name": "Python 3 (ipykernel)",
157 | "language": "python",
158 | "name": "python3"
159 | },
160 | "language_info": {
161 | "codemirror_mode": {
162 | "name": "ipython",
163 | "version": 3
164 | },
165 | "file_extension": ".py",
166 | "mimetype": "text/x-python",
167 | "name": "python",
168 | "nbconvert_exporter": "python",
169 | "pygments_lexer": "ipython3",
170 | "version": "3.10.13"
171 | },
172 | "vscode": {
173 | "interpreter": {
174 | "hash": "585a938ec471c889bf0cce0aed741a99eaf47ca09c0fa8393793bc5bfe77ba11"
175 | }
176 | }
177 | },
178 | "nbformat": 4,
179 | "nbformat_minor": 5
180 | }
181 |
--------------------------------------------------------------------------------
/cases/jinyong_selenium.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "id": "0a06d755",
7 | "metadata": {},
8 | "outputs": [],
9 | "source": [
10 | "'''匯入套件'''\n",
11 | "from selenium import webdriver\n",
12 | "from selenium.webdriver.chrome.service import Service\n",
13 | "from webdriver_manager.chrome import ChromeDriverManager\n",
14 | "from selenium.common.exceptions import TimeoutException\n",
15 | "from selenium.webdriver.support.ui import WebDriverWait\n",
16 | "from selenium.webdriver.support import expected_conditions as EC\n",
17 | "from selenium.webdriver.common.by import By\n",
18 | "import json, os, time, re\n",
19 | "from pprint import pprint\n",
20 | "from urllib import parse\n",
21 | "\n",
22 | "# 隨機取得 User-Agent\n",
23 | "from fake_useragent import UserAgent\n",
24 | "ua = UserAgent()\n",
25 | "\n",
26 | "'''設定 Chrome 瀏覽器開啟時的狀態'''\n",
27 | "my_options = webdriver.ChromeOptions()\n",
28 | "my_options.add_argument('--start-maximized')\n",
29 | "my_options.add_argument('--incognito')\n",
30 | "my_options.add_argument('--disable-popup-blocking')\n",
31 | "my_options.add_argument(f'--user-agent={ua.random}')\n",
32 | "\n",
33 | "'''建立操控 Chrome 瀏覽器的變數'''\n",
34 | "# 使用 Chrome 的 WebDriver\n",
35 | "driver = webdriver.Chrome(\n",
36 | " options = my_options,\n",
37 | " service = Service(ChromeDriverManager().install())\n",
38 | ")\n",
39 | "'''\n",
40 | "補充: 沒有特別設定，只要電腦有安裝 Chrome，就可以直接使用\n",
41 | "driver = webdriver.Chrome()\n",
42 | "'''\n",
43 | "\n",
44 | "'''放置 金庸小說 metadata 的資訊'''\n",
45 | "listData = []\n",
46 | "\n",
47 | "'''小庸小說的網址'''\n",
48 | "url = 'https://www.bookwormzz.com/ky'\n",
49 | "\n",
50 | "# 沒有放置 txt 檔的資料夾，就建立起來\n",
51 | "folderPath = 'jinyong'\n",
52 | "if not os.path.exists(folderPath):\n",
53 | " os.makedirs(folderPath)"
54 | ]
55 | },
56 | {
57 | "cell_type": "code",
58 | "execution_count": null,
59 | "id": "9c5a6200",
60 | "metadata": {},
61 | "outputs": [],
62 | "source": [
63 | "# 取得小說的主要連結\n",
64 | "def getMainLinks():\n",
65 | " # 走訪首頁\n",
66 | " driver.get(url)\n",
67 | " \n",
68 | " # 取得主要連結\n",
69 | " a_elms = driver.find_elements(By.CSS_SELECTOR, 'a[data-ajax=\"false\"]')\n",
70 | " \n",
71 | " # 整理主要連結資訊\n",
72 | " for a in a_elms:\n",
73 | " listData.append({\n",
74 | " \"title\": a.get_attribute('innerText'),\n",
75 | " \"link\": parse.unquote( a.get_attribute('href') ) + \"#book_toc\",\n",
76 | " \"sub\": [] # 為了放置各個章回小說的內頁資料，下一個步驟會用到\n",
77 | " })\n",
78 | "\n",
79 | "# 取得所有章回小說的連結\n",
80 | "def getSubLinks():\n",
81 | " for i in range( len(listData) ):\n",
82 | " # 走訪章回小說內頁\n",
83 | " driver.get(listData[i][\"link\"])\n",
84 | " \n",
85 | " # 若是走訪網頁時，等待不到特定的元素，視為沒有資料，continue 到 for 的下一個 index 去\n",
86 | " try:\n",
87 | " # 等待元素\n",
88 | " WebDriverWait(driver, 5).until(\n",
89 | " EC.presence_of_element_located(\n",
90 | " (By.CSS_SELECTOR, 'div[data-theme=\"b\"][data-content-theme=\"c\"] a[rel=\"external\"]')\n",
91 | " )\n",
92 | " )\n",
93 | " \n",
94 | " # 整理章回小說\n",
95 | " a_elms = driver.find_elements(By.CSS_SELECTOR, 'div[data-theme=\"b\"][data-content-theme=\"c\"] a[rel=\"external\"]')\n",
96 | " for a in a_elms:\n",
97 | " listData[i][\"sub\"].append({\n",
98 | " \"sub_title\": a.get_attribute(\"innerText\"),\n",
99 | " \"sub_link\": parse.unquote( a.get_attribute(\"href\") )\n",
100 | " })\n",
101 | " except TimeoutException as e:\n",
102 | " continue\n",
103 | "\n",
104 | "# 建立金庸小說的 json 檔\n",
105 | "def saveJson():\n",
106 | " with open(f\"{folderPath}/jinyong.json\", \"w\", encoding=\"utf-8\") as file:\n",
107 | " file.write( json.dumps(listData, ensure_ascii=False) )\n",
108 | "\n",
109 | "# 將金庸小說所有章回的內容，各自寫到 txt 與 json 中\n",
110 | "def writeTxt(): \n",
111 | " # 稍候建立 train.json 前的程式變數\n",
112 | " listContent = []\n",
113 | "\n",
114 | " # 開啟 金庸小說 metadata 的 json 檔\n",
115 | " with open(f\"{folderPath}/jinyong.json\", \"r\", encoding=\"utf-8\") as file:\n",
116 | " strJson = file.read()\n",
117 | "\n",
118 | " # 走訪所有章回的小說文字內容\n",
119 | " listResult = json.loads(strJson)\n",
120 | " for i in range( len(listResult) ):\n",
121 | " for j in range( len(listResult[i][\"sub\"]) ):\n",
122 | " # 走訪內頁\n",
123 | " driver.get( listResult[i]['sub'][j]['sub_link'] )\n",
124 | " div = driver.find_element(By.CSS_SELECTOR, 'div#html > div')\n",
125 | " \n",
126 | " # 取得內文\n",
127 | " strContent = div.get_attribute('innerText')\n",
128 | " \n",
129 | " # 資料預處理\n",
130 | " strContent = re.sub(r\" |\\r|\\n| |\\s\", '', strContent)\n",
131 | "\n",
132 | " # 決定 txt 的檔案名稱\n",
133 | " fileName = f\"{listResult[i]['title']}_{listResult[i]['sub'][j]['sub_title']}.txt\"\n",
134 | "\n",
135 | " # 將小說內容存到 txt 中\n",
136 | " with open(f\"{folderPath}/{fileName}\", \"w\", encoding=\"utf-8\") as file:\n",
137 | " file.write( strContent )\n",
138 | "\n",
139 | " # 額外將小說內容放到 list 當中，建立 train.json\n",
140 | " listContent.append(strContent)\n",
141 | "\n",
142 | " # 延伸之後的教學，在此建立訓練資料\n",
143 | " with open(f\"{folderPath}/train.json\", \"w\", encoding=\"utf-8\") as file:\n",
144 | " file.write( json.dumps(listContent, ensure_ascii=False) )\n",
145 | "\n",
146 | "# 關閉瀏覽器\n",
147 | "def close():\n",
148 | " driver.quit()"
149 | ]
150 | },
151 | {
152 | "cell_type": "code",
153 | "execution_count": null,
154 | "id": "d36a7209",
155 | "metadata": {},
156 | "outputs": [],
157 | "source": [
158 | "# 主程式\n",
159 | "if __name__ == \"__main__\":\n",
160 | " time1 = time.time()\n",
161 | " getMainLinks()\n",
162 | " getSubLinks()\n",
163 | " saveJson()\n",
164 | " writeTxt()\n",
165 | " close()\n",
166 | " print(f\"執行總花費時間: {time.time() - time1}\")"
167 | ]
168 | }
169 | ],
170 | "metadata": {
171 | "kernelspec": {
172 | "display_name": "Python 3 (ipykernel)",
173 | "language": "python",
174 | "name": "python3"
175 | },
176 | "language_info": {
177 | "codemirror_mode": {
178 | "name": "ipython",
179 | "version": 3
180 | },
181 | "file_extension": ".py",
182 | "mimetype": "text/x-python",
183 | "name": "python",
184 | "nbconvert_exporter": "python",
185 | "pygments_lexer": "ipython3",
186 | "version": "3.10.13"
187 | }
188 | },
189 | "nbformat": 4,
190 | "nbformat_minor": 5
191 | }
192 |
--------------------------------------------------------------------------------
/cases/leaflet/README.md:
--------------------------------------------------------------------------------
1 | # Leaflet.js - 對手機平台友善的互動式地圖開放源始碼函式庫 (JavaScript Library)
2 | [官方網站](https://leafletjs.com/)
3 |
4 | ## 1. 基礎用法
5 |
6 | ### 1.1 建立一專案資料夾，並在資料夾中建立 html 網頁 (以 VS code 為例)
7 | - 建立資料夾，例如 **gis**，並將 **gis** 資料夾拖入 vs code，或是以 vs code 開啟資料夾視窗。
8 | - 建立 html 檔，例如 **index.html**。
9 | - 開啟 index.html，輸入 **!**，提示文字出現選，選擇「!」，自動建立 html 樣版。
10 | ```html
11 |
12 |
13 |
14 |
15 |
16 |
17 | Document
18 |
19 |
20 |
21 |
22 |
23 | ```
24 |
25 | ### 1.2 按下 Tutorials 連結，選擇 Leaflet Quick Start Guide
26 | - 放置 Leaflet CSS (CDN) 檔案在 index.html 的 `` 之前：
27 | ```html
28 |
29 | ```
30 | - 放置 Leaflet JavaScript (CDN) 檔案在 ` ` 之前：
31 | ```html
32 |