26 | 27 | The Dormouse's story 28 | 29 |
30 |31 | Once upon a time there were three little sisters; and their names were 32 | 33 | Elsie 34 | 35 | , 36 | 37 | Lacie 38 | 39 | and 40 | 41 | Tillie 42 | 43 | ; 44 | and they lived at the bottom of a well. 45 |
46 |47 | ... 48 |
49 | 50 | 51 | ''' -------------------------------------------------------------------------------- /Beautiful Soup 爬虫/002.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 简单的bs4库的使用 3 | ''' 4 | 5 | import bs4 6 | 7 | 8 | #首先我们先将html文件已lxml的方式做成一锅汤 9 | soup = bs4.BeautifulSoup(open('Beautiful Soup 爬虫/demo.html'),'lxml') 10 | 11 | #找到head标签里的内容 12 | #print (soup.head) 13 | 14 | #找到所有的text内容 15 | #print (soup.text) 16 | 17 | #找到第一个a标签 18 | #print (soup.a) 19 | 20 | #找到所有a标签 21 | #print (soup.find_all('a')) 22 | 23 | #找到a标签下的所有子节点，一列表方式返回 24 | #print(soup.a.contents) 25 | 26 | #通过.children生成器，我们可以循环迭代出每一个子节点 27 | #tag = soup.body 28 | #for s in tag.children: 29 | # print(s) 30 | 31 | #通过迭代找到所的string 32 | for i in soup.strings: 33 | print(i) -------------------------------------------------------------------------------- /Beautiful Soup 爬虫/003.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 通过Python来学习re库 --- 正则表达式相关内容 3 | 4 | ''' 5 | 6 | import re 7 | 8 | test = 'python is the best language , pretty good !' 9 | 10 | p = re.findall('p+',test) 11 | 12 | print(p) 13 | 14 | 15 | ''' 16 | re.search(pattern, string, flags=0) 17 | 在一个字符串中搜索匹配正则表达式的第一个位置 18 | 返回match对象 19 | ∙ pattern : 正则表达式的字符串或原生字符串表示 20 | ∙ string : 待匹配字符串 21 | ∙ flags : 正则表达式使用时的控制标记 22 | ''' 23 | 24 | str1 = 'hello , world ,life is short ,use Python .WHAT? ' 25 | 26 | a = re.search(r'\w+',str1) 27 | print(a.group()) # hello 28 | 29 | b = re.search(r'w.+D',str1,re.I) 30 | print(b.group()) 31 | 32 | 33 | ''' 34 | re.findall(pattern, string, flags=0) 35 | 搜索字符串，以列表类型返回全部能匹配的子串 36 | ∙ pattern : 正则表达式的字符串或原生字符串表示 37 | ∙ string : 待匹配字符串 38 | ∙ flags : 正则表达式使用时的控制标记 39 | ''' 40 | 41 | c = re.findall(r'\w+',str1) 42 | print (c) 43 | #['hello', 'world', 'life', 'is', 'short', 'use', 'Python', 'WHAT'] 44 | 45 | str2 = 'hssso' 46 | re1 = re.compile(r'h.{3}o') 47 | print(re1.findall(str1)) 48 | print(re1.findall(str2)) 49 | # ['hello'] 50 | #['hssso'] 51 | 52 | ''' 53 | match 对象的属性 54 | 55 | .string : 待匹配的文本 56 | .re : 匹配时使用的patter对象(正则表达式) 57 | .pos : 正则表达式搜索文本的开始位置 58 | .endpos : 正则表达式搜索文本的结束位置 59 | ''' 60 | d = re.search(r'e.+d',str1) 61 | print(d.group()) # ello , world 62 | print (d.string) # hello , world ,life is short ,use Python .WHAT? 63 | print (d.re) # re.compile('e.+d') 64 | print (d.pos) # 0 65 | print (d.endpos) # 48 66 | -------------------------------------------------------------------------------- /Beautiful Soup 爬虫/baidutieba.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 抓取百度贴吧---生活大爆炸吧的基本内容 3 | 爬虫线路： requests - bs4 4 | Python版本： 3.6 5 | OS： mac os 12.12.4 6 | ''' 7 | 8 | import requests 9 | import time 10 | from bs4 import BeautifulSoup 11 | 12 | # 首先我们写好抓取网页的函数 13 | 14 | 15 | def get_html(url): 16 | try: 17 | r = requests.get(url, timeout=30) 18 | r.raise_for_status() 19 | # 这里我们知道百度贴吧的编码是utf-8，所以手动设置的。爬去其他的页面时建议使用： 20 | # r.encoding = r.apparent_encoding 21 | r.encoding = 'utf-8' 22 | return r.text 23 | except: 24 | return " ERROR " 25 | 26 | 27 | def get_content(url): 28 | ''' 29 | 分析贴吧的网页文件，整理信息，保存在列表变量中 30 | ''' 31 | 32 | # 初始化一个列表来保存所有的帖子信息： 33 | comments = [] 34 | # 首先，我们把需要爬取信息的网页下载到本地 35 | html = get_html(url) 36 | 37 | # 我们来做一锅汤 38 | soup = BeautifulSoup(html, 'lxml') 39 | 40 | # 按照之前的分析，我们找到所有具有‘ j_thread_list clearfix’属性的li标签。返回一个列表类型。 41 | liTags = soup.find_all('li', attrs={'class': ' j_thread_list clearfix'}) 42 | 43 | # 通过循环找到每个帖子里的我们需要的信息： 44 | for li in liTags: 45 | # 初始化一个字典来存储文章信息 46 | comment = {} 47 | # 这里使用一个try except 防止爬虫找不到信息从而停止运行 48 | try: 49 | # 开始筛选信息，并保存到字典中 50 | comment['title'] = li.find( 51 | 'a', attrs={'class': 'j_th_tit '}).text.strip() 52 | comment['last_reply_data'] = li.find('span',attrs={'class':'threadlist_reply_date pull_right j_reply_data'}).text.strip() 53 | comments.append(comment) 54 | except: 55 | print('出了点小问题') 56 | 57 | return comments 58 | 59 | 60 | def Out2File(dict): 61 | ''' 62 | 将爬取到的文件写入到本地 63 | 保存到当前目录的 TTBT.txt文件中。 64 | 65 | ''' 66 | with open('TTBT.txt', 'a+') as f: 67 | for comment in dict: 68 | f.write('标题： {} \t 链接：{} \t 发帖人：{} \t 发帖时间：{} \t 回复数量： {} \n'.format( 69 | comment['title'], comment['last_reply_data'])) 70 | 71 | print('当前页面爬取完成') 72 | 73 | 74 | def main(base_url, deep): 75 | url_list = [] 76 | # 将所有需要爬去的url存入列表 77 | for i in range(0, deep): 78 | url_list.append(base_url + '&pn=' + str(50 * i)) 79 | print('所有的网页已经下载到本地！ 开始筛选信息。。。。') 80 | 81 | # 循环写入所有的数据 82 | for url in url_list: 83 | content = get_content(url) 84 | Out2File(content) 85 | print('所有的信息都已经保存完毕！') 86 | 87 | 88 | base_url = 'http://tieba.baidu.com/f?kw=%E7%94%9F%E6%B4%BB%E5%A4%A7%E7%88%86%E7%82%B8&ie=utf-8' 89 | # 设置需要爬取的页码数量 90 | deep = 1 91 | 92 | if __name__ == '__main__': 93 | main(base_url, deep) 94 | -------------------------------------------------------------------------------- /Beautiful Soup 爬虫/bocai.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 爬取Dota菠菜结果信息 3 | 使用 requests --- bs4 线路 4 | Python版本： 3.6 5 | OS： mac os 12.12.4 6 | ''' 7 | 8 | import requests 9 | import bs4 10 | 11 | def get_html(url): 12 | try: 13 | r = requests.get(url, timeout=30) 14 | r.raise_for_status() 15 | r.encoding = r.apparent_encoding 16 | return r.text 17 | except: 18 | return " ERROR " 19 | 20 | def print_result(url): 21 | ''' 22 | 查询比赛结果，并格式化输出！ 23 | ''' 24 | html = get_html(url) 25 | soup = bs4.BeautifulSoup(html,'lxml') 26 | match_list = soup.find_all('div', attrs={'class': 'matchmain bisai_qukuai'}) 27 | for match in match_list: 28 | time = match.find('div', attrs={'class': 'whenm'}).text.strip() 29 | teamname = match.find_all('span', attrs={'class': 'team_name'}) 30 | 31 | 32 | #由于网站的构造问题，队名有的时候会不显示，所以我们需要过滤掉一些注释,方法如下: 33 | if teamname[0].string[0:3] == 'php': 34 | team1_name = "暂无队名" 35 | else: 36 | team1_name = teamname[0].string 37 | 38 | # 这里我们采用了css选择器：比原来的属性选择更加方便 39 | team1_support_level = match.find('span', class_='team_number_green').string 40 | 41 | team2_name = teamname[1].string 42 | team2_support_level = match.find('span', class_='team_number_red').string 43 | 44 | print('比赛时间：{}，\n 队伍一：{} 胜率 {}\n 队伍二：{} 胜率 {} \n'.format(time,team1_name,team1_support_level,team2_name,team2_support_level)) 45 | 46 | 47 | 48 | def main(): 49 | url= 'http://dota2bocai.com/match' 50 | print_result(url) 51 | 52 | if __name__ == '__main__': 53 | main() -------------------------------------------------------------------------------- /Beautiful Soup 爬虫/demo.html: -------------------------------------------------------------------------------- 1 | 2 |The Dormouse's story
5 | 6 |Once upon a time there were three little sisters; and their names were 7 | Elsie, 8 | Lacie and 9 | Tillie; 10 | and they lived at the bottom of a well.
11 | 12 |...
13 | -------------------------------------------------------------------------------- /Beautiful Soup 爬虫/dianying.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 爬取最新电影排行榜单 3 | url：http://dianying.2345.com/top/ 4 | 使用 requests --- bs4 线路 5 | Python版本： 3.6 6 | OS： mac os 12.12.4 7 | ''' 8 | 9 | import requests 10 | import bs4 11 | 12 | 13 | def get_html(url): 14 | try: 15 | r = requests.get(url, timeout=30) 16 | r.raise_for_status 17 | # 该网站采用gbk编码！ 18 | r.encoding = 'gbk' 19 | return r.text 20 | except: 21 | return "someting wrong" 22 | 23 | 24 | def get_content(url): 25 | html = get_html(url) 26 | soup = bs4.BeautifulSoup(html, 'lxml') 27 | 28 | # 找到电影排行榜的ul列表 29 | movies_list = soup.find('ul', class_='picList clearfix') 30 | movies = movies_list.find_all('li') 31 | 32 | for top in movies: 33 | #找到图片连接， 34 | img_url=top.find('img')['src'] 35 | 36 | 37 | name = top.find('span',class_='sTit').a.text 38 | #这里做一个异常捕获，防止没有上映时间的出现 39 | try: 40 | time = top.find('span',class_='sIntro').text 41 | except: 42 | time = "暂无上映时间" 43 | 44 | #这里用bs4库迭代找出“pACtor”的所有子孙节点，即每一位演员解决了名字分割的问题 45 | actors = top.find('p',class_='pActor') 46 | actor= '' 47 | for act in actors.contents: 48 | actor = actor + act.string +' ' 49 | #找到影片简介 50 | intro = top.find('p',class_='pTxt pIntroShow').text 51 | 52 | print("片名：{}\t{}\n{}\n{} \n \n ".format(name,time,actor,intro) ) 53 | 54 | #我们来吧图片下载下来： 55 | with open('/Users/ehco/Desktop/img/'+name+'.png','wb+') as f: 56 | f.write(requests.get(img_url).content) 57 | 58 | 59 | def main(): 60 | url = 'http://dianying.2345.com/top/' 61 | get_content(url) 62 | 63 | if __name__ == "__main__": 64 | main() -------------------------------------------------------------------------------- /Beautiful Soup 爬虫/kouhong.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Author Ehco1996 3 | Time 2017-11-10 4 | 5 | 如何暗示男朋友给自己买火红 6 | ''' 7 | 8 | from bs4 import BeautifulSoup 9 | import requests 10 | import os 11 | 12 | 13 | def get_html_text(url): 14 | try: 15 | r = requests.get(url, timeout=10) 16 | r.raise_for_status 17 | return r.text 18 | except: 19 | return -1 20 | 21 | 22 | def parse_img(html): 23 | data = [] 24 | soup = BeautifulSoup(html, 'lxml') 25 | img_list = soup.find_all('img') 26 | for img in img_list: 27 | data.append({ 28 | 'src': img['src'], 29 | 'name': img['alt'].replace(' ', '').replace('/', '') 30 | }) 31 | return data 32 | 33 | 34 | def get_img_response(url): 35 | try: 36 | r = requests.get(url, stream=True) 37 | r.raise_for_status 38 | return r.content 39 | except: 40 | return -1 41 | 42 | 43 | def download_img(data): 44 | curr_dir = os.path.dirname(os.path.realpath(__file__)) + '/img/' 45 | if not os.path.exists('img'): 46 | os.mkdir('img') 47 | for img in data: 48 | path = os.path.join(curr_dir, img['name'] + '.jpg') 49 | with open(path, 'wb') as f: 50 | f.write(get_img_response(img['src'])) 51 | 52 | 53 | def main(): 54 | html = get_html_text( 55 | 'https://www.1688.com/pic/-.html?spm=a261b.8768355.searchbar.5.oUjRZK&keywords=%BF%DA%BA%EC') 56 | if html != -1: 57 | img_data = parse_img(html) 58 | download_img(img_data) 59 | 60 | 61 | if __name__ == '__main__': 62 | main() 63 | -------------------------------------------------------------------------------- /Beautiful Soup 爬虫/parse_station.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import re 3 | 4 | #关闭https证书验证警告 5 | requests.packages.urllib3.disable_warnings() 6 | 7 | url = 'https://kyfw.12306.cn/otn/resources/js/framework/station_name.js?station_version=1.9018' 8 | r = requests.get(url,verify=False) 9 | pattern = u'([\u4e00-\u9fa5]+)\|([A-Z]+)' 10 | result = re.findall(pattern,r.text) 11 | station = dict(result) 12 | 13 | print(station) -------------------------------------------------------------------------------- /Beautiful Soup 爬虫/qiubai.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | 4 | 5 | def get_html_text(url): 6 | try: 7 | r = requests.get(url, timeout=3) 8 | r.raise_for_status() 9 | r.encoding = r.apparent_encoding 10 | return r.text 11 | except: 12 | return 'something wrong' 13 | 14 | 15 | def get_jokes(url): 16 | ''' 17 | 返回当前url页面的糗百的 18 | 段子作者，主体，热评 19 | 返回类型：列表 20 | ''' 21 | joke_list = [] 22 | 23 | html = get_html_text(url) 24 | soup = BeautifulSoup(html, 'lxml') 25 | 26 | articles = soup.find_all('div', class_='article block untagged mb15') 27 | 28 | for article in articles: 29 | body = article.find('span').text 30 | author = article.find('img')['alt'] 31 | try: 32 | comment = article.find( 33 | 'div', class_='main-text').contents[0].replace('\n', '') 34 | except: 35 | comment = '暂时没有热评' 36 | 37 | joke = '作者：{}\n{}\n\n热评{}'.format(author, body, comment) 38 | joke_list.append(joke) 39 | 40 | return joke_list 41 | 42 | 43 | # test: 44 | 45 | url = 'https://www.qiushibaike.com/8hr' 46 | 47 | a = get_jokes(url) 48 | print(a[1]) 49 | -------------------------------------------------------------------------------- /Beautiful Soup 爬虫/反爬虫.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import random 3 | 4 | 5 | def get_html(url): 6 | try: 7 | r = requests.get(url, timeout=30) 8 | print(r.headers) 9 | r.raise_for_status 10 | r.encoding = r.apparent_encoding 11 | 12 | return r.status_code 13 | except: 14 | return "Someting Wrong！" 15 | 16 | 17 | def get_agent(): 18 | ''' 19 | 模拟header的user-agent字段， 20 | 返回一个随机的user-agent字典类型的键值对 21 | ''' 22 | agents = ['Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;', 23 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv,2.0.1) Gecko/20100101 Firefox/4.0.1', 24 | 'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11', 25 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11', 26 | 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)'] 27 | fakeheader = {} 28 | fakeheader['User-agent'] = agents[random.randint(0, len(agents))] 29 | return fakeheader 30 | 31 | 32 | def get_proxy(): 33 | ''' 34 | 简答模拟代理池 35 | 返回一个字典类型的键值对， 36 | ''' 37 | proxy = ["http://116.211.143.11:80", 38 | "http://183.1.86.235:8118", 39 | "http://183.32.88.244:808", 40 | "http://121.40.42.35:9999", 41 | "http://222.94.148.210:808"] 42 | fakepxs = {} 43 | fakepxs['http'] = proxy[random.randint(0, len(proxy))] 44 | return fakepxs 45 | 46 | 47 | print(get_html('https://zhuanlan.zhihu.com')) -------------------------------------------------------------------------------- /Beautiful Soup 爬虫/沪铜数据.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 获取新浪网 沪铜CUO历史交易记录 3 | 从1999-01-01 到 2017-06-15 4 | 网址：http://vip.stock.finance.sina.com.cn/q/view/vFutures_History.php?page=1&breed=CU0&start=1999-01-01&end=2017-06-15&jys=shfe&pz=CU&hy=CU0&type=inner&name=%A1%E4%A8%AE%26%23182%3B11109 5 | ''' 6 | 7 | import requests 8 | from bs4 import BeautifulSoup 9 | 10 | def get_html_text(url): 11 | try: 12 | r = requests.get(url,timeout=3) 13 | r.raise_for_status() 14 | r.encoding = r.apparent_encoding 15 | return r.text 16 | except: 17 | return 'something wrong' 18 | 19 | 20 | 21 | def get_one_data(url): 22 | data = [] 23 | html = get_html_text(url) 24 | soup = BeautifulSoup(html,'lxml') 25 | days = soup.find('div',class_='historyList') 26 | columns = days.find_all('tr') 27 | 28 | ''' 29 | test area: 30 | info = columns[2].find_all('td') 31 | date = info[0].text 32 | close_price = info[1].text 33 | print(date,close_price) 34 | ''' 35 | 36 | for i in range(2,len(columns)): 37 | try: 38 | info = columns[i].find_all('td') 39 | date = info[0].text 40 | close_price = info[1].text 41 | data.append(date+' : '+close_price) 42 | except: 43 | continue 44 | 45 | return data 46 | 47 | def W2File(data): 48 | with open('cuo_data.txt','a+') as f: 49 | for one in data: 50 | f.write(one+'\n') 51 | 52 | print('数据写入完毕！') 53 | 54 | #url = 'http://vip.stock.finance.sina.com.cn/q/view/vFutures_History.php?page=1&breed=CU0&start=1999-01-01&end=2017-06-15&jys=shfe&pz=CU&hy=CU0&type=inner&name=%A1%E4%A8%AE%26%23182%3B11109' 55 | urls = [] 56 | 57 | for i in range(1,77): 58 | urls.append('http://vip.stock.finance.sina.com.cn/q/view/vFutures_History.php?page='+str(i)+'&breed=CU0&start=1999-01-01&end=2017-06-15&jys=shfe&pz=CU&hy=CU0&type=inner&name=%A1%E4%A8%AE%26%23182%3B11109') 59 | 60 | 61 | for url in urls: 62 | data = get_one_data(url) 63 | W2File(data) -------------------------------------------------------------------------------- /Google-Image/spider.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Google Image Spider 3 | 4 | ''' 5 | 6 | import requests 7 | from bs4 import BeautifulSoup 8 | 9 | 10 | SEARCHRUL = 'https://www.google.com/search?&safe=off&q={}&tbm=isch&tbs=itp:photo,isz:l' 11 | 12 | 13 | def get_html_text(url): 14 | '''获取网页的原始text''' 15 | headers = {} 16 | headers['User-Agent'] = "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17" 17 | try: 18 | r = requests.get(url, timeout=9, headers=headers) 19 | r.raise_for_status 20 | r.encoding = r.apparent_encoding 21 | return r.text 22 | except: 23 | return 'error' 24 | 25 | 26 | def parse_img_url(q, nums): 27 | ''' 28 | 解析返回搜索图片的原始链接 29 | q ： 搜索关键词 30 | nums： 返回的结果数量 最大值20 31 | ''' 32 | links = [] 33 | # 防止越界 34 | if nums > 20 or nums > 0: 35 | num = 20 36 | 37 | url = SEARCHRUL.format(q) 38 | print(url) 39 | html = get_html_text(url) 40 | if html != 'error': 41 | soup = BeautifulSoup(html, 'lxml') 42 | content = soup.find_all('div', class_='rg_meta', limit=nums) 43 | for link in content: 44 | rec = eval(link.text) 45 | links.append(rec['ou']) 46 | return links 47 | else: 48 | return 'error' 49 | 50 | 51 | res = parse_img_url('test', 15) 52 | 53 | for url in res: 54 | print(url) 55 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Python-crawler 2 | 3 | > 由于代码是比较早之前写的，抓取的网站目录结构可能有所变动 4 | > 所以部分代码可能不能使用了，欢迎正在学习爬虫的大家给这个项目**提PR** 5 | > 让更多的代码能跑起来~ 6 | 7 | 从零开始系统化的学习写Python爬虫。 8 | 主要是记录一下自己写Python爬虫的经过与心得。 9 | 同时也是为了分享一下如何能更高效率的学习写爬虫。 10 | IDE：Vscode Python版本: 3.6 11 | 12 | * 知乎专栏：https://zhuanlan.zhihu.com/Ehco-python 13 | 14 | 详细学习路径： 15 | ### 一：Beautiful Soup 爬虫 16 | 17 | * requests库的安装与使用 https://zhuanlan.zhihu.com/p/26681429 18 | * 安装beautiful soup 爬虫环境 https://zhuanlan.zhihu.com/p/26683864 19 | * beautiful soup 的解析器 https://zhuanlan.zhihu.com/p/26691931 20 | * re库 正则表达式的使用 https://zhuanlan.zhihu.com/p/26701898 21 | * bs4 爬虫实践： 获取百度贴吧的内容 https://zhuanlan.zhihu.com/p/26722495 22 | * bs4 爬虫实践： 获取双色球中奖信息 https://zhuanlan.zhihu.com/p/26747717 23 | * bs4 爬虫实践： 排行榜小说批量下载 https://zhuanlan.zhihu.com/p/26756909 24 | * bs4 爬虫实践： 获取电影信息 https://zhuanlan.zhihu.com/p/26786056 25 | * bs4 爬虫实践： 悦音台mv排行榜与反爬虫技术 https://zhuanlan.zhihu.com/p/26809626 26 | ------ 27 | ### 二： Scrapy 爬虫框架 28 | 29 | * Scrapy 爬虫框架的安装与基本介绍 https://zhuanlan.zhihu.com/p/26832971 30 | * Scrapy 选择器和基本使用 https://zhuanlan.zhihu.com/p/26854842 31 | * Scrapy 爬虫实践：天气预报&数据存储 https://zhuanlan.zhihu.com/p/26885412 32 | * Scrapy 爬虫实践：代理的爬取和验证 https://zhuanlan.zhihu.com/p/26939527 33 | * Scrapy 爬虫实践：糗事百科&爬虫攻防 https://zhuanlan.zhihu.com/p/26980300 34 | * Scrapy 爬虫实践：重构排行榜小说爬虫&Mysql数据库 https://zhuanlan.zhihu.com/p/27027200 35 | ------ 36 | ### 三： 浏览器模拟爬虫 37 | 38 | * Selenium模拟浏览器 https://zhuanlan.zhihu.com/p/27115580 39 | * 爬虫实践：获取快代理 https://zhuanlan.zhihu.com/p/27150025 40 | * 爬虫实践：漫画批量下载 https://zhuanlan.zhihu.com/p/27155429 41 | 42 | ### 四： 练手项目 43 | 44 | * 爬虫实践：螺纹钢数据&Cookies https://zhuanlan.zhihu.com/p/27232687 45 | * 爬虫实践：登录正方教务系统 https://zhuanlan.zhihu.com/p/27256315 46 | * 爬虫应用： requests+django实现微信公众号后台 https://zhuanlan.zhihu.com/p/27625233 47 | * 爬虫应用： 12306火车票信息查询 https://zhuanlan.zhihu.com/p/27969976 48 | * 爬虫应用： 利用斗鱼Api抓取弹幕 https://zhuanlan.zhihu.com/p/28164017 49 | * 爬虫应用： 获取支付宝账单信息 https://zhuanlan.zhihu.com/p/28537306 50 | * 爬虫应用：IT之家热门段子（评论）爬取 https://zhuanlan.zhihu.com/p/28806210 51 | * 爬虫应用：一号店 商品信息查询程序 https://zhuanlan.zhihu.com/p/28982497 52 | * 爬虫应用：搜狗输入法词库抓取 https://zhuanlan.zhihu.com/p/31186373 53 | * 爬虫应用：复古网盘游戏抓取 https://zhuanlan.zhihu.com/p/32420131 54 | * 爬虫应用：自动填写问卷星 https://zhuanlan.zhihu.com/p/36224375 55 | * 爬虫应用：腾讯漫画下载~ https://zhuanlan.zhihu.com/p/39578774 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /Scrapy 爬虫框架/001.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 简单介绍scrapycXPATH选择器的使用 3 | ''' 4 | from scrapy.selector import Selector 5 | 6 | xml = ''' 7 | 8 | 9 |