',str(get.content,'gbk',errors='ignore'))
12 | for url in article_url:
13 | tittle=['default']
14 | getpage=requests.get(str(base_url)+str(url))
15 | tittle=re.findall(r'
(.*)
',str(getpage.content,'gbk',errors='ignore'))
16 | file=tittle[0]
17 | if os.path.exists(file)==False:
18 | os.makedirs(file)
19 | tree=etree.HTML(str(getpage.content,'gbk',errors='ignore'))
20 | img_url=tree.xpath('//input/@data-src')
21 | filename=1
22 | for download_url in img_url:
23 | headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36Name','Referer':'https://t66y.com'}
24 | req=requests.get(url=download_url,headers=headers)
25 | file_path='./'+file+"/"+str(filename)+'.jpg'
26 | with open(file_path,'wb') as f:
27 | print('开始下载：'+file+"----第"+str(filename)+"张图片")
28 | f.write(req.content)
29 | filename+=1
30 | print(file+"--下载完成")
31 | except:
32 | print("崩了，兄弟")
33 |
34 |
35 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2019 Liu Y
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # 单线程爬虫，下载的图片文件全部保存在了项目下，下载过的会自动跳
2 |
3 | 请在python 3下运行
4 |
5 | 需要的库：
6 | ```bash
7 | requests
8 | lxml
9 | ```
10 |
11 | 运行：
12 | ```bash
13 | python 1024.py
14 | ```
15 |
16 | 停止:
17 | ```bash
18 | ctrl+c
19 | ```
20 | 如果网络不好，小概率会崩掉，重新运行就OK
21 |
22 |
23 | 
24 |
25 | # 默认爬的是达盖尔的旗帜，可以自己修改想要爬的分区
26 |
27 | ```bash
28 | 把page_url里的pid=16修改成pid=8就可以爬取新时代区了
29 | ```
30 |
31 |
32 |
--------------------------------------------------------------------------------
/demo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/6yi/1024-web-crawler/83f939a53a921c49fbbacb5c5b8000c88b1da858/demo.png
--------------------------------------------------------------------------------