├── .gitignore
├── LICENSE
├── README.md
└── ygdy8Movie.py


/.gitignore:
--------------------------------------------------------------------------------
 1 | # Byte-compiled / optimized / DLL files
 2 | __pycache__/
 3 | *.py[cod]
 4 | *$py.class
 5 | 
 6 | # C extensions
 7 | *.so
 8 | 
 9 | # Distribution / packaging
10 | .Python
11 | env/
12 | build/
13 | develop-eggs/
14 | dist/
15 | downloads/
16 | eggs/
17 | .eggs/
18 | lib/
19 | lib64/
20 | parts/
21 | sdist/
22 | var/
23 | *.egg-info/
24 | .installed.cfg
25 | *.egg
26 | 
27 | # PyInstaller
28 | #  Usually these files are written by a python script from a template
29 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
30 | *.manifest
31 | *.spec
32 | 
33 | # Installer logs
34 | pip-log.txt
35 | pip-delete-this-directory.txt
36 | 
37 | # Unit test / coverage reports
38 | htmlcov/
39 | .tox/
40 | .coverage
41 | .coverage.*
42 | .cache
43 | nosetests.xml
44 | coverage.xml
45 | *,cover
46 | .hypothesis/
47 | 
48 | # Translations
49 | *.mo
50 | *.pot
51 | 
52 | # Django stuff:
53 | *.log
54 | 
55 | # Sphinx documentation
56 | docs/_build/
57 | 
58 | # PyBuilder
59 | target/
60 | 
61 | #Ipython Notebook
62 | .ipynb_checkpoints
63 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2016 LijunSong
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Python抓取电影天堂电影信息
  2 |  
  3 | 最近想做一个电影资讯的App，由于豆瓣不对外开发API了，正愁怎么录入数据呢。最近在学`Python` 想想那就用它来抓取电影信息吧。
  4 | 
  5 | <font color="red">此项目仅供学习</font>
  6 | 
  7 | > Python2.7
  8 | > 
  9 | > Mac OS
 10 | 
 11 | 抓取的是电影天堂里面最新电影的页面。链接地址：`http://www.dytt8.net/html/gndy/dyzz/index.html`
 12 | 
 13 | 
 14 | #### 获取页面的中电影详情页链接
 15 | 
 16 | ```python
 17 | import urllib2
 18 | import os
 19 | import re
 20 | import string
 21 | 
 22 | 
 23 | # 电影URL集合
 24 | movieUrls = []
 25 | 
 26 | 
 27 | # 获取电影列表
 28 | def queryMovieList():
 29 | 
 30 | 	url = 'http://www.dytt8.net/html/gndy/dyzz/index.html'   
 31 | 	conent = urllib2.urlopen(url)
 32 | 	conent =  conent.read()
 33 | 	conent = conent.decode('gb2312','ignore').encode('utf-8','ignore')  
 34 | 	pattern = re.compile ('<div class="title_all"><h1><font color=#008800>.*?</a>></font></h1></div>'+
 35 | 	                      '(.*?)<td height="25" align="center" bgcolor="#F4FAE2"> ',re.S)
 36 | 	items = re.findall(pattern,conent) 
 37 | 	
 38 | 	str = ''.join(items)
 39 | 	pattern = re.compile ('<a href="(.*?)" class="ulink">(.*?)</a>.*?<td colspan.*?>(.*?)</td>',re.S)
 40 | 	news = re.findall(pattern, str)
 41 | 
 42 | 	for  j in news:
 43 | 	   
 44 |         	movieUrls.append('http://www.dytt8.net'+j[0])
 45 | 
 46 | ```
 47 | 
 48 | #### 抓取详情页中的电影数据
 49 | 
 50 | ```python
 51 | def queryMovieInfo(movieUrls):
 52 | 
 53 | 	for index, item in enumerate(movieUrls):
 54 | 
 55 | 		print('电影URL: ' + item)
 56 | 
 57 | 		conent = urllib2.urlopen(item)
 58 | 		conent = conent.read()
 59 | 		conent = conent.decode('gb2312','ignore').encode('utf-8','ignore')  
 60 | 
 61 | 
 62 | 		movieName = re.findall(r'<div class="title_all"><h1><font color=#07519a>(.*?)</font></h1></div>', conent, re.S)
 63 | 		if (len(movieName) > 0):
 64 | 			movieName = movieName[0] + ""
 65 | 			# 截取名称
 66 | 			movieName = movieName[movieName.find("《") + 3:movieName.find("》")]
 67 | 		else:
 68 | 			movieName = ""
 69 | 
 70 | 		print("电影名称: " + movieName.strip())
 71 | 
 72 | 		movieContent = re.findall(r'<div class="co_content8">(.*?)</tbody>',conent , re.S)
 73 | 
 74 | 
 75 | 		pattern = re.compile('<ul>(.*?)<tr>', re.S)
 76 | 		movieDate = re.findall(pattern,movieContent[0])
 77 | 
 78 | 		if (len(movieDate) > 0):
 79 | 			movieDate = movieDate[0].strip() + ''
 80 | 		else:
 81 | 			movieDate = ""
 82 | 
 83 | 		print("电影发布时间: " + movieDate[-10:])
 84 | 
 85 | 		pattern = re.compile('<br /><br />(.*?)<br /><br /><img')
 86 | 		movieInfo = re.findall(pattern, movieContent[0])
 87 | 
 88 | 		if (len(movieInfo) > 0):
 89 | 			movieInfo = movieInfo[0]+''
 90 | 
 91 | 			# 删除<br />标签
 92 | 			movieInfo = movieInfo.replace("<br />","")
 93 | 
 94 | 			# 根据 ◎ 符号拆分
 95 | 
 96 | 			movieInfo = movieInfo.split('◎')
 97 | 
 98 | 		else:
 99 | 			movieInfo = ""
100 | 
101 | 		print("电影基础信息: ")
102 | 
103 | 		for item in movieInfo:
104 | 			print(item)
105 | 
106 | 
107 | 		# 电影海报
108 | 		pattern = re.compile('<img.*? src="(.*?)".*? />', re.S)		
109 | 		movieImg = re.findall(pattern,movieContent[0])
110 | 
111 | 		if (len(movieImg) > 0):
112 | 			movieImg = movieImg[0]
113 | 		else:
114 | 			movieImg = ""
115 |  
116 | 		print("电影海报: " + movieImg)
117 | 
118 | 		pattern = re.compile('<td style="WORD-WRAP: break-word" bgcolor="#fdfddf"><a href="(.*?)">.*?</a></td>', re.S)
119 | 		movieDownUrl = re.findall(pattern,movieContent[0])
120 | 
121 | 		if (len(movieDownUrl) > 0):
122 | 			movieDownUrl = movieDownUrl[0]
123 | 		else:
124 | 			movieDownUrl = ""
125 | 
126 | 		print("电影下载地址：" + movieDownUrl + "")
127 | 
128 | 		print("------------------------------------------------\n\n\n")
129 | 	
130 | ```
131 | 
132 | #### 执行抓取
133 | 
134 | ```python
135 | 
136 | if __name__=='__main__':
137 | 
138 | 
139 |      print("开始抓取电影数据");
140 | 	 
141 |      queryMovieList()
142 | 
143 |      print(len(movieUrls))
144 | 
145 | 
146 |      queryMovieInfo(movieUrls)
147 | 
148 |      print("结束抓取电影数据")
149 | 
150 | ```
151 | 
152 | #### 总结
153 | 
154 | 学好正则表达式很重要,很重要,很重要!!!! Python的语法好有感觉, 对比Java ...
155 | 
156 | 
157 | 


--------------------------------------------------------------------------------
/ygdy8Movie.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | #coding:utf-8
  3 | 
  4 | """
  5 | 	抓取电影天堂的电影数据，以及下载地址
  6 | 	Date: 2016-04-05
  7 | 	Language: Python2.7.11
  8 | 	Anthor: Lijun Song
  9 | """
 10 | 
 11 | import urllib2
 12 | import os
 13 | import re
 14 | import string
 15 | import mysql 
 16 | 
 17 | 
 18 | # 电影URL集合
 19 | movieUrls = []
 20 | 
 21 | 
 22 | # 获取电影列表
 23 | def queryMovieList():
 24 | 
 25 | 	url = 'http://www.dytt8.net/html/gndy/dyzz/index.html'   
 26 | 	conent = urllib2.urlopen(url)
 27 | 	conent =  conent.read()
 28 | 	conent = conent.decode('gb2312','ignore').encode('utf-8','ignore')  
 29 | 	pattern = re.compile ('<div class="title_all"><h1><font color=#008800>.*?</a>></font></h1></div>'+
 30 | 	                      '(.*?)<td height="25" align="center" bgcolor="#F4FAE2"> ',re.S)
 31 | 	items = re.findall(pattern,conent) 
 32 | 	
 33 | 	str = ''.join(items)
 34 | 	pattern = re.compile ('<a href="(.*?)" class="ulink">(.*?)</a>.*?<td colspan.*?>(.*?)</td>',re.S)
 35 | 	news = re.findall(pattern, str)
 36 | 
 37 | 
 38 | 	for  j in news:
 39 |         	movieUrls.append('http://www.dytt8.net'+j[0])
 40 | 
 41 | 
 42 | def queryMovieInfo(movieUrls):
 43 | 
 44 | 	for index, item in enumerate(movieUrls):
 45 | 
 46 | 		print('电影URL: ' + item)
 47 | 
 48 | 		conent = urllib2.urlopen(item)
 49 | 		conent = conent.read()
 50 | 		conent = conent.decode('gb2312','ignore').encode('utf-8','ignore')  
 51 | 
 52 | 
 53 | 		movieName = re.findall(r'<div class="title_all"><h1><font color=#07519a>(.*?)</font></h1></div>', conent, re.S)
 54 | 		if (len(movieName) > 0):
 55 | 			movieName = movieName[0] + ""
 56 | 			# 截取名称
 57 | 			movieName = movieName[movieName.find("《") + 3:movieName.find("》")]
 58 | 		else:
 59 | 			movieName = ""
 60 | 
 61 | 		print("电影名称: " + movieName.strip())
 62 | 
 63 | 		movieContent = re.findall(r'<div class="co_content8">(.*?)</tbody>',conent , re.S)
 64 | 
 65 | 
 66 | 		pattern = re.compile('<ul>(.*?)<tr>', re.S)
 67 | 		movieDate = re.findall(pattern,movieContent[0])
 68 | 
 69 | 		if (len(movieDate) > 0):
 70 | 			movieDate = movieDate[0].strip() + ''
 71 | 		else:
 72 | 			movieDate = ""
 73 | 
 74 | 		print("电影发布时间: " + movieDate[-10:])
 75 | 
 76 | 		pattern = re.compile('<br /><br />(.*?)<br /><br /><img')
 77 | 		movieInfo = re.findall(pattern, movieContent[0])
 78 | 
 79 | 		if (len(movieInfo) > 0):
 80 | 			movieInfo = movieInfo[0]+''
 81 | 
 82 | 			# 删除<br />标签
 83 | 			movieInfo = movieInfo.replace("<br />","")
 84 | 
 85 | 			# 根据 ◎ 符号拆分
 86 | 
 87 | 			movieInfo = movieInfo.split('◎')
 88 | 
 89 | 		else:
 90 | 			movieInfo = ""
 91 | 
 92 | 		print("电影基础信息: ")
 93 | 
 94 | 		for item in movieInfo:
 95 | 			print(item)
 96 | 
 97 | 
 98 | 		# 电影海报
 99 | 		pattern = re.compile('<img.*? src="(.*?)".*? />', re.S)		
100 | 		movieImg = re.findall(pattern,movieContent[0])
101 | 
102 | 		if (len(movieImg) > 0):
103 | 			movieImg = movieImg[0]
104 | 		else:
105 | 			movieImg = ""
106 |  
107 | 		print("电影海报: " + movieImg)
108 | 
109 | 		pattern = re.compile('<td style="WORD-WRAP: break-word" bgcolor="#fdfddf"><a href="(.*?)">.*?</a></td>', re.S)
110 | 		movieDownUrl = re.findall(pattern,movieContent[0])
111 | 
112 | 		if (len(movieDownUrl) > 0):
113 | 			movieDownUrl = movieDownUrl[0]
114 | 		else:
115 | 			movieDownUrl = ""
116 | 
117 | 		print("电影下载地址：" + movieDownUrl + "")
118 | 
119 | 		print("------------------------------------------------\n\n\n")
120 | 	
121 | 
122 | if __name__=='__main__':
123 | 
124 | 
125 |      print("开始抓取电影数据");
126 | 	 
127 |      queryMovieList()
128 | 
129 |      print(len(movieUrls))
130 | 
131 | 
132 |      queryMovieInfo(movieUrls)
133 | 
134 |      print("结束抓取电影数据")
135 | 


--------------------------------------------------------------------------------